xref: /illumos-gate/usr/src/uts/intel/io/vmm/vmm.c (revision fdad6fbf87b201fdb96a704fc41fa8be1e4efbc8)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2011 NetApp, Inc.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 /*
29  * This file and its contents are supplied under the terms of the
30  * Common Development and Distribution License ("CDDL"), version 1.0.
31  * You may only use this file in accordance with the terms of version
32  * 1.0 of the CDDL.
33  *
34  * A full copy of the text of the CDDL should have accompanied this
35  * source.  A copy of the CDDL is also available via the Internet at
36  * http://www.illumos.org/license/CDDL.
37  */
38 /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */
39 
40 /*
41  * Copyright 2015 Pluribus Networks Inc.
42  * Copyright 2018 Joyent, Inc.
43  * Copyright 2025 Oxide Computer Company
44  * Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
45  */
46 
47 
48 #include <sys/cdefs.h>
49 
50 #include <sys/param.h>
51 #include <sys/systm.h>
52 #include <sys/kernel.h>
53 #include <sys/module.h>
54 #include <sys/sysctl.h>
55 #include <sys/kmem.h>
56 #include <sys/pcpu.h>
57 #include <sys/mutex.h>
58 #include <sys/proc.h>
59 #include <sys/rwlock.h>
60 #include <sys/sched.h>
61 #include <sys/systm.h>
62 #include <sys/sunddi.h>
63 #include <sys/hma.h>
64 #include <sys/archsystm.h>
65 
66 #include <machine/md_var.h>
67 #include <x86/psl.h>
68 #include <x86/apicreg.h>
69 
70 #include <machine/specialreg.h>
71 #include <machine/vmm.h>
72 #include <machine/vmm_dev.h>
73 #include <machine/vmparam.h>
74 #include <sys/vmm_instruction_emul.h>
75 #include <sys/vmm_vm.h>
76 #include <sys/vmm_gpt.h>
77 #include <sys/vmm_data.h>
78 
79 #include "vmm_ioport.h"
80 #include "vmm_host.h"
81 #include "vmm_util.h"
82 #include "vatpic.h"
83 #include "vatpit.h"
84 #include "vhpet.h"
85 #include "vioapic.h"
86 #include "vlapic.h"
87 #include "vpmtmr.h"
88 #include "vrtc.h"
89 #include "vmm_stat.h"
90 #include "vmm_lapic.h"
91 
92 #include "io/ppt.h"
93 #include "io/iommu.h"
94 
95 struct vlapic;
96 
97 /* Flags for vtc_status */
98 #define	VTCS_FPU_RESTORED	1 /* guest FPU restored, host FPU saved */
99 #define	VTCS_FPU_CTX_CRITICAL	2 /* in ctx where FPU restore cannot be lazy */
100 
101 typedef struct vm_thread_ctx {
102 	struct vm	*vtc_vm;
103 	int		vtc_vcpuid;
104 	uint_t		vtc_status;
105 	enum vcpu_ustate vtc_ustate;
106 } vm_thread_ctx_t;
107 
108 #define	VMM_MTRR_VAR_MAX 10
109 #define	VMM_MTRR_DEF_MASK \
110 	(MTRR_DEF_ENABLE | MTRR_DEF_FIXED_ENABLE | MTRR_DEF_TYPE)
111 #define	VMM_MTRR_PHYSBASE_MASK (MTRR_PHYSBASE_PHYSBASE | MTRR_PHYSBASE_TYPE)
112 #define	VMM_MTRR_PHYSMASK_MASK (MTRR_PHYSMASK_PHYSMASK | MTRR_PHYSMASK_VALID)
113 struct vm_mtrr {
114 	uint64_t def_type;
115 	uint64_t fixed4k[8];
116 	uint64_t fixed16k[2];
117 	uint64_t fixed64k;
118 	struct {
119 		uint64_t base;
120 		uint64_t mask;
121 	} var[VMM_MTRR_VAR_MAX];
122 };
123 
124 /*
125  * Initialization:
126  * (a) allocated when vcpu is created
127  * (i) initialized when vcpu is created and when it is reinitialized
128  * (o) initialized the first time the vcpu is created
129  * (x) initialized before use
130  */
131 struct vcpu {
132 	/* (o) protects state, run_state, hostcpu, sipi_vector */
133 	kmutex_t	lock;
134 
135 	enum vcpu_state	state;		/* (o) vcpu state */
136 	enum vcpu_run_state run_state;	/* (i) vcpu init/sipi/run state */
137 	kcondvar_t	vcpu_cv;	/* (o) cpu waiter cv */
138 	kcondvar_t	state_cv;	/* (o) IDLE-transition cv */
139 	int		hostcpu;	/* (o) vcpu's current host cpu */
140 	int		lastloccpu;	/* (o) last host cpu localized to */
141 	bool		reqidle;	/* (i) request vcpu to idle */
142 	bool		reqconsist;	/* (i) req. vcpu exit when consistent */
143 	bool		reqbarrier;	/* (i) request vcpu exit barrier */
144 	struct vlapic	*vlapic;	/* (i) APIC device model */
145 	enum x2apic_state x2apic_state;	/* (i) APIC mode */
146 	uint64_t	exit_intinfo;	/* (i) events pending at VM exit */
147 	uint64_t	exc_pending;	/* (i) exception pending */
148 	bool		nmi_pending;	/* (i) NMI pending */
149 	bool		extint_pending;	/* (i) INTR pending */
150 
151 	uint8_t		sipi_vector;	/* (i) SIPI vector */
152 	hma_fpu_t	*guestfpu;	/* (a,i) guest fpu state */
153 	uint64_t	guest_xcr0;	/* (i) guest %xcr0 register */
154 	void		*stats;		/* (a,i) statistics */
155 	struct vm_exit	exitinfo;	/* (x) exit reason and collateral */
156 	uint64_t	nextrip;	/* (x) next instruction to execute */
157 	struct vie	*vie_ctx;	/* (x) instruction emulation context */
158 	vm_client_t	*vmclient;	/* (a) VM-system client */
159 	uint64_t	tsc_offset;	/* (x) vCPU TSC offset */
160 	struct vm_mtrr	mtrr;		/* (i) vcpu's MTRR */
161 	vcpu_cpuid_config_t cpuid_cfg;	/* (x) cpuid configuration */
162 
163 	enum vcpu_ustate ustate;	/* (i) microstate for the vcpu */
164 	hrtime_t	ustate_when;	/* (i) time of last ustate change */
165 	uint64_t ustate_total[VU_MAX];	/* (o) total time spent in ustates */
166 	vm_thread_ctx_t	vtc;		/* (o) thread state for ctxops */
167 	struct ctxop	*ctxop;		/* (o) ctxop storage for vcpu */
168 };
169 
170 #define	vcpu_lock(v)		mutex_enter(&((v)->lock))
171 #define	vcpu_unlock(v)		mutex_exit(&((v)->lock))
172 #define	vcpu_assert_locked(v)	ASSERT(MUTEX_HELD(&((v)->lock)))
173 
174 struct mem_seg {
175 	size_t	len;
176 	bool	sysmem;
177 	vm_object_t *object;
178 };
179 #define	VM_MAX_MEMSEGS	5
180 
181 struct mem_map {
182 	vm_paddr_t	gpa;
183 	size_t		len;
184 	vm_ooffset_t	segoff;
185 	int		segid;
186 	int		prot;
187 	int		flags;
188 };
189 #define	VM_MAX_MEMMAPS	8
190 
191 /*
192  * Initialization:
193  * (o) initialized the first time the VM is created
194  * (i) initialized when VM is created and when it is reinitialized
195  * (x) initialized before use
196  */
197 struct vm {
198 	void		*cookie;		/* (i) cpu-specific data */
199 	void		*iommu;			/* (x) iommu-specific data */
200 	struct vhpet	*vhpet;			/* (i) virtual HPET */
201 	struct vioapic	*vioapic;		/* (i) virtual ioapic */
202 	struct vatpic	*vatpic;		/* (i) virtual atpic */
203 	struct vatpit	*vatpit;		/* (i) virtual atpit */
204 	struct vpmtmr	*vpmtmr;		/* (i) virtual ACPI PM timer */
205 	struct vrtc	*vrtc;			/* (o) virtual RTC */
206 	volatile cpuset_t active_cpus;		/* (i) active vcpus */
207 	volatile cpuset_t debug_cpus;		/* (i) vcpus stopped for dbg */
208 	volatile cpuset_t halted_cpus;		/* (x) cpus in a hard halt */
209 	int		suspend_how;		/* (i) stop VM execution */
210 	int		suspend_source;		/* (i) src vcpuid of suspend */
211 	hrtime_t	suspend_when;		/* (i) time suspend asserted */
212 	struct mem_map	mem_maps[VM_MAX_MEMMAPS]; /* (i) guest address space */
213 	struct mem_seg	mem_segs[VM_MAX_MEMSEGS]; /* (o) guest memory regions */
214 	struct vmspace	*vmspace;		/* (o) guest's address space */
215 	struct vcpu	vcpu[VM_MAXCPU];	/* (i) guest vcpus */
216 	/* The following describe the vm cpu topology */
217 	uint16_t	sockets;		/* (o) num of sockets */
218 	uint16_t	cores;			/* (o) num of cores/socket */
219 	uint16_t	threads;		/* (o) num of threads/core */
220 	uint16_t	maxcpus;		/* (o) max pluggable cpus */
221 
222 	hrtime_t	boot_hrtime;		/* (i) hrtime at VM boot */
223 
224 	/* TSC and TSC scaling related values */
225 	uint64_t	tsc_offset;		/* (i) VM-wide TSC offset */
226 	uint64_t	guest_freq;		/* (i) guest TSC Frequency */
227 	uint64_t	freq_multiplier;	/* (i) guest/host TSC Ratio */
228 
229 	struct ioport_config ioports;		/* (o) ioport handling */
230 
231 	bool		mem_transient;		/* (o) alloc transient memory */
232 	bool		is_paused;		/* (i) instance is paused */
233 };
234 
235 static int vmm_initialized;
236 static uint64_t vmm_host_freq;
237 
238 
239 static void
nullop_panic(void)240 nullop_panic(void)
241 {
242 	panic("null vmm operation call");
243 }
244 
245 /* Do not allow use of an un-set `ops` to do anything but panic */
246 static struct vmm_ops vmm_ops_null = {
247 	.init		= (vmm_init_func_t)nullop_panic,
248 	.cleanup	= (vmm_cleanup_func_t)nullop_panic,
249 	.resume		= (vmm_resume_func_t)nullop_panic,
250 	.vminit		= (vmi_init_func_t)nullop_panic,
251 	.vmrun		= (vmi_run_func_t)nullop_panic,
252 	.vmcleanup	= (vmi_cleanup_func_t)nullop_panic,
253 	.vmgetreg	= (vmi_get_register_t)nullop_panic,
254 	.vmsetreg	= (vmi_set_register_t)nullop_panic,
255 	.vmgetdesc	= (vmi_get_desc_t)nullop_panic,
256 	.vmsetdesc	= (vmi_set_desc_t)nullop_panic,
257 	.vmgetcap	= (vmi_get_cap_t)nullop_panic,
258 	.vmsetcap	= (vmi_set_cap_t)nullop_panic,
259 	.vlapic_init	= (vmi_vlapic_init)nullop_panic,
260 	.vlapic_cleanup	= (vmi_vlapic_cleanup)nullop_panic,
261 	.vmpause	= (vmi_pause_t)nullop_panic,
262 	.vmsavectx	= (vmi_savectx)nullop_panic,
263 	.vmrestorectx	= (vmi_restorectx)nullop_panic,
264 	.vmgetmsr	= (vmi_get_msr_t)nullop_panic,
265 	.vmsetmsr	= (vmi_set_msr_t)nullop_panic,
266 	.vmfreqratio	= (vmi_freqratio_t)nullop_panic,
267 	.fr_fracsize	= 0,
268 	.fr_intsize	= 0,
269 };
270 
271 static struct vmm_ops *ops = &vmm_ops_null;
272 static vmm_pte_ops_t *pte_ops = NULL;
273 
274 #define	VMM_INIT()			((*ops->init)())
275 #define	VMM_CLEANUP()			((*ops->cleanup)())
276 #define	VMM_RESUME()			((*ops->resume)())
277 
278 #define	VMINIT(vm)		((*ops->vminit)(vm))
279 #define	VMRUN(vmi, vcpu, rip)	((*ops->vmrun)(vmi, vcpu, rip))
280 #define	VMCLEANUP(vmi)			((*ops->vmcleanup)(vmi))
281 
282 #define	VMGETREG(vmi, vcpu, num, rv)	((*ops->vmgetreg)(vmi, vcpu, num, rv))
283 #define	VMSETREG(vmi, vcpu, num, val)	((*ops->vmsetreg)(vmi, vcpu, num, val))
284 #define	VMGETDESC(vmi, vcpu, num, dsc)	((*ops->vmgetdesc)(vmi, vcpu, num, dsc))
285 #define	VMSETDESC(vmi, vcpu, num, dsc)	((*ops->vmsetdesc)(vmi, vcpu, num, dsc))
286 #define	VMGETCAP(vmi, vcpu, num, rv)	((*ops->vmgetcap)(vmi, vcpu, num, rv))
287 #define	VMSETCAP(vmi, vcpu, num, val)	((*ops->vmsetcap)(vmi, vcpu, num, val))
288 #define	VLAPIC_INIT(vmi, vcpu)		((*ops->vlapic_init)(vmi, vcpu))
289 #define	VLAPIC_CLEANUP(vmi, vlapic)	((*ops->vlapic_cleanup)(vmi, vlapic))
290 
291 #define	fpu_start_emulating()	load_cr0(rcr0() | CR0_TS)
292 #define	fpu_stop_emulating()	clts()
293 
294 SDT_PROVIDER_DEFINE(vmm);
295 
296 SYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
297     NULL);
298 
299 /*
300  * Halt the guest if all vcpus are executing a HLT instruction with
301  * interrupts disabled.
302  */
303 int halt_detection_enabled = 1;
304 
305 /* Trap into hypervisor on all guest exceptions and reflect them back */
306 int trace_guest_exceptions;
307 
308 /* Trap WBINVD and ignore it */
309 int trap_wbinvd = 1;
310 
311 static void vm_free_memmap(struct vm *vm, int ident);
312 static bool sysmem_mapping(struct vm *vm, struct mem_map *mm);
313 static void vcpu_notify_event_locked(struct vcpu *vcpu, vcpu_notify_t);
314 static bool vcpu_sleep_bailout_checks(struct vm *vm, int vcpuid);
315 static int vcpu_vector_sipi(struct vm *vm, int vcpuid, uint8_t vector);
316 static bool vm_is_suspended(struct vm *, struct vm_exit *);
317 
318 static void vmm_savectx(void *);
319 static void vmm_restorectx(void *);
320 static const struct ctxop_template vmm_ctxop_tpl = {
321 	.ct_rev		= CTXOP_TPL_REV,
322 	.ct_save	= vmm_savectx,
323 	.ct_restore	= vmm_restorectx,
324 };
325 
326 static uint64_t calc_tsc_offset(uint64_t base_host_tsc, uint64_t base_guest_tsc,
327     uint64_t mult);
328 static uint64_t calc_guest_tsc(uint64_t host_tsc, uint64_t mult,
329     uint64_t offset);
330 
331 /* functions implemented in vmm_time_support.S */
332 uint64_t calc_freq_multiplier(uint64_t guest_hz, uint64_t host_hz,
333     uint32_t frac_size);
334 uint64_t scale_tsc(uint64_t tsc, uint64_t multiplier, uint32_t frac_size);
335 
336 #ifdef KTR
337 static const char *
vcpu_state2str(enum vcpu_state state)338 vcpu_state2str(enum vcpu_state state)
339 {
340 
341 	switch (state) {
342 	case VCPU_IDLE:
343 		return ("idle");
344 	case VCPU_FROZEN:
345 		return ("frozen");
346 	case VCPU_RUNNING:
347 		return ("running");
348 	case VCPU_SLEEPING:
349 		return ("sleeping");
350 	default:
351 		return ("unknown");
352 	}
353 }
354 #endif
355 
356 static void
vcpu_cleanup(struct vm * vm,int i,bool destroy)357 vcpu_cleanup(struct vm *vm, int i, bool destroy)
358 {
359 	struct vcpu *vcpu = &vm->vcpu[i];
360 
361 	VLAPIC_CLEANUP(vm->cookie, vcpu->vlapic);
362 	if (destroy) {
363 		vmm_stat_free(vcpu->stats);
364 
365 		vcpu_cpuid_cleanup(&vcpu->cpuid_cfg);
366 
367 		hma_fpu_free(vcpu->guestfpu);
368 		vcpu->guestfpu = NULL;
369 
370 		vie_free(vcpu->vie_ctx);
371 		vcpu->vie_ctx = NULL;
372 
373 		vmc_destroy(vcpu->vmclient);
374 		vcpu->vmclient = NULL;
375 
376 		ctxop_free(vcpu->ctxop);
377 		mutex_destroy(&vcpu->lock);
378 	}
379 }
380 
381 static void
vcpu_init(struct vm * vm,int vcpu_id,bool create)382 vcpu_init(struct vm *vm, int vcpu_id, bool create)
383 {
384 	struct vcpu *vcpu;
385 
386 	KASSERT(vcpu_id >= 0 && vcpu_id < vm->maxcpus,
387 	    ("vcpu_init: invalid vcpu %d", vcpu_id));
388 
389 	vcpu = &vm->vcpu[vcpu_id];
390 
391 	if (create) {
392 		mutex_init(&vcpu->lock, NULL, MUTEX_ADAPTIVE, NULL);
393 
394 		vcpu->state = VCPU_IDLE;
395 		vcpu->hostcpu = NOCPU;
396 		vcpu->lastloccpu = NOCPU;
397 		vcpu->guestfpu = hma_fpu_alloc(KM_SLEEP);
398 		vcpu->stats = vmm_stat_alloc();
399 		vcpu->vie_ctx = vie_alloc();
400 		vcpu_cpuid_init(&vcpu->cpuid_cfg);
401 
402 		vcpu->ustate = VU_INIT;
403 		vcpu->ustate_when = gethrtime();
404 
405 		vcpu->vtc.vtc_vm = vm;
406 		vcpu->vtc.vtc_vcpuid = vcpu_id;
407 		vcpu->ctxop = ctxop_allocate(&vmm_ctxop_tpl, &vcpu->vtc);
408 	} else {
409 		vie_reset(vcpu->vie_ctx);
410 		bzero(&vcpu->exitinfo, sizeof (vcpu->exitinfo));
411 		vcpu_ustate_change(vm, vcpu_id, VU_INIT);
412 		bzero(&vcpu->mtrr, sizeof (vcpu->mtrr));
413 	}
414 
415 	vcpu->run_state = VRS_HALT;
416 	vcpu->vlapic = VLAPIC_INIT(vm->cookie, vcpu_id);
417 	(void) vm_set_x2apic_state(vm, vcpu_id, X2APIC_DISABLED);
418 	vcpu->reqidle = false;
419 	vcpu->reqconsist = false;
420 	vcpu->reqbarrier = false;
421 	vcpu->exit_intinfo = 0;
422 	vcpu->nmi_pending = false;
423 	vcpu->extint_pending = false;
424 	vcpu->exc_pending = 0;
425 	vcpu->guest_xcr0 = XFEATURE_ENABLED_X87;
426 	(void) hma_fpu_init(vcpu->guestfpu);
427 	vmm_stat_init(vcpu->stats);
428 	vcpu->tsc_offset = 0;
429 }
430 
431 int
vcpu_trace_exceptions(struct vm * vm,int vcpuid)432 vcpu_trace_exceptions(struct vm *vm, int vcpuid)
433 {
434 	return (trace_guest_exceptions);
435 }
436 
437 int
vcpu_trap_wbinvd(struct vm * vm,int vcpuid)438 vcpu_trap_wbinvd(struct vm *vm, int vcpuid)
439 {
440 	return (trap_wbinvd);
441 }
442 
443 struct vm_exit *
vm_exitinfo(struct vm * vm,int cpuid)444 vm_exitinfo(struct vm *vm, int cpuid)
445 {
446 	struct vcpu *vcpu;
447 
448 	if (cpuid < 0 || cpuid >= vm->maxcpus)
449 		panic("vm_exitinfo: invalid cpuid %d", cpuid);
450 
451 	vcpu = &vm->vcpu[cpuid];
452 
453 	return (&vcpu->exitinfo);
454 }
455 
456 struct vie *
vm_vie_ctx(struct vm * vm,int cpuid)457 vm_vie_ctx(struct vm *vm, int cpuid)
458 {
459 	if (cpuid < 0 || cpuid >= vm->maxcpus)
460 		panic("vm_vie_ctx: invalid cpuid %d", cpuid);
461 
462 	return (vm->vcpu[cpuid].vie_ctx);
463 }
464 
465 static int
vmm_init(void)466 vmm_init(void)
467 {
468 	vmm_host_state_init();
469 	vmm_host_freq = unscalehrtime(NANOSEC);
470 
471 	if (vmm_is_intel()) {
472 		ops = &vmm_ops_intel;
473 		pte_ops = &ept_pte_ops;
474 	} else if (vmm_is_svm()) {
475 		ops = &vmm_ops_amd;
476 		pte_ops = &rvi_pte_ops;
477 	} else {
478 		return (ENXIO);
479 	}
480 
481 	return (VMM_INIT());
482 }
483 
484 int
vmm_mod_load()485 vmm_mod_load()
486 {
487 	int	error;
488 
489 	VERIFY(vmm_initialized == 0);
490 
491 	error = vmm_init();
492 	if (error == 0)
493 		vmm_initialized = 1;
494 
495 	return (error);
496 }
497 
498 int
vmm_mod_unload()499 vmm_mod_unload()
500 {
501 	int	error;
502 
503 	VERIFY(vmm_initialized == 1);
504 
505 	error = VMM_CLEANUP();
506 	if (error)
507 		return (error);
508 	vmm_initialized = 0;
509 
510 	return (0);
511 }
512 
513 /*
514  * Create a test IOMMU domain to see if the host system has necessary hardware
515  * and drivers to do so.
516  */
517 bool
vmm_check_iommu(void)518 vmm_check_iommu(void)
519 {
520 	void *domain;
521 	const size_t arb_test_sz = (1UL << 32);
522 
523 	domain = iommu_create_domain(arb_test_sz);
524 	if (domain == NULL) {
525 		return (false);
526 	}
527 	iommu_destroy_domain(domain);
528 	return (true);
529 }
530 
531 static void
vm_init(struct vm * vm,bool create)532 vm_init(struct vm *vm, bool create)
533 {
534 	int i;
535 
536 	vm->cookie = VMINIT(vm);
537 	vm->iommu = NULL;
538 	vm->vioapic = vioapic_init(vm);
539 	vm->vhpet = vhpet_init(vm);
540 	vm->vatpic = vatpic_init(vm);
541 	vm->vatpit = vatpit_init(vm);
542 	vm->vpmtmr = vpmtmr_init(vm);
543 	if (create)
544 		vm->vrtc = vrtc_init(vm);
545 
546 	vm_inout_init(vm, &vm->ioports);
547 
548 	CPU_ZERO(&vm->active_cpus);
549 	CPU_ZERO(&vm->debug_cpus);
550 
551 	vm->suspend_how = 0;
552 	vm->suspend_source = 0;
553 	vm->suspend_when = 0;
554 
555 	for (i = 0; i < vm->maxcpus; i++)
556 		vcpu_init(vm, i, create);
557 
558 	/*
559 	 * Configure VM time-related data, including:
560 	 * - VM-wide TSC offset
561 	 * - boot_hrtime
562 	 * - guest_freq (same as host at boot time)
563 	 * - freq_multiplier (used for scaling)
564 	 *
565 	 * This data is configured such that the call to vm_init() represents
566 	 * the boot time (when the TSC(s) read 0).  Each vCPU will have its own
567 	 * offset from this, which is altered if/when the guest writes to
568 	 * MSR_TSC.
569 	 *
570 	 * Further changes to this data may occur if userspace writes to the
571 	 * time data.
572 	 */
573 	const uint64_t boot_tsc = rdtsc_offset();
574 
575 	/* Convert the boot TSC reading to hrtime */
576 	vm->boot_hrtime = (hrtime_t)boot_tsc;
577 	scalehrtime(&vm->boot_hrtime);
578 
579 	/* Guest frequency is the same as the host at boot time */
580 	vm->guest_freq = vmm_host_freq;
581 
582 	/* no scaling needed if guest_freq == host_freq */
583 	vm->freq_multiplier = VM_TSCM_NOSCALE;
584 
585 	/* configure VM-wide offset: initial guest TSC is 0 at boot */
586 	vm->tsc_offset = calc_tsc_offset(boot_tsc, 0, vm->freq_multiplier);
587 }
588 
589 /*
590  * The default CPU topology is a single thread per package.
591  */
592 uint_t cores_per_package = 1;
593 uint_t threads_per_core = 1;
594 
595 int
vm_create(uint64_t flags,struct vm ** retvm)596 vm_create(uint64_t flags, struct vm **retvm)
597 {
598 	struct vm *vm;
599 	struct vmspace *vmspace;
600 
601 	/*
602 	 * If vmm.ko could not be successfully initialized then don't attempt
603 	 * to create the virtual machine.
604 	 */
605 	if (!vmm_initialized)
606 		return (ENXIO);
607 
608 	bool track_dirty = (flags & VCF_TRACK_DIRTY) != 0;
609 	if (track_dirty && !pte_ops->vpeo_hw_ad_supported())
610 		return (ENOTSUP);
611 
612 	vmspace = vmspace_alloc(VM_MAXUSER_ADDRESS, pte_ops, track_dirty);
613 	if (vmspace == NULL)
614 		return (ENOMEM);
615 
616 	vm = kmem_zalloc(sizeof (struct vm), KM_SLEEP);
617 
618 	vm->vmspace = vmspace;
619 	vm->mem_transient = (flags & VCF_RESERVOIR_MEM) == 0;
620 	for (uint_t i = 0; i < VM_MAXCPU; i++) {
621 		vm->vcpu[i].vmclient = vmspace_client_alloc(vmspace);
622 	}
623 
624 	vm->sockets = 1;
625 	vm->cores = cores_per_package;	/* XXX backwards compatibility */
626 	vm->threads = threads_per_core;	/* XXX backwards compatibility */
627 	vm->maxcpus = VM_MAXCPU;	/* XXX temp to keep code working */
628 
629 	vm_init(vm, true);
630 
631 	*retvm = vm;
632 	return (0);
633 }
634 
635 void
vm_get_topology(struct vm * vm,uint16_t * sockets,uint16_t * cores,uint16_t * threads,uint16_t * maxcpus)636 vm_get_topology(struct vm *vm, uint16_t *sockets, uint16_t *cores,
637     uint16_t *threads, uint16_t *maxcpus)
638 {
639 	*sockets = vm->sockets;
640 	*cores = vm->cores;
641 	*threads = vm->threads;
642 	*maxcpus = vm->maxcpus;
643 }
644 
645 uint16_t
vm_get_maxcpus(struct vm * vm)646 vm_get_maxcpus(struct vm *vm)
647 {
648 	return (vm->maxcpus);
649 }
650 
651 int
vm_set_topology(struct vm * vm,uint16_t sockets,uint16_t cores,uint16_t threads,uint16_t maxcpus)652 vm_set_topology(struct vm *vm, uint16_t sockets, uint16_t cores,
653     uint16_t threads, uint16_t maxcpus)
654 {
655 	if (maxcpus != 0)
656 		return (EINVAL);	/* XXX remove when supported */
657 	if ((sockets * cores * threads) > vm->maxcpus)
658 		return (EINVAL);
659 	/* XXX need to check sockets * cores * threads == vCPU, how? */
660 	vm->sockets = sockets;
661 	vm->cores = cores;
662 	vm->threads = threads;
663 	vm->maxcpus = VM_MAXCPU;	/* XXX temp to keep code working */
664 	return (0);
665 }
666 
667 static void
vm_cleanup(struct vm * vm,bool destroy)668 vm_cleanup(struct vm *vm, bool destroy)
669 {
670 	struct mem_map *mm;
671 	int i;
672 
673 	ppt_unassign_all(vm);
674 
675 	if (vm->iommu != NULL)
676 		iommu_destroy_domain(vm->iommu);
677 
678 	/*
679 	 * Devices which attach their own ioport hooks should be cleaned up
680 	 * first so they can tear down those registrations.
681 	 */
682 	vpmtmr_cleanup(vm->vpmtmr);
683 
684 	vm_inout_cleanup(vm, &vm->ioports);
685 
686 	if (destroy)
687 		vrtc_cleanup(vm->vrtc);
688 	else
689 		vrtc_reset(vm->vrtc);
690 
691 	vatpit_cleanup(vm->vatpit);
692 	vhpet_cleanup(vm->vhpet);
693 	vatpic_cleanup(vm->vatpic);
694 	vioapic_cleanup(vm->vioapic);
695 
696 	for (i = 0; i < vm->maxcpus; i++)
697 		vcpu_cleanup(vm, i, destroy);
698 
699 	VMCLEANUP(vm->cookie);
700 
701 	/*
702 	 * System memory is removed from the guest address space only when
703 	 * the VM is destroyed. This is because the mapping remains the same
704 	 * across VM reset.
705 	 *
706 	 * Device memory can be relocated by the guest (e.g. using PCI BARs)
707 	 * so those mappings are removed on a VM reset.
708 	 */
709 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
710 		mm = &vm->mem_maps[i];
711 		if (destroy || !sysmem_mapping(vm, mm)) {
712 			vm_free_memmap(vm, i);
713 		} else {
714 			/*
715 			 * We need to reset the IOMMU flag so this mapping can
716 			 * be reused when a VM is rebooted. Since the IOMMU
717 			 * domain has already been destroyed we can just reset
718 			 * the flag here.
719 			 */
720 			mm->flags &= ~VM_MEMMAP_F_IOMMU;
721 		}
722 	}
723 
724 	if (destroy) {
725 		for (i = 0; i < VM_MAX_MEMSEGS; i++)
726 			vm_free_memseg(vm, i);
727 
728 		vmspace_destroy(vm->vmspace);
729 		vm->vmspace = NULL;
730 	}
731 }
732 
733 void
vm_destroy(struct vm * vm)734 vm_destroy(struct vm *vm)
735 {
736 	vm_cleanup(vm, true);
737 	kmem_free(vm, sizeof (*vm));
738 }
739 
740 int
vm_reinit(struct vm * vm,uint64_t flags)741 vm_reinit(struct vm *vm, uint64_t flags)
742 {
743 	vm_cleanup(vm, false);
744 	vm_init(vm, false);
745 	return (0);
746 }
747 
748 bool
vm_is_paused(struct vm * vm)749 vm_is_paused(struct vm *vm)
750 {
751 	return (vm->is_paused);
752 }
753 
754 int
vm_pause_instance(struct vm * vm)755 vm_pause_instance(struct vm *vm)
756 {
757 	if (vm->is_paused) {
758 		return (EALREADY);
759 	}
760 	vm->is_paused = true;
761 
762 	for (uint_t i = 0; i < vm->maxcpus; i++) {
763 		struct vcpu *vcpu = &vm->vcpu[i];
764 
765 		if (!CPU_ISSET(i, &vm->active_cpus)) {
766 			continue;
767 		}
768 		vlapic_pause(vcpu->vlapic);
769 
770 		/*
771 		 * vCPU-specific pause logic includes stashing any
772 		 * to-be-injected events in exit_intinfo where it can be
773 		 * accessed in a manner generic to the backend.
774 		 */
775 		ops->vmpause(vm->cookie, i);
776 	}
777 	vhpet_pause(vm->vhpet);
778 	vatpit_pause(vm->vatpit);
779 	vrtc_pause(vm->vrtc);
780 
781 	return (0);
782 }
783 
784 int
vm_resume_instance(struct vm * vm)785 vm_resume_instance(struct vm *vm)
786 {
787 	if (!vm->is_paused) {
788 		return (EALREADY);
789 	}
790 	vm->is_paused = false;
791 
792 	vrtc_resume(vm->vrtc);
793 	vatpit_resume(vm->vatpit);
794 	vhpet_resume(vm->vhpet);
795 	for (uint_t i = 0; i < vm->maxcpus; i++) {
796 		struct vcpu *vcpu = &vm->vcpu[i];
797 
798 		if (!CPU_ISSET(i, &vm->active_cpus)) {
799 			continue;
800 		}
801 		vlapic_resume(vcpu->vlapic);
802 	}
803 
804 	return (0);
805 }
806 
807 int
vm_map_mmio(struct vm * vm,vm_paddr_t gpa,size_t len,vm_paddr_t hpa)808 vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa)
809 {
810 	vm_object_t *obj;
811 
812 	if ((obj = vmm_mmio_alloc(vm->vmspace, gpa, len, hpa)) == NULL)
813 		return (ENOMEM);
814 	else
815 		return (0);
816 }
817 
818 int
vm_unmap_mmio(struct vm * vm,vm_paddr_t gpa,size_t len)819 vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len)
820 {
821 	return (vmspace_unmap(vm->vmspace, gpa, len));
822 }
823 
824 /*
825  * Return 'true' if 'gpa' is allocated in the guest address space.
826  *
827  * This function is called in the context of a running vcpu which acts as
828  * an implicit lock on 'vm->mem_maps[]'.
829  */
830 bool
vm_mem_allocated(struct vm * vm,int vcpuid,vm_paddr_t gpa)831 vm_mem_allocated(struct vm *vm, int vcpuid, vm_paddr_t gpa)
832 {
833 	struct mem_map *mm;
834 	int i;
835 
836 #ifdef INVARIANTS
837 	int hostcpu, state;
838 	state = vcpu_get_state(vm, vcpuid, &hostcpu);
839 	KASSERT(state == VCPU_RUNNING && hostcpu == curcpu,
840 	    ("%s: invalid vcpu state %d/%d", __func__, state, hostcpu));
841 #endif
842 
843 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
844 		mm = &vm->mem_maps[i];
845 		if (mm->len != 0 && gpa >= mm->gpa && gpa < mm->gpa + mm->len)
846 			return (true);		/* 'gpa' is sysmem or devmem */
847 	}
848 
849 	if (ppt_is_mmio(vm, gpa))
850 		return (true);			/* 'gpa' is pci passthru mmio */
851 
852 	return (false);
853 }
854 
855 int
vm_alloc_memseg(struct vm * vm,int ident,size_t len,bool sysmem)856 vm_alloc_memseg(struct vm *vm, int ident, size_t len, bool sysmem)
857 {
858 	struct mem_seg *seg;
859 	vm_object_t *obj;
860 
861 	if (ident < 0 || ident >= VM_MAX_MEMSEGS)
862 		return (EINVAL);
863 
864 	if (len == 0 || (len & PAGE_MASK))
865 		return (EINVAL);
866 
867 	seg = &vm->mem_segs[ident];
868 	if (seg->object != NULL) {
869 		if (seg->len == len && seg->sysmem == sysmem)
870 			return (EEXIST);
871 		else
872 			return (EINVAL);
873 	}
874 
875 	obj = vm_object_mem_allocate(len, vm->mem_transient);
876 	if (obj == NULL)
877 		return (ENOMEM);
878 
879 	seg->len = len;
880 	seg->object = obj;
881 	seg->sysmem = sysmem;
882 	return (0);
883 }
884 
885 int
vm_get_memseg(struct vm * vm,int ident,size_t * len,bool * sysmem,vm_object_t ** objptr)886 vm_get_memseg(struct vm *vm, int ident, size_t *len, bool *sysmem,
887     vm_object_t **objptr)
888 {
889 	struct mem_seg *seg;
890 
891 	if (ident < 0 || ident >= VM_MAX_MEMSEGS)
892 		return (EINVAL);
893 
894 	seg = &vm->mem_segs[ident];
895 	if (len)
896 		*len = seg->len;
897 	if (sysmem)
898 		*sysmem = seg->sysmem;
899 	if (objptr)
900 		*objptr = seg->object;
901 	return (0);
902 }
903 
904 void
vm_free_memseg(struct vm * vm,int ident)905 vm_free_memseg(struct vm *vm, int ident)
906 {
907 	struct mem_seg *seg;
908 
909 	KASSERT(ident >= 0 && ident < VM_MAX_MEMSEGS,
910 	    ("%s: invalid memseg ident %d", __func__, ident));
911 
912 	seg = &vm->mem_segs[ident];
913 	if (seg->object != NULL) {
914 		vm_object_release(seg->object);
915 		bzero(seg, sizeof (struct mem_seg));
916 	}
917 }
918 
919 int
vm_mmap_memseg(struct vm * vm,vm_paddr_t gpa,int segid,vm_ooffset_t first,size_t len,int prot,int flags)920 vm_mmap_memseg(struct vm *vm, vm_paddr_t gpa, int segid, vm_ooffset_t first,
921     size_t len, int prot, int flags)
922 {
923 	struct mem_seg *seg;
924 	struct mem_map *m, *map;
925 	vm_ooffset_t last;
926 	int i, error;
927 
928 	if (prot == 0 || (prot & ~(PROT_ALL)) != 0)
929 		return (EINVAL);
930 
931 	if (flags & ~VM_MEMMAP_F_WIRED)
932 		return (EINVAL);
933 
934 	if (segid < 0 || segid >= VM_MAX_MEMSEGS)
935 		return (EINVAL);
936 
937 	seg = &vm->mem_segs[segid];
938 	if (seg->object == NULL)
939 		return (EINVAL);
940 
941 	last = first + len;
942 	if (first < 0 || first >= last || last > seg->len)
943 		return (EINVAL);
944 
945 	if ((gpa | first | last) & PAGE_MASK)
946 		return (EINVAL);
947 
948 	map = NULL;
949 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
950 		m = &vm->mem_maps[i];
951 		if (m->len == 0) {
952 			map = m;
953 			break;
954 		}
955 	}
956 
957 	if (map == NULL)
958 		return (ENOSPC);
959 
960 	error = vmspace_map(vm->vmspace, seg->object, first, gpa, len, prot);
961 	if (error != 0)
962 		return (EFAULT);
963 
964 	vm_object_reference(seg->object);
965 
966 	if ((flags & VM_MEMMAP_F_WIRED) != 0) {
967 		error = vmspace_populate(vm->vmspace, gpa, len);
968 		if (error != 0) {
969 			VERIFY0(vmspace_unmap(vm->vmspace, gpa, len));
970 			return (EFAULT);
971 		}
972 	}
973 
974 	map->gpa = gpa;
975 	map->len = len;
976 	map->segoff = first;
977 	map->segid = segid;
978 	map->prot = prot;
979 	map->flags = flags;
980 	return (0);
981 }
982 
983 int
vm_munmap_memseg(struct vm * vm,vm_paddr_t gpa,size_t len)984 vm_munmap_memseg(struct vm *vm, vm_paddr_t gpa, size_t len)
985 {
986 	struct mem_map *m;
987 	int i;
988 
989 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
990 		m = &vm->mem_maps[i];
991 		if (m->gpa == gpa && m->len == len &&
992 		    (m->flags & VM_MEMMAP_F_IOMMU) == 0) {
993 			vm_free_memmap(vm, i);
994 			return (0);
995 		}
996 	}
997 
998 	return (EINVAL);
999 }
1000 
1001 int
vm_mmap_getnext(struct vm * vm,vm_paddr_t * gpa,int * segid,vm_ooffset_t * segoff,size_t * len,int * prot,int * flags)1002 vm_mmap_getnext(struct vm *vm, vm_paddr_t *gpa, int *segid,
1003     vm_ooffset_t *segoff, size_t *len, int *prot, int *flags)
1004 {
1005 	struct mem_map *mm, *mmnext;
1006 	int i;
1007 
1008 	mmnext = NULL;
1009 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
1010 		mm = &vm->mem_maps[i];
1011 		if (mm->len == 0 || mm->gpa < *gpa)
1012 			continue;
1013 		if (mmnext == NULL || mm->gpa < mmnext->gpa)
1014 			mmnext = mm;
1015 	}
1016 
1017 	if (mmnext != NULL) {
1018 		*gpa = mmnext->gpa;
1019 		if (segid)
1020 			*segid = mmnext->segid;
1021 		if (segoff)
1022 			*segoff = mmnext->segoff;
1023 		if (len)
1024 			*len = mmnext->len;
1025 		if (prot)
1026 			*prot = mmnext->prot;
1027 		if (flags)
1028 			*flags = mmnext->flags;
1029 		return (0);
1030 	} else {
1031 		return (ENOENT);
1032 	}
1033 }
1034 
1035 static void
vm_free_memmap(struct vm * vm,int ident)1036 vm_free_memmap(struct vm *vm, int ident)
1037 {
1038 	struct mem_map *mm;
1039 	int error;
1040 
1041 	mm = &vm->mem_maps[ident];
1042 	if (mm->len) {
1043 		error = vmspace_unmap(vm->vmspace, mm->gpa, mm->len);
1044 		VERIFY0(error);
1045 		bzero(mm, sizeof (struct mem_map));
1046 	}
1047 }
1048 
1049 static __inline bool
sysmem_mapping(struct vm * vm,struct mem_map * mm)1050 sysmem_mapping(struct vm *vm, struct mem_map *mm)
1051 {
1052 
1053 	if (mm->len != 0 && vm->mem_segs[mm->segid].sysmem)
1054 		return (true);
1055 	else
1056 		return (false);
1057 }
1058 
1059 vm_paddr_t
vmm_sysmem_maxaddr(struct vm * vm)1060 vmm_sysmem_maxaddr(struct vm *vm)
1061 {
1062 	struct mem_map *mm;
1063 	vm_paddr_t maxaddr;
1064 	int i;
1065 
1066 	maxaddr = 0;
1067 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
1068 		mm = &vm->mem_maps[i];
1069 		if (sysmem_mapping(vm, mm)) {
1070 			if (maxaddr < mm->gpa + mm->len)
1071 				maxaddr = mm->gpa + mm->len;
1072 		}
1073 	}
1074 	return (maxaddr);
1075 }
1076 
1077 static void
vm_iommu_modify(struct vm * vm,bool map)1078 vm_iommu_modify(struct vm *vm, bool map)
1079 {
1080 	int i, sz;
1081 	vm_paddr_t gpa, hpa;
1082 	struct mem_map *mm;
1083 	vm_client_t *vmc;
1084 
1085 	sz = PAGE_SIZE;
1086 	vmc = vmspace_client_alloc(vm->vmspace);
1087 
1088 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
1089 		mm = &vm->mem_maps[i];
1090 		if (!sysmem_mapping(vm, mm))
1091 			continue;
1092 
1093 		if (map) {
1094 			KASSERT((mm->flags & VM_MEMMAP_F_IOMMU) == 0,
1095 			    ("iommu map found invalid memmap %lx/%lx/%x",
1096 			    mm->gpa, mm->len, mm->flags));
1097 			if ((mm->flags & VM_MEMMAP_F_WIRED) == 0)
1098 				continue;
1099 			mm->flags |= VM_MEMMAP_F_IOMMU;
1100 		} else {
1101 			if ((mm->flags & VM_MEMMAP_F_IOMMU) == 0)
1102 				continue;
1103 			mm->flags &= ~VM_MEMMAP_F_IOMMU;
1104 			KASSERT((mm->flags & VM_MEMMAP_F_WIRED) != 0,
1105 			    ("iommu unmap found invalid memmap %lx/%lx/%x",
1106 			    mm->gpa, mm->len, mm->flags));
1107 		}
1108 
1109 		gpa = mm->gpa;
1110 		while (gpa < mm->gpa + mm->len) {
1111 			vm_page_t *vmp;
1112 
1113 			vmp = vmc_hold(vmc, gpa, PROT_WRITE);
1114 			ASSERT(vmp != NULL);
1115 			hpa = ((uintptr_t)vmp_get_pfn(vmp) << PAGESHIFT);
1116 			(void) vmp_release(vmp);
1117 
1118 			/*
1119 			 * When originally ported from FreeBSD, the logic for
1120 			 * adding memory to the guest domain would
1121 			 * simultaneously remove it from the host domain.  The
1122 			 * justification for that is not clear, and FreeBSD has
1123 			 * subsequently changed the behavior to not remove the
1124 			 * memory from the host domain.
1125 			 *
1126 			 * Leaving the guest memory in the host domain for the
1127 			 * life of the VM is necessary to make it available for
1128 			 * DMA, such as through viona in the TX path.
1129 			 */
1130 			if (map) {
1131 				iommu_create_mapping(vm->iommu, gpa, hpa, sz);
1132 			} else {
1133 				iommu_remove_mapping(vm->iommu, gpa, sz);
1134 			}
1135 
1136 			gpa += PAGE_SIZE;
1137 		}
1138 	}
1139 	vmc_destroy(vmc);
1140 
1141 	/*
1142 	 * Invalidate the cached translations associated with the domain
1143 	 * from which pages were removed.
1144 	 */
1145 	iommu_invalidate_tlb(vm->iommu);
1146 }
1147 
1148 int
vm_unassign_pptdev(struct vm * vm,int pptfd)1149 vm_unassign_pptdev(struct vm *vm, int pptfd)
1150 {
1151 	int error;
1152 
1153 	error = ppt_unassign_device(vm, pptfd);
1154 	if (error)
1155 		return (error);
1156 
1157 	if (ppt_assigned_devices(vm) == 0)
1158 		vm_iommu_modify(vm, false);
1159 
1160 	return (0);
1161 }
1162 
1163 int
vm_assign_pptdev(struct vm * vm,int pptfd)1164 vm_assign_pptdev(struct vm *vm, int pptfd)
1165 {
1166 	int error;
1167 	vm_paddr_t maxaddr;
1168 
1169 	/* Set up the IOMMU to do the 'gpa' to 'hpa' translation */
1170 	if (ppt_assigned_devices(vm) == 0) {
1171 		KASSERT(vm->iommu == NULL,
1172 		    ("vm_assign_pptdev: iommu must be NULL"));
1173 		maxaddr = vmm_sysmem_maxaddr(vm);
1174 		vm->iommu = iommu_create_domain(maxaddr);
1175 		if (vm->iommu == NULL)
1176 			return (ENXIO);
1177 		vm_iommu_modify(vm, true);
1178 	}
1179 
1180 	error = ppt_assign_device(vm, pptfd);
1181 	return (error);
1182 }
1183 
1184 int
vm_get_register(struct vm * vm,int vcpuid,int reg,uint64_t * retval)1185 vm_get_register(struct vm *vm, int vcpuid, int reg, uint64_t *retval)
1186 {
1187 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
1188 		return (EINVAL);
1189 
1190 	if (reg >= VM_REG_LAST)
1191 		return (EINVAL);
1192 
1193 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
1194 	switch (reg) {
1195 	case VM_REG_GUEST_XCR0:
1196 		*retval = vcpu->guest_xcr0;
1197 		return (0);
1198 	default:
1199 		return (VMGETREG(vm->cookie, vcpuid, reg, retval));
1200 	}
1201 }
1202 
1203 int
vm_set_register(struct vm * vm,int vcpuid,int reg,uint64_t val)1204 vm_set_register(struct vm *vm, int vcpuid, int reg, uint64_t val)
1205 {
1206 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
1207 		return (EINVAL);
1208 
1209 	if (reg >= VM_REG_LAST)
1210 		return (EINVAL);
1211 
1212 	int error;
1213 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
1214 	switch (reg) {
1215 	case VM_REG_GUEST_RIP:
1216 		error = VMSETREG(vm->cookie, vcpuid, reg, val);
1217 		if (error == 0) {
1218 			vcpu->nextrip = val;
1219 		}
1220 		return (error);
1221 	case VM_REG_GUEST_XCR0:
1222 		if (!validate_guest_xcr0(val, vmm_get_host_xcr0())) {
1223 			return (EINVAL);
1224 		}
1225 		vcpu->guest_xcr0 = val;
1226 		return (0);
1227 	default:
1228 		return (VMSETREG(vm->cookie, vcpuid, reg, val));
1229 	}
1230 }
1231 
1232 static bool
is_descriptor_table(int reg)1233 is_descriptor_table(int reg)
1234 {
1235 	switch (reg) {
1236 	case VM_REG_GUEST_IDTR:
1237 	case VM_REG_GUEST_GDTR:
1238 		return (true);
1239 	default:
1240 		return (false);
1241 	}
1242 }
1243 
1244 static bool
is_segment_register(int reg)1245 is_segment_register(int reg)
1246 {
1247 	switch (reg) {
1248 	case VM_REG_GUEST_ES:
1249 	case VM_REG_GUEST_CS:
1250 	case VM_REG_GUEST_SS:
1251 	case VM_REG_GUEST_DS:
1252 	case VM_REG_GUEST_FS:
1253 	case VM_REG_GUEST_GS:
1254 	case VM_REG_GUEST_TR:
1255 	case VM_REG_GUEST_LDTR:
1256 		return (true);
1257 	default:
1258 		return (false);
1259 	}
1260 }
1261 
1262 int
vm_get_seg_desc(struct vm * vm,int vcpu,int reg,struct seg_desc * desc)1263 vm_get_seg_desc(struct vm *vm, int vcpu, int reg, struct seg_desc *desc)
1264 {
1265 
1266 	if (vcpu < 0 || vcpu >= vm->maxcpus)
1267 		return (EINVAL);
1268 
1269 	if (!is_segment_register(reg) && !is_descriptor_table(reg))
1270 		return (EINVAL);
1271 
1272 	return (VMGETDESC(vm->cookie, vcpu, reg, desc));
1273 }
1274 
1275 int
vm_set_seg_desc(struct vm * vm,int vcpu,int reg,const struct seg_desc * desc)1276 vm_set_seg_desc(struct vm *vm, int vcpu, int reg, const struct seg_desc *desc)
1277 {
1278 	if (vcpu < 0 || vcpu >= vm->maxcpus)
1279 		return (EINVAL);
1280 
1281 	if (!is_segment_register(reg) && !is_descriptor_table(reg))
1282 		return (EINVAL);
1283 
1284 	return (VMSETDESC(vm->cookie, vcpu, reg, desc));
1285 }
1286 
1287 static int
translate_hma_xsave_result(hma_fpu_xsave_result_t res)1288 translate_hma_xsave_result(hma_fpu_xsave_result_t res)
1289 {
1290 	switch (res) {
1291 	case HFXR_OK:
1292 		return (0);
1293 	case HFXR_NO_SPACE:
1294 		return (ENOSPC);
1295 	case HFXR_BAD_ALIGN:
1296 	case HFXR_UNSUP_FMT:
1297 	case HFXR_UNSUP_FEAT:
1298 	case HFXR_INVALID_DATA:
1299 		return (EINVAL);
1300 	default:
1301 		panic("unexpected xsave result");
1302 	}
1303 }
1304 
1305 int
vm_get_fpu(struct vm * vm,int vcpuid,void * buf,size_t len)1306 vm_get_fpu(struct vm *vm, int vcpuid, void *buf, size_t len)
1307 {
1308 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
1309 		return (EINVAL);
1310 
1311 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
1312 	hma_fpu_xsave_result_t res;
1313 
1314 	res = hma_fpu_get_xsave_state(vcpu->guestfpu, buf, len);
1315 	return (translate_hma_xsave_result(res));
1316 }
1317 
1318 int
vm_set_fpu(struct vm * vm,int vcpuid,void * buf,size_t len)1319 vm_set_fpu(struct vm *vm, int vcpuid, void *buf, size_t len)
1320 {
1321 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
1322 		return (EINVAL);
1323 
1324 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
1325 	hma_fpu_xsave_result_t res;
1326 
1327 	res = hma_fpu_set_xsave_state(vcpu->guestfpu, buf, len);
1328 	return (translate_hma_xsave_result(res));
1329 }
1330 
1331 int
vm_get_run_state(struct vm * vm,int vcpuid,uint32_t * state,uint8_t * sipi_vec)1332 vm_get_run_state(struct vm *vm, int vcpuid, uint32_t *state, uint8_t *sipi_vec)
1333 {
1334 	struct vcpu *vcpu;
1335 
1336 	if (vcpuid < 0 || vcpuid >= vm->maxcpus) {
1337 		return (EINVAL);
1338 	}
1339 
1340 	vcpu = &vm->vcpu[vcpuid];
1341 
1342 	vcpu_lock(vcpu);
1343 	*state = vcpu->run_state;
1344 	*sipi_vec = vcpu->sipi_vector;
1345 	vcpu_unlock(vcpu);
1346 
1347 	return (0);
1348 }
1349 
1350 int
vm_set_run_state(struct vm * vm,int vcpuid,uint32_t state,uint8_t sipi_vec)1351 vm_set_run_state(struct vm *vm, int vcpuid, uint32_t state, uint8_t sipi_vec)
1352 {
1353 	struct vcpu *vcpu;
1354 
1355 	if (vcpuid < 0 || vcpuid >= vm->maxcpus) {
1356 		return (EINVAL);
1357 	}
1358 	if (!VRS_IS_VALID(state)) {
1359 		return (EINVAL);
1360 	}
1361 
1362 	vcpu = &vm->vcpu[vcpuid];
1363 
1364 	vcpu_lock(vcpu);
1365 	vcpu->run_state = state;
1366 	vcpu->sipi_vector = sipi_vec;
1367 	vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
1368 	vcpu_unlock(vcpu);
1369 
1370 	return (0);
1371 }
1372 
1373 int
vm_track_dirty_pages(struct vm * vm,uint64_t gpa,size_t len,uint8_t * bitmap)1374 vm_track_dirty_pages(struct vm *vm, uint64_t gpa, size_t len, uint8_t *bitmap)
1375 {
1376 	ASSERT0(gpa & PAGEOFFSET);
1377 	ASSERT0(len & PAGEOFFSET);
1378 
1379 	/*
1380 	 * The only difference in expectations between this legacy interface and
1381 	 * an equivalent call to vm_npt_do_operation() is the check for
1382 	 * dirty-page-tracking being enabled on the vmspace.
1383 	 */
1384 	if (!vmspace_get_tracking(vm->vmspace)) {
1385 		return (EPERM);
1386 	}
1387 
1388 	vmspace_bits_operate(vm->vmspace, gpa, len,
1389 	    VBO_RESET_DIRTY | VBO_FLAG_BITMAP_OUT, bitmap);
1390 	return (0);
1391 }
1392 
1393 int
vm_npt_do_operation(struct vm * vm,uint64_t gpa,size_t len,uint32_t oper,uint8_t * bitmap,int * rvalp)1394 vm_npt_do_operation(struct vm *vm, uint64_t gpa, size_t len, uint32_t oper,
1395     uint8_t *bitmap, int *rvalp)
1396 {
1397 	ASSERT0(gpa & PAGEOFFSET);
1398 	ASSERT0(len & PAGEOFFSET);
1399 
1400 	/*
1401 	 * For now, the bits defined in vmm_dev.h are meant to match up 1:1 with
1402 	 * those in vmm_vm.h
1403 	 */
1404 	CTASSERT(VNO_OP_RESET_DIRTY == VBO_RESET_DIRTY);
1405 	CTASSERT(VNO_OP_SET_DIRTY == VBO_SET_DIRTY);
1406 	CTASSERT(VNO_OP_GET_DIRTY == VBO_GET_DIRTY);
1407 	CTASSERT(VNO_FLAG_BITMAP_IN == VBO_FLAG_BITMAP_IN);
1408 	CTASSERT(VNO_FLAG_BITMAP_OUT == VBO_FLAG_BITMAP_OUT);
1409 
1410 	const uint32_t oper_only =
1411 	    oper & ~(VNO_FLAG_BITMAP_IN | VNO_FLAG_BITMAP_OUT);
1412 	switch (oper_only) {
1413 	case VNO_OP_RESET_DIRTY:
1414 	case VNO_OP_SET_DIRTY:
1415 	case VNO_OP_GET_DIRTY:
1416 		if (len == 0) {
1417 			break;
1418 		}
1419 		vmspace_bits_operate(vm->vmspace, gpa, len, oper, bitmap);
1420 		break;
1421 	case VNO_OP_GET_TRACK_DIRTY:
1422 		ASSERT3P(rvalp, !=, NULL);
1423 		*rvalp = vmspace_get_tracking(vm->vmspace) ? 1 : 0;
1424 		break;
1425 	case VNO_OP_EN_TRACK_DIRTY:
1426 		return (vmspace_set_tracking(vm->vmspace, true));
1427 	case VNO_OP_DIS_TRACK_DIRTY:
1428 		return (vmspace_set_tracking(vm->vmspace, false));
1429 	default:
1430 		return (EINVAL);
1431 	}
1432 	return (0);
1433 }
1434 
1435 static void
restore_guest_fpustate(struct vcpu * vcpu)1436 restore_guest_fpustate(struct vcpu *vcpu)
1437 {
1438 	/* Save host FPU and restore guest FPU */
1439 	fpu_stop_emulating();
1440 	hma_fpu_start_guest(vcpu->guestfpu);
1441 
1442 	/* restore guest XCR0 if XSAVE is enabled in the host */
1443 	if (rcr4() & CR4_XSAVE)
1444 		load_xcr(0, vcpu->guest_xcr0);
1445 
1446 	/*
1447 	 * The FPU is now "dirty" with the guest's state so turn on emulation
1448 	 * to trap any access to the FPU by the host.
1449 	 */
1450 	fpu_start_emulating();
1451 }
1452 
1453 static void
save_guest_fpustate(struct vcpu * vcpu)1454 save_guest_fpustate(struct vcpu *vcpu)
1455 {
1456 
1457 	if ((rcr0() & CR0_TS) == 0)
1458 		panic("fpu emulation not enabled in host!");
1459 
1460 	/* save guest XCR0 and restore host XCR0 */
1461 	if (rcr4() & CR4_XSAVE) {
1462 		vcpu->guest_xcr0 = rxcr(0);
1463 		load_xcr(0, vmm_get_host_xcr0());
1464 	}
1465 
1466 	/* save guest FPU and restore host FPU */
1467 	fpu_stop_emulating();
1468 	hma_fpu_stop_guest(vcpu->guestfpu);
1469 	/*
1470 	 * When the host state has been restored, we should not re-enable
1471 	 * CR0.TS on illumos for eager FPU.
1472 	 */
1473 }
1474 
1475 static int
vcpu_set_state_locked(struct vm * vm,int vcpuid,enum vcpu_state newstate,bool from_idle)1476 vcpu_set_state_locked(struct vm *vm, int vcpuid, enum vcpu_state newstate,
1477     bool from_idle)
1478 {
1479 	struct vcpu *vcpu;
1480 	int error;
1481 
1482 	vcpu = &vm->vcpu[vcpuid];
1483 	vcpu_assert_locked(vcpu);
1484 
1485 	/*
1486 	 * State transitions from the vmmdev_ioctl() must always begin from
1487 	 * the VCPU_IDLE state. This guarantees that there is only a single
1488 	 * ioctl() operating on a vcpu at any point.
1489 	 */
1490 	if (from_idle) {
1491 		while (vcpu->state != VCPU_IDLE) {
1492 			vcpu->reqidle = true;
1493 			vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
1494 			cv_wait(&vcpu->state_cv, &vcpu->lock);
1495 			vcpu->reqidle = false;
1496 		}
1497 	} else {
1498 		KASSERT(vcpu->state != VCPU_IDLE, ("invalid transition from "
1499 		    "vcpu idle state"));
1500 	}
1501 
1502 	if (vcpu->state == VCPU_RUNNING) {
1503 		KASSERT(vcpu->hostcpu == curcpu, ("curcpu %d and hostcpu %d "
1504 		    "mismatch for running vcpu", curcpu, vcpu->hostcpu));
1505 	} else {
1506 		KASSERT(vcpu->hostcpu == NOCPU, ("Invalid hostcpu %d for a "
1507 		    "vcpu that is not running", vcpu->hostcpu));
1508 	}
1509 
1510 	/*
1511 	 * The following state transitions are allowed:
1512 	 * IDLE -> FROZEN -> IDLE
1513 	 * FROZEN -> RUNNING -> FROZEN
1514 	 * FROZEN -> SLEEPING -> FROZEN
1515 	 */
1516 	switch (vcpu->state) {
1517 	case VCPU_IDLE:
1518 	case VCPU_RUNNING:
1519 	case VCPU_SLEEPING:
1520 		error = (newstate != VCPU_FROZEN);
1521 		break;
1522 	case VCPU_FROZEN:
1523 		error = (newstate == VCPU_FROZEN);
1524 		break;
1525 	default:
1526 		error = 1;
1527 		break;
1528 	}
1529 
1530 	if (error)
1531 		return (EBUSY);
1532 
1533 	vcpu->state = newstate;
1534 	if (newstate == VCPU_RUNNING)
1535 		vcpu->hostcpu = curcpu;
1536 	else
1537 		vcpu->hostcpu = NOCPU;
1538 
1539 	if (newstate == VCPU_IDLE) {
1540 		cv_broadcast(&vcpu->state_cv);
1541 	}
1542 
1543 	return (0);
1544 }
1545 
1546 static void
vcpu_require_state(struct vm * vm,int vcpuid,enum vcpu_state newstate)1547 vcpu_require_state(struct vm *vm, int vcpuid, enum vcpu_state newstate)
1548 {
1549 	int error;
1550 
1551 	if ((error = vcpu_set_state(vm, vcpuid, newstate, false)) != 0)
1552 		panic("Error %d setting state to %d\n", error, newstate);
1553 }
1554 
1555 static void
vcpu_require_state_locked(struct vm * vm,int vcpuid,enum vcpu_state newstate)1556 vcpu_require_state_locked(struct vm *vm, int vcpuid, enum vcpu_state newstate)
1557 {
1558 	int error;
1559 
1560 	if ((error = vcpu_set_state_locked(vm, vcpuid, newstate, false)) != 0)
1561 		panic("Error %d setting state to %d", error, newstate);
1562 }
1563 
1564 /*
1565  * Emulate a guest 'hlt' by sleeping until the vcpu is ready to run.
1566  */
1567 static int
vm_handle_hlt(struct vm * vm,int vcpuid,bool intr_disabled)1568 vm_handle_hlt(struct vm *vm, int vcpuid, bool intr_disabled)
1569 {
1570 	struct vcpu *vcpu;
1571 	int vcpu_halted, vm_halted;
1572 	bool userspace_exit = false;
1573 
1574 	KASSERT(!CPU_ISSET(vcpuid, &vm->halted_cpus), ("vcpu already halted"));
1575 
1576 	vcpu = &vm->vcpu[vcpuid];
1577 	vcpu_halted = 0;
1578 	vm_halted = 0;
1579 
1580 	vcpu_lock(vcpu);
1581 	while (1) {
1582 		/*
1583 		 * Do a final check for pending interrupts (including NMI and
1584 		 * INIT) before putting this thread to sleep.
1585 		 */
1586 		if (vm_nmi_pending(vm, vcpuid))
1587 			break;
1588 		if (vcpu_run_state_pending(vm, vcpuid))
1589 			break;
1590 		if (!intr_disabled) {
1591 			if (vm_extint_pending(vm, vcpuid) ||
1592 			    vlapic_pending_intr(vcpu->vlapic, NULL)) {
1593 				break;
1594 			}
1595 		}
1596 
1597 		/*
1598 		 * Also check for software events which would cause a wake-up.
1599 		 * This will set the appropriate exitcode directly, rather than
1600 		 * requiring a trip through VM_RUN().
1601 		 */
1602 		if (vcpu_sleep_bailout_checks(vm, vcpuid)) {
1603 			userspace_exit = true;
1604 			break;
1605 		}
1606 
1607 		/*
1608 		 * Some Linux guests implement "halt" by having all vcpus
1609 		 * execute HLT with interrupts disabled. 'halted_cpus' keeps
1610 		 * track of the vcpus that have entered this state. When all
1611 		 * vcpus enter the halted state the virtual machine is halted.
1612 		 */
1613 		if (intr_disabled) {
1614 			if (!vcpu_halted && halt_detection_enabled) {
1615 				vcpu_halted = 1;
1616 				CPU_SET_ATOMIC(vcpuid, &vm->halted_cpus);
1617 			}
1618 			if (CPU_CMP(&vm->halted_cpus, &vm->active_cpus) == 0) {
1619 				vm_halted = 1;
1620 				break;
1621 			}
1622 		}
1623 
1624 		vcpu_ustate_change(vm, vcpuid, VU_IDLE);
1625 		vcpu_require_state_locked(vm, vcpuid, VCPU_SLEEPING);
1626 		(void) cv_wait_sig(&vcpu->vcpu_cv, &vcpu->lock);
1627 		vcpu_require_state_locked(vm, vcpuid, VCPU_FROZEN);
1628 		vcpu_ustate_change(vm, vcpuid, VU_EMU_KERN);
1629 	}
1630 
1631 	if (vcpu_halted)
1632 		CPU_CLR_ATOMIC(vcpuid, &vm->halted_cpus);
1633 
1634 	vcpu_unlock(vcpu);
1635 
1636 	if (vm_halted) {
1637 		(void) vm_suspend(vm, VM_SUSPEND_HALT, -1);
1638 	}
1639 
1640 	return (userspace_exit ? -1 : 0);
1641 }
1642 
1643 static int
vm_handle_paging(struct vm * vm,int vcpuid)1644 vm_handle_paging(struct vm *vm, int vcpuid)
1645 {
1646 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
1647 	vm_client_t *vmc = vcpu->vmclient;
1648 	struct vm_exit *vme = &vcpu->exitinfo;
1649 	const int ftype = vme->u.paging.fault_type;
1650 
1651 	ASSERT0(vme->inst_length);
1652 	ASSERT(ftype == PROT_READ || ftype == PROT_WRITE || ftype == PROT_EXEC);
1653 
1654 	if (vmc_fault(vmc, vme->u.paging.gpa, ftype) != 0) {
1655 		/*
1656 		 * If the fault cannot be serviced, kick it out to userspace for
1657 		 * handling (or more likely, halting the instance).
1658 		 */
1659 		return (-1);
1660 	}
1661 
1662 	return (0);
1663 }
1664 
1665 int
vm_service_mmio_read(struct vm * vm,int cpuid,uint64_t gpa,uint64_t * rval,int rsize)1666 vm_service_mmio_read(struct vm *vm, int cpuid, uint64_t gpa, uint64_t *rval,
1667     int rsize)
1668 {
1669 	int err = ESRCH;
1670 
1671 	if (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE) {
1672 		struct vlapic *vlapic = vm_lapic(vm, cpuid);
1673 
1674 		err = vlapic_mmio_read(vlapic, gpa, rval, rsize);
1675 	} else if (gpa >= VIOAPIC_BASE && gpa < VIOAPIC_BASE + VIOAPIC_SIZE) {
1676 		err = vioapic_mmio_read(vm, cpuid, gpa, rval, rsize);
1677 	} else if (gpa >= VHPET_BASE && gpa < VHPET_BASE + VHPET_SIZE) {
1678 		err = vhpet_mmio_read(vm, cpuid, gpa, rval, rsize);
1679 	}
1680 
1681 	return (err);
1682 }
1683 
1684 int
vm_service_mmio_write(struct vm * vm,int cpuid,uint64_t gpa,uint64_t wval,int wsize)1685 vm_service_mmio_write(struct vm *vm, int cpuid, uint64_t gpa, uint64_t wval,
1686     int wsize)
1687 {
1688 	int err = ESRCH;
1689 
1690 	if (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE) {
1691 		struct vlapic *vlapic = vm_lapic(vm, cpuid);
1692 
1693 		err = vlapic_mmio_write(vlapic, gpa, wval, wsize);
1694 	} else if (gpa >= VIOAPIC_BASE && gpa < VIOAPIC_BASE + VIOAPIC_SIZE) {
1695 		err = vioapic_mmio_write(vm, cpuid, gpa, wval, wsize);
1696 	} else if (gpa >= VHPET_BASE && gpa < VHPET_BASE + VHPET_SIZE) {
1697 		err = vhpet_mmio_write(vm, cpuid, gpa, wval, wsize);
1698 	}
1699 
1700 	return (err);
1701 }
1702 
1703 static int
vm_handle_mmio_emul(struct vm * vm,int vcpuid)1704 vm_handle_mmio_emul(struct vm *vm, int vcpuid)
1705 {
1706 	struct vie *vie;
1707 	struct vcpu *vcpu;
1708 	struct vm_exit *vme;
1709 	uint64_t inst_addr;
1710 	int error, fault, cs_d;
1711 
1712 	vcpu = &vm->vcpu[vcpuid];
1713 	vme = &vcpu->exitinfo;
1714 	vie = vcpu->vie_ctx;
1715 
1716 	KASSERT(vme->inst_length == 0, ("%s: invalid inst_length %d",
1717 	    __func__, vme->inst_length));
1718 
1719 	inst_addr = vme->rip + vme->u.mmio_emul.cs_base;
1720 	cs_d = vme->u.mmio_emul.cs_d;
1721 
1722 	/* Fetch the faulting instruction */
1723 	if (vie_needs_fetch(vie)) {
1724 		error = vie_fetch_instruction(vie, vm, vcpuid, inst_addr,
1725 		    &fault);
1726 		if (error != 0) {
1727 			return (error);
1728 		} else if (fault) {
1729 			/*
1730 			 * If a fault during instruction fetch was encountered,
1731 			 * it will have asserted that the appropriate exception
1732 			 * be injected at next entry.
1733 			 * No further work is required.
1734 			 */
1735 			return (0);
1736 		}
1737 	}
1738 
1739 	if (vie_decode_instruction(vie, vm, vcpuid, cs_d) != 0) {
1740 		/* Dump (unrecognized) instruction bytes in userspace */
1741 		vie_fallback_exitinfo(vie, vme);
1742 		return (-1);
1743 	}
1744 	if (vme->u.mmio_emul.gla != VIE_INVALID_GLA &&
1745 	    vie_verify_gla(vie, vm, vcpuid, vme->u.mmio_emul.gla) != 0) {
1746 		/* Decoded GLA does not match GLA from VM exit state */
1747 		vie_fallback_exitinfo(vie, vme);
1748 		return (-1);
1749 	}
1750 
1751 repeat:
1752 	error = vie_emulate_mmio(vie, vm, vcpuid);
1753 	if (error < 0) {
1754 		/*
1755 		 * MMIO not handled by any of the in-kernel-emulated devices, so
1756 		 * make a trip out to userspace for it.
1757 		 */
1758 		vie_exitinfo(vie, vme);
1759 	} else if (error == EAGAIN) {
1760 		/*
1761 		 * Continue emulating the rep-prefixed instruction, which has
1762 		 * not completed its iterations.
1763 		 *
1764 		 * In case this can be emulated in-kernel and has a high
1765 		 * repetition count (causing a tight spin), it should be
1766 		 * deferential to yield conditions.
1767 		 */
1768 		if (!vcpu_should_yield(vm, vcpuid)) {
1769 			goto repeat;
1770 		} else {
1771 			/*
1772 			 * Defer to the contending load by making a trip to
1773 			 * userspace with a no-op (BOGUS) exit reason.
1774 			 */
1775 			vie_reset(vie);
1776 			vme->exitcode = VM_EXITCODE_BOGUS;
1777 			return (-1);
1778 		}
1779 	} else if (error == 0) {
1780 		/* Update %rip now that instruction has been emulated */
1781 		vie_advance_pc(vie, &vcpu->nextrip);
1782 	}
1783 	return (error);
1784 }
1785 
1786 static int
vm_handle_inout(struct vm * vm,int vcpuid,struct vm_exit * vme)1787 vm_handle_inout(struct vm *vm, int vcpuid, struct vm_exit *vme)
1788 {
1789 	struct vcpu *vcpu;
1790 	struct vie *vie;
1791 	int err;
1792 
1793 	vcpu = &vm->vcpu[vcpuid];
1794 	vie = vcpu->vie_ctx;
1795 
1796 repeat:
1797 	err = vie_emulate_inout(vie, vm, vcpuid);
1798 
1799 	if (err < 0) {
1800 		/*
1801 		 * In/out not handled by any of the in-kernel-emulated devices,
1802 		 * so make a trip out to userspace for it.
1803 		 */
1804 		vie_exitinfo(vie, vme);
1805 		return (err);
1806 	} else if (err == EAGAIN) {
1807 		/*
1808 		 * Continue emulating the rep-prefixed ins/outs, which has not
1809 		 * completed its iterations.
1810 		 *
1811 		 * In case this can be emulated in-kernel and has a high
1812 		 * repetition count (causing a tight spin), it should be
1813 		 * deferential to yield conditions.
1814 		 */
1815 		if (!vcpu_should_yield(vm, vcpuid)) {
1816 			goto repeat;
1817 		} else {
1818 			/*
1819 			 * Defer to the contending load by making a trip to
1820 			 * userspace with a no-op (BOGUS) exit reason.
1821 			 */
1822 			vie_reset(vie);
1823 			vme->exitcode = VM_EXITCODE_BOGUS;
1824 			return (-1);
1825 		}
1826 	} else if (err != 0) {
1827 		/* Emulation failure.  Bail all the way out to userspace. */
1828 		vme->exitcode = VM_EXITCODE_INST_EMUL;
1829 		bzero(&vme->u.inst_emul, sizeof (vme->u.inst_emul));
1830 		return (-1);
1831 	}
1832 
1833 	vie_advance_pc(vie, &vcpu->nextrip);
1834 	return (0);
1835 }
1836 
1837 static int
vm_handle_inst_emul(struct vm * vm,int vcpuid)1838 vm_handle_inst_emul(struct vm *vm, int vcpuid)
1839 {
1840 	struct vie *vie;
1841 	struct vcpu *vcpu;
1842 	struct vm_exit *vme;
1843 	uint64_t cs_base;
1844 	int error, fault, cs_d;
1845 
1846 	vcpu = &vm->vcpu[vcpuid];
1847 	vme = &vcpu->exitinfo;
1848 	vie = vcpu->vie_ctx;
1849 
1850 	vie_cs_info(vie, vm, vcpuid, &cs_base, &cs_d);
1851 
1852 	/* Fetch the faulting instruction */
1853 	ASSERT(vie_needs_fetch(vie));
1854 	error = vie_fetch_instruction(vie, vm, vcpuid, vme->rip + cs_base,
1855 	    &fault);
1856 	if (error != 0) {
1857 		return (error);
1858 	} else if (fault) {
1859 		/*
1860 		 * If a fault during instruction fetch was encounted, it will
1861 		 * have asserted that the appropriate exception be injected at
1862 		 * next entry.  No further work is required.
1863 		 */
1864 		return (0);
1865 	}
1866 
1867 	if (vie_decode_instruction(vie, vm, vcpuid, cs_d) != 0) {
1868 		/* Dump (unrecognized) instruction bytes in userspace */
1869 		vie_fallback_exitinfo(vie, vme);
1870 		return (-1);
1871 	}
1872 
1873 	error = vie_emulate_other(vie, vm, vcpuid);
1874 	if (error != 0) {
1875 		/*
1876 		 * Instruction emulation was unable to complete successfully, so
1877 		 * kick it out to userspace for handling.
1878 		 */
1879 		vie_fallback_exitinfo(vie, vme);
1880 	} else {
1881 		/* Update %rip now that instruction has been emulated */
1882 		vie_advance_pc(vie, &vcpu->nextrip);
1883 	}
1884 	return (error);
1885 }
1886 
1887 static int
vm_handle_run_state(struct vm * vm,int vcpuid)1888 vm_handle_run_state(struct vm *vm, int vcpuid)
1889 {
1890 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
1891 	bool handled = false;
1892 
1893 	vcpu_lock(vcpu);
1894 	while (1) {
1895 		if ((vcpu->run_state & VRS_PEND_INIT) != 0) {
1896 			vcpu_unlock(vcpu);
1897 			VERIFY0(vcpu_arch_reset(vm, vcpuid, true));
1898 			vcpu_lock(vcpu);
1899 
1900 			vcpu->run_state &= ~(VRS_RUN | VRS_PEND_INIT);
1901 			vcpu->run_state |= VRS_INIT;
1902 		}
1903 
1904 		if ((vcpu->run_state & (VRS_INIT | VRS_RUN | VRS_PEND_SIPI)) ==
1905 		    (VRS_INIT | VRS_PEND_SIPI)) {
1906 			const uint8_t vector = vcpu->sipi_vector;
1907 
1908 			vcpu_unlock(vcpu);
1909 			VERIFY0(vcpu_vector_sipi(vm, vcpuid, vector));
1910 			vcpu_lock(vcpu);
1911 
1912 			vcpu->run_state &= ~VRS_PEND_SIPI;
1913 			vcpu->run_state |= VRS_RUN;
1914 		}
1915 
1916 		/*
1917 		 * If the vCPU is now in the running state, there is no need to
1918 		 * wait for anything prior to re-entry.
1919 		 */
1920 		if ((vcpu->run_state & VRS_RUN) != 0) {
1921 			handled = true;
1922 			break;
1923 		}
1924 
1925 		/*
1926 		 * Also check for software events which would cause a wake-up.
1927 		 * This will set the appropriate exitcode directly, rather than
1928 		 * requiring a trip through VM_RUN().
1929 		 */
1930 		if (vcpu_sleep_bailout_checks(vm, vcpuid)) {
1931 			break;
1932 		}
1933 
1934 		vcpu_ustate_change(vm, vcpuid, VU_IDLE);
1935 		vcpu_require_state_locked(vm, vcpuid, VCPU_SLEEPING);
1936 		(void) cv_wait_sig(&vcpu->vcpu_cv, &vcpu->lock);
1937 		vcpu_require_state_locked(vm, vcpuid, VCPU_FROZEN);
1938 		vcpu_ustate_change(vm, vcpuid, VU_EMU_KERN);
1939 	}
1940 	vcpu_unlock(vcpu);
1941 
1942 	return (handled ? 0 : -1);
1943 }
1944 
1945 static int
vm_rdmtrr(const struct vm_mtrr * mtrr,uint32_t num,uint64_t * val)1946 vm_rdmtrr(const struct vm_mtrr *mtrr, uint32_t num, uint64_t *val)
1947 {
1948 	switch (num) {
1949 	case MSR_MTRRcap:
1950 		*val = MTRR_CAP_WC | MTRR_CAP_FIXED | VMM_MTRR_VAR_MAX;
1951 		break;
1952 	case MSR_MTRRdefType:
1953 		*val = mtrr->def_type;
1954 		break;
1955 	case MSR_MTRR4kBase ... MSR_MTRR4kBase + 7:
1956 		*val = mtrr->fixed4k[num - MSR_MTRR4kBase];
1957 		break;
1958 	case MSR_MTRR16kBase ... MSR_MTRR16kBase + 1:
1959 		*val = mtrr->fixed16k[num - MSR_MTRR16kBase];
1960 		break;
1961 	case MSR_MTRR64kBase:
1962 		*val = mtrr->fixed64k;
1963 		break;
1964 	case MSR_MTRRVarBase ... MSR_MTRRVarBase + (VMM_MTRR_VAR_MAX * 2) - 1: {
1965 		uint_t offset = num - MSR_MTRRVarBase;
1966 		if (offset % 2 == 0) {
1967 			*val = mtrr->var[offset / 2].base;
1968 		} else {
1969 			*val = mtrr->var[offset / 2].mask;
1970 		}
1971 		break;
1972 	}
1973 	default:
1974 		return (EINVAL);
1975 	}
1976 
1977 	return (0);
1978 }
1979 
1980 static int
vm_wrmtrr(struct vm_mtrr * mtrr,uint32_t num,uint64_t val)1981 vm_wrmtrr(struct vm_mtrr *mtrr, uint32_t num, uint64_t val)
1982 {
1983 	switch (num) {
1984 	case MSR_MTRRcap:
1985 		/* MTRRCAP is read only */
1986 		return (EPERM);
1987 	case MSR_MTRRdefType:
1988 		if (val & ~VMM_MTRR_DEF_MASK) {
1989 			/* generate #GP on writes to reserved fields */
1990 			return (EINVAL);
1991 		}
1992 		mtrr->def_type = val;
1993 		break;
1994 	case MSR_MTRR4kBase ... MSR_MTRR4kBase + 7:
1995 		mtrr->fixed4k[num - MSR_MTRR4kBase] = val;
1996 		break;
1997 	case MSR_MTRR16kBase ... MSR_MTRR16kBase + 1:
1998 		mtrr->fixed16k[num - MSR_MTRR16kBase] = val;
1999 		break;
2000 	case MSR_MTRR64kBase:
2001 		mtrr->fixed64k = val;
2002 		break;
2003 	case MSR_MTRRVarBase ... MSR_MTRRVarBase + (VMM_MTRR_VAR_MAX * 2) - 1: {
2004 		uint_t offset = num - MSR_MTRRVarBase;
2005 		if (offset % 2 == 0) {
2006 			if (val & ~VMM_MTRR_PHYSBASE_MASK) {
2007 				/* generate #GP on writes to reserved fields */
2008 				return (EINVAL);
2009 			}
2010 			mtrr->var[offset / 2].base = val;
2011 		} else {
2012 			if (val & ~VMM_MTRR_PHYSMASK_MASK) {
2013 				/* generate #GP on writes to reserved fields */
2014 				return (EINVAL);
2015 			}
2016 			mtrr->var[offset / 2].mask = val;
2017 		}
2018 		break;
2019 	}
2020 	default:
2021 		return (EINVAL);
2022 	}
2023 
2024 	return (0);
2025 }
2026 
2027 static bool
is_mtrr_msr(uint32_t msr)2028 is_mtrr_msr(uint32_t msr)
2029 {
2030 	switch (msr) {
2031 	case MSR_MTRRcap:
2032 	case MSR_MTRRdefType:
2033 	case MSR_MTRR4kBase ... MSR_MTRR4kBase + 7:
2034 	case MSR_MTRR16kBase ... MSR_MTRR16kBase + 1:
2035 	case MSR_MTRR64kBase:
2036 	case MSR_MTRRVarBase ... MSR_MTRRVarBase + (VMM_MTRR_VAR_MAX * 2) - 1:
2037 		return (true);
2038 	default:
2039 		return (false);
2040 	}
2041 }
2042 
2043 static int
vm_handle_rdmsr(struct vm * vm,int vcpuid,struct vm_exit * vme)2044 vm_handle_rdmsr(struct vm *vm, int vcpuid, struct vm_exit *vme)
2045 {
2046 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
2047 	const uint32_t code = vme->u.msr.code;
2048 	uint64_t val = 0;
2049 
2050 	switch (code) {
2051 	case MSR_MCG_CAP:
2052 	case MSR_MCG_STATUS:
2053 		val = 0;
2054 		break;
2055 
2056 	case MSR_MTRRcap:
2057 	case MSR_MTRRdefType:
2058 	case MSR_MTRR4kBase ... MSR_MTRR4kBase + 7:
2059 	case MSR_MTRR16kBase ... MSR_MTRR16kBase + 1:
2060 	case MSR_MTRR64kBase:
2061 	case MSR_MTRRVarBase ... MSR_MTRRVarBase + (VMM_MTRR_VAR_MAX * 2) - 1:
2062 		if (vm_rdmtrr(&vcpu->mtrr, code, &val) != 0)
2063 			vm_inject_gp(vm, vcpuid);
2064 		break;
2065 
2066 	case MSR_TSC:
2067 		/*
2068 		 * Get the guest TSC, applying necessary vCPU offsets.
2069 		 *
2070 		 * In all likelihood, this should always be handled in guest
2071 		 * context by VMX/SVM rather than taking an exit.  (Both VMX and
2072 		 * SVM pass through read-only access to MSR_TSC to the guest.)
2073 		 *
2074 		 * The VM-wide TSC offset and per-vCPU offset are included in
2075 		 * the calculations of vcpu_tsc_offset(), so this is sufficient
2076 		 * to use as the offset in our calculations.
2077 		 *
2078 		 * No physical offset is requested of vcpu_tsc_offset() since
2079 		 * rdtsc_offset() takes care of that instead.
2080 		 */
2081 		val = calc_guest_tsc(rdtsc_offset(), vm->freq_multiplier,
2082 		    vcpu_tsc_offset(vm, vcpuid, false));
2083 		break;
2084 
2085 	default:
2086 		/*
2087 		 * Anything not handled at this point will be kicked out to
2088 		 * userspace for attempted processing there.
2089 		 */
2090 		return (-1);
2091 	}
2092 
2093 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_RAX,
2094 	    val & 0xffffffff));
2095 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_RDX,
2096 	    val >> 32));
2097 	return (0);
2098 }
2099 
2100 static int
vm_handle_wrmsr(struct vm * vm,int vcpuid,struct vm_exit * vme)2101 vm_handle_wrmsr(struct vm *vm, int vcpuid, struct vm_exit *vme)
2102 {
2103 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
2104 	const uint32_t code = vme->u.msr.code;
2105 	const uint64_t val = vme->u.msr.wval;
2106 
2107 	switch (code) {
2108 	case MSR_MCG_CAP:
2109 	case MSR_MCG_STATUS:
2110 		/* Ignore writes */
2111 		break;
2112 
2113 	case MSR_MTRRcap:
2114 	case MSR_MTRRdefType:
2115 	case MSR_MTRR4kBase ... MSR_MTRR4kBase + 7:
2116 	case MSR_MTRR16kBase ... MSR_MTRR16kBase + 1:
2117 	case MSR_MTRR64kBase:
2118 	case MSR_MTRRVarBase ... MSR_MTRRVarBase + (VMM_MTRR_VAR_MAX * 2) - 1:
2119 		if (vm_wrmtrr(&vcpu->mtrr, code, val) != 0)
2120 			vm_inject_gp(vm, vcpuid);
2121 		break;
2122 
2123 	case MSR_TSC:
2124 		/*
2125 		 * The effect of writing the TSC MSR is that a subsequent read
2126 		 * of the TSC would report that value written (plus any time
2127 		 * elapsed between the write and the read).
2128 		 *
2129 		 * To calculate that per-vCPU offset, we can work backwards from
2130 		 * the guest TSC at the time of write:
2131 		 *
2132 		 * value = current guest TSC + vCPU offset
2133 		 *
2134 		 * so therefore:
2135 		 *
2136 		 * value - current guest TSC = vCPU offset
2137 		 */
2138 		vcpu->tsc_offset = val - calc_guest_tsc(rdtsc_offset(),
2139 		    vm->freq_multiplier, vm->tsc_offset);
2140 		break;
2141 
2142 	default:
2143 		/*
2144 		 * Anything not handled at this point will be kicked out to
2145 		 * userspace for attempted processing there.
2146 		 */
2147 		return (-1);
2148 	}
2149 
2150 	return (0);
2151 }
2152 
2153 /*
2154  * Has a suspend event been asserted on the VM?
2155  *
2156  * The reason and (in the case of a triple-fault) source vcpuid are optionally
2157  * returned if such a state is present.
2158  */
2159 static bool
vm_is_suspended(struct vm * vm,struct vm_exit * vme)2160 vm_is_suspended(struct vm *vm, struct vm_exit *vme)
2161 {
2162 	const int val = vm->suspend_how;
2163 	if (val == 0) {
2164 		return (false);
2165 	} else {
2166 		if (vme != NULL) {
2167 			vme->exitcode = VM_EXITCODE_SUSPENDED;
2168 			vme->u.suspended.how = val;
2169 			vme->u.suspended.source = vm->suspend_source;
2170 			/*
2171 			 * Normalize suspend event time and, on the off chance
2172 			 * that it was recorded as occuring prior to VM boot,
2173 			 * clamp it to a minimum of 0.
2174 			 */
2175 			vme->u.suspended.when = (uint64_t)
2176 			    MAX(vm_normalize_hrtime(vm, vm->suspend_when), 0);
2177 		}
2178 		return (true);
2179 	}
2180 }
2181 
2182 int
vm_suspend(struct vm * vm,enum vm_suspend_how how,int source)2183 vm_suspend(struct vm *vm, enum vm_suspend_how how, int source)
2184 {
2185 	if (how <= VM_SUSPEND_NONE || how >= VM_SUSPEND_LAST) {
2186 		return (EINVAL);
2187 	}
2188 
2189 	/*
2190 	 * Although the common case of calling vm_suspend() is via
2191 	 * ioctl(VM_SUSPEND), where all the vCPUs will be held in the frozen
2192 	 * state, it can also be called by a running vCPU to indicate a
2193 	 * triple-fault.  In the latter case, there is no exclusion from a
2194 	 * racing vm_suspend() from a different vCPU, so assertion of the
2195 	 * suspended state must be performed carefully.
2196 	 *
2197 	 * The `suspend_when` is set first via atomic cmpset to pick a "winner"
2198 	 * of the suspension race, followed by population of 'suspend_source'.
2199 	 * Only after those are done, and a membar is emitted will 'suspend_how'
2200 	 * be set, which makes the suspended state visible to any vCPU checking
2201 	 * for it.  That order will prevent an incomplete suspend state (between
2202 	 * 'how', 'source', and 'when') from being observed.
2203 	 */
2204 	const hrtime_t now = gethrtime();
2205 	if (atomic_cmpset_long((ulong_t *)&vm->suspend_when, 0, now) == 0) {
2206 		return (EALREADY);
2207 	}
2208 	vm->suspend_source = source;
2209 	membar_producer();
2210 	vm->suspend_how = how;
2211 
2212 	/* Notify all active vcpus that they are now suspended. */
2213 	for (uint_t i = 0; i < vm->maxcpus; i++) {
2214 		struct vcpu *vcpu = &vm->vcpu[i];
2215 
2216 		vcpu_lock(vcpu);
2217 
2218 		if (!CPU_ISSET(i, &vm->active_cpus)) {
2219 			/*
2220 			 * vCPUs not already marked as active can be ignored,
2221 			 * since they cannot become marked as active unless the
2222 			 * VM is reinitialized, clearing the suspended state.
2223 			 */
2224 			vcpu_unlock(vcpu);
2225 			continue;
2226 		}
2227 
2228 		switch (vcpu->state) {
2229 		case VCPU_IDLE:
2230 		case VCPU_FROZEN:
2231 			/*
2232 			 * vCPUs not locked by in-kernel activity can be
2233 			 * immediately marked as suspended: The ustate is moved
2234 			 * back to VU_INIT, since no further guest work will
2235 			 * occur while the VM is in this state.
2236 			 *
2237 			 * A FROZEN vCPU may still change its ustate on the way
2238 			 * out of the kernel, but a subsequent check at the end
2239 			 * of vm_run() should be adequate to fix it up.
2240 			 */
2241 			vcpu_ustate_change(vm, i, VU_INIT);
2242 			break;
2243 		default:
2244 			/*
2245 			 * Any vCPUs which are running or waiting in-kernel
2246 			 * (such as in HLT) are notified to pick up the newly
2247 			 * suspended state.
2248 			 */
2249 			vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
2250 			break;
2251 		}
2252 		vcpu_unlock(vcpu);
2253 	}
2254 	return (0);
2255 }
2256 
2257 void
vm_exit_run_state(struct vm * vm,int vcpuid,uint64_t rip)2258 vm_exit_run_state(struct vm *vm, int vcpuid, uint64_t rip)
2259 {
2260 	struct vm_exit *vmexit;
2261 
2262 	vmexit = vm_exitinfo(vm, vcpuid);
2263 	vmexit->rip = rip;
2264 	vmexit->inst_length = 0;
2265 	vmexit->exitcode = VM_EXITCODE_RUN_STATE;
2266 	vmm_stat_incr(vm, vcpuid, VMEXIT_RUN_STATE, 1);
2267 }
2268 
2269 /*
2270  * Some vmm resources, such as the lapic, may have CPU-specific resources
2271  * allocated to them which would benefit from migration onto the host CPU which
2272  * is processing the vcpu state.
2273  */
2274 static void
vm_localize_resources(struct vm * vm,struct vcpu * vcpu)2275 vm_localize_resources(struct vm *vm, struct vcpu *vcpu)
2276 {
2277 	/*
2278 	 * Localizing cyclic resources requires acquisition of cpu_lock, and
2279 	 * doing so with kpreempt disabled is a recipe for deadlock disaster.
2280 	 */
2281 	VERIFY(curthread->t_preempt == 0);
2282 
2283 	/*
2284 	 * Do not bother with localization if this vCPU is about to return to
2285 	 * the host CPU it was last localized to.
2286 	 */
2287 	if (vcpu->lastloccpu == curcpu)
2288 		return;
2289 
2290 	/*
2291 	 * Localize system-wide resources to the primary boot vCPU.  While any
2292 	 * of the other vCPUs may access them, it keeps the potential interrupt
2293 	 * footprint constrained to CPUs involved with this instance.
2294 	 */
2295 	if (vcpu == &vm->vcpu[0]) {
2296 		vhpet_localize_resources(vm->vhpet);
2297 		vrtc_localize_resources(vm->vrtc);
2298 		vatpit_localize_resources(vm->vatpit);
2299 	}
2300 
2301 	vlapic_localize_resources(vcpu->vlapic);
2302 
2303 	vcpu->lastloccpu = curcpu;
2304 }
2305 
2306 static void
vmm_savectx(void * arg)2307 vmm_savectx(void *arg)
2308 {
2309 	vm_thread_ctx_t *vtc = arg;
2310 	struct vm *vm = vtc->vtc_vm;
2311 	const int vcpuid = vtc->vtc_vcpuid;
2312 
2313 	if (ops->vmsavectx != NULL) {
2314 		ops->vmsavectx(vm->cookie, vcpuid);
2315 	}
2316 
2317 	/*
2318 	 * Account for going off-cpu, unless the vCPU is idled, where being
2319 	 * off-cpu is the explicit point.
2320 	 */
2321 	if (vm->vcpu[vcpuid].ustate != VU_IDLE) {
2322 		vtc->vtc_ustate = vm->vcpu[vcpuid].ustate;
2323 		vcpu_ustate_change(vm, vcpuid, VU_SCHED);
2324 	}
2325 
2326 	/*
2327 	 * If the CPU holds the restored guest FPU state, save it and restore
2328 	 * the host FPU state before this thread goes off-cpu.
2329 	 */
2330 	if ((vtc->vtc_status & VTCS_FPU_RESTORED) != 0) {
2331 		struct vcpu *vcpu = &vm->vcpu[vcpuid];
2332 
2333 		save_guest_fpustate(vcpu);
2334 		vtc->vtc_status &= ~VTCS_FPU_RESTORED;
2335 	}
2336 }
2337 
2338 static void
vmm_restorectx(void * arg)2339 vmm_restorectx(void *arg)
2340 {
2341 	vm_thread_ctx_t *vtc = arg;
2342 	struct vm *vm = vtc->vtc_vm;
2343 	const int vcpuid = vtc->vtc_vcpuid;
2344 
2345 	/* Complete microstate accounting for vCPU being off-cpu */
2346 	if (vm->vcpu[vcpuid].ustate != VU_IDLE) {
2347 		vcpu_ustate_change(vm, vcpuid, vtc->vtc_ustate);
2348 	}
2349 
2350 	/*
2351 	 * When coming back on-cpu, only restore the guest FPU status if the
2352 	 * thread is in a context marked as requiring it.  This should be rare,
2353 	 * occurring only when a future logic error results in a voluntary
2354 	 * sleep during the VMRUN critical section.
2355 	 *
2356 	 * The common case will result in elision of the guest FPU state
2357 	 * restoration, deferring that action until it is clearly necessary
2358 	 * during vm_run.
2359 	 */
2360 	VERIFY((vtc->vtc_status & VTCS_FPU_RESTORED) == 0);
2361 	if ((vtc->vtc_status & VTCS_FPU_CTX_CRITICAL) != 0) {
2362 		struct vcpu *vcpu = &vm->vcpu[vcpuid];
2363 
2364 		restore_guest_fpustate(vcpu);
2365 		vtc->vtc_status |= VTCS_FPU_RESTORED;
2366 	}
2367 
2368 	if (ops->vmrestorectx != NULL) {
2369 		ops->vmrestorectx(vm->cookie, vcpuid);
2370 	}
2371 
2372 }
2373 
2374 /* Convenience defines for parsing vm_entry`cmd values */
2375 #define	VEC_MASK_FLAGS	(VEC_FLAG_EXIT_CONSISTENT)
2376 #define	VEC_MASK_CMD	(~VEC_MASK_FLAGS)
2377 
2378 static int
vm_entry_actions(struct vm * vm,int vcpuid,const struct vm_entry * entry,struct vm_exit * vme)2379 vm_entry_actions(struct vm *vm, int vcpuid, const struct vm_entry *entry,
2380     struct vm_exit *vme)
2381 {
2382 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
2383 	struct vie *vie = vcpu->vie_ctx;
2384 	int err = 0;
2385 
2386 	const uint_t cmd = entry->cmd & VEC_MASK_CMD;
2387 	const uint_t flags = entry->cmd & VEC_MASK_FLAGS;
2388 
2389 	switch (cmd) {
2390 	case VEC_DEFAULT:
2391 		break;
2392 	case VEC_DISCARD_INSTR:
2393 		vie_reset(vie);
2394 		break;
2395 	case VEC_FULFILL_MMIO:
2396 		err = vie_fulfill_mmio(vie, &entry->u.mmio);
2397 		if (err == 0) {
2398 			err = vie_emulate_mmio(vie, vm, vcpuid);
2399 			if (err == 0) {
2400 				vie_advance_pc(vie, &vcpu->nextrip);
2401 			} else if (err < 0) {
2402 				vie_exitinfo(vie, vme);
2403 			} else if (err == EAGAIN) {
2404 				/*
2405 				 * Clear the instruction emulation state in
2406 				 * order to re-enter VM context and continue
2407 				 * this 'rep <instruction>'
2408 				 */
2409 				vie_reset(vie);
2410 				err = 0;
2411 			}
2412 		}
2413 		break;
2414 	case VEC_FULFILL_INOUT:
2415 		err = vie_fulfill_inout(vie, &entry->u.inout);
2416 		if (err == 0) {
2417 			err = vie_emulate_inout(vie, vm, vcpuid);
2418 			if (err == 0) {
2419 				vie_advance_pc(vie, &vcpu->nextrip);
2420 			} else if (err < 0) {
2421 				vie_exitinfo(vie, vme);
2422 			} else if (err == EAGAIN) {
2423 				/*
2424 				 * Clear the instruction emulation state in
2425 				 * order to re-enter VM context and continue
2426 				 * this 'rep ins/outs'
2427 				 */
2428 				vie_reset(vie);
2429 				err = 0;
2430 			}
2431 		}
2432 		break;
2433 	default:
2434 		return (EINVAL);
2435 	}
2436 
2437 	/*
2438 	 * Pay heed to requests for exit-when-vCPU-is-consistent requests, at
2439 	 * least when we are not immediately bound for another exit due to
2440 	 * multi-part instruction emulation or related causes.
2441 	 */
2442 	if ((flags & VEC_FLAG_EXIT_CONSISTENT) != 0 && err == 0) {
2443 		vcpu->reqconsist = true;
2444 	}
2445 
2446 	return (err);
2447 }
2448 
2449 static int
vm_loop_checks(struct vm * vm,int vcpuid,struct vm_exit * vme)2450 vm_loop_checks(struct vm *vm, int vcpuid, struct vm_exit *vme)
2451 {
2452 	struct vie *vie;
2453 
2454 	vie = vm->vcpu[vcpuid].vie_ctx;
2455 
2456 	if (vie_pending(vie)) {
2457 		/*
2458 		 * Userspace has not fulfilled the pending needs of the
2459 		 * instruction emulation, so bail back out.
2460 		 */
2461 		vie_exitinfo(vie, vme);
2462 		return (-1);
2463 	}
2464 
2465 	return (0);
2466 }
2467 
2468 int
vm_run(struct vm * vm,int vcpuid,const struct vm_entry * entry)2469 vm_run(struct vm *vm, int vcpuid, const struct vm_entry *entry)
2470 {
2471 	int error;
2472 	struct vcpu *vcpu;
2473 	struct vm_exit *vme;
2474 	bool intr_disabled;
2475 	int affinity_type = CPU_CURRENT;
2476 
2477 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2478 		return (EINVAL);
2479 	if (!CPU_ISSET(vcpuid, &vm->active_cpus))
2480 		return (EINVAL);
2481 	if (vm->is_paused) {
2482 		return (EBUSY);
2483 	}
2484 
2485 	vcpu = &vm->vcpu[vcpuid];
2486 	vme = &vcpu->exitinfo;
2487 
2488 	vcpu_ustate_change(vm, vcpuid, VU_EMU_KERN);
2489 
2490 	vcpu->vtc.vtc_status = 0;
2491 	ctxop_attach(curthread, vcpu->ctxop);
2492 
2493 	error = vm_entry_actions(vm, vcpuid, entry, vme);
2494 	if (error != 0) {
2495 		goto exit;
2496 	}
2497 
2498 restart:
2499 	error = vm_loop_checks(vm, vcpuid, vme);
2500 	if (error != 0) {
2501 		goto exit;
2502 	}
2503 
2504 	thread_affinity_set(curthread, affinity_type);
2505 	/*
2506 	 * Resource localization should happen after the CPU affinity for the
2507 	 * thread has been set to ensure that access from restricted contexts,
2508 	 * such as VMX-accelerated APIC operations, can occur without inducing
2509 	 * cyclic cross-calls.
2510 	 *
2511 	 * This must be done prior to disabling kpreempt via critical_enter().
2512 	 */
2513 	vm_localize_resources(vm, vcpu);
2514 	affinity_type = CPU_CURRENT;
2515 	critical_enter();
2516 
2517 	/* Force a trip through update_sregs to reload %fs/%gs and friends */
2518 	PCB_SET_UPDATE_SEGS(&ttolwp(curthread)->lwp_pcb);
2519 
2520 	if ((vcpu->vtc.vtc_status & VTCS_FPU_RESTORED) == 0) {
2521 		restore_guest_fpustate(vcpu);
2522 		vcpu->vtc.vtc_status |= VTCS_FPU_RESTORED;
2523 	}
2524 	vcpu->vtc.vtc_status |= VTCS_FPU_CTX_CRITICAL;
2525 
2526 	vcpu_require_state(vm, vcpuid, VCPU_RUNNING);
2527 	error = VMRUN(vm->cookie, vcpuid, vcpu->nextrip);
2528 	vcpu_require_state(vm, vcpuid, VCPU_FROZEN);
2529 
2530 	/*
2531 	 * Once clear of the delicate contexts comprising the VM_RUN handler,
2532 	 * thread CPU affinity can be loosened while other processing occurs.
2533 	 */
2534 	vcpu->vtc.vtc_status &= ~VTCS_FPU_CTX_CRITICAL;
2535 	thread_affinity_clear(curthread);
2536 	critical_exit();
2537 
2538 	if (error != 0) {
2539 		/* Communicate out any error from VMRUN() above */
2540 		goto exit;
2541 	}
2542 
2543 	vcpu->nextrip = vme->rip + vme->inst_length;
2544 	switch (vme->exitcode) {
2545 	case VM_EXITCODE_RUN_STATE:
2546 		error = vm_handle_run_state(vm, vcpuid);
2547 		break;
2548 	case VM_EXITCODE_IOAPIC_EOI:
2549 		vioapic_process_eoi(vm, vcpuid,
2550 		    vme->u.ioapic_eoi.vector);
2551 		break;
2552 	case VM_EXITCODE_HLT:
2553 		intr_disabled = ((vme->u.hlt.rflags & PSL_I) == 0);
2554 		error = vm_handle_hlt(vm, vcpuid, intr_disabled);
2555 		break;
2556 	case VM_EXITCODE_PAGING:
2557 		error = vm_handle_paging(vm, vcpuid);
2558 		break;
2559 	case VM_EXITCODE_MMIO_EMUL:
2560 		error = vm_handle_mmio_emul(vm, vcpuid);
2561 		break;
2562 	case VM_EXITCODE_INOUT:
2563 		error = vm_handle_inout(vm, vcpuid, vme);
2564 		break;
2565 	case VM_EXITCODE_INST_EMUL:
2566 		error = vm_handle_inst_emul(vm, vcpuid);
2567 		break;
2568 	case VM_EXITCODE_MONITOR:
2569 	case VM_EXITCODE_MWAIT:
2570 	case VM_EXITCODE_VMINSN:
2571 		vm_inject_ud(vm, vcpuid);
2572 		break;
2573 	case VM_EXITCODE_RDMSR:
2574 		error = vm_handle_rdmsr(vm, vcpuid, vme);
2575 		break;
2576 	case VM_EXITCODE_WRMSR:
2577 		error = vm_handle_wrmsr(vm, vcpuid, vme);
2578 		break;
2579 	case VM_EXITCODE_HT:
2580 		affinity_type = CPU_BEST;
2581 		break;
2582 	case VM_EXITCODE_MTRAP:
2583 		VERIFY0(vm_suspend_cpu(vm, vcpuid));
2584 		error = -1;
2585 		break;
2586 	default:
2587 		/* handled in userland */
2588 		error = -1;
2589 		break;
2590 	}
2591 
2592 	if (error == 0) {
2593 		/* VM exit conditions handled in-kernel, continue running */
2594 		goto restart;
2595 	}
2596 
2597 exit:
2598 	kpreempt_disable();
2599 	ctxop_detach(curthread, vcpu->ctxop);
2600 	/* Make sure all of the needed vCPU context state is saved */
2601 	vmm_savectx(&vcpu->vtc);
2602 	kpreempt_enable();
2603 
2604 	/*
2605 	 * Bill time in userspace against VU_EMU_USER, unless the VM is
2606 	 * suspended, in which case VU_INIT is the choice.
2607 	 */
2608 	vcpu_ustate_change(vm, vcpuid,
2609 	    vm_is_suspended(vm, NULL) ? VU_INIT : VU_EMU_USER);
2610 
2611 	return (error);
2612 }
2613 
2614 int
vm_restart_instruction(void * arg,int vcpuid)2615 vm_restart_instruction(void *arg, int vcpuid)
2616 {
2617 	struct vm *vm;
2618 	struct vcpu *vcpu;
2619 	enum vcpu_state state;
2620 	uint64_t rip;
2621 	int error;
2622 
2623 	vm = arg;
2624 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2625 		return (EINVAL);
2626 
2627 	vcpu = &vm->vcpu[vcpuid];
2628 	state = vcpu_get_state(vm, vcpuid, NULL);
2629 	if (state == VCPU_RUNNING) {
2630 		/*
2631 		 * When a vcpu is "running" the next instruction is determined
2632 		 * by adding 'rip' and 'inst_length' in the vcpu's 'exitinfo'.
2633 		 * Thus setting 'inst_length' to zero will cause the current
2634 		 * instruction to be restarted.
2635 		 */
2636 		vcpu->exitinfo.inst_length = 0;
2637 	} else if (state == VCPU_FROZEN) {
2638 		/*
2639 		 * When a vcpu is "frozen" it is outside the critical section
2640 		 * around VMRUN() and 'nextrip' points to the next instruction.
2641 		 * Thus instruction restart is achieved by setting 'nextrip'
2642 		 * to the vcpu's %rip.
2643 		 */
2644 		error = vm_get_register(vm, vcpuid, VM_REG_GUEST_RIP, &rip);
2645 		KASSERT(!error, ("%s: error %d getting rip", __func__, error));
2646 		vcpu->nextrip = rip;
2647 	} else {
2648 		panic("%s: invalid state %d", __func__, state);
2649 	}
2650 	return (0);
2651 }
2652 
2653 int
vm_exit_intinfo(struct vm * vm,int vcpuid,uint64_t info)2654 vm_exit_intinfo(struct vm *vm, int vcpuid, uint64_t info)
2655 {
2656 	struct vcpu *vcpu;
2657 
2658 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2659 		return (EINVAL);
2660 
2661 	vcpu = &vm->vcpu[vcpuid];
2662 
2663 	if (VM_INTINFO_PENDING(info)) {
2664 		const uint32_t type = VM_INTINFO_TYPE(info);
2665 		const uint8_t vector = VM_INTINFO_VECTOR(info);
2666 
2667 		if (type == VM_INTINFO_NMI && vector != IDT_NMI)
2668 			return (EINVAL);
2669 		if (type == VM_INTINFO_HWEXCP && vector >= 32)
2670 			return (EINVAL);
2671 		if (info & VM_INTINFO_MASK_RSVD)
2672 			return (EINVAL);
2673 	} else {
2674 		info = 0;
2675 	}
2676 	vcpu->exit_intinfo = info;
2677 	return (0);
2678 }
2679 
2680 enum exc_class {
2681 	EXC_BENIGN,
2682 	EXC_CONTRIBUTORY,
2683 	EXC_PAGEFAULT
2684 };
2685 
2686 #define	IDT_VE	20	/* Virtualization Exception (Intel specific) */
2687 
2688 static enum exc_class
exception_class(uint64_t info)2689 exception_class(uint64_t info)
2690 {
2691 	ASSERT(VM_INTINFO_PENDING(info));
2692 
2693 	/* Table 6-4, "Interrupt and Exception Classes", Intel SDM, Vol 3 */
2694 	switch (VM_INTINFO_TYPE(info)) {
2695 	case VM_INTINFO_HWINTR:
2696 	case VM_INTINFO_SWINTR:
2697 	case VM_INTINFO_NMI:
2698 		return (EXC_BENIGN);
2699 	default:
2700 		/*
2701 		 * Hardware exception.
2702 		 *
2703 		 * SVM and VT-x use identical type values to represent NMI,
2704 		 * hardware interrupt and software interrupt.
2705 		 *
2706 		 * SVM uses type '3' for all exceptions. VT-x uses type '3'
2707 		 * for exceptions except #BP and #OF. #BP and #OF use a type
2708 		 * value of '5' or '6'. Therefore we don't check for explicit
2709 		 * values of 'type' to classify 'intinfo' into a hardware
2710 		 * exception.
2711 		 */
2712 		break;
2713 	}
2714 
2715 	switch (VM_INTINFO_VECTOR(info)) {
2716 	case IDT_PF:
2717 	case IDT_VE:
2718 		return (EXC_PAGEFAULT);
2719 	case IDT_DE:
2720 	case IDT_TS:
2721 	case IDT_NP:
2722 	case IDT_SS:
2723 	case IDT_GP:
2724 		return (EXC_CONTRIBUTORY);
2725 	default:
2726 		return (EXC_BENIGN);
2727 	}
2728 }
2729 
2730 /*
2731  * Fetch event pending injection into the guest, if one exists.
2732  *
2733  * Returns true if an event is to be injected (which is placed in `retinfo`).
2734  */
2735 bool
vm_entry_intinfo(struct vm * vm,int vcpuid,uint64_t * retinfo)2736 vm_entry_intinfo(struct vm *vm, int vcpuid, uint64_t *retinfo)
2737 {
2738 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
2739 	const uint64_t info1 = vcpu->exit_intinfo;
2740 	vcpu->exit_intinfo = 0;
2741 	const uint64_t info2 = vcpu->exc_pending;
2742 	vcpu->exc_pending = 0;
2743 
2744 	if (VM_INTINFO_PENDING(info1) && VM_INTINFO_PENDING(info2)) {
2745 		/*
2746 		 * If an exception occurs while attempting to call the
2747 		 * double-fault handler the processor enters shutdown mode
2748 		 * (aka triple fault).
2749 		 */
2750 		if (VM_INTINFO_TYPE(info1) == VM_INTINFO_HWEXCP &&
2751 		    VM_INTINFO_VECTOR(info1) == IDT_DF) {
2752 			(void) vm_suspend(vm, VM_SUSPEND_TRIPLEFAULT, vcpuid);
2753 			*retinfo = 0;
2754 			return (false);
2755 		}
2756 		/*
2757 		 * "Conditions for Generating a Double Fault"
2758 		 *  Intel SDM, Vol3, Table 6-5
2759 		 */
2760 		const enum exc_class exc1 = exception_class(info1);
2761 		const enum exc_class exc2 = exception_class(info2);
2762 		if ((exc1 == EXC_CONTRIBUTORY && exc2 == EXC_CONTRIBUTORY) ||
2763 		    (exc1 == EXC_PAGEFAULT && exc2 != EXC_BENIGN)) {
2764 			/* Convert nested fault into a double fault. */
2765 			*retinfo =
2766 			    VM_INTINFO_VALID |
2767 			    VM_INTINFO_DEL_ERRCODE |
2768 			    VM_INTINFO_HWEXCP |
2769 			    IDT_DF;
2770 		} else {
2771 			/* Handle exceptions serially */
2772 			vcpu->exit_intinfo = info1;
2773 			*retinfo = info2;
2774 		}
2775 		return (true);
2776 	} else if (VM_INTINFO_PENDING(info1)) {
2777 		*retinfo = info1;
2778 		return (true);
2779 	} else if (VM_INTINFO_PENDING(info2)) {
2780 		*retinfo = info2;
2781 		return (true);
2782 	}
2783 
2784 	return (false);
2785 }
2786 
2787 int
vm_get_intinfo(struct vm * vm,int vcpuid,uint64_t * info1,uint64_t * info2)2788 vm_get_intinfo(struct vm *vm, int vcpuid, uint64_t *info1, uint64_t *info2)
2789 {
2790 	struct vcpu *vcpu;
2791 
2792 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2793 		return (EINVAL);
2794 
2795 	vcpu = &vm->vcpu[vcpuid];
2796 	*info1 = vcpu->exit_intinfo;
2797 	*info2 = vcpu->exc_pending;
2798 	return (0);
2799 }
2800 
2801 int
vm_inject_exception(struct vm * vm,int vcpuid,uint8_t vector,bool errcode_valid,uint32_t errcode,bool restart_instruction)2802 vm_inject_exception(struct vm *vm, int vcpuid, uint8_t vector,
2803     bool errcode_valid, uint32_t errcode, bool restart_instruction)
2804 {
2805 	struct vcpu *vcpu;
2806 	uint64_t regval;
2807 	int error;
2808 
2809 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2810 		return (EINVAL);
2811 
2812 	if (vector >= 32)
2813 		return (EINVAL);
2814 
2815 	/*
2816 	 * NMIs are to be injected via their own specialized path using
2817 	 * vm_inject_nmi().
2818 	 */
2819 	if (vector == IDT_NMI) {
2820 		return (EINVAL);
2821 	}
2822 
2823 	/*
2824 	 * A double fault exception should never be injected directly into
2825 	 * the guest. It is a derived exception that results from specific
2826 	 * combinations of nested faults.
2827 	 */
2828 	if (vector == IDT_DF) {
2829 		return (EINVAL);
2830 	}
2831 
2832 	vcpu = &vm->vcpu[vcpuid];
2833 
2834 	if (VM_INTINFO_PENDING(vcpu->exc_pending)) {
2835 		/* Unable to inject exception due to one already pending */
2836 		return (EBUSY);
2837 	}
2838 
2839 	if (errcode_valid) {
2840 		/*
2841 		 * Exceptions don't deliver an error code in real mode.
2842 		 */
2843 		error = vm_get_register(vm, vcpuid, VM_REG_GUEST_CR0, &regval);
2844 		VERIFY0(error);
2845 		if ((regval & CR0_PE) == 0) {
2846 			errcode_valid = false;
2847 		}
2848 	}
2849 
2850 	/*
2851 	 * From section 26.6.1 "Interruptibility State" in Intel SDM:
2852 	 *
2853 	 * Event blocking by "STI" or "MOV SS" is cleared after guest executes
2854 	 * one instruction or incurs an exception.
2855 	 */
2856 	error = vm_set_register(vm, vcpuid, VM_REG_GUEST_INTR_SHADOW, 0);
2857 	VERIFY0(error);
2858 
2859 	if (restart_instruction) {
2860 		VERIFY0(vm_restart_instruction(vm, vcpuid));
2861 	}
2862 
2863 	uint64_t val = VM_INTINFO_VALID | VM_INTINFO_HWEXCP | vector;
2864 	if (errcode_valid) {
2865 		val |= VM_INTINFO_DEL_ERRCODE;
2866 		val |= (uint64_t)errcode << VM_INTINFO_SHIFT_ERRCODE;
2867 	}
2868 	vcpu->exc_pending = val;
2869 	return (0);
2870 }
2871 
2872 void
vm_inject_ud(struct vm * vm,int vcpuid)2873 vm_inject_ud(struct vm *vm, int vcpuid)
2874 {
2875 	VERIFY0(vm_inject_exception(vm, vcpuid, IDT_UD, false, 0, true));
2876 }
2877 
2878 void
vm_inject_gp(struct vm * vm,int vcpuid)2879 vm_inject_gp(struct vm *vm, int vcpuid)
2880 {
2881 	VERIFY0(vm_inject_exception(vm, vcpuid, IDT_GP, true, 0, true));
2882 }
2883 
2884 void
vm_inject_ac(struct vm * vm,int vcpuid,uint32_t errcode)2885 vm_inject_ac(struct vm *vm, int vcpuid, uint32_t errcode)
2886 {
2887 	VERIFY0(vm_inject_exception(vm, vcpuid, IDT_AC, true, errcode, true));
2888 }
2889 
2890 void
vm_inject_ss(struct vm * vm,int vcpuid,uint32_t errcode)2891 vm_inject_ss(struct vm *vm, int vcpuid, uint32_t errcode)
2892 {
2893 	VERIFY0(vm_inject_exception(vm, vcpuid, IDT_SS, true, errcode, true));
2894 }
2895 
2896 void
vm_inject_pf(struct vm * vm,int vcpuid,uint32_t errcode,uint64_t cr2)2897 vm_inject_pf(struct vm *vm, int vcpuid, uint32_t errcode, uint64_t cr2)
2898 {
2899 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_CR2, cr2));
2900 	VERIFY0(vm_inject_exception(vm, vcpuid, IDT_PF, true, errcode, true));
2901 }
2902 
2903 static VMM_STAT(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu");
2904 
2905 int
vm_inject_nmi(struct vm * vm,int vcpuid)2906 vm_inject_nmi(struct vm *vm, int vcpuid)
2907 {
2908 	struct vcpu *vcpu;
2909 
2910 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2911 		return (EINVAL);
2912 
2913 	vcpu = &vm->vcpu[vcpuid];
2914 
2915 	vcpu->nmi_pending = true;
2916 	vcpu_notify_event(vm, vcpuid);
2917 	return (0);
2918 }
2919 
2920 bool
vm_nmi_pending(struct vm * vm,int vcpuid)2921 vm_nmi_pending(struct vm *vm, int vcpuid)
2922 {
2923 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
2924 
2925 	return (vcpu->nmi_pending);
2926 }
2927 
2928 void
vm_nmi_clear(struct vm * vm,int vcpuid)2929 vm_nmi_clear(struct vm *vm, int vcpuid)
2930 {
2931 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
2932 
2933 	ASSERT(vcpu->nmi_pending);
2934 
2935 	vcpu->nmi_pending = false;
2936 	vmm_stat_incr(vm, vcpuid, VCPU_NMI_COUNT, 1);
2937 }
2938 
2939 static VMM_STAT(VCPU_EXTINT_COUNT, "number of ExtINTs delivered to vcpu");
2940 
2941 int
vm_inject_extint(struct vm * vm,int vcpuid)2942 vm_inject_extint(struct vm *vm, int vcpuid)
2943 {
2944 	struct vcpu *vcpu;
2945 
2946 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2947 		return (EINVAL);
2948 
2949 	vcpu = &vm->vcpu[vcpuid];
2950 
2951 	vcpu->extint_pending = true;
2952 	vcpu_notify_event(vm, vcpuid);
2953 	return (0);
2954 }
2955 
2956 bool
vm_extint_pending(struct vm * vm,int vcpuid)2957 vm_extint_pending(struct vm *vm, int vcpuid)
2958 {
2959 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
2960 
2961 	return (vcpu->extint_pending);
2962 }
2963 
2964 void
vm_extint_clear(struct vm * vm,int vcpuid)2965 vm_extint_clear(struct vm *vm, int vcpuid)
2966 {
2967 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
2968 
2969 	ASSERT(vcpu->extint_pending);
2970 
2971 	vcpu->extint_pending = false;
2972 	vmm_stat_incr(vm, vcpuid, VCPU_EXTINT_COUNT, 1);
2973 }
2974 
2975 int
vm_inject_init(struct vm * vm,int vcpuid)2976 vm_inject_init(struct vm *vm, int vcpuid)
2977 {
2978 	struct vcpu *vcpu;
2979 
2980 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2981 		return (EINVAL);
2982 
2983 	vcpu = &vm->vcpu[vcpuid];
2984 	vcpu_lock(vcpu);
2985 	vcpu->run_state |= VRS_PEND_INIT;
2986 	/*
2987 	 * As part of queuing the INIT request, clear any pending SIPI.  It
2988 	 * would not otherwise survive across the reset of the vCPU when it
2989 	 * undergoes the requested INIT.  We would not want it to linger when it
2990 	 * could be mistaken as a subsequent (after the INIT) SIPI request.
2991 	 */
2992 	vcpu->run_state &= ~VRS_PEND_SIPI;
2993 	vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
2994 
2995 	vcpu_unlock(vcpu);
2996 	return (0);
2997 }
2998 
2999 int
vm_inject_sipi(struct vm * vm,int vcpuid,uint8_t vector)3000 vm_inject_sipi(struct vm *vm, int vcpuid, uint8_t vector)
3001 {
3002 	struct vcpu *vcpu;
3003 
3004 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3005 		return (EINVAL);
3006 
3007 	vcpu = &vm->vcpu[vcpuid];
3008 	vcpu_lock(vcpu);
3009 	vcpu->run_state |= VRS_PEND_SIPI;
3010 	vcpu->sipi_vector = vector;
3011 	/* SIPI is only actionable if the CPU is waiting in INIT state */
3012 	if ((vcpu->run_state & (VRS_INIT | VRS_RUN)) == VRS_INIT) {
3013 		vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
3014 	}
3015 	vcpu_unlock(vcpu);
3016 	return (0);
3017 }
3018 
3019 bool
vcpu_run_state_pending(struct vm * vm,int vcpuid)3020 vcpu_run_state_pending(struct vm *vm, int vcpuid)
3021 {
3022 	struct vcpu *vcpu;
3023 
3024 	ASSERT(vcpuid >= 0 && vcpuid < vm->maxcpus);
3025 	vcpu = &vm->vcpu[vcpuid];
3026 
3027 	/* Of interest: vCPU not in running state or with pending INIT */
3028 	return ((vcpu->run_state & (VRS_RUN | VRS_PEND_INIT)) != VRS_RUN);
3029 }
3030 
3031 int
vcpu_arch_reset(struct vm * vm,int vcpuid,bool init_only)3032 vcpu_arch_reset(struct vm *vm, int vcpuid, bool init_only)
3033 {
3034 	struct seg_desc desc;
3035 	const enum vm_reg_name clear_regs[] = {
3036 		VM_REG_GUEST_CR2,
3037 		VM_REG_GUEST_CR3,
3038 		VM_REG_GUEST_CR4,
3039 		VM_REG_GUEST_RAX,
3040 		VM_REG_GUEST_RBX,
3041 		VM_REG_GUEST_RCX,
3042 		VM_REG_GUEST_RSI,
3043 		VM_REG_GUEST_RDI,
3044 		VM_REG_GUEST_RBP,
3045 		VM_REG_GUEST_RSP,
3046 		VM_REG_GUEST_R8,
3047 		VM_REG_GUEST_R9,
3048 		VM_REG_GUEST_R10,
3049 		VM_REG_GUEST_R11,
3050 		VM_REG_GUEST_R12,
3051 		VM_REG_GUEST_R13,
3052 		VM_REG_GUEST_R14,
3053 		VM_REG_GUEST_R15,
3054 		VM_REG_GUEST_DR0,
3055 		VM_REG_GUEST_DR1,
3056 		VM_REG_GUEST_DR2,
3057 		VM_REG_GUEST_DR3,
3058 		VM_REG_GUEST_EFER,
3059 	};
3060 	const enum vm_reg_name data_segs[] = {
3061 		VM_REG_GUEST_SS,
3062 		VM_REG_GUEST_DS,
3063 		VM_REG_GUEST_ES,
3064 		VM_REG_GUEST_FS,
3065 		VM_REG_GUEST_GS,
3066 	};
3067 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
3068 
3069 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3070 		return (EINVAL);
3071 
3072 	for (uint_t i = 0; i < nitems(clear_regs); i++) {
3073 		VERIFY0(vm_set_register(vm, vcpuid, clear_regs[i], 0));
3074 	}
3075 
3076 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_RFLAGS, 2));
3077 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_RIP, 0xfff0));
3078 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_CR0, 0x60000010));
3079 
3080 	/*
3081 	 * The prescribed contents of %rdx differ slightly between the Intel and
3082 	 * AMD architectural definitions.  The former expects the Extended Model
3083 	 * in bits 16-19 where the latter expects all the Family, Model, and
3084 	 * Stepping be there.  Common boot ROMs appear to disregard this
3085 	 * anyways, so we stick with a compromise value similar to what is
3086 	 * spelled out in the Intel SDM.
3087 	 */
3088 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_RDX, 0x600));
3089 
3090 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_DR6, 0xffff0ff0));
3091 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_DR7, 0x400));
3092 
3093 	/* CS: Present, R/W, Accessed */
3094 	desc.access = 0x0093;
3095 	desc.base = 0xffff0000;
3096 	desc.limit = 0xffff;
3097 	VERIFY0(vm_set_seg_desc(vm, vcpuid, VM_REG_GUEST_CS, &desc));
3098 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_CS, 0xf000));
3099 
3100 	/* SS, DS, ES, FS, GS: Present, R/W, Accessed */
3101 	desc.access = 0x0093;
3102 	desc.base = 0;
3103 	desc.limit = 0xffff;
3104 	for (uint_t i = 0; i < nitems(data_segs); i++) {
3105 		VERIFY0(vm_set_seg_desc(vm, vcpuid, data_segs[i], &desc));
3106 		VERIFY0(vm_set_register(vm, vcpuid, data_segs[i], 0));
3107 	}
3108 
3109 	/* GDTR, IDTR */
3110 	desc.base = 0;
3111 	desc.limit = 0xffff;
3112 	VERIFY0(vm_set_seg_desc(vm, vcpuid, VM_REG_GUEST_GDTR, &desc));
3113 	VERIFY0(vm_set_seg_desc(vm, vcpuid, VM_REG_GUEST_IDTR, &desc));
3114 
3115 	/* LDTR: Present, LDT */
3116 	desc.access = 0x0082;
3117 	desc.base = 0;
3118 	desc.limit = 0xffff;
3119 	VERIFY0(vm_set_seg_desc(vm, vcpuid, VM_REG_GUEST_LDTR, &desc));
3120 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_LDTR, 0));
3121 
3122 	/* TR: Present, 32-bit TSS */
3123 	desc.access = 0x008b;
3124 	desc.base = 0;
3125 	desc.limit = 0xffff;
3126 	VERIFY0(vm_set_seg_desc(vm, vcpuid, VM_REG_GUEST_TR, &desc));
3127 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_TR, 0));
3128 
3129 	vlapic_reset(vm_lapic(vm, vcpuid));
3130 
3131 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_INTR_SHADOW, 0));
3132 
3133 	vcpu->exit_intinfo = 0;
3134 	vcpu->exc_pending = 0;
3135 	vcpu->nmi_pending = false;
3136 	vcpu->extint_pending = 0;
3137 
3138 	/*
3139 	 * A CPU reset caused by power-on or system reset clears more state than
3140 	 * one which is trigged from an INIT IPI.
3141 	 */
3142 	if (!init_only) {
3143 		vcpu->guest_xcr0 = XFEATURE_ENABLED_X87;
3144 		(void) hma_fpu_init(vcpu->guestfpu);
3145 
3146 		/* XXX: clear MSRs and other pieces */
3147 		bzero(&vcpu->mtrr, sizeof (vcpu->mtrr));
3148 	}
3149 
3150 	return (0);
3151 }
3152 
3153 static int
vcpu_vector_sipi(struct vm * vm,int vcpuid,uint8_t vector)3154 vcpu_vector_sipi(struct vm *vm, int vcpuid, uint8_t vector)
3155 {
3156 	struct seg_desc desc;
3157 
3158 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3159 		return (EINVAL);
3160 
3161 	/* CS: Present, R/W, Accessed */
3162 	desc.access = 0x0093;
3163 	desc.base = (uint64_t)vector << 12;
3164 	desc.limit = 0xffff;
3165 	VERIFY0(vm_set_seg_desc(vm, vcpuid, VM_REG_GUEST_CS, &desc));
3166 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_CS,
3167 	    (uint64_t)vector << 8));
3168 
3169 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_RIP, 0));
3170 
3171 	return (0);
3172 }
3173 
3174 int
vm_get_capability(struct vm * vm,int vcpu,int type,int * retval)3175 vm_get_capability(struct vm *vm, int vcpu, int type, int *retval)
3176 {
3177 	if (vcpu < 0 || vcpu >= vm->maxcpus)
3178 		return (EINVAL);
3179 
3180 	if (type < 0 || type >= VM_CAP_MAX)
3181 		return (EINVAL);
3182 
3183 	return (VMGETCAP(vm->cookie, vcpu, type, retval));
3184 }
3185 
3186 int
vm_set_capability(struct vm * vm,int vcpu,int type,int val)3187 vm_set_capability(struct vm *vm, int vcpu, int type, int val)
3188 {
3189 	if (vcpu < 0 || vcpu >= vm->maxcpus)
3190 		return (EINVAL);
3191 
3192 	if (type < 0 || type >= VM_CAP_MAX)
3193 		return (EINVAL);
3194 
3195 	return (VMSETCAP(vm->cookie, vcpu, type, val));
3196 }
3197 
3198 vcpu_cpuid_config_t *
vm_cpuid_config(struct vm * vm,int vcpuid)3199 vm_cpuid_config(struct vm *vm, int vcpuid)
3200 {
3201 	ASSERT3S(vcpuid, >=, 0);
3202 	ASSERT3S(vcpuid, <, VM_MAXCPU);
3203 
3204 	return (&vm->vcpu[vcpuid].cpuid_cfg);
3205 }
3206 
3207 struct vlapic *
vm_lapic(struct vm * vm,int cpu)3208 vm_lapic(struct vm *vm, int cpu)
3209 {
3210 	ASSERT3S(cpu, >=, 0);
3211 	ASSERT3S(cpu, <, VM_MAXCPU);
3212 
3213 	return (vm->vcpu[cpu].vlapic);
3214 }
3215 
3216 struct vioapic *
vm_ioapic(struct vm * vm)3217 vm_ioapic(struct vm *vm)
3218 {
3219 
3220 	return (vm->vioapic);
3221 }
3222 
3223 struct vhpet *
vm_hpet(struct vm * vm)3224 vm_hpet(struct vm *vm)
3225 {
3226 
3227 	return (vm->vhpet);
3228 }
3229 
3230 void *
vm_iommu_domain(struct vm * vm)3231 vm_iommu_domain(struct vm *vm)
3232 {
3233 
3234 	return (vm->iommu);
3235 }
3236 
3237 int
vcpu_set_state(struct vm * vm,int vcpuid,enum vcpu_state newstate,bool from_idle)3238 vcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state newstate,
3239     bool from_idle)
3240 {
3241 	int error;
3242 	struct vcpu *vcpu;
3243 
3244 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3245 		panic("vcpu_set_state: invalid vcpuid %d", vcpuid);
3246 
3247 	vcpu = &vm->vcpu[vcpuid];
3248 
3249 	vcpu_lock(vcpu);
3250 	error = vcpu_set_state_locked(vm, vcpuid, newstate, from_idle);
3251 	vcpu_unlock(vcpu);
3252 
3253 	return (error);
3254 }
3255 
3256 enum vcpu_state
vcpu_get_state(struct vm * vm,int vcpuid,int * hostcpu)3257 vcpu_get_state(struct vm *vm, int vcpuid, int *hostcpu)
3258 {
3259 	struct vcpu *vcpu;
3260 	enum vcpu_state state;
3261 
3262 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3263 		panic("vcpu_get_state: invalid vcpuid %d", vcpuid);
3264 
3265 	vcpu = &vm->vcpu[vcpuid];
3266 
3267 	vcpu_lock(vcpu);
3268 	state = vcpu->state;
3269 	if (hostcpu != NULL)
3270 		*hostcpu = vcpu->hostcpu;
3271 	vcpu_unlock(vcpu);
3272 
3273 	return (state);
3274 }
3275 
3276 /*
3277  * Calculate the TSC offset for a vCPU, applying physical CPU adjustments if
3278  * requested. The offset calculations include the VM-wide TSC offset.
3279  */
3280 uint64_t
vcpu_tsc_offset(struct vm * vm,int vcpuid,bool phys_adj)3281 vcpu_tsc_offset(struct vm *vm, int vcpuid, bool phys_adj)
3282 {
3283 	ASSERT(vcpuid >= 0 && vcpuid < vm->maxcpus);
3284 
3285 	uint64_t vcpu_off = vm->tsc_offset + vm->vcpu[vcpuid].tsc_offset;
3286 
3287 	if (phys_adj) {
3288 		/* Include any offset for the current physical CPU too */
3289 		vcpu_off += vmm_host_tsc_delta();
3290 	}
3291 
3292 	return (vcpu_off);
3293 }
3294 
3295 uint64_t
vm_get_freq_multiplier(struct vm * vm)3296 vm_get_freq_multiplier(struct vm *vm)
3297 {
3298 	return (vm->freq_multiplier);
3299 }
3300 
3301 /* Normalize hrtime against the boot time for a VM */
3302 hrtime_t
vm_normalize_hrtime(struct vm * vm,hrtime_t hrt)3303 vm_normalize_hrtime(struct vm *vm, hrtime_t hrt)
3304 {
3305 	/* To avoid underflow/overflow UB, perform math as unsigned */
3306 	return ((hrtime_t)((uint64_t)hrt - (uint64_t)vm->boot_hrtime));
3307 }
3308 
3309 /* Denormalize hrtime against the boot time for a VM */
3310 hrtime_t
vm_denormalize_hrtime(struct vm * vm,hrtime_t hrt)3311 vm_denormalize_hrtime(struct vm *vm, hrtime_t hrt)
3312 {
3313 	/* To avoid underflow/overflow UB, perform math as unsigned */
3314 	return ((hrtime_t)((uint64_t)hrt + (uint64_t)vm->boot_hrtime));
3315 }
3316 
3317 int
vm_activate_cpu(struct vm * vm,int vcpuid)3318 vm_activate_cpu(struct vm *vm, int vcpuid)
3319 {
3320 
3321 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3322 		return (EINVAL);
3323 
3324 	if (CPU_ISSET(vcpuid, &vm->active_cpus))
3325 		return (EBUSY);
3326 
3327 	if (vm_is_suspended(vm, NULL)) {
3328 		return (EBUSY);
3329 	}
3330 
3331 	CPU_SET_ATOMIC(vcpuid, &vm->active_cpus);
3332 
3333 	/*
3334 	 * It is possible that this vCPU was undergoing activation at the same
3335 	 * time that the VM was being suspended.
3336 	 */
3337 	if (vm_is_suspended(vm, NULL)) {
3338 		return (EBUSY);
3339 	}
3340 
3341 	return (0);
3342 }
3343 
3344 int
vm_suspend_cpu(struct vm * vm,int vcpuid)3345 vm_suspend_cpu(struct vm *vm, int vcpuid)
3346 {
3347 	int i;
3348 
3349 	if (vcpuid < -1 || vcpuid >= vm->maxcpus)
3350 		return (EINVAL);
3351 
3352 	if (vcpuid == -1) {
3353 		vm->debug_cpus = vm->active_cpus;
3354 		for (i = 0; i < vm->maxcpus; i++) {
3355 			if (CPU_ISSET(i, &vm->active_cpus))
3356 				vcpu_notify_event(vm, i);
3357 		}
3358 	} else {
3359 		if (!CPU_ISSET(vcpuid, &vm->active_cpus))
3360 			return (EINVAL);
3361 
3362 		CPU_SET_ATOMIC(vcpuid, &vm->debug_cpus);
3363 		vcpu_notify_event(vm, vcpuid);
3364 	}
3365 	return (0);
3366 }
3367 
3368 int
vm_resume_cpu(struct vm * vm,int vcpuid)3369 vm_resume_cpu(struct vm *vm, int vcpuid)
3370 {
3371 
3372 	if (vcpuid < -1 || vcpuid >= vm->maxcpus)
3373 		return (EINVAL);
3374 
3375 	if (vcpuid == -1) {
3376 		CPU_ZERO(&vm->debug_cpus);
3377 	} else {
3378 		if (!CPU_ISSET(vcpuid, &vm->debug_cpus))
3379 			return (EINVAL);
3380 
3381 		CPU_CLR_ATOMIC(vcpuid, &vm->debug_cpus);
3382 	}
3383 	return (0);
3384 }
3385 
3386 static bool
vcpu_bailout_checks(struct vm * vm,int vcpuid)3387 vcpu_bailout_checks(struct vm *vm, int vcpuid)
3388 {
3389 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
3390 	struct vm_exit *vme = &vcpu->exitinfo;
3391 
3392 	ASSERT(vcpuid >= 0 && vcpuid < vm->maxcpus);
3393 
3394 	/*
3395 	 * Check if VM is suspended, only passing the 'vm_exit *' to be
3396 	 * populated if this check is being performed as part of entry.
3397 	 */
3398 	if (vm_is_suspended(vm, vme)) {
3399 		/* Confirm exit details are as expected */
3400 		VERIFY3S(vme->exitcode, ==, VM_EXITCODE_SUSPENDED);
3401 		VERIFY(vme->u.suspended.how > VM_SUSPEND_NONE &&
3402 		    vme->u.suspended.how < VM_SUSPEND_LAST);
3403 
3404 		return (true);
3405 	}
3406 	if (vcpu->reqidle) {
3407 		/*
3408 		 * Another thread is trying to lock this vCPU and is waiting for
3409 		 * it to enter the VCPU_IDLE state.  Take a lap with a BOGUS
3410 		 * exit to allow other thread(s) access to this vCPU.
3411 		 */
3412 		vme->exitcode = VM_EXITCODE_BOGUS;
3413 		vmm_stat_incr(vm, vcpuid, VMEXIT_REQIDLE, 1);
3414 		return (true);
3415 	}
3416 	if (vcpu->reqbarrier) {
3417 		/*
3418 		 * Similar to 'reqidle', userspace has requested that this vCPU
3419 		 * be pushed to a barrier by exiting to userspace.  Take that
3420 		 * lap with BOGUS and clear the flag.
3421 		 */
3422 		vme->exitcode = VM_EXITCODE_BOGUS;
3423 		vcpu->reqbarrier = false;
3424 		return (true);
3425 	}
3426 	if (vcpu->reqconsist) {
3427 		/*
3428 		 * We only expect exit-when-consistent requests to be asserted
3429 		 * during entry, not as an otherwise spontaneous condition.  As
3430 		 * such, we do not count it among the exit statistics, and emit
3431 		 * the expected BOGUS exitcode, while clearing the request.
3432 		 */
3433 		vme->exitcode = VM_EXITCODE_BOGUS;
3434 		vcpu->reqconsist = false;
3435 		return (true);
3436 	}
3437 	if (vcpu_should_yield(vm, vcpuid)) {
3438 		vme->exitcode = VM_EXITCODE_BOGUS;
3439 		vmm_stat_incr(vm, vcpuid, VMEXIT_ASTPENDING, 1);
3440 		return (true);
3441 	}
3442 	if (CPU_ISSET(vcpuid, &vm->debug_cpus)) {
3443 		vme->exitcode = VM_EXITCODE_DEBUG;
3444 		return (true);
3445 	}
3446 
3447 	return (false);
3448 }
3449 
3450 static bool
vcpu_sleep_bailout_checks(struct vm * vm,int vcpuid)3451 vcpu_sleep_bailout_checks(struct vm *vm, int vcpuid)
3452 {
3453 	if (vcpu_bailout_checks(vm, vcpuid)) {
3454 		struct vcpu *vcpu = &vm->vcpu[vcpuid];
3455 		struct vm_exit *vme = &vcpu->exitinfo;
3456 
3457 		/*
3458 		 * Bail-out check done prior to sleeping (in vCPU contexts like
3459 		 * HLT or wait-for-SIPI) expect that %rip is already populated
3460 		 * in the vm_exit structure, and we would only modify the
3461 		 * exitcode and clear the inst_length.
3462 		 */
3463 		vme->inst_length = 0;
3464 		return (true);
3465 	}
3466 	return (false);
3467 }
3468 
3469 bool
vcpu_entry_bailout_checks(struct vm * vm,int vcpuid,uint64_t rip)3470 vcpu_entry_bailout_checks(struct vm *vm, int vcpuid, uint64_t rip)
3471 {
3472 	if (vcpu_bailout_checks(vm, vcpuid)) {
3473 		struct vcpu *vcpu = &vm->vcpu[vcpuid];
3474 		struct vm_exit *vme = &vcpu->exitinfo;
3475 
3476 		/*
3477 		 * Bail-out checks done as part of VM entry require an updated
3478 		 * %rip to populate the vm_exit struct if any of the conditions
3479 		 * of interest are matched in the check.
3480 		 */
3481 		vme->rip = rip;
3482 		vme->inst_length = 0;
3483 		return (true);
3484 	}
3485 	return (false);
3486 }
3487 
3488 int
vm_vcpu_barrier(struct vm * vm,int vcpuid)3489 vm_vcpu_barrier(struct vm *vm, int vcpuid)
3490 {
3491 	if (vcpuid >= 0 && vcpuid < vm->maxcpus) {
3492 		struct vcpu *vcpu = &vm->vcpu[vcpuid];
3493 
3494 		/* Push specified vCPU to barrier */
3495 		vcpu_lock(vcpu);
3496 		if (CPU_ISSET(vcpuid, &vm->active_cpus)) {
3497 			vcpu->reqbarrier = true;
3498 			vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
3499 		}
3500 		vcpu_unlock(vcpu);
3501 
3502 		return (0);
3503 	} else if (vcpuid == -1) {
3504 		/* Push all (active) vCPUs to barrier */
3505 		for (int i = 0; i < vm->maxcpus; i++) {
3506 			struct vcpu *vcpu = &vm->vcpu[i];
3507 
3508 			vcpu_lock(vcpu);
3509 			if (CPU_ISSET(vcpuid, &vm->active_cpus)) {
3510 				vcpu->reqbarrier = true;
3511 				vcpu_notify_event_locked(vcpu,
3512 				    VCPU_NOTIFY_EXIT);
3513 			}
3514 			vcpu_unlock(vcpu);
3515 		}
3516 
3517 		return (0);
3518 	} else {
3519 		return (EINVAL);
3520 	}
3521 }
3522 
3523 cpuset_t
vm_active_cpus(struct vm * vm)3524 vm_active_cpus(struct vm *vm)
3525 {
3526 	return (vm->active_cpus);
3527 }
3528 
3529 cpuset_t
vm_debug_cpus(struct vm * vm)3530 vm_debug_cpus(struct vm *vm)
3531 {
3532 	return (vm->debug_cpus);
3533 }
3534 
3535 void *
vcpu_stats(struct vm * vm,int vcpuid)3536 vcpu_stats(struct vm *vm, int vcpuid)
3537 {
3538 
3539 	return (vm->vcpu[vcpuid].stats);
3540 }
3541 
3542 int
vm_get_x2apic_state(struct vm * vm,int vcpuid,enum x2apic_state * state)3543 vm_get_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state *state)
3544 {
3545 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3546 		return (EINVAL);
3547 
3548 	*state = vm->vcpu[vcpuid].x2apic_state;
3549 
3550 	return (0);
3551 }
3552 
3553 int
vm_set_x2apic_state(struct vm * vm,int vcpuid,enum x2apic_state state)3554 vm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
3555 {
3556 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3557 		return (EINVAL);
3558 
3559 	if (state >= X2APIC_STATE_LAST)
3560 		return (EINVAL);
3561 
3562 	vm->vcpu[vcpuid].x2apic_state = state;
3563 
3564 	vlapic_set_x2apic_state(vm, vcpuid, state);
3565 
3566 	return (0);
3567 }
3568 
3569 /*
3570  * This function is called to ensure that a vcpu "sees" a pending event
3571  * as soon as possible:
3572  * - If the vcpu thread is sleeping then it is woken up.
3573  * - If the vcpu is running on a different host_cpu then an IPI will be directed
3574  *   to the host_cpu to cause the vcpu to trap into the hypervisor.
3575  */
3576 static void
vcpu_notify_event_locked(struct vcpu * vcpu,vcpu_notify_t ntype)3577 vcpu_notify_event_locked(struct vcpu *vcpu, vcpu_notify_t ntype)
3578 {
3579 	int hostcpu;
3580 
3581 	ASSERT(ntype == VCPU_NOTIFY_APIC || VCPU_NOTIFY_EXIT);
3582 
3583 	hostcpu = vcpu->hostcpu;
3584 	if (vcpu->state == VCPU_RUNNING) {
3585 		KASSERT(hostcpu != NOCPU, ("vcpu running on invalid hostcpu"));
3586 		if (hostcpu != curcpu) {
3587 			if (ntype == VCPU_NOTIFY_APIC) {
3588 				vlapic_post_intr(vcpu->vlapic, hostcpu);
3589 			} else {
3590 				poke_cpu(hostcpu);
3591 			}
3592 		} else {
3593 			/*
3594 			 * If the 'vcpu' is running on 'curcpu' then it must
3595 			 * be sending a notification to itself (e.g. SELF_IPI).
3596 			 * The pending event will be picked up when the vcpu
3597 			 * transitions back to guest context.
3598 			 */
3599 		}
3600 	} else {
3601 		KASSERT(hostcpu == NOCPU, ("vcpu state %d not consistent "
3602 		    "with hostcpu %d", vcpu->state, hostcpu));
3603 		if (vcpu->state == VCPU_SLEEPING) {
3604 			cv_signal(&vcpu->vcpu_cv);
3605 		}
3606 	}
3607 }
3608 
3609 void
vcpu_notify_event(struct vm * vm,int vcpuid)3610 vcpu_notify_event(struct vm *vm, int vcpuid)
3611 {
3612 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
3613 
3614 	vcpu_lock(vcpu);
3615 	vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
3616 	vcpu_unlock(vcpu);
3617 }
3618 
3619 void
vcpu_notify_event_type(struct vm * vm,int vcpuid,vcpu_notify_t ntype)3620 vcpu_notify_event_type(struct vm *vm, int vcpuid, vcpu_notify_t ntype)
3621 {
3622 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
3623 
3624 	if (ntype == VCPU_NOTIFY_NONE) {
3625 		return;
3626 	}
3627 
3628 	vcpu_lock(vcpu);
3629 	vcpu_notify_event_locked(vcpu, ntype);
3630 	vcpu_unlock(vcpu);
3631 }
3632 
3633 void
vcpu_ustate_change(struct vm * vm,int vcpuid,enum vcpu_ustate ustate)3634 vcpu_ustate_change(struct vm *vm, int vcpuid, enum vcpu_ustate ustate)
3635 {
3636 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
3637 	const hrtime_t now = gethrtime();
3638 
3639 	ASSERT3S(ustate, <, VU_MAX);
3640 	ASSERT3S(ustate, >=, VU_INIT);
3641 
3642 	if (ustate == vcpu->ustate) {
3643 		return;
3644 	}
3645 
3646 	const hrtime_t delta = now - vcpu->ustate_when;
3647 	vcpu->ustate_total[vcpu->ustate] += delta;
3648 
3649 	membar_producer();
3650 
3651 	vcpu->ustate_when = now;
3652 	vcpu->ustate = ustate;
3653 }
3654 
3655 void *
vm_get_cookie(struct vm * vm)3656 vm_get_cookie(struct vm *vm)
3657 {
3658 	return (vm->cookie);
3659 }
3660 
3661 struct vmspace *
vm_get_vmspace(struct vm * vm)3662 vm_get_vmspace(struct vm *vm)
3663 {
3664 
3665 	return (vm->vmspace);
3666 }
3667 
3668 struct vm_client *
vm_get_vmclient(struct vm * vm,int vcpuid)3669 vm_get_vmclient(struct vm *vm, int vcpuid)
3670 {
3671 	return (vm->vcpu[vcpuid].vmclient);
3672 }
3673 
3674 int
vm_apicid2vcpuid(struct vm * vm,int apicid)3675 vm_apicid2vcpuid(struct vm *vm, int apicid)
3676 {
3677 	/*
3678 	 * XXX apic id is assumed to be numerically identical to vcpu id
3679 	 */
3680 	return (apicid);
3681 }
3682 
3683 struct vatpic *
vm_atpic(struct vm * vm)3684 vm_atpic(struct vm *vm)
3685 {
3686 	return (vm->vatpic);
3687 }
3688 
3689 struct vatpit *
vm_atpit(struct vm * vm)3690 vm_atpit(struct vm *vm)
3691 {
3692 	return (vm->vatpit);
3693 }
3694 
3695 struct vpmtmr *
vm_pmtmr(struct vm * vm)3696 vm_pmtmr(struct vm *vm)
3697 {
3698 
3699 	return (vm->vpmtmr);
3700 }
3701 
3702 struct vrtc *
vm_rtc(struct vm * vm)3703 vm_rtc(struct vm *vm)
3704 {
3705 
3706 	return (vm->vrtc);
3707 }
3708 
3709 enum vm_reg_name
vm_segment_name(int seg)3710 vm_segment_name(int seg)
3711 {
3712 	static enum vm_reg_name seg_names[] = {
3713 		VM_REG_GUEST_ES,
3714 		VM_REG_GUEST_CS,
3715 		VM_REG_GUEST_SS,
3716 		VM_REG_GUEST_DS,
3717 		VM_REG_GUEST_FS,
3718 		VM_REG_GUEST_GS
3719 	};
3720 
3721 	KASSERT(seg >= 0 && seg < nitems(seg_names),
3722 	    ("%s: invalid segment encoding %d", __func__, seg));
3723 	return (seg_names[seg]);
3724 }
3725 
3726 void
vm_copy_teardown(struct vm * vm,int vcpuid,struct vm_copyinfo * copyinfo,uint_t num_copyinfo)3727 vm_copy_teardown(struct vm *vm, int vcpuid, struct vm_copyinfo *copyinfo,
3728     uint_t num_copyinfo)
3729 {
3730 	for (uint_t idx = 0; idx < num_copyinfo; idx++) {
3731 		if (copyinfo[idx].cookie != NULL) {
3732 			(void) vmp_release((vm_page_t *)copyinfo[idx].cookie);
3733 		}
3734 	}
3735 	bzero(copyinfo, num_copyinfo * sizeof (struct vm_copyinfo));
3736 }
3737 
3738 int
vm_copy_setup(struct vm * vm,int vcpuid,struct vm_guest_paging * paging,uint64_t gla,size_t len,int prot,struct vm_copyinfo * copyinfo,uint_t num_copyinfo,int * fault)3739 vm_copy_setup(struct vm *vm, int vcpuid, struct vm_guest_paging *paging,
3740     uint64_t gla, size_t len, int prot, struct vm_copyinfo *copyinfo,
3741     uint_t num_copyinfo, int *fault)
3742 {
3743 	uint_t idx, nused;
3744 	size_t n, off, remaining;
3745 	vm_client_t *vmc = vm_get_vmclient(vm, vcpuid);
3746 
3747 	bzero(copyinfo, sizeof (struct vm_copyinfo) * num_copyinfo);
3748 
3749 	nused = 0;
3750 	remaining = len;
3751 	while (remaining > 0) {
3752 		uint64_t gpa;
3753 		int error;
3754 
3755 		if (nused >= num_copyinfo)
3756 			return (EFAULT);
3757 		error = vm_gla2gpa(vm, vcpuid, paging, gla, prot, &gpa, fault);
3758 		if (error || *fault)
3759 			return (error);
3760 		off = gpa & PAGEOFFSET;
3761 		n = min(remaining, PAGESIZE - off);
3762 		copyinfo[nused].gpa = gpa;
3763 		copyinfo[nused].len = n;
3764 		remaining -= n;
3765 		gla += n;
3766 		nused++;
3767 	}
3768 
3769 	for (idx = 0; idx < nused; idx++) {
3770 		vm_page_t *vmp;
3771 		caddr_t hva;
3772 
3773 		vmp = vmc_hold(vmc, copyinfo[idx].gpa & PAGEMASK, prot);
3774 		if (vmp == NULL) {
3775 			break;
3776 		}
3777 		if ((prot & PROT_WRITE) != 0) {
3778 			hva = (caddr_t)vmp_get_writable(vmp);
3779 		} else {
3780 			hva = (caddr_t)vmp_get_readable(vmp);
3781 		}
3782 		copyinfo[idx].hva = hva + (copyinfo[idx].gpa & PAGEOFFSET);
3783 		copyinfo[idx].cookie = vmp;
3784 		copyinfo[idx].prot = prot;
3785 	}
3786 
3787 	if (idx != nused) {
3788 		vm_copy_teardown(vm, vcpuid, copyinfo, num_copyinfo);
3789 		return (EFAULT);
3790 	} else {
3791 		*fault = 0;
3792 		return (0);
3793 	}
3794 }
3795 
3796 void
vm_copyin(struct vm * vm,int vcpuid,struct vm_copyinfo * copyinfo,void * kaddr,size_t len)3797 vm_copyin(struct vm *vm, int vcpuid, struct vm_copyinfo *copyinfo, void *kaddr,
3798     size_t len)
3799 {
3800 	char *dst;
3801 	int idx;
3802 
3803 	dst = kaddr;
3804 	idx = 0;
3805 	while (len > 0) {
3806 		ASSERT(copyinfo[idx].prot & PROT_READ);
3807 
3808 		bcopy(copyinfo[idx].hva, dst, copyinfo[idx].len);
3809 		len -= copyinfo[idx].len;
3810 		dst += copyinfo[idx].len;
3811 		idx++;
3812 	}
3813 }
3814 
3815 void
vm_copyout(struct vm * vm,int vcpuid,const void * kaddr,struct vm_copyinfo * copyinfo,size_t len)3816 vm_copyout(struct vm *vm, int vcpuid, const void *kaddr,
3817     struct vm_copyinfo *copyinfo, size_t len)
3818 {
3819 	const char *src;
3820 	int idx;
3821 
3822 	src = kaddr;
3823 	idx = 0;
3824 	while (len > 0) {
3825 		ASSERT(copyinfo[idx].prot & PROT_WRITE);
3826 
3827 		bcopy(src, copyinfo[idx].hva, copyinfo[idx].len);
3828 		len -= copyinfo[idx].len;
3829 		src += copyinfo[idx].len;
3830 		idx++;
3831 	}
3832 }
3833 
3834 /*
3835  * Return the amount of in-use and wired memory for the VM. Since
3836  * these are global stats, only return the values with for vCPU 0
3837  */
3838 VMM_STAT_DECLARE(VMM_MEM_RESIDENT);
3839 
3840 static void
vm_get_rescnt(struct vm * vm,int vcpu,struct vmm_stat_type * stat)3841 vm_get_rescnt(struct vm *vm, int vcpu, struct vmm_stat_type *stat)
3842 {
3843 	if (vcpu == 0) {
3844 		vmm_stat_set(vm, vcpu, VMM_MEM_RESIDENT,
3845 		    PAGE_SIZE * vmspace_resident_count(vm->vmspace));
3846 	}
3847 }
3848 
3849 VMM_STAT_FUNC(VMM_MEM_RESIDENT, "Resident memory", vm_get_rescnt);
3850 
3851 int
vm_ioport_access(struct vm * vm,int vcpuid,bool in,uint16_t port,uint8_t bytes,uint32_t * val)3852 vm_ioport_access(struct vm *vm, int vcpuid, bool in, uint16_t port,
3853     uint8_t bytes, uint32_t *val)
3854 {
3855 	return (vm_inout_access(&vm->ioports, in, port, bytes, val));
3856 }
3857 
3858 /*
3859  * bhyve-internal interfaces to attach or detach IO port handlers.
3860  * Must be called with VM write lock held for safety.
3861  */
3862 int
vm_ioport_attach(struct vm * vm,uint16_t port,ioport_handler_t func,void * arg,void ** cookie)3863 vm_ioport_attach(struct vm *vm, uint16_t port, ioport_handler_t func, void *arg,
3864     void **cookie)
3865 {
3866 	int err;
3867 	err = vm_inout_attach(&vm->ioports, port, IOPF_DEFAULT, func, arg);
3868 	if (err == 0) {
3869 		*cookie = (void *)IOP_GEN_COOKIE(func, arg, port);
3870 	}
3871 	return (err);
3872 }
3873 int
vm_ioport_detach(struct vm * vm,void ** cookie,ioport_handler_t * old_func,void ** old_arg)3874 vm_ioport_detach(struct vm *vm, void **cookie, ioport_handler_t *old_func,
3875     void **old_arg)
3876 {
3877 	uint16_t port = IOP_PORT_FROM_COOKIE((uintptr_t)*cookie);
3878 	int err;
3879 
3880 	err = vm_inout_detach(&vm->ioports, port, false, old_func, old_arg);
3881 	if (err == 0) {
3882 		*cookie = NULL;
3883 	}
3884 	return (err);
3885 }
3886 
3887 /*
3888  * External driver interfaces to attach or detach IO port handlers.
3889  * Must be called with VM write lock held for safety.
3890  */
3891 int
vm_ioport_hook(struct vm * vm,uint16_t port,ioport_handler_t func,void * arg,void ** cookie)3892 vm_ioport_hook(struct vm *vm, uint16_t port, ioport_handler_t func,
3893     void *arg, void **cookie)
3894 {
3895 	int err;
3896 
3897 	if (port == 0) {
3898 		return (EINVAL);
3899 	}
3900 
3901 	err = vm_inout_attach(&vm->ioports, port, IOPF_DRV_HOOK, func, arg);
3902 	if (err == 0) {
3903 		*cookie = (void *)IOP_GEN_COOKIE(func, arg, port);
3904 	}
3905 	return (err);
3906 }
3907 void
vm_ioport_unhook(struct vm * vm,void ** cookie)3908 vm_ioport_unhook(struct vm *vm, void **cookie)
3909 {
3910 	uint16_t port = IOP_PORT_FROM_COOKIE((uintptr_t)*cookie);
3911 	ioport_handler_t old_func;
3912 	void *old_arg;
3913 	int err;
3914 
3915 	err = vm_inout_detach(&vm->ioports, port, true, &old_func, &old_arg);
3916 
3917 	/* ioport-hook-using drivers are expected to be well-behaved */
3918 	VERIFY0(err);
3919 	VERIFY(IOP_GEN_COOKIE(old_func, old_arg, port) == (uintptr_t)*cookie);
3920 
3921 	*cookie = NULL;
3922 }
3923 
3924 int
vmm_kstat_update_vcpu(struct kstat * ksp,int rw)3925 vmm_kstat_update_vcpu(struct kstat *ksp, int rw)
3926 {
3927 	struct vm *vm = ksp->ks_private;
3928 	vmm_vcpu_kstats_t *vvk = ksp->ks_data;
3929 	const int vcpuid = vvk->vvk_vcpu.value.ui32;
3930 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
3931 
3932 	ASSERT3U(vcpuid, <, VM_MAXCPU);
3933 
3934 	vvk->vvk_time_init.value.ui64 = vcpu->ustate_total[VU_INIT];
3935 	vvk->vvk_time_run.value.ui64 = vcpu->ustate_total[VU_RUN];
3936 	vvk->vvk_time_idle.value.ui64 = vcpu->ustate_total[VU_IDLE];
3937 	vvk->vvk_time_emu_kern.value.ui64 = vcpu->ustate_total[VU_EMU_KERN];
3938 	vvk->vvk_time_emu_user.value.ui64 = vcpu->ustate_total[VU_EMU_USER];
3939 	vvk->vvk_time_sched.value.ui64 = vcpu->ustate_total[VU_SCHED];
3940 
3941 	return (0);
3942 }
3943 
3944 SET_DECLARE(vmm_data_version_entries, const vmm_data_version_entry_t);
3945 
3946 static int
vmm_data_find(const vmm_data_req_t * req,const vmm_data_version_entry_t ** resp)3947 vmm_data_find(const vmm_data_req_t *req, const vmm_data_version_entry_t **resp)
3948 {
3949 	const vmm_data_version_entry_t **vdpp, *vdp;
3950 
3951 	ASSERT(resp != NULL);
3952 	ASSERT(req->vdr_result_len != NULL);
3953 
3954 	SET_FOREACH(vdpp, vmm_data_version_entries) {
3955 		vdp = *vdpp;
3956 		if (vdp->vdve_class != req->vdr_class ||
3957 		    vdp->vdve_version != req->vdr_version) {
3958 			continue;
3959 		}
3960 
3961 		/*
3962 		 * Enforce any data length expectation expressed by the provider
3963 		 * for this data.
3964 		 */
3965 		if (vdp->vdve_len_expect != 0 &&
3966 		    vdp->vdve_len_expect > req->vdr_len) {
3967 			*req->vdr_result_len = vdp->vdve_len_expect;
3968 			return (ENOSPC);
3969 		}
3970 
3971 		/*
3972 		 * Make sure that the provided vcpuid is acceptable for the
3973 		 * backend handler.
3974 		 */
3975 		if (vdp->vdve_readf != NULL || vdp->vdve_writef != NULL) {
3976 			/*
3977 			 * While it is tempting to demand the -1 sentinel value
3978 			 * in vcpuid here, that expectation was not established
3979 			 * for early consumers, so it is ignored.
3980 			 */
3981 		} else if (vdp->vdve_vcpu_readf != NULL ||
3982 		    vdp->vdve_vcpu_writef != NULL) {
3983 			/*
3984 			 * Per-vCPU handlers which permit "wildcard" access will
3985 			 * accept a vcpuid of -1 (for VM-wide data), while all
3986 			 * others expect vcpuid [0, VM_MAXCPU).
3987 			 */
3988 			const int llimit = vdp->vdve_vcpu_wildcard ? -1 : 0;
3989 			if (req->vdr_vcpuid < llimit ||
3990 			    req->vdr_vcpuid >= VM_MAXCPU) {
3991 				return (EINVAL);
3992 			}
3993 		} else {
3994 			/*
3995 			 * A provider with neither VM-wide nor per-vCPU handlers
3996 			 * is completely unexpected.  Such a situation should be
3997 			 * made into a compile-time error.  Bail out for now,
3998 			 * rather than punishing the user with a panic.
3999 			 */
4000 			return (EINVAL);
4001 		}
4002 
4003 
4004 		*resp = vdp;
4005 		return (0);
4006 	}
4007 	return (EINVAL);
4008 }
4009 
4010 static void *
vmm_data_from_class(const vmm_data_req_t * req,struct vm * vm)4011 vmm_data_from_class(const vmm_data_req_t *req, struct vm *vm)
4012 {
4013 	switch (req->vdr_class) {
4014 	case VDC_REGISTER:
4015 	case VDC_MSR:
4016 	case VDC_FPU:
4017 	case VDC_LAPIC:
4018 	case VDC_VMM_ARCH:
4019 		/*
4020 		 * These have per-CPU handling which is dispatched outside
4021 		 * vmm_data_version_entries listing.
4022 		 */
4023 		panic("Unexpected per-vcpu class %u", req->vdr_class);
4024 		break;
4025 
4026 	case VDC_IOAPIC:
4027 		return (vm->vioapic);
4028 	case VDC_ATPIT:
4029 		return (vm->vatpit);
4030 	case VDC_ATPIC:
4031 		return (vm->vatpic);
4032 	case VDC_HPET:
4033 		return (vm->vhpet);
4034 	case VDC_PM_TIMER:
4035 		return (vm->vpmtmr);
4036 	case VDC_RTC:
4037 		return (vm->vrtc);
4038 	case VDC_VMM_TIME:
4039 		return (vm);
4040 	case VDC_VERSION:
4041 		/*
4042 		 * Play along with all of the other classes which need backup
4043 		 * data, even though version info does not require it.
4044 		 */
4045 		return (vm);
4046 
4047 	default:
4048 		/* The data class will have been validated by now */
4049 		panic("Unexpected class %u", req->vdr_class);
4050 	}
4051 }
4052 
4053 const uint32_t default_msr_iter[] = {
4054 	/*
4055 	 * Although EFER is also available via the get/set-register interface,
4056 	 * we include it in the default list of emitted MSRs.
4057 	 */
4058 	MSR_EFER,
4059 
4060 	/*
4061 	 * While gsbase and fsbase are accessible via the MSR accessors, they
4062 	 * are not included in MSR iteration since they are covered by the
4063 	 * segment descriptor interface too.
4064 	 */
4065 	MSR_KGSBASE,
4066 
4067 	MSR_STAR,
4068 	MSR_LSTAR,
4069 	MSR_CSTAR,
4070 	MSR_SF_MASK,
4071 
4072 	MSR_SYSENTER_CS_MSR,
4073 	MSR_SYSENTER_ESP_MSR,
4074 	MSR_SYSENTER_EIP_MSR,
4075 
4076 	MSR_PAT,
4077 
4078 	MSR_TSC,
4079 
4080 	MSR_MTRRcap,
4081 	MSR_MTRRdefType,
4082 	MSR_MTRR4kBase, MSR_MTRR4kBase + 1, MSR_MTRR4kBase + 2,
4083 	MSR_MTRR4kBase + 3, MSR_MTRR4kBase + 4, MSR_MTRR4kBase + 5,
4084 	MSR_MTRR4kBase + 6, MSR_MTRR4kBase + 7,
4085 	MSR_MTRR16kBase, MSR_MTRR16kBase + 1,
4086 	MSR_MTRR64kBase,
4087 };
4088 
4089 static int
vmm_data_read_msr(struct vm * vm,int vcpuid,uint32_t msr,uint64_t * value)4090 vmm_data_read_msr(struct vm *vm, int vcpuid, uint32_t msr, uint64_t *value)
4091 {
4092 	int err = 0;
4093 
4094 	switch (msr) {
4095 	case MSR_TSC:
4096 		/*
4097 		 * The vmm-data interface for MSRs provides access to the
4098 		 * per-vCPU offset of the TSC, when reading/writing MSR_TSC.
4099 		 *
4100 		 * The VM-wide offset (and scaling) of the guest TSC is accessed
4101 		 * via the VMM_TIME data class.
4102 		 */
4103 		*value = vm->vcpu[vcpuid].tsc_offset;
4104 		return (0);
4105 
4106 	default:
4107 		if (is_mtrr_msr(msr)) {
4108 			err = vm_rdmtrr(&vm->vcpu[vcpuid].mtrr, msr, value);
4109 		} else {
4110 			err = ops->vmgetmsr(vm->cookie, vcpuid, msr, value);
4111 		}
4112 		break;
4113 	}
4114 
4115 	return (err);
4116 }
4117 
4118 static int
vmm_data_write_msr(struct vm * vm,int vcpuid,uint32_t msr,uint64_t value)4119 vmm_data_write_msr(struct vm *vm, int vcpuid, uint32_t msr, uint64_t value)
4120 {
4121 	int err = 0;
4122 
4123 	switch (msr) {
4124 	case MSR_TSC:
4125 		/* See vmm_data_read_msr() for more detail */
4126 		vm->vcpu[vcpuid].tsc_offset = value;
4127 		return (0);
4128 	case MSR_MTRRcap: {
4129 		/*
4130 		 * MTRRcap is read-only.  If the desired value matches the
4131 		 * existing one, consider it a success.
4132 		 */
4133 		uint64_t comp;
4134 		err = vm_rdmtrr(&vm->vcpu[vcpuid].mtrr, msr, &comp);
4135 		if (err == 0 && comp != value) {
4136 			return (EINVAL);
4137 		}
4138 		break;
4139 	}
4140 	default:
4141 		if (is_mtrr_msr(msr)) {
4142 			/* MTRRcap is already handled above */
4143 			ASSERT3U(msr, !=, MSR_MTRRcap);
4144 
4145 			err = vm_wrmtrr(&vm->vcpu[vcpuid].mtrr, msr, value);
4146 		} else {
4147 			err = ops->vmsetmsr(vm->cookie, vcpuid, msr, value);
4148 		}
4149 		break;
4150 	}
4151 
4152 	return (err);
4153 }
4154 
4155 static int
vmm_data_read_msrs(struct vm * vm,int vcpuid,const vmm_data_req_t * req)4156 vmm_data_read_msrs(struct vm *vm, int vcpuid, const vmm_data_req_t *req)
4157 {
4158 	VERIFY3U(req->vdr_class, ==, VDC_MSR);
4159 	VERIFY3U(req->vdr_version, ==, 1);
4160 
4161 	struct vdi_field_entry_v1 *entryp = req->vdr_data;
4162 
4163 	/* Specific MSRs requested */
4164 	if ((req->vdr_flags & VDX_FLAG_READ_COPYIN) != 0) {
4165 		const uint_t count =
4166 		    req->vdr_len / sizeof (struct vdi_field_entry_v1);
4167 
4168 		for (uint_t i = 0; i < count; i++, entryp++) {
4169 			int err = vmm_data_read_msr(vm, vcpuid,
4170 			    entryp->vfe_ident, &entryp->vfe_value);
4171 
4172 			if (err != 0) {
4173 				return (err);
4174 			}
4175 		}
4176 
4177 		*req->vdr_result_len =
4178 		    count * sizeof (struct vdi_field_entry_v1);
4179 		return (0);
4180 	}
4181 
4182 	/*
4183 	 * If specific MSRs are not requested, try to provide all those which we
4184 	 * know about instead.
4185 	 */
4186 	const uint_t num_msrs = nitems(default_msr_iter) +
4187 	    (VMM_MTRR_VAR_MAX * 2);
4188 	const uint32_t output_len =
4189 	    num_msrs * sizeof (struct vdi_field_entry_v1);
4190 
4191 	*req->vdr_result_len = output_len;
4192 	if (req->vdr_len < output_len) {
4193 		return (ENOSPC);
4194 	}
4195 
4196 	/* Output the MSRs in the default list */
4197 	for (uint_t i = 0; i < nitems(default_msr_iter); i++, entryp++) {
4198 		entryp->vfe_ident = default_msr_iter[i];
4199 
4200 		/* All of these MSRs are expected to work */
4201 		VERIFY0(vmm_data_read_msr(vm, vcpuid, entryp->vfe_ident,
4202 		    &entryp->vfe_value));
4203 	}
4204 
4205 	/* Output the variable MTRRs */
4206 	for (uint_t i = 0; i < (VMM_MTRR_VAR_MAX * 2); i++, entryp++) {
4207 		entryp->vfe_ident = MSR_MTRRVarBase + i;
4208 
4209 		/* All of these MSRs are expected to work */
4210 		VERIFY0(vmm_data_read_msr(vm, vcpuid, entryp->vfe_ident,
4211 		    &entryp->vfe_value));
4212 	}
4213 	return (0);
4214 }
4215 
4216 static int
vmm_data_write_msrs(struct vm * vm,int vcpuid,const vmm_data_req_t * req)4217 vmm_data_write_msrs(struct vm *vm, int vcpuid, const vmm_data_req_t *req)
4218 {
4219 	VERIFY3U(req->vdr_class, ==, VDC_MSR);
4220 	VERIFY3U(req->vdr_version, ==, 1);
4221 
4222 	const struct vdi_field_entry_v1 *entryp = req->vdr_data;
4223 	const uint_t entry_count =
4224 	    req->vdr_len / sizeof (struct vdi_field_entry_v1);
4225 
4226 	/*
4227 	 * First make sure that all of the MSRs can be manipulated.
4228 	 * For now, this check is done by going though the getmsr handler
4229 	 */
4230 	for (uint_t i = 0; i < entry_count; i++, entryp++) {
4231 		const uint64_t msr = entryp->vfe_ident;
4232 		uint64_t val;
4233 
4234 		if (vmm_data_read_msr(vm, vcpuid, msr, &val) != 0) {
4235 			return (EINVAL);
4236 		}
4237 	}
4238 
4239 	/*
4240 	 * Fairly confident that all of the 'set' operations are at least
4241 	 * targeting valid MSRs, continue on.
4242 	 */
4243 	entryp = req->vdr_data;
4244 	for (uint_t i = 0; i < entry_count; i++, entryp++) {
4245 		int err = vmm_data_write_msr(vm, vcpuid, entryp->vfe_ident,
4246 		    entryp->vfe_value);
4247 
4248 		if (err != 0) {
4249 			return (err);
4250 		}
4251 	}
4252 	*req->vdr_result_len = entry_count * sizeof (struct vdi_field_entry_v1);
4253 
4254 	return (0);
4255 }
4256 
4257 static const vmm_data_version_entry_t msr_v1 = {
4258 	.vdve_class = VDC_MSR,
4259 	.vdve_version = 1,
4260 	.vdve_len_per_item = sizeof (struct vdi_field_entry_v1),
4261 	.vdve_vcpu_readf = vmm_data_read_msrs,
4262 	.vdve_vcpu_writef = vmm_data_write_msrs,
4263 };
4264 VMM_DATA_VERSION(msr_v1);
4265 
4266 static const uint32_t vmm_arch_v1_fields[] = {
4267 	VAI_VM_IS_PAUSED,
4268 };
4269 
4270 static const uint32_t vmm_arch_v1_vcpu_fields[] = {
4271 	VAI_PEND_NMI,
4272 	VAI_PEND_EXTINT,
4273 	VAI_PEND_EXCP,
4274 	VAI_PEND_INTINFO,
4275 };
4276 
4277 static bool
vmm_read_arch_field(struct vm * vm,int vcpuid,uint32_t ident,uint64_t * valp)4278 vmm_read_arch_field(struct vm *vm, int vcpuid, uint32_t ident, uint64_t *valp)
4279 {
4280 	ASSERT(valp != NULL);
4281 
4282 	if (vcpuid == -1) {
4283 		switch (ident) {
4284 		case VAI_VM_IS_PAUSED:
4285 			*valp = vm->is_paused ? 1 : 0;
4286 			return (true);
4287 		default:
4288 			break;
4289 		}
4290 	} else {
4291 		VERIFY(vcpuid >= 0 && vcpuid <= VM_MAXCPU);
4292 
4293 		struct vcpu *vcpu = &vm->vcpu[vcpuid];
4294 		switch (ident) {
4295 		case VAI_PEND_NMI:
4296 			*valp = vcpu->nmi_pending != 0 ? 1 : 0;
4297 			return (true);
4298 		case VAI_PEND_EXTINT:
4299 			*valp = vcpu->extint_pending != 0 ? 1 : 0;
4300 			return (true);
4301 		case VAI_PEND_EXCP:
4302 			*valp = vcpu->exc_pending;
4303 			return (true);
4304 		case VAI_PEND_INTINFO:
4305 			*valp = vcpu->exit_intinfo;
4306 			return (true);
4307 		default:
4308 			break;
4309 		}
4310 	}
4311 	return (false);
4312 }
4313 
4314 static int
vmm_data_read_varch(struct vm * vm,int vcpuid,const vmm_data_req_t * req)4315 vmm_data_read_varch(struct vm *vm, int vcpuid, const vmm_data_req_t *req)
4316 {
4317 	VERIFY3U(req->vdr_class, ==, VDC_VMM_ARCH);
4318 	VERIFY3U(req->vdr_version, ==, 1);
4319 
4320 	/* per-vCPU fields are handled separately from VM-wide ones */
4321 	if (vcpuid != -1 && (vcpuid < 0 || vcpuid >= VM_MAXCPU)) {
4322 		return (EINVAL);
4323 	}
4324 
4325 	struct vdi_field_entry_v1 *entryp = req->vdr_data;
4326 
4327 	/* Specific fields requested */
4328 	if ((req->vdr_flags & VDX_FLAG_READ_COPYIN) != 0) {
4329 		const uint_t count =
4330 		    req->vdr_len / sizeof (struct vdi_field_entry_v1);
4331 
4332 		for (uint_t i = 0; i < count; i++, entryp++) {
4333 			if (!vmm_read_arch_field(vm, vcpuid, entryp->vfe_ident,
4334 			    &entryp->vfe_value)) {
4335 				return (EINVAL);
4336 			}
4337 		}
4338 		*req->vdr_result_len =
4339 		    count * sizeof (struct vdi_field_entry_v1);
4340 		return (0);
4341 	}
4342 
4343 	/* Emit all of the possible values */
4344 	const uint32_t *idents;
4345 	uint_t ident_count;
4346 
4347 	if (vcpuid == -1) {
4348 		idents = vmm_arch_v1_fields;
4349 		ident_count = nitems(vmm_arch_v1_fields);
4350 	} else {
4351 		idents = vmm_arch_v1_vcpu_fields;
4352 		ident_count = nitems(vmm_arch_v1_vcpu_fields);
4353 
4354 	}
4355 
4356 	const uint32_t total_size =
4357 	    ident_count * sizeof (struct vdi_field_entry_v1);
4358 
4359 	*req->vdr_result_len = total_size;
4360 	if (req->vdr_len < total_size) {
4361 		return (ENOSPC);
4362 	}
4363 	for (uint_t i = 0; i < ident_count; i++, entryp++) {
4364 		entryp->vfe_ident = idents[i];
4365 		VERIFY(vmm_read_arch_field(vm, vcpuid, entryp->vfe_ident,
4366 		    &entryp->vfe_value));
4367 	}
4368 	return (0);
4369 }
4370 
4371 static int
vmm_data_write_varch_vcpu(struct vm * vm,int vcpuid,const vmm_data_req_t * req)4372 vmm_data_write_varch_vcpu(struct vm *vm, int vcpuid, const vmm_data_req_t *req)
4373 {
4374 	VERIFY3U(req->vdr_class, ==, VDC_VMM_ARCH);
4375 	VERIFY3U(req->vdr_version, ==, 1);
4376 
4377 	if (vcpuid < 0 || vcpuid >= VM_MAXCPU) {
4378 		return (EINVAL);
4379 	}
4380 
4381 	const struct vdi_field_entry_v1 *entryp = req->vdr_data;
4382 	const uint_t entry_count =
4383 	    req->vdr_len / sizeof (struct vdi_field_entry_v1);
4384 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
4385 
4386 	for (uint_t i = 0; i < entry_count; i++, entryp++) {
4387 		const uint64_t val = entryp->vfe_value;
4388 
4389 		switch (entryp->vfe_ident) {
4390 		case VAI_PEND_NMI:
4391 			vcpu->nmi_pending = (val != 0);
4392 			break;
4393 		case VAI_PEND_EXTINT:
4394 			vcpu->extint_pending = (val != 0);
4395 			break;
4396 		case VAI_PEND_EXCP:
4397 			if (!VM_INTINFO_PENDING(val)) {
4398 				vcpu->exc_pending = 0;
4399 			} else if (VM_INTINFO_TYPE(val) != VM_INTINFO_HWEXCP ||
4400 			    (val & VM_INTINFO_MASK_RSVD) != 0) {
4401 				/* reject improperly-formed hw exception */
4402 				return (EINVAL);
4403 			} else {
4404 				vcpu->exc_pending = val;
4405 			}
4406 			break;
4407 		case VAI_PEND_INTINFO:
4408 			if (vm_exit_intinfo(vm, vcpuid, val) != 0) {
4409 				return (EINVAL);
4410 			}
4411 			break;
4412 		default:
4413 			return (EINVAL);
4414 		}
4415 	}
4416 
4417 	*req->vdr_result_len = entry_count * sizeof (struct vdi_field_entry_v1);
4418 	return (0);
4419 }
4420 
4421 static int
vmm_data_write_varch(struct vm * vm,int vcpuid,const vmm_data_req_t * req)4422 vmm_data_write_varch(struct vm *vm, int vcpuid, const vmm_data_req_t *req)
4423 {
4424 	VERIFY3U(req->vdr_class, ==, VDC_VMM_ARCH);
4425 	VERIFY3U(req->vdr_version, ==, 1);
4426 
4427 	/* per-vCPU fields are handled separately from VM-wide ones */
4428 	if (vcpuid != -1) {
4429 		return (vmm_data_write_varch_vcpu(vm, vcpuid, req));
4430 	}
4431 
4432 	const struct vdi_field_entry_v1 *entryp = req->vdr_data;
4433 	const uint_t entry_count =
4434 	    req->vdr_len / sizeof (struct vdi_field_entry_v1);
4435 
4436 	if (entry_count > 0) {
4437 		if (entryp->vfe_ident == VAI_VM_IS_PAUSED) {
4438 			/*
4439 			 * The VM_PAUSE and VM_RESUME ioctls are the officially
4440 			 * sanctioned mechanisms for setting the is-paused state
4441 			 * of the VM.
4442 			 */
4443 			return (EPERM);
4444 		} else {
4445 			/* no other valid arch entries at this time */
4446 			return (EINVAL);
4447 		}
4448 	}
4449 
4450 	*req->vdr_result_len = entry_count * sizeof (struct vdi_field_entry_v1);
4451 	return (0);
4452 }
4453 
4454 static const vmm_data_version_entry_t vmm_arch_v1 = {
4455 	.vdve_class = VDC_VMM_ARCH,
4456 	.vdve_version = 1,
4457 	.vdve_len_per_item = sizeof (struct vdi_field_entry_v1),
4458 	.vdve_vcpu_readf = vmm_data_read_varch,
4459 	.vdve_vcpu_writef = vmm_data_write_varch,
4460 
4461 	/*
4462 	 * Handlers for VMM_ARCH can process VM-wide (vcpuid == -1) entries in
4463 	 * addition to vCPU specific ones.
4464 	 */
4465 	.vdve_vcpu_wildcard = true,
4466 };
4467 VMM_DATA_VERSION(vmm_arch_v1);
4468 
4469 
4470 /*
4471  * GUEST TIME SUPPORT
4472  *
4473  * Broadly, there are two categories of functionality related to time passing in
4474  * the guest: the guest's TSC and timers used by emulated devices.
4475  *
4476  * ---------------------------
4477  * GUEST TSC "VIRTUALIZATION"
4478  * ---------------------------
4479  *
4480  * The TSC can be read either via an instruction (rdtsc/rdtscp) or by reading
4481  * the TSC MSR.
4482  *
4483  * When a guest reads the TSC via its MSR, the guest will exit and we emulate
4484  * the rdmsr. More typically, the guest reads the TSC via a rdtsc(p)
4485  * instruction. Both SVM and VMX support virtualizing the guest TSC in hardware
4486  * -- that is, a guest will not generally exit on a rdtsc instruction.
4487  *
4488  * To support hardware-virtualized guest TSC, both SVM and VMX provide two knobs
4489  * for the hypervisor to adjust the guest's view of the TSC:
4490  * - TSC offset
4491  * - TSC frequency multiplier (also called "frequency ratio")
4492  *
4493  * When a guest calls rdtsc(p), the TSC value it sees is the sum of:
4494  *     guest_tsc = (host TSC, scaled according to frequency multiplier)
4495  *		    + (TSC offset, programmed by hypervisor)
4496  *
4497  * See the discussions of the TSC offset and frequency multiplier below for more
4498  * details on each of these.
4499  *
4500  * --------------------
4501  * TSC OFFSET OVERVIEW
4502  * --------------------
4503  *
4504  * The TSC offset is a value added to the host TSC (which may be scaled first)
4505  * to provide the guest TSC. This offset addition is generally done by hardware,
4506  * but may be used in emulating the TSC if necessary.
4507  *
4508  * Recall that general formula for calculating the guest TSC is:
4509  *
4510  *	guest_tsc = (host TSC, scaled if needed) + TSC offset
4511  *
4512  * Intuitively, the TSC offset is simply an offset of the host's TSC to make the
4513  * guest's view of the TSC appear correct: The guest TSC should be 0 at boot and
4514  * monotonically increase at a roughly constant frequency. Thus in the simplest
4515  * case, the TSC offset is just the negated value of the host TSC when the guest
4516  * was booted, assuming they have the same frequencies.
4517  *
4518  * In practice, there are several factors that can make calculating the TSC
4519  * offset more complicated, including:
4520  *
4521  * (1) the physical CPU the guest is running on
4522  * (2) whether the guest has written to the TSC of that vCPU
4523  * (3) differing host and guest frequencies, like after a live migration
4524  * (4) a guest running on a different system than where it was booted, like
4525  *     after a live migration
4526  *
4527  * We will explore each of these factors individually. See below for a
4528  * summary.
4529  *
4530  *
4531  * (1) Physical CPU offsets
4532  *
4533  * The system maintains a set of per-CPU offsets to the TSC to provide a
4534  * consistent view of the TSC regardless of the CPU a thread is running on.
4535  * These offsets are included automatically as a part of rdtsc_offset().
4536  *
4537  * The per-CPU offset must be included as a part reading the host TSC when
4538  * calculating the offset before running the guest on a given CPU.
4539  *
4540  *
4541  * (2) Guest TSC writes (vCPU offsets)
4542  *
4543  * The TSC is a writable MSR. When a guest writes to the TSC, this operation
4544  * should result in the TSC, when read from that vCPU, shows the value written,
4545  * plus whatever time has elapsed since the read.
4546  *
4547  * To support this, when the guest writes to the TSC, we store an additional
4548  * vCPU offset calculated to make future reads of the TSC map to what the guest
4549  * expects.
4550  *
4551  *
4552  * (3) Differing host and guest frequencies (host TSC scaling)
4553  *
4554  * A guest has the same frequency of its host when it boots, but it may be
4555  * migrated to a machine with a different TSC frequency. Systems expect that
4556  * their TSC frequency does not change. To support this fiction in which a guest
4557  * is running on hardware of a different TSC frequency, the hypervisor  can
4558  * program a "frequency multiplier" that represents the ratio of guest/host
4559  * frequency.
4560  *
4561  * Any time a host TSC is used in calculations for the offset, it should be
4562  * "scaled" according to this multiplier, and the hypervisor should program the
4563  * multiplier before running a guest so that the hardware virtualization of the
4564  * TSC functions properly. Similarly, the multiplier should be used in any TSC
4565  * emulation.
4566  *
4567  * See below for more details about the frequency multiplier.
4568  *
4569  *
4570  * (4) Guest running on a system it did not boot on ("base guest TSC")
4571  *
4572  * When a guest boots, its TSC offset is simply the negated host TSC at the time
4573  * it booted. If a guest is migrated from a source host to a target host, the
4574  * TSC offset from the source host is no longer useful for several reasons:
4575  * - the target host TSC has no relationship to the source host TSC
4576  * - the guest did not boot on the target system, so the TSC of the target host
4577  *   is not sufficient to describe how long the guest has been running prior to
4578  *   migration
4579  * - the target system may have a different TSC frequency than the source system
4580  *
4581  * Ignoring the issue of frequency differences for a moment, let's consider how
4582  * to re-align the guest TSC with the host TSC of the target host. Intuitively,
4583  * for the guest to see the correct TSC, we still want to add some offset to the
4584  * host TSC that offsets how long this guest has been running on
4585  * the system.
4586  *
4587  * An example here might be helpful. Consider a source host and target host,
4588  * both with TSC frequencies of 1GHz. On the source host, the guest and host TSC
4589  * values might look like:
4590  *
4591  *  +----------------------------------------------------------------------+
4592  *  | Event                 | source host TSC  | guest TSC                 |
4593  *  ------------------------------------------------------------------------
4594  *  | guest boot  (t=0s)    | 5000000000       | 5000000000 + -5000000000  |
4595  *  |                       |                  | 0			   |
4596  *  ------------------------------------------------------------------------
4597  *  | guest rdtsc (t=10s))  | 15000000000      | 15000000000 + -5000000000 |
4598  *  |                       |                  | 10000000000		   |
4599  *  ------------------------------------------------------------------------
4600  *  | migration   (t=15s)   | 20000000000      | 20000000000 + -5000000000 |
4601  *  |                       |                  | 15000000000		   |
4602  *  +----------------------------------------------------------------------+
4603  *
4604  * Ignoring the time it takes for a guest to physically migrate machines, on the
4605  * target host, we would expect the TSC to continue functioning as such:
4606  *
4607  *  +----------------------------------------------------------------------+
4608  *  | Event                 | target host TSC  | guest TSC                 |
4609  *  ------------------------------------------------------------------------
4610  *  | guest migrate (t=15s) | 300000000000     | 15000000000		   |
4611  *  ------------------------------------------------------------------------
4612  *  | guest rdtsc (t=20s))  | 305000000000     | 20000000000		   |
4613  *  ------------------------------------------------------------------------
4614  *
4615  * In order to produce a correct TSC value here, we can calculate a new
4616  * "effective" boot TSC that maps to what the host TSC would've been had it been
4617  * booted on the target. We add that to the guest TSC when it began to run on
4618  * this machine, and negate them both to get a new offset. In this example, the
4619  * effective boot TSC is: -(300000000000 - 15000000000) = -285000000000.
4620  *
4621  *  +-------------------------------------------------------------------------+
4622  *  | Event                 | target host TSC  | guest TSC                    |
4623  *  ---------------------------------------------------------------------------
4624  *  | guest "boot" (t=0s)   | 285000000000     | 285000000000 + -285000000000 |
4625  *  |                       |                  | 0			      |
4626  *  ---------------------------------------------------------------------------
4627  *  | guest migrate (t=15s) | 300000000000     | 300000000000 + -285000000000 |
4628  *  |                       |                  | 15000000000		      |
4629  *  ---------------------------------------------------------------------------
4630  *  | guest rdtsc (t=20s))  | 305000000000     | 305000000000 + -285000000000 |
4631  *  |                       |                  | 20000000000		      |
4632  *  --------------------------------------------------------------------------+
4633  *
4634  * To support the offset calculation following a migration, the VMM data time
4635  * interface allows callers to set a "base guest TSC", which is the TSC value of
4636  * the guest when it began running on the host. The current guest TSC can be
4637  * requested via a read of the time data. See below for details on that
4638  * interface.
4639  *
4640  * Frequency differences between the host and the guest are accounted for when
4641  * scaling the host TSC. See below for details on the frequency multiplier.
4642  *
4643  *
4644  * --------------------
4645  * TSC OFFSET SUMMARY
4646  * --------------------
4647  *
4648  * Factoring in all of the components to the TSC above, the TSC offset that is
4649  * programmed by the hypervisor before running a given vCPU is:
4650  *
4651  * offset = -((base host TSC, scaled if needed) - base_guest_tsc) + vCPU offset
4652  *
4653  * This offset is stored in two pieces. Per-vCPU offsets are stored with the
4654  * given vCPU and added in when programming the offset. The rest of the offset
4655  * is stored as a VM-wide offset, and computed either at boot or when the time
4656  * data is written to.
4657  *
4658  * It is safe to add the vCPU offset and the VM-wide offsets together because
4659  * the vCPU offset is in terms of the guest TSC. The host TSC is scaled before
4660  * using it in calculations, so all TSC values are applicable to the same
4661  * frequency.
4662  *
4663  * Note: Though both the VM-wide offset and per-vCPU offsets may be negative, we
4664  * store them as unsigned values and perform all offsetting math unsigned. This
4665  * is to avoid UB from signed overflow.
4666  *
4667  * -------------------------
4668  * TSC FREQUENCY MULTIPLIER
4669  * -------------------------
4670  *
4671  * In order to account for frequency differences between the host and guest, SVM
4672  * and VMX provide an interface to set a "frequency multiplier" (or "frequency
4673  * ratio") representing guest to host frequency. In a hardware-virtualized read
4674  * of the TSC, the host TSC is scaled using this multiplier prior to adding the
4675  * programmed TSC offset.
4676  *
4677  * Both platforms represent the ratio as a fixed point number, where the lower
4678  * bits are used as a fractional component, and some number of the upper bits
4679  * are used as the integer component.
4680  *
4681  * Some example multipliers, for a platform with FRAC fractional bits in the
4682  * multiplier:
4683  * - guest frequency == host: 1 << FRAC
4684  * - guest frequency is 2x host: 1 << (FRAC + 1)
4685  * - guest frequency is 0.5x host: 1 << (FRAC - 1), as the highest-order
4686  *   fractional bit represents 1/2
4687  * - guest frequency is 2.5x host: (1 << FRAC) | (1 << (FRAC - 1))
4688  * and so on.
4689  *
4690  * In general, the frequency multiplier is calculated as follows:
4691  *		(guest_hz * (1 << FRAC_SIZE)) / host_hz
4692  *
4693  * The multiplier should be used any time the host TSC value is used in
4694  * calculations with the guest TSC (and their frequencies differ). The function
4695  * `vmm_scale_tsc` is intended to be used for these purposes, as it will scale
4696  * the host TSC only if needed.
4697  *
4698  * The multiplier should also be programmed by the hypervisor before the guest
4699  * is run.
4700  *
4701  *
4702  * ----------------------------
4703  * DEVICE TIMERS (BOOT_HRTIME)
4704  * ----------------------------
4705  *
4706  * Emulated devices use timers to do things such as scheduling periodic events.
4707  * These timers are scheduled relative to the hrtime of the host. When device
4708  * state is exported or imported, we use boot_hrtime to normalize these timers
4709  * against the host hrtime. The boot_hrtime represents the hrtime of the host
4710  * when the guest was booted.
4711  *
4712  * If a guest is migrated to a different machine, boot_hrtime must be adjusted
4713  * to match the hrtime of when the guest was effectively booted on the target
4714  * host. This allows timers to continue functioning when device state is
4715  * imported on the target.
4716  *
4717  *
4718  * ------------------------
4719  * VMM DATA TIME INTERFACE
4720  * ------------------------
4721  *
4722  * In order to facilitate live migrations of guests, we provide an interface,
4723  * via the VMM data read/write ioctls, for userspace to make changes to the
4724  * guest's view of the TSC and device timers, allowing these features to
4725  * continue functioning after a migration.
4726  *
4727  * The interface was designed to expose the minimal amount of data needed for a
4728  * userspace component to make adjustments to the guest's view of time (e.g., to
4729  * account for time passing in a live migration). At a minimum, such a program
4730  * needs:
4731  * - the current guest TSC
4732  * - guest TSC frequency
4733  * - guest's boot_hrtime
4734  * - timestamps of when this data was taken (hrtime for hrtime calculations, and
4735  *   wall clock time for computing time deltas between machines)
4736  *
4737  * The wall clock time is provided for consumers to make adjustments to the
4738  * guest TSC and boot_hrtime based on deltas observed during migrations. It may
4739  * be prudent for consumers to use this data only in circumstances where the
4740  * source and target have well-synchronized wall clocks, but nothing in the
4741  * interface depends on this assumption.
4742  *
4743  * On writes, consumers write back:
4744  * - the base guest TSC (used for TSC offset calculations)
4745  * - desired boot_hrtime
4746  * - guest_frequency (cannot change)
4747  * - hrtime of when this data was adjusted
4748  * - (wall clock time on writes is ignored)
4749  *
4750  * The interface will adjust the input guest TSC slightly, based on the input
4751  * hrtime, to account for latency between userspace calculations and application
4752  * of the data on the kernel side. This amounts to adding a small amount of
4753  * additional "uptime" for the guest.
4754  *
4755  * After the adjustments, the interface updates the VM-wide TSC offset and
4756  * boot_hrtime. Per-vCPU offsets are not adjusted, as those are already in terms
4757  * of the guest TSC and can be exported/imported via the MSR VMM data interface.
4758  *
4759  *
4760  * --------------------------------
4761  * SUPPORTED PLATFORMS AND CAVEATS
4762  * --------------------------------
4763  *
4764  * While both VMX and SVM offer TSC scaling as a feature, at this time only SVM
4765  * is supported by bhyve.
4766  *
4767  * The time data interface is designed such that Intel support can be added
4768  * easily, and all other aspects of the time interface should work on Intel.
4769  * (Without frequency control though, in practice, doing live migrations of
4770  * guests on Intel will not work for time-related things, as two machines
4771  * rarely have exactly the same frequency).
4772  *
4773  * Additionally, while on both SVM and VMX the frequency multiplier is a fixed
4774  * point number, each uses a different number of fractional and integer bits for
4775  * the multiplier. As such, calculating the multiplier and fractional bit size
4776  * is requested via the vmm_ops.
4777  *
4778  * Care should be taken to set reasonable limits for ratios based on the
4779  * platform, as the difference in fractional bits can lead to slightly different
4780  * tradeoffs in terms of representable ratios and potentially overflowing
4781  * calculations.
4782  */
4783 
4784 /*
4785  * Scales the TSC if needed, based on the input frequency multiplier.
4786  */
4787 static uint64_t
vmm_scale_tsc(uint64_t tsc,uint64_t mult)4788 vmm_scale_tsc(uint64_t tsc, uint64_t mult)
4789 {
4790 	const uint32_t frac_size = ops->fr_fracsize;
4791 
4792 	if (mult != VM_TSCM_NOSCALE) {
4793 		VERIFY3U(frac_size, >, 0);
4794 		return (scale_tsc(tsc, mult, frac_size));
4795 	} else {
4796 		return (tsc);
4797 	}
4798 }
4799 
4800 /*
4801  * Calculate the frequency multiplier, which represents the ratio of
4802  * guest_hz / host_hz. The frequency multiplier is a fixed point number with
4803  * `frac_sz` fractional bits (fractional bits begin at bit 0).
4804  *
4805  * See comment for "calc_freq_multiplier" in "vmm_time_support.S" for more
4806  * information about valid input to this function.
4807  */
4808 uint64_t
vmm_calc_freq_multiplier(uint64_t guest_hz,uint64_t host_hz,uint32_t frac_size)4809 vmm_calc_freq_multiplier(uint64_t guest_hz, uint64_t host_hz,
4810     uint32_t frac_size)
4811 {
4812 	VERIFY3U(guest_hz, !=, 0);
4813 	VERIFY3U(frac_size, >, 0);
4814 	VERIFY3U(frac_size, <, 64);
4815 
4816 	return (calc_freq_multiplier(guest_hz, host_hz, frac_size));
4817 }
4818 
4819 /*
4820  * Calculate the guest VM-wide TSC offset.
4821  *
4822  * offset = - ((base host TSC, scaled if needed) - base_guest_tsc)
4823  *
4824  * The base_host_tsc and the base_guest_tsc are the TSC values of the host
4825  * (read on the system) and the guest (calculated) at the same point in time.
4826  * This allows us to fix the guest TSC at this point in time as a base, either
4827  * following boot (guest TSC = 0), or a change to the guest's time data from
4828  * userspace (such as in the case of a migration).
4829  */
4830 static uint64_t
calc_tsc_offset(uint64_t base_host_tsc,uint64_t base_guest_tsc,uint64_t mult)4831 calc_tsc_offset(uint64_t base_host_tsc, uint64_t base_guest_tsc, uint64_t mult)
4832 {
4833 	const uint64_t htsc_scaled = vmm_scale_tsc(base_host_tsc, mult);
4834 	if (htsc_scaled > base_guest_tsc) {
4835 		return ((uint64_t)(- (int64_t)(htsc_scaled - base_guest_tsc)));
4836 	} else {
4837 		return (base_guest_tsc - htsc_scaled);
4838 	}
4839 }
4840 
4841 /*
4842  * Calculate an estimate of the guest TSC.
4843  *
4844  * guest_tsc = (host TSC, scaled if needed) + offset
4845  */
4846 static uint64_t
calc_guest_tsc(uint64_t host_tsc,uint64_t mult,uint64_t offset)4847 calc_guest_tsc(uint64_t host_tsc, uint64_t mult, uint64_t offset)
4848 {
4849 	return (vmm_scale_tsc(host_tsc, mult) + offset);
4850 }
4851 
4852 /*
4853  * Take a non-atomic "snapshot" of the current:
4854  * - TSC
4855  * - hrtime
4856  * - wall clock time
4857  */
4858 static void
vmm_time_snapshot(uint64_t * tsc,hrtime_t * hrtime,timespec_t * hrestime)4859 vmm_time_snapshot(uint64_t *tsc, hrtime_t *hrtime, timespec_t *hrestime)
4860 {
4861 	/*
4862 	 * Disable interrupts while we take the readings: In the absence of a
4863 	 * mechanism to convert hrtime to hrestime, we want the time between
4864 	 * each of these measurements to be as small as possible.
4865 	 */
4866 	ulong_t iflag = intr_clear();
4867 
4868 	hrtime_t hrt = gethrtimeunscaledf();
4869 	*tsc = (uint64_t)hrt;
4870 	*hrtime = hrt;
4871 	scalehrtime(hrtime);
4872 	gethrestime(hrestime);
4873 
4874 	intr_restore(iflag);
4875 }
4876 
4877 /*
4878  * Read VMM Time data
4879  *
4880  * Provides:
4881  * - the current guest TSC and TSC frequency
4882  * - guest boot_hrtime
4883  * - timestamps of the read (hrtime and wall clock time)
4884  */
4885 static int
vmm_data_read_vmm_time(void * arg,const vmm_data_req_t * req)4886 vmm_data_read_vmm_time(void *arg, const vmm_data_req_t *req)
4887 {
4888 	VERIFY3U(req->vdr_class, ==, VDC_VMM_TIME);
4889 	VERIFY3U(req->vdr_version, ==, 1);
4890 	VERIFY3U(req->vdr_len, >=, sizeof (struct vdi_time_info_v1));
4891 
4892 	struct vm *vm = arg;
4893 	struct vdi_time_info_v1 *out = req->vdr_data;
4894 
4895 	/*
4896 	 * Since write operations on VMM_TIME data are strict about vcpuid
4897 	 * (see: vmm_data_write_vmm_time()), read operations should be as well.
4898 	 */
4899 	if (req->vdr_vcpuid != -1) {
4900 		return (EINVAL);
4901 	}
4902 
4903 	/* Take a snapshot of this point in time */
4904 	uint64_t tsc;
4905 	hrtime_t hrtime;
4906 	timespec_t hrestime;
4907 	vmm_time_snapshot(&tsc, &hrtime, &hrestime);
4908 
4909 	/* Write the output values */
4910 	out->vt_guest_freq = vm->guest_freq;
4911 
4912 	/*
4913 	 * Use only the VM-wide TSC offset for calculating the guest TSC,
4914 	 * ignoring per-vCPU offsets. This value is provided as a "base" guest
4915 	 * TSC at the time of the read; per-vCPU offsets are factored in as
4916 	 * needed elsewhere, either when running the vCPU or if the guest reads
4917 	 * the TSC via rdmsr.
4918 	 */
4919 	out->vt_guest_tsc = calc_guest_tsc(tsc, vm->freq_multiplier,
4920 	    vm->tsc_offset);
4921 	out->vt_boot_hrtime = vm->boot_hrtime;
4922 	out->vt_hrtime = hrtime;
4923 	out->vt_hres_sec = hrestime.tv_sec;
4924 	out->vt_hres_ns = hrestime.tv_nsec;
4925 
4926 	return (0);
4927 }
4928 
4929 /*
4930  * Modify VMM Time data related values
4931  *
4932  * This interface serves to allow guests' TSC and device timers to continue
4933  * functioning across live migrations. On a successful write, the VM-wide TSC
4934  * offset and boot_hrtime of the guest are updated.
4935  *
4936  * The interface requires an hrtime of the system at which the caller wrote
4937  * this data; this allows us to adjust the TSC and boot_hrtime slightly to
4938  * account for time passing between the userspace call and application
4939  * of the data here.
4940  *
4941  * There are several possibilities for invalid input, including:
4942  * - a requested guest frequency of 0, or a frequency otherwise unsupported by
4943  *   the underlying platform
4944  * - hrtime or boot_hrtime values that appear to be from the future
4945  * - the requested frequency does not match the host, and this system does not
4946  *   have hardware TSC scaling support
4947  */
4948 static int
vmm_data_write_vmm_time(void * arg,const vmm_data_req_t * req)4949 vmm_data_write_vmm_time(void *arg, const vmm_data_req_t *req)
4950 {
4951 	VERIFY3U(req->vdr_class, ==, VDC_VMM_TIME);
4952 	VERIFY3U(req->vdr_version, ==, 1);
4953 	VERIFY3U(req->vdr_len, >=, sizeof (struct vdi_time_info_v1));
4954 
4955 	struct vm *vm = arg;
4956 	const struct vdi_time_info_v1 *src = req->vdr_data;
4957 
4958 	/*
4959 	 * While vcpuid values != -1 are tolerated by the vmm_data machinery for
4960 	 * VM-wide endpoints, the time-related data is more strict: It relies on
4961 	 * write-locking the VM (implied by the vcpuid -1) to prevent vCPUs or
4962 	 * other bits from observing inconsistent values while the state is
4963 	 * being written.
4964 	 */
4965 	if (req->vdr_vcpuid != -1) {
4966 		return (EINVAL);
4967 	}
4968 
4969 	/*
4970 	 * Platform-specific checks will verify the requested frequency against
4971 	 * the supported range further, but a frequency of 0 is never valid.
4972 	 */
4973 	if (src->vt_guest_freq == 0) {
4974 		return (EINVAL);
4975 	}
4976 
4977 	/*
4978 	 * Check whether the request frequency is supported and get the
4979 	 * frequency multiplier.
4980 	 */
4981 	uint64_t mult = VM_TSCM_NOSCALE;
4982 	freqratio_res_t res = ops->vmfreqratio(src->vt_guest_freq,
4983 	    vmm_host_freq, &mult);
4984 	switch (res) {
4985 	case FR_SCALING_NOT_SUPPORTED:
4986 		/*
4987 		 * This system doesn't support TSC scaling, and the guest/host
4988 		 * frequencies differ
4989 		 */
4990 		return (EPERM);
4991 	case FR_OUT_OF_RANGE:
4992 		/* Requested frequency ratio is too small/large */
4993 		return (EINVAL);
4994 	case FR_SCALING_NOT_NEEDED:
4995 		/* Host and guest frequencies are the same */
4996 		VERIFY3U(mult, ==, VM_TSCM_NOSCALE);
4997 		break;
4998 	case FR_VALID:
4999 		VERIFY3U(mult, !=, VM_TSCM_NOSCALE);
5000 		break;
5001 	}
5002 
5003 	/*
5004 	 * Find (and validate) the hrtime delta between the input request and
5005 	 * when we received it so that we can bump the TSC to account for time
5006 	 * passing.
5007 	 *
5008 	 * We ignore the hrestime as input, as this is a field that
5009 	 * exists for reads.
5010 	 */
5011 	uint64_t tsc;
5012 	hrtime_t hrtime;
5013 	timespec_t hrestime;
5014 	vmm_time_snapshot(&tsc, &hrtime, &hrestime);
5015 	if ((src->vt_hrtime > hrtime) || (src->vt_boot_hrtime > hrtime)) {
5016 		/*
5017 		 * The caller has passed in an hrtime / boot_hrtime from the
5018 		 * future.
5019 		 */
5020 		return (EINVAL);
5021 	}
5022 	hrtime_t hrt_delta = hrtime - src->vt_hrtime;
5023 
5024 	/* Calculate guest TSC adjustment */
5025 	const uint64_t host_ticks = unscalehrtime(hrt_delta);
5026 	const uint64_t guest_ticks = vmm_scale_tsc(host_ticks,
5027 	    vm->freq_multiplier);
5028 	const uint64_t base_guest_tsc = src->vt_guest_tsc + guest_ticks;
5029 
5030 	/* Update guest time data */
5031 	vm->freq_multiplier = mult;
5032 	vm->guest_freq = src->vt_guest_freq;
5033 	vm->boot_hrtime = src->vt_boot_hrtime;
5034 	vm->tsc_offset = calc_tsc_offset(tsc, base_guest_tsc,
5035 	    vm->freq_multiplier);
5036 
5037 	return (0);
5038 }
5039 
5040 static const vmm_data_version_entry_t vmm_time_v1 = {
5041 	.vdve_class = VDC_VMM_TIME,
5042 	.vdve_version = 1,
5043 	.vdve_len_expect = sizeof (struct vdi_time_info_v1),
5044 	.vdve_readf = vmm_data_read_vmm_time,
5045 	.vdve_writef = vmm_data_write_vmm_time,
5046 };
5047 VMM_DATA_VERSION(vmm_time_v1);
5048 
5049 
5050 static int
vmm_data_read_versions(void * arg,const vmm_data_req_t * req)5051 vmm_data_read_versions(void *arg, const vmm_data_req_t *req)
5052 {
5053 	VERIFY3U(req->vdr_class, ==, VDC_VERSION);
5054 	VERIFY3U(req->vdr_version, ==, 1);
5055 
5056 	const uint32_t total_size = SET_COUNT(vmm_data_version_entries) *
5057 	    sizeof (struct vdi_version_entry_v1);
5058 
5059 	/* Make sure there is room for all of the entries */
5060 	*req->vdr_result_len = total_size;
5061 	if (req->vdr_len < *req->vdr_result_len) {
5062 		return (ENOSPC);
5063 	}
5064 
5065 	struct vdi_version_entry_v1 *entryp = req->vdr_data;
5066 	const vmm_data_version_entry_t **vdpp;
5067 	SET_FOREACH(vdpp, vmm_data_version_entries) {
5068 		const vmm_data_version_entry_t *vdp = *vdpp;
5069 
5070 		entryp->vve_class = vdp->vdve_class;
5071 		entryp->vve_version = vdp->vdve_version;
5072 		entryp->vve_len_expect = vdp->vdve_len_expect;
5073 		entryp->vve_len_per_item = vdp->vdve_len_per_item;
5074 		entryp++;
5075 	}
5076 	return (0);
5077 }
5078 
5079 static int
vmm_data_write_versions(void * arg,const vmm_data_req_t * req)5080 vmm_data_write_versions(void *arg, const vmm_data_req_t *req)
5081 {
5082 	/* Writing to the version information makes no sense */
5083 	return (EPERM);
5084 }
5085 
5086 static const vmm_data_version_entry_t versions_v1 = {
5087 	.vdve_class = VDC_VERSION,
5088 	.vdve_version = 1,
5089 	.vdve_len_per_item = sizeof (struct vdi_version_entry_v1),
5090 	.vdve_readf = vmm_data_read_versions,
5091 	.vdve_writef = vmm_data_write_versions,
5092 };
5093 VMM_DATA_VERSION(versions_v1);
5094 
5095 int
vmm_data_read(struct vm * vm,const vmm_data_req_t * req)5096 vmm_data_read(struct vm *vm, const vmm_data_req_t *req)
5097 {
5098 	int err = 0;
5099 
5100 	const vmm_data_version_entry_t *entry = NULL;
5101 	err = vmm_data_find(req, &entry);
5102 	if (err != 0) {
5103 		return (err);
5104 	}
5105 	ASSERT(entry != NULL);
5106 
5107 	if (entry->vdve_readf != NULL) {
5108 		void *datap = vmm_data_from_class(req, vm);
5109 
5110 		err = entry->vdve_readf(datap, req);
5111 	} else if (entry->vdve_vcpu_readf != NULL) {
5112 		err = entry->vdve_vcpu_readf(vm, req->vdr_vcpuid, req);
5113 	} else {
5114 		err = EINVAL;
5115 	}
5116 
5117 	/*
5118 	 * Successful reads of fixed-length data should populate the length of
5119 	 * that result.
5120 	 */
5121 	if (err == 0 && entry->vdve_len_expect != 0) {
5122 		*req->vdr_result_len = entry->vdve_len_expect;
5123 	}
5124 
5125 	return (err);
5126 }
5127 
5128 int
vmm_data_write(struct vm * vm,const vmm_data_req_t * req)5129 vmm_data_write(struct vm *vm, const vmm_data_req_t *req)
5130 {
5131 	int err = 0;
5132 
5133 	const vmm_data_version_entry_t *entry = NULL;
5134 	err = vmm_data_find(req, &entry);
5135 	if (err != 0) {
5136 		return (err);
5137 	}
5138 	ASSERT(entry != NULL);
5139 
5140 	if (entry->vdve_writef != NULL) {
5141 		void *datap = vmm_data_from_class(req, vm);
5142 
5143 		err = entry->vdve_writef(datap, req);
5144 	} else if (entry->vdve_vcpu_writef != NULL) {
5145 		err = entry->vdve_vcpu_writef(vm, req->vdr_vcpuid, req);
5146 	} else {
5147 		err = EINVAL;
5148 	}
5149 
5150 	/*
5151 	 * Successful writes of fixed-length data should populate the length of
5152 	 * that result.
5153 	 */
5154 	if (err == 0 && entry->vdve_len_expect != 0) {
5155 		*req->vdr_result_len = entry->vdve_len_expect;
5156 	}
5157 
5158 	return (err);
5159 }
5160