1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011 NetApp, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28 /*
29 * This file and its contents are supplied under the terms of the
30 * Common Development and Distribution License ("CDDL"), version 1.0.
31 * You may only use this file in accordance with the terms of version
32 * 1.0 of the CDDL.
33 *
34 * A full copy of the text of the CDDL should have accompanied this
35 * source. A copy of the CDDL is also available via the Internet at
36 * http://www.illumos.org/license/CDDL.
37 */
38 /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */
39
40 /*
41 * Copyright 2015 Pluribus Networks Inc.
42 * Copyright 2018 Joyent, Inc.
43 * Copyright 2025 Oxide Computer Company
44 * Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
45 */
46
47
48 #include <sys/cdefs.h>
49
50 #include <sys/param.h>
51 #include <sys/systm.h>
52 #include <sys/kernel.h>
53 #include <sys/module.h>
54 #include <sys/sysctl.h>
55 #include <sys/kmem.h>
56 #include <sys/pcpu.h>
57 #include <sys/mutex.h>
58 #include <sys/proc.h>
59 #include <sys/rwlock.h>
60 #include <sys/sched.h>
61 #include <sys/systm.h>
62 #include <sys/sunddi.h>
63 #include <sys/hma.h>
64 #include <sys/archsystm.h>
65
66 #include <machine/md_var.h>
67 #include <x86/psl.h>
68 #include <x86/apicreg.h>
69
70 #include <machine/specialreg.h>
71 #include <machine/vmm.h>
72 #include <machine/vmm_dev.h>
73 #include <machine/vmparam.h>
74 #include <sys/vmm_instruction_emul.h>
75 #include <sys/vmm_vm.h>
76 #include <sys/vmm_gpt.h>
77 #include <sys/vmm_data.h>
78
79 #include "vmm_ioport.h"
80 #include "vmm_host.h"
81 #include "vmm_util.h"
82 #include "vatpic.h"
83 #include "vatpit.h"
84 #include "vhpet.h"
85 #include "vioapic.h"
86 #include "vlapic.h"
87 #include "vpmtmr.h"
88 #include "vrtc.h"
89 #include "vmm_stat.h"
90 #include "vmm_lapic.h"
91
92 #include "io/ppt.h"
93 #include "io/iommu.h"
94
95 struct vlapic;
96
97 /* Flags for vtc_status */
98 #define VTCS_FPU_RESTORED 1 /* guest FPU restored, host FPU saved */
99 #define VTCS_FPU_CTX_CRITICAL 2 /* in ctx where FPU restore cannot be lazy */
100
101 typedef struct vm_thread_ctx {
102 struct vm *vtc_vm;
103 int vtc_vcpuid;
104 uint_t vtc_status;
105 enum vcpu_ustate vtc_ustate;
106 } vm_thread_ctx_t;
107
108 #define VMM_MTRR_VAR_MAX 10
109 #define VMM_MTRR_DEF_MASK \
110 (MTRR_DEF_ENABLE | MTRR_DEF_FIXED_ENABLE | MTRR_DEF_TYPE)
111 #define VMM_MTRR_PHYSBASE_MASK (MTRR_PHYSBASE_PHYSBASE | MTRR_PHYSBASE_TYPE)
112 #define VMM_MTRR_PHYSMASK_MASK (MTRR_PHYSMASK_PHYSMASK | MTRR_PHYSMASK_VALID)
113 struct vm_mtrr {
114 uint64_t def_type;
115 uint64_t fixed4k[8];
116 uint64_t fixed16k[2];
117 uint64_t fixed64k;
118 struct {
119 uint64_t base;
120 uint64_t mask;
121 } var[VMM_MTRR_VAR_MAX];
122 };
123
124 /*
125 * Initialization:
126 * (a) allocated when vcpu is created
127 * (i) initialized when vcpu is created and when it is reinitialized
128 * (o) initialized the first time the vcpu is created
129 * (x) initialized before use
130 */
131 struct vcpu {
132 /* (o) protects state, run_state, hostcpu, sipi_vector */
133 kmutex_t lock;
134
135 enum vcpu_state state; /* (o) vcpu state */
136 enum vcpu_run_state run_state; /* (i) vcpu init/sipi/run state */
137 kcondvar_t vcpu_cv; /* (o) cpu waiter cv */
138 kcondvar_t state_cv; /* (o) IDLE-transition cv */
139 int hostcpu; /* (o) vcpu's current host cpu */
140 int lastloccpu; /* (o) last host cpu localized to */
141 bool reqidle; /* (i) request vcpu to idle */
142 bool reqconsist; /* (i) req. vcpu exit when consistent */
143 bool reqbarrier; /* (i) request vcpu exit barrier */
144 struct vlapic *vlapic; /* (i) APIC device model */
145 enum x2apic_state x2apic_state; /* (i) APIC mode */
146 uint64_t exit_intinfo; /* (i) events pending at VM exit */
147 uint64_t exc_pending; /* (i) exception pending */
148 bool nmi_pending; /* (i) NMI pending */
149 bool extint_pending; /* (i) INTR pending */
150
151 uint8_t sipi_vector; /* (i) SIPI vector */
152 hma_fpu_t *guestfpu; /* (a,i) guest fpu state */
153 uint64_t guest_xcr0; /* (i) guest %xcr0 register */
154 void *stats; /* (a,i) statistics */
155 struct vm_exit exitinfo; /* (x) exit reason and collateral */
156 uint64_t nextrip; /* (x) next instruction to execute */
157 struct vie *vie_ctx; /* (x) instruction emulation context */
158 vm_client_t *vmclient; /* (a) VM-system client */
159 uint64_t tsc_offset; /* (x) vCPU TSC offset */
160 struct vm_mtrr mtrr; /* (i) vcpu's MTRR */
161 vcpu_cpuid_config_t cpuid_cfg; /* (x) cpuid configuration */
162
163 enum vcpu_ustate ustate; /* (i) microstate for the vcpu */
164 hrtime_t ustate_when; /* (i) time of last ustate change */
165 uint64_t ustate_total[VU_MAX]; /* (o) total time spent in ustates */
166 vm_thread_ctx_t vtc; /* (o) thread state for ctxops */
167 struct ctxop *ctxop; /* (o) ctxop storage for vcpu */
168 };
169
170 #define vcpu_lock(v) mutex_enter(&((v)->lock))
171 #define vcpu_unlock(v) mutex_exit(&((v)->lock))
172 #define vcpu_assert_locked(v) ASSERT(MUTEX_HELD(&((v)->lock)))
173
174 struct mem_seg {
175 size_t len;
176 bool sysmem;
177 vm_object_t *object;
178 };
179 #define VM_MAX_MEMSEGS 5
180
181 struct mem_map {
182 vm_paddr_t gpa;
183 size_t len;
184 vm_ooffset_t segoff;
185 int segid;
186 int prot;
187 int flags;
188 };
189 #define VM_MAX_MEMMAPS 8
190
191 /*
192 * Initialization:
193 * (o) initialized the first time the VM is created
194 * (i) initialized when VM is created and when it is reinitialized
195 * (x) initialized before use
196 */
197 struct vm {
198 void *cookie; /* (i) cpu-specific data */
199 void *iommu; /* (x) iommu-specific data */
200 struct vhpet *vhpet; /* (i) virtual HPET */
201 struct vioapic *vioapic; /* (i) virtual ioapic */
202 struct vatpic *vatpic; /* (i) virtual atpic */
203 struct vatpit *vatpit; /* (i) virtual atpit */
204 struct vpmtmr *vpmtmr; /* (i) virtual ACPI PM timer */
205 struct vrtc *vrtc; /* (o) virtual RTC */
206 volatile cpuset_t active_cpus; /* (i) active vcpus */
207 volatile cpuset_t debug_cpus; /* (i) vcpus stopped for dbg */
208 volatile cpuset_t halted_cpus; /* (x) cpus in a hard halt */
209 int suspend_how; /* (i) stop VM execution */
210 int suspend_source; /* (i) src vcpuid of suspend */
211 hrtime_t suspend_when; /* (i) time suspend asserted */
212 struct mem_map mem_maps[VM_MAX_MEMMAPS]; /* (i) guest address space */
213 struct mem_seg mem_segs[VM_MAX_MEMSEGS]; /* (o) guest memory regions */
214 struct vmspace *vmspace; /* (o) guest's address space */
215 struct vcpu vcpu[VM_MAXCPU]; /* (i) guest vcpus */
216 /* The following describe the vm cpu topology */
217 uint16_t sockets; /* (o) num of sockets */
218 uint16_t cores; /* (o) num of cores/socket */
219 uint16_t threads; /* (o) num of threads/core */
220 uint16_t maxcpus; /* (o) max pluggable cpus */
221
222 hrtime_t boot_hrtime; /* (i) hrtime at VM boot */
223
224 /* TSC and TSC scaling related values */
225 uint64_t tsc_offset; /* (i) VM-wide TSC offset */
226 uint64_t guest_freq; /* (i) guest TSC Frequency */
227 uint64_t freq_multiplier; /* (i) guest/host TSC Ratio */
228
229 struct ioport_config ioports; /* (o) ioport handling */
230
231 bool mem_transient; /* (o) alloc transient memory */
232 bool is_paused; /* (i) instance is paused */
233 };
234
235 static int vmm_initialized;
236 static uint64_t vmm_host_freq;
237
238
239 static void
nullop_panic(void)240 nullop_panic(void)
241 {
242 panic("null vmm operation call");
243 }
244
245 /* Do not allow use of an un-set `ops` to do anything but panic */
246 static struct vmm_ops vmm_ops_null = {
247 .init = (vmm_init_func_t)nullop_panic,
248 .cleanup = (vmm_cleanup_func_t)nullop_panic,
249 .resume = (vmm_resume_func_t)nullop_panic,
250 .vminit = (vmi_init_func_t)nullop_panic,
251 .vmrun = (vmi_run_func_t)nullop_panic,
252 .vmcleanup = (vmi_cleanup_func_t)nullop_panic,
253 .vmgetreg = (vmi_get_register_t)nullop_panic,
254 .vmsetreg = (vmi_set_register_t)nullop_panic,
255 .vmgetdesc = (vmi_get_desc_t)nullop_panic,
256 .vmsetdesc = (vmi_set_desc_t)nullop_panic,
257 .vmgetcap = (vmi_get_cap_t)nullop_panic,
258 .vmsetcap = (vmi_set_cap_t)nullop_panic,
259 .vlapic_init = (vmi_vlapic_init)nullop_panic,
260 .vlapic_cleanup = (vmi_vlapic_cleanup)nullop_panic,
261 .vmpause = (vmi_pause_t)nullop_panic,
262 .vmsavectx = (vmi_savectx)nullop_panic,
263 .vmrestorectx = (vmi_restorectx)nullop_panic,
264 .vmgetmsr = (vmi_get_msr_t)nullop_panic,
265 .vmsetmsr = (vmi_set_msr_t)nullop_panic,
266 .vmfreqratio = (vmi_freqratio_t)nullop_panic,
267 .fr_fracsize = 0,
268 .fr_intsize = 0,
269 };
270
271 static struct vmm_ops *ops = &vmm_ops_null;
272 static vmm_pte_ops_t *pte_ops = NULL;
273
274 #define VMM_INIT() ((*ops->init)())
275 #define VMM_CLEANUP() ((*ops->cleanup)())
276 #define VMM_RESUME() ((*ops->resume)())
277
278 #define VMINIT(vm) ((*ops->vminit)(vm))
279 #define VMRUN(vmi, vcpu, rip) ((*ops->vmrun)(vmi, vcpu, rip))
280 #define VMCLEANUP(vmi) ((*ops->vmcleanup)(vmi))
281
282 #define VMGETREG(vmi, vcpu, num, rv) ((*ops->vmgetreg)(vmi, vcpu, num, rv))
283 #define VMSETREG(vmi, vcpu, num, val) ((*ops->vmsetreg)(vmi, vcpu, num, val))
284 #define VMGETDESC(vmi, vcpu, num, dsc) ((*ops->vmgetdesc)(vmi, vcpu, num, dsc))
285 #define VMSETDESC(vmi, vcpu, num, dsc) ((*ops->vmsetdesc)(vmi, vcpu, num, dsc))
286 #define VMGETCAP(vmi, vcpu, num, rv) ((*ops->vmgetcap)(vmi, vcpu, num, rv))
287 #define VMSETCAP(vmi, vcpu, num, val) ((*ops->vmsetcap)(vmi, vcpu, num, val))
288 #define VLAPIC_INIT(vmi, vcpu) ((*ops->vlapic_init)(vmi, vcpu))
289 #define VLAPIC_CLEANUP(vmi, vlapic) ((*ops->vlapic_cleanup)(vmi, vlapic))
290
291 #define fpu_start_emulating() load_cr0(rcr0() | CR0_TS)
292 #define fpu_stop_emulating() clts()
293
294 SDT_PROVIDER_DEFINE(vmm);
295
296 SYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
297 NULL);
298
299 /*
300 * Halt the guest if all vcpus are executing a HLT instruction with
301 * interrupts disabled.
302 */
303 int halt_detection_enabled = 1;
304
305 /* Trap into hypervisor on all guest exceptions and reflect them back */
306 int trace_guest_exceptions;
307
308 /* Trap WBINVD and ignore it */
309 int trap_wbinvd = 1;
310
311 static void vm_free_memmap(struct vm *vm, int ident);
312 static bool sysmem_mapping(struct vm *vm, struct mem_map *mm);
313 static void vcpu_notify_event_locked(struct vcpu *vcpu, vcpu_notify_t);
314 static bool vcpu_sleep_bailout_checks(struct vm *vm, int vcpuid);
315 static int vcpu_vector_sipi(struct vm *vm, int vcpuid, uint8_t vector);
316 static bool vm_is_suspended(struct vm *, struct vm_exit *);
317
318 static void vmm_savectx(void *);
319 static void vmm_restorectx(void *);
320 static const struct ctxop_template vmm_ctxop_tpl = {
321 .ct_rev = CTXOP_TPL_REV,
322 .ct_save = vmm_savectx,
323 .ct_restore = vmm_restorectx,
324 };
325
326 static uint64_t calc_tsc_offset(uint64_t base_host_tsc, uint64_t base_guest_tsc,
327 uint64_t mult);
328 static uint64_t calc_guest_tsc(uint64_t host_tsc, uint64_t mult,
329 uint64_t offset);
330
331 /* functions implemented in vmm_time_support.S */
332 uint64_t calc_freq_multiplier(uint64_t guest_hz, uint64_t host_hz,
333 uint32_t frac_size);
334 uint64_t scale_tsc(uint64_t tsc, uint64_t multiplier, uint32_t frac_size);
335
336 #ifdef KTR
337 static const char *
vcpu_state2str(enum vcpu_state state)338 vcpu_state2str(enum vcpu_state state)
339 {
340
341 switch (state) {
342 case VCPU_IDLE:
343 return ("idle");
344 case VCPU_FROZEN:
345 return ("frozen");
346 case VCPU_RUNNING:
347 return ("running");
348 case VCPU_SLEEPING:
349 return ("sleeping");
350 default:
351 return ("unknown");
352 }
353 }
354 #endif
355
356 static void
vcpu_cleanup(struct vm * vm,int i,bool destroy)357 vcpu_cleanup(struct vm *vm, int i, bool destroy)
358 {
359 struct vcpu *vcpu = &vm->vcpu[i];
360
361 VLAPIC_CLEANUP(vm->cookie, vcpu->vlapic);
362 if (destroy) {
363 vmm_stat_free(vcpu->stats);
364
365 vcpu_cpuid_cleanup(&vcpu->cpuid_cfg);
366
367 hma_fpu_free(vcpu->guestfpu);
368 vcpu->guestfpu = NULL;
369
370 vie_free(vcpu->vie_ctx);
371 vcpu->vie_ctx = NULL;
372
373 vmc_destroy(vcpu->vmclient);
374 vcpu->vmclient = NULL;
375
376 ctxop_free(vcpu->ctxop);
377 mutex_destroy(&vcpu->lock);
378 }
379 }
380
381 static void
vcpu_init(struct vm * vm,int vcpu_id,bool create)382 vcpu_init(struct vm *vm, int vcpu_id, bool create)
383 {
384 struct vcpu *vcpu;
385
386 KASSERT(vcpu_id >= 0 && vcpu_id < vm->maxcpus,
387 ("vcpu_init: invalid vcpu %d", vcpu_id));
388
389 vcpu = &vm->vcpu[vcpu_id];
390
391 if (create) {
392 mutex_init(&vcpu->lock, NULL, MUTEX_ADAPTIVE, NULL);
393
394 vcpu->state = VCPU_IDLE;
395 vcpu->hostcpu = NOCPU;
396 vcpu->lastloccpu = NOCPU;
397 vcpu->guestfpu = hma_fpu_alloc(KM_SLEEP);
398 vcpu->stats = vmm_stat_alloc();
399 vcpu->vie_ctx = vie_alloc();
400 vcpu_cpuid_init(&vcpu->cpuid_cfg);
401
402 vcpu->ustate = VU_INIT;
403 vcpu->ustate_when = gethrtime();
404
405 vcpu->vtc.vtc_vm = vm;
406 vcpu->vtc.vtc_vcpuid = vcpu_id;
407 vcpu->ctxop = ctxop_allocate(&vmm_ctxop_tpl, &vcpu->vtc);
408 } else {
409 vie_reset(vcpu->vie_ctx);
410 bzero(&vcpu->exitinfo, sizeof (vcpu->exitinfo));
411 vcpu_ustate_change(vm, vcpu_id, VU_INIT);
412 bzero(&vcpu->mtrr, sizeof (vcpu->mtrr));
413 }
414
415 vcpu->run_state = VRS_HALT;
416 vcpu->vlapic = VLAPIC_INIT(vm->cookie, vcpu_id);
417 (void) vm_set_x2apic_state(vm, vcpu_id, X2APIC_DISABLED);
418 vcpu->reqidle = false;
419 vcpu->reqconsist = false;
420 vcpu->reqbarrier = false;
421 vcpu->exit_intinfo = 0;
422 vcpu->nmi_pending = false;
423 vcpu->extint_pending = false;
424 vcpu->exc_pending = 0;
425 vcpu->guest_xcr0 = XFEATURE_ENABLED_X87;
426 (void) hma_fpu_init(vcpu->guestfpu);
427 vmm_stat_init(vcpu->stats);
428 vcpu->tsc_offset = 0;
429 }
430
431 int
vcpu_trace_exceptions(struct vm * vm,int vcpuid)432 vcpu_trace_exceptions(struct vm *vm, int vcpuid)
433 {
434 return (trace_guest_exceptions);
435 }
436
437 int
vcpu_trap_wbinvd(struct vm * vm,int vcpuid)438 vcpu_trap_wbinvd(struct vm *vm, int vcpuid)
439 {
440 return (trap_wbinvd);
441 }
442
443 struct vm_exit *
vm_exitinfo(struct vm * vm,int cpuid)444 vm_exitinfo(struct vm *vm, int cpuid)
445 {
446 struct vcpu *vcpu;
447
448 if (cpuid < 0 || cpuid >= vm->maxcpus)
449 panic("vm_exitinfo: invalid cpuid %d", cpuid);
450
451 vcpu = &vm->vcpu[cpuid];
452
453 return (&vcpu->exitinfo);
454 }
455
456 struct vie *
vm_vie_ctx(struct vm * vm,int cpuid)457 vm_vie_ctx(struct vm *vm, int cpuid)
458 {
459 if (cpuid < 0 || cpuid >= vm->maxcpus)
460 panic("vm_vie_ctx: invalid cpuid %d", cpuid);
461
462 return (vm->vcpu[cpuid].vie_ctx);
463 }
464
465 static int
vmm_init(void)466 vmm_init(void)
467 {
468 vmm_host_state_init();
469 vmm_host_freq = unscalehrtime(NANOSEC);
470
471 if (vmm_is_intel()) {
472 ops = &vmm_ops_intel;
473 pte_ops = &ept_pte_ops;
474 } else if (vmm_is_svm()) {
475 ops = &vmm_ops_amd;
476 pte_ops = &rvi_pte_ops;
477 } else {
478 return (ENXIO);
479 }
480
481 return (VMM_INIT());
482 }
483
484 int
vmm_mod_load()485 vmm_mod_load()
486 {
487 int error;
488
489 VERIFY(vmm_initialized == 0);
490
491 error = vmm_init();
492 if (error == 0)
493 vmm_initialized = 1;
494
495 return (error);
496 }
497
498 int
vmm_mod_unload()499 vmm_mod_unload()
500 {
501 int error;
502
503 VERIFY(vmm_initialized == 1);
504
505 error = VMM_CLEANUP();
506 if (error)
507 return (error);
508 vmm_initialized = 0;
509
510 return (0);
511 }
512
513 /*
514 * Create a test IOMMU domain to see if the host system has necessary hardware
515 * and drivers to do so.
516 */
517 bool
vmm_check_iommu(void)518 vmm_check_iommu(void)
519 {
520 void *domain;
521 const size_t arb_test_sz = (1UL << 32);
522
523 domain = iommu_create_domain(arb_test_sz);
524 if (domain == NULL) {
525 return (false);
526 }
527 iommu_destroy_domain(domain);
528 return (true);
529 }
530
531 static void
vm_init(struct vm * vm,bool create)532 vm_init(struct vm *vm, bool create)
533 {
534 int i;
535
536 vm->cookie = VMINIT(vm);
537 vm->iommu = NULL;
538 vm->vioapic = vioapic_init(vm);
539 vm->vhpet = vhpet_init(vm);
540 vm->vatpic = vatpic_init(vm);
541 vm->vatpit = vatpit_init(vm);
542 vm->vpmtmr = vpmtmr_init(vm);
543 if (create)
544 vm->vrtc = vrtc_init(vm);
545
546 vm_inout_init(vm, &vm->ioports);
547
548 CPU_ZERO(&vm->active_cpus);
549 CPU_ZERO(&vm->debug_cpus);
550
551 vm->suspend_how = 0;
552 vm->suspend_source = 0;
553 vm->suspend_when = 0;
554
555 for (i = 0; i < vm->maxcpus; i++)
556 vcpu_init(vm, i, create);
557
558 /*
559 * Configure VM time-related data, including:
560 * - VM-wide TSC offset
561 * - boot_hrtime
562 * - guest_freq (same as host at boot time)
563 * - freq_multiplier (used for scaling)
564 *
565 * This data is configured such that the call to vm_init() represents
566 * the boot time (when the TSC(s) read 0). Each vCPU will have its own
567 * offset from this, which is altered if/when the guest writes to
568 * MSR_TSC.
569 *
570 * Further changes to this data may occur if userspace writes to the
571 * time data.
572 */
573 const uint64_t boot_tsc = rdtsc_offset();
574
575 /* Convert the boot TSC reading to hrtime */
576 vm->boot_hrtime = (hrtime_t)boot_tsc;
577 scalehrtime(&vm->boot_hrtime);
578
579 /* Guest frequency is the same as the host at boot time */
580 vm->guest_freq = vmm_host_freq;
581
582 /* no scaling needed if guest_freq == host_freq */
583 vm->freq_multiplier = VM_TSCM_NOSCALE;
584
585 /* configure VM-wide offset: initial guest TSC is 0 at boot */
586 vm->tsc_offset = calc_tsc_offset(boot_tsc, 0, vm->freq_multiplier);
587 }
588
589 /*
590 * The default CPU topology is a single thread per package.
591 */
592 uint_t cores_per_package = 1;
593 uint_t threads_per_core = 1;
594
595 int
vm_create(uint64_t flags,struct vm ** retvm)596 vm_create(uint64_t flags, struct vm **retvm)
597 {
598 struct vm *vm;
599 struct vmspace *vmspace;
600
601 /*
602 * If vmm.ko could not be successfully initialized then don't attempt
603 * to create the virtual machine.
604 */
605 if (!vmm_initialized)
606 return (ENXIO);
607
608 bool track_dirty = (flags & VCF_TRACK_DIRTY) != 0;
609 if (track_dirty && !pte_ops->vpeo_hw_ad_supported())
610 return (ENOTSUP);
611
612 vmspace = vmspace_alloc(VM_MAXUSER_ADDRESS, pte_ops, track_dirty);
613 if (vmspace == NULL)
614 return (ENOMEM);
615
616 vm = kmem_zalloc(sizeof (struct vm), KM_SLEEP);
617
618 vm->vmspace = vmspace;
619 vm->mem_transient = (flags & VCF_RESERVOIR_MEM) == 0;
620 for (uint_t i = 0; i < VM_MAXCPU; i++) {
621 vm->vcpu[i].vmclient = vmspace_client_alloc(vmspace);
622 }
623
624 vm->sockets = 1;
625 vm->cores = cores_per_package; /* XXX backwards compatibility */
626 vm->threads = threads_per_core; /* XXX backwards compatibility */
627 vm->maxcpus = VM_MAXCPU; /* XXX temp to keep code working */
628
629 vm_init(vm, true);
630
631 *retvm = vm;
632 return (0);
633 }
634
635 void
vm_get_topology(struct vm * vm,uint16_t * sockets,uint16_t * cores,uint16_t * threads,uint16_t * maxcpus)636 vm_get_topology(struct vm *vm, uint16_t *sockets, uint16_t *cores,
637 uint16_t *threads, uint16_t *maxcpus)
638 {
639 *sockets = vm->sockets;
640 *cores = vm->cores;
641 *threads = vm->threads;
642 *maxcpus = vm->maxcpus;
643 }
644
645 uint16_t
vm_get_maxcpus(struct vm * vm)646 vm_get_maxcpus(struct vm *vm)
647 {
648 return (vm->maxcpus);
649 }
650
651 int
vm_set_topology(struct vm * vm,uint16_t sockets,uint16_t cores,uint16_t threads,uint16_t maxcpus)652 vm_set_topology(struct vm *vm, uint16_t sockets, uint16_t cores,
653 uint16_t threads, uint16_t maxcpus)
654 {
655 if (maxcpus != 0)
656 return (EINVAL); /* XXX remove when supported */
657 if ((sockets * cores * threads) > vm->maxcpus)
658 return (EINVAL);
659 /* XXX need to check sockets * cores * threads == vCPU, how? */
660 vm->sockets = sockets;
661 vm->cores = cores;
662 vm->threads = threads;
663 vm->maxcpus = VM_MAXCPU; /* XXX temp to keep code working */
664 return (0);
665 }
666
667 static void
vm_cleanup(struct vm * vm,bool destroy)668 vm_cleanup(struct vm *vm, bool destroy)
669 {
670 struct mem_map *mm;
671 int i;
672
673 ppt_unassign_all(vm);
674
675 if (vm->iommu != NULL)
676 iommu_destroy_domain(vm->iommu);
677
678 /*
679 * Devices which attach their own ioport hooks should be cleaned up
680 * first so they can tear down those registrations.
681 */
682 vpmtmr_cleanup(vm->vpmtmr);
683
684 vm_inout_cleanup(vm, &vm->ioports);
685
686 if (destroy)
687 vrtc_cleanup(vm->vrtc);
688 else
689 vrtc_reset(vm->vrtc);
690
691 vatpit_cleanup(vm->vatpit);
692 vhpet_cleanup(vm->vhpet);
693 vatpic_cleanup(vm->vatpic);
694 vioapic_cleanup(vm->vioapic);
695
696 for (i = 0; i < vm->maxcpus; i++)
697 vcpu_cleanup(vm, i, destroy);
698
699 VMCLEANUP(vm->cookie);
700
701 /*
702 * System memory is removed from the guest address space only when
703 * the VM is destroyed. This is because the mapping remains the same
704 * across VM reset.
705 *
706 * Device memory can be relocated by the guest (e.g. using PCI BARs)
707 * so those mappings are removed on a VM reset.
708 */
709 for (i = 0; i < VM_MAX_MEMMAPS; i++) {
710 mm = &vm->mem_maps[i];
711 if (destroy || !sysmem_mapping(vm, mm)) {
712 vm_free_memmap(vm, i);
713 } else {
714 /*
715 * We need to reset the IOMMU flag so this mapping can
716 * be reused when a VM is rebooted. Since the IOMMU
717 * domain has already been destroyed we can just reset
718 * the flag here.
719 */
720 mm->flags &= ~VM_MEMMAP_F_IOMMU;
721 }
722 }
723
724 if (destroy) {
725 for (i = 0; i < VM_MAX_MEMSEGS; i++)
726 vm_free_memseg(vm, i);
727
728 vmspace_destroy(vm->vmspace);
729 vm->vmspace = NULL;
730 }
731 }
732
733 void
vm_destroy(struct vm * vm)734 vm_destroy(struct vm *vm)
735 {
736 vm_cleanup(vm, true);
737 kmem_free(vm, sizeof (*vm));
738 }
739
740 int
vm_reinit(struct vm * vm,uint64_t flags)741 vm_reinit(struct vm *vm, uint64_t flags)
742 {
743 vm_cleanup(vm, false);
744 vm_init(vm, false);
745 return (0);
746 }
747
748 bool
vm_is_paused(struct vm * vm)749 vm_is_paused(struct vm *vm)
750 {
751 return (vm->is_paused);
752 }
753
754 int
vm_pause_instance(struct vm * vm)755 vm_pause_instance(struct vm *vm)
756 {
757 if (vm->is_paused) {
758 return (EALREADY);
759 }
760 vm->is_paused = true;
761
762 for (uint_t i = 0; i < vm->maxcpus; i++) {
763 struct vcpu *vcpu = &vm->vcpu[i];
764
765 if (!CPU_ISSET(i, &vm->active_cpus)) {
766 continue;
767 }
768 vlapic_pause(vcpu->vlapic);
769
770 /*
771 * vCPU-specific pause logic includes stashing any
772 * to-be-injected events in exit_intinfo where it can be
773 * accessed in a manner generic to the backend.
774 */
775 ops->vmpause(vm->cookie, i);
776 }
777 vhpet_pause(vm->vhpet);
778 vatpit_pause(vm->vatpit);
779 vrtc_pause(vm->vrtc);
780
781 return (0);
782 }
783
784 int
vm_resume_instance(struct vm * vm)785 vm_resume_instance(struct vm *vm)
786 {
787 if (!vm->is_paused) {
788 return (EALREADY);
789 }
790 vm->is_paused = false;
791
792 vrtc_resume(vm->vrtc);
793 vatpit_resume(vm->vatpit);
794 vhpet_resume(vm->vhpet);
795 for (uint_t i = 0; i < vm->maxcpus; i++) {
796 struct vcpu *vcpu = &vm->vcpu[i];
797
798 if (!CPU_ISSET(i, &vm->active_cpus)) {
799 continue;
800 }
801 vlapic_resume(vcpu->vlapic);
802 }
803
804 return (0);
805 }
806
807 int
vm_map_mmio(struct vm * vm,vm_paddr_t gpa,size_t len,vm_paddr_t hpa)808 vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa)
809 {
810 vm_object_t *obj;
811
812 if ((obj = vmm_mmio_alloc(vm->vmspace, gpa, len, hpa)) == NULL)
813 return (ENOMEM);
814 else
815 return (0);
816 }
817
818 int
vm_unmap_mmio(struct vm * vm,vm_paddr_t gpa,size_t len)819 vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len)
820 {
821 return (vmspace_unmap(vm->vmspace, gpa, len));
822 }
823
824 /*
825 * Return 'true' if 'gpa' is allocated in the guest address space.
826 *
827 * This function is called in the context of a running vcpu which acts as
828 * an implicit lock on 'vm->mem_maps[]'.
829 */
830 bool
vm_mem_allocated(struct vm * vm,int vcpuid,vm_paddr_t gpa)831 vm_mem_allocated(struct vm *vm, int vcpuid, vm_paddr_t gpa)
832 {
833 struct mem_map *mm;
834 int i;
835
836 #ifdef INVARIANTS
837 int hostcpu, state;
838 state = vcpu_get_state(vm, vcpuid, &hostcpu);
839 KASSERT(state == VCPU_RUNNING && hostcpu == curcpu,
840 ("%s: invalid vcpu state %d/%d", __func__, state, hostcpu));
841 #endif
842
843 for (i = 0; i < VM_MAX_MEMMAPS; i++) {
844 mm = &vm->mem_maps[i];
845 if (mm->len != 0 && gpa >= mm->gpa && gpa < mm->gpa + mm->len)
846 return (true); /* 'gpa' is sysmem or devmem */
847 }
848
849 if (ppt_is_mmio(vm, gpa))
850 return (true); /* 'gpa' is pci passthru mmio */
851
852 return (false);
853 }
854
855 int
vm_alloc_memseg(struct vm * vm,int ident,size_t len,bool sysmem)856 vm_alloc_memseg(struct vm *vm, int ident, size_t len, bool sysmem)
857 {
858 struct mem_seg *seg;
859 vm_object_t *obj;
860
861 if (ident < 0 || ident >= VM_MAX_MEMSEGS)
862 return (EINVAL);
863
864 if (len == 0 || (len & PAGE_MASK))
865 return (EINVAL);
866
867 seg = &vm->mem_segs[ident];
868 if (seg->object != NULL) {
869 if (seg->len == len && seg->sysmem == sysmem)
870 return (EEXIST);
871 else
872 return (EINVAL);
873 }
874
875 obj = vm_object_mem_allocate(len, vm->mem_transient);
876 if (obj == NULL)
877 return (ENOMEM);
878
879 seg->len = len;
880 seg->object = obj;
881 seg->sysmem = sysmem;
882 return (0);
883 }
884
885 int
vm_get_memseg(struct vm * vm,int ident,size_t * len,bool * sysmem,vm_object_t ** objptr)886 vm_get_memseg(struct vm *vm, int ident, size_t *len, bool *sysmem,
887 vm_object_t **objptr)
888 {
889 struct mem_seg *seg;
890
891 if (ident < 0 || ident >= VM_MAX_MEMSEGS)
892 return (EINVAL);
893
894 seg = &vm->mem_segs[ident];
895 if (len)
896 *len = seg->len;
897 if (sysmem)
898 *sysmem = seg->sysmem;
899 if (objptr)
900 *objptr = seg->object;
901 return (0);
902 }
903
904 void
vm_free_memseg(struct vm * vm,int ident)905 vm_free_memseg(struct vm *vm, int ident)
906 {
907 struct mem_seg *seg;
908
909 KASSERT(ident >= 0 && ident < VM_MAX_MEMSEGS,
910 ("%s: invalid memseg ident %d", __func__, ident));
911
912 seg = &vm->mem_segs[ident];
913 if (seg->object != NULL) {
914 vm_object_release(seg->object);
915 bzero(seg, sizeof (struct mem_seg));
916 }
917 }
918
919 int
vm_mmap_memseg(struct vm * vm,vm_paddr_t gpa,int segid,vm_ooffset_t first,size_t len,int prot,int flags)920 vm_mmap_memseg(struct vm *vm, vm_paddr_t gpa, int segid, vm_ooffset_t first,
921 size_t len, int prot, int flags)
922 {
923 struct mem_seg *seg;
924 struct mem_map *m, *map;
925 vm_ooffset_t last;
926 int i, error;
927
928 if (prot == 0 || (prot & ~(PROT_ALL)) != 0)
929 return (EINVAL);
930
931 if (flags & ~VM_MEMMAP_F_WIRED)
932 return (EINVAL);
933
934 if (segid < 0 || segid >= VM_MAX_MEMSEGS)
935 return (EINVAL);
936
937 seg = &vm->mem_segs[segid];
938 if (seg->object == NULL)
939 return (EINVAL);
940
941 last = first + len;
942 if (first < 0 || first >= last || last > seg->len)
943 return (EINVAL);
944
945 if ((gpa | first | last) & PAGE_MASK)
946 return (EINVAL);
947
948 map = NULL;
949 for (i = 0; i < VM_MAX_MEMMAPS; i++) {
950 m = &vm->mem_maps[i];
951 if (m->len == 0) {
952 map = m;
953 break;
954 }
955 }
956
957 if (map == NULL)
958 return (ENOSPC);
959
960 error = vmspace_map(vm->vmspace, seg->object, first, gpa, len, prot);
961 if (error != 0)
962 return (EFAULT);
963
964 vm_object_reference(seg->object);
965
966 if ((flags & VM_MEMMAP_F_WIRED) != 0) {
967 error = vmspace_populate(vm->vmspace, gpa, len);
968 if (error != 0) {
969 VERIFY0(vmspace_unmap(vm->vmspace, gpa, len));
970 return (EFAULT);
971 }
972 }
973
974 map->gpa = gpa;
975 map->len = len;
976 map->segoff = first;
977 map->segid = segid;
978 map->prot = prot;
979 map->flags = flags;
980 return (0);
981 }
982
983 int
vm_munmap_memseg(struct vm * vm,vm_paddr_t gpa,size_t len)984 vm_munmap_memseg(struct vm *vm, vm_paddr_t gpa, size_t len)
985 {
986 struct mem_map *m;
987 int i;
988
989 for (i = 0; i < VM_MAX_MEMMAPS; i++) {
990 m = &vm->mem_maps[i];
991 if (m->gpa == gpa && m->len == len &&
992 (m->flags & VM_MEMMAP_F_IOMMU) == 0) {
993 vm_free_memmap(vm, i);
994 return (0);
995 }
996 }
997
998 return (EINVAL);
999 }
1000
1001 int
vm_mmap_getnext(struct vm * vm,vm_paddr_t * gpa,int * segid,vm_ooffset_t * segoff,size_t * len,int * prot,int * flags)1002 vm_mmap_getnext(struct vm *vm, vm_paddr_t *gpa, int *segid,
1003 vm_ooffset_t *segoff, size_t *len, int *prot, int *flags)
1004 {
1005 struct mem_map *mm, *mmnext;
1006 int i;
1007
1008 mmnext = NULL;
1009 for (i = 0; i < VM_MAX_MEMMAPS; i++) {
1010 mm = &vm->mem_maps[i];
1011 if (mm->len == 0 || mm->gpa < *gpa)
1012 continue;
1013 if (mmnext == NULL || mm->gpa < mmnext->gpa)
1014 mmnext = mm;
1015 }
1016
1017 if (mmnext != NULL) {
1018 *gpa = mmnext->gpa;
1019 if (segid)
1020 *segid = mmnext->segid;
1021 if (segoff)
1022 *segoff = mmnext->segoff;
1023 if (len)
1024 *len = mmnext->len;
1025 if (prot)
1026 *prot = mmnext->prot;
1027 if (flags)
1028 *flags = mmnext->flags;
1029 return (0);
1030 } else {
1031 return (ENOENT);
1032 }
1033 }
1034
1035 static void
vm_free_memmap(struct vm * vm,int ident)1036 vm_free_memmap(struct vm *vm, int ident)
1037 {
1038 struct mem_map *mm;
1039 int error;
1040
1041 mm = &vm->mem_maps[ident];
1042 if (mm->len) {
1043 error = vmspace_unmap(vm->vmspace, mm->gpa, mm->len);
1044 VERIFY0(error);
1045 bzero(mm, sizeof (struct mem_map));
1046 }
1047 }
1048
1049 static __inline bool
sysmem_mapping(struct vm * vm,struct mem_map * mm)1050 sysmem_mapping(struct vm *vm, struct mem_map *mm)
1051 {
1052
1053 if (mm->len != 0 && vm->mem_segs[mm->segid].sysmem)
1054 return (true);
1055 else
1056 return (false);
1057 }
1058
1059 vm_paddr_t
vmm_sysmem_maxaddr(struct vm * vm)1060 vmm_sysmem_maxaddr(struct vm *vm)
1061 {
1062 struct mem_map *mm;
1063 vm_paddr_t maxaddr;
1064 int i;
1065
1066 maxaddr = 0;
1067 for (i = 0; i < VM_MAX_MEMMAPS; i++) {
1068 mm = &vm->mem_maps[i];
1069 if (sysmem_mapping(vm, mm)) {
1070 if (maxaddr < mm->gpa + mm->len)
1071 maxaddr = mm->gpa + mm->len;
1072 }
1073 }
1074 return (maxaddr);
1075 }
1076
1077 static void
vm_iommu_modify(struct vm * vm,bool map)1078 vm_iommu_modify(struct vm *vm, bool map)
1079 {
1080 int i, sz;
1081 vm_paddr_t gpa, hpa;
1082 struct mem_map *mm;
1083 vm_client_t *vmc;
1084
1085 sz = PAGE_SIZE;
1086 vmc = vmspace_client_alloc(vm->vmspace);
1087
1088 for (i = 0; i < VM_MAX_MEMMAPS; i++) {
1089 mm = &vm->mem_maps[i];
1090 if (!sysmem_mapping(vm, mm))
1091 continue;
1092
1093 if (map) {
1094 KASSERT((mm->flags & VM_MEMMAP_F_IOMMU) == 0,
1095 ("iommu map found invalid memmap %lx/%lx/%x",
1096 mm->gpa, mm->len, mm->flags));
1097 if ((mm->flags & VM_MEMMAP_F_WIRED) == 0)
1098 continue;
1099 mm->flags |= VM_MEMMAP_F_IOMMU;
1100 } else {
1101 if ((mm->flags & VM_MEMMAP_F_IOMMU) == 0)
1102 continue;
1103 mm->flags &= ~VM_MEMMAP_F_IOMMU;
1104 KASSERT((mm->flags & VM_MEMMAP_F_WIRED) != 0,
1105 ("iommu unmap found invalid memmap %lx/%lx/%x",
1106 mm->gpa, mm->len, mm->flags));
1107 }
1108
1109 gpa = mm->gpa;
1110 while (gpa < mm->gpa + mm->len) {
1111 vm_page_t *vmp;
1112
1113 vmp = vmc_hold(vmc, gpa, PROT_WRITE);
1114 ASSERT(vmp != NULL);
1115 hpa = ((uintptr_t)vmp_get_pfn(vmp) << PAGESHIFT);
1116 (void) vmp_release(vmp);
1117
1118 /*
1119 * When originally ported from FreeBSD, the logic for
1120 * adding memory to the guest domain would
1121 * simultaneously remove it from the host domain. The
1122 * justification for that is not clear, and FreeBSD has
1123 * subsequently changed the behavior to not remove the
1124 * memory from the host domain.
1125 *
1126 * Leaving the guest memory in the host domain for the
1127 * life of the VM is necessary to make it available for
1128 * DMA, such as through viona in the TX path.
1129 */
1130 if (map) {
1131 iommu_create_mapping(vm->iommu, gpa, hpa, sz);
1132 } else {
1133 iommu_remove_mapping(vm->iommu, gpa, sz);
1134 }
1135
1136 gpa += PAGE_SIZE;
1137 }
1138 }
1139 vmc_destroy(vmc);
1140
1141 /*
1142 * Invalidate the cached translations associated with the domain
1143 * from which pages were removed.
1144 */
1145 iommu_invalidate_tlb(vm->iommu);
1146 }
1147
1148 int
vm_unassign_pptdev(struct vm * vm,int pptfd)1149 vm_unassign_pptdev(struct vm *vm, int pptfd)
1150 {
1151 int error;
1152
1153 error = ppt_unassign_device(vm, pptfd);
1154 if (error)
1155 return (error);
1156
1157 if (ppt_assigned_devices(vm) == 0)
1158 vm_iommu_modify(vm, false);
1159
1160 return (0);
1161 }
1162
1163 int
vm_assign_pptdev(struct vm * vm,int pptfd)1164 vm_assign_pptdev(struct vm *vm, int pptfd)
1165 {
1166 int error;
1167 vm_paddr_t maxaddr;
1168
1169 /* Set up the IOMMU to do the 'gpa' to 'hpa' translation */
1170 if (ppt_assigned_devices(vm) == 0) {
1171 KASSERT(vm->iommu == NULL,
1172 ("vm_assign_pptdev: iommu must be NULL"));
1173 maxaddr = vmm_sysmem_maxaddr(vm);
1174 vm->iommu = iommu_create_domain(maxaddr);
1175 if (vm->iommu == NULL)
1176 return (ENXIO);
1177 vm_iommu_modify(vm, true);
1178 }
1179
1180 error = ppt_assign_device(vm, pptfd);
1181 return (error);
1182 }
1183
1184 int
vm_get_register(struct vm * vm,int vcpuid,int reg,uint64_t * retval)1185 vm_get_register(struct vm *vm, int vcpuid, int reg, uint64_t *retval)
1186 {
1187 if (vcpuid < 0 || vcpuid >= vm->maxcpus)
1188 return (EINVAL);
1189
1190 if (reg >= VM_REG_LAST)
1191 return (EINVAL);
1192
1193 struct vcpu *vcpu = &vm->vcpu[vcpuid];
1194 switch (reg) {
1195 case VM_REG_GUEST_XCR0:
1196 *retval = vcpu->guest_xcr0;
1197 return (0);
1198 default:
1199 return (VMGETREG(vm->cookie, vcpuid, reg, retval));
1200 }
1201 }
1202
1203 int
vm_set_register(struct vm * vm,int vcpuid,int reg,uint64_t val)1204 vm_set_register(struct vm *vm, int vcpuid, int reg, uint64_t val)
1205 {
1206 if (vcpuid < 0 || vcpuid >= vm->maxcpus)
1207 return (EINVAL);
1208
1209 if (reg >= VM_REG_LAST)
1210 return (EINVAL);
1211
1212 int error;
1213 struct vcpu *vcpu = &vm->vcpu[vcpuid];
1214 switch (reg) {
1215 case VM_REG_GUEST_RIP:
1216 error = VMSETREG(vm->cookie, vcpuid, reg, val);
1217 if (error == 0) {
1218 vcpu->nextrip = val;
1219 }
1220 return (error);
1221 case VM_REG_GUEST_XCR0:
1222 if (!validate_guest_xcr0(val, vmm_get_host_xcr0())) {
1223 return (EINVAL);
1224 }
1225 vcpu->guest_xcr0 = val;
1226 return (0);
1227 default:
1228 return (VMSETREG(vm->cookie, vcpuid, reg, val));
1229 }
1230 }
1231
1232 static bool
is_descriptor_table(int reg)1233 is_descriptor_table(int reg)
1234 {
1235 switch (reg) {
1236 case VM_REG_GUEST_IDTR:
1237 case VM_REG_GUEST_GDTR:
1238 return (true);
1239 default:
1240 return (false);
1241 }
1242 }
1243
1244 static bool
is_segment_register(int reg)1245 is_segment_register(int reg)
1246 {
1247 switch (reg) {
1248 case VM_REG_GUEST_ES:
1249 case VM_REG_GUEST_CS:
1250 case VM_REG_GUEST_SS:
1251 case VM_REG_GUEST_DS:
1252 case VM_REG_GUEST_FS:
1253 case VM_REG_GUEST_GS:
1254 case VM_REG_GUEST_TR:
1255 case VM_REG_GUEST_LDTR:
1256 return (true);
1257 default:
1258 return (false);
1259 }
1260 }
1261
1262 int
vm_get_seg_desc(struct vm * vm,int vcpu,int reg,struct seg_desc * desc)1263 vm_get_seg_desc(struct vm *vm, int vcpu, int reg, struct seg_desc *desc)
1264 {
1265
1266 if (vcpu < 0 || vcpu >= vm->maxcpus)
1267 return (EINVAL);
1268
1269 if (!is_segment_register(reg) && !is_descriptor_table(reg))
1270 return (EINVAL);
1271
1272 return (VMGETDESC(vm->cookie, vcpu, reg, desc));
1273 }
1274
1275 int
vm_set_seg_desc(struct vm * vm,int vcpu,int reg,const struct seg_desc * desc)1276 vm_set_seg_desc(struct vm *vm, int vcpu, int reg, const struct seg_desc *desc)
1277 {
1278 if (vcpu < 0 || vcpu >= vm->maxcpus)
1279 return (EINVAL);
1280
1281 if (!is_segment_register(reg) && !is_descriptor_table(reg))
1282 return (EINVAL);
1283
1284 return (VMSETDESC(vm->cookie, vcpu, reg, desc));
1285 }
1286
1287 static int
translate_hma_xsave_result(hma_fpu_xsave_result_t res)1288 translate_hma_xsave_result(hma_fpu_xsave_result_t res)
1289 {
1290 switch (res) {
1291 case HFXR_OK:
1292 return (0);
1293 case HFXR_NO_SPACE:
1294 return (ENOSPC);
1295 case HFXR_BAD_ALIGN:
1296 case HFXR_UNSUP_FMT:
1297 case HFXR_UNSUP_FEAT:
1298 case HFXR_INVALID_DATA:
1299 return (EINVAL);
1300 default:
1301 panic("unexpected xsave result");
1302 }
1303 }
1304
1305 int
vm_get_fpu(struct vm * vm,int vcpuid,void * buf,size_t len)1306 vm_get_fpu(struct vm *vm, int vcpuid, void *buf, size_t len)
1307 {
1308 if (vcpuid < 0 || vcpuid >= vm->maxcpus)
1309 return (EINVAL);
1310
1311 struct vcpu *vcpu = &vm->vcpu[vcpuid];
1312 hma_fpu_xsave_result_t res;
1313
1314 res = hma_fpu_get_xsave_state(vcpu->guestfpu, buf, len);
1315 return (translate_hma_xsave_result(res));
1316 }
1317
1318 int
vm_set_fpu(struct vm * vm,int vcpuid,void * buf,size_t len)1319 vm_set_fpu(struct vm *vm, int vcpuid, void *buf, size_t len)
1320 {
1321 if (vcpuid < 0 || vcpuid >= vm->maxcpus)
1322 return (EINVAL);
1323
1324 struct vcpu *vcpu = &vm->vcpu[vcpuid];
1325 hma_fpu_xsave_result_t res;
1326
1327 res = hma_fpu_set_xsave_state(vcpu->guestfpu, buf, len);
1328 return (translate_hma_xsave_result(res));
1329 }
1330
1331 int
vm_get_run_state(struct vm * vm,int vcpuid,uint32_t * state,uint8_t * sipi_vec)1332 vm_get_run_state(struct vm *vm, int vcpuid, uint32_t *state, uint8_t *sipi_vec)
1333 {
1334 struct vcpu *vcpu;
1335
1336 if (vcpuid < 0 || vcpuid >= vm->maxcpus) {
1337 return (EINVAL);
1338 }
1339
1340 vcpu = &vm->vcpu[vcpuid];
1341
1342 vcpu_lock(vcpu);
1343 *state = vcpu->run_state;
1344 *sipi_vec = vcpu->sipi_vector;
1345 vcpu_unlock(vcpu);
1346
1347 return (0);
1348 }
1349
1350 int
vm_set_run_state(struct vm * vm,int vcpuid,uint32_t state,uint8_t sipi_vec)1351 vm_set_run_state(struct vm *vm, int vcpuid, uint32_t state, uint8_t sipi_vec)
1352 {
1353 struct vcpu *vcpu;
1354
1355 if (vcpuid < 0 || vcpuid >= vm->maxcpus) {
1356 return (EINVAL);
1357 }
1358 if (!VRS_IS_VALID(state)) {
1359 return (EINVAL);
1360 }
1361
1362 vcpu = &vm->vcpu[vcpuid];
1363
1364 vcpu_lock(vcpu);
1365 vcpu->run_state = state;
1366 vcpu->sipi_vector = sipi_vec;
1367 vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
1368 vcpu_unlock(vcpu);
1369
1370 return (0);
1371 }
1372
1373 int
vm_track_dirty_pages(struct vm * vm,uint64_t gpa,size_t len,uint8_t * bitmap)1374 vm_track_dirty_pages(struct vm *vm, uint64_t gpa, size_t len, uint8_t *bitmap)
1375 {
1376 ASSERT0(gpa & PAGEOFFSET);
1377 ASSERT0(len & PAGEOFFSET);
1378
1379 /*
1380 * The only difference in expectations between this legacy interface and
1381 * an equivalent call to vm_npt_do_operation() is the check for
1382 * dirty-page-tracking being enabled on the vmspace.
1383 */
1384 if (!vmspace_get_tracking(vm->vmspace)) {
1385 return (EPERM);
1386 }
1387
1388 vmspace_bits_operate(vm->vmspace, gpa, len,
1389 VBO_RESET_DIRTY | VBO_FLAG_BITMAP_OUT, bitmap);
1390 return (0);
1391 }
1392
1393 int
vm_npt_do_operation(struct vm * vm,uint64_t gpa,size_t len,uint32_t oper,uint8_t * bitmap,int * rvalp)1394 vm_npt_do_operation(struct vm *vm, uint64_t gpa, size_t len, uint32_t oper,
1395 uint8_t *bitmap, int *rvalp)
1396 {
1397 ASSERT0(gpa & PAGEOFFSET);
1398 ASSERT0(len & PAGEOFFSET);
1399
1400 /*
1401 * For now, the bits defined in vmm_dev.h are meant to match up 1:1 with
1402 * those in vmm_vm.h
1403 */
1404 CTASSERT(VNO_OP_RESET_DIRTY == VBO_RESET_DIRTY);
1405 CTASSERT(VNO_OP_SET_DIRTY == VBO_SET_DIRTY);
1406 CTASSERT(VNO_OP_GET_DIRTY == VBO_GET_DIRTY);
1407 CTASSERT(VNO_FLAG_BITMAP_IN == VBO_FLAG_BITMAP_IN);
1408 CTASSERT(VNO_FLAG_BITMAP_OUT == VBO_FLAG_BITMAP_OUT);
1409
1410 const uint32_t oper_only =
1411 oper & ~(VNO_FLAG_BITMAP_IN | VNO_FLAG_BITMAP_OUT);
1412 switch (oper_only) {
1413 case VNO_OP_RESET_DIRTY:
1414 case VNO_OP_SET_DIRTY:
1415 case VNO_OP_GET_DIRTY:
1416 if (len == 0) {
1417 break;
1418 }
1419 vmspace_bits_operate(vm->vmspace, gpa, len, oper, bitmap);
1420 break;
1421 case VNO_OP_GET_TRACK_DIRTY:
1422 ASSERT3P(rvalp, !=, NULL);
1423 *rvalp = vmspace_get_tracking(vm->vmspace) ? 1 : 0;
1424 break;
1425 case VNO_OP_EN_TRACK_DIRTY:
1426 return (vmspace_set_tracking(vm->vmspace, true));
1427 case VNO_OP_DIS_TRACK_DIRTY:
1428 return (vmspace_set_tracking(vm->vmspace, false));
1429 default:
1430 return (EINVAL);
1431 }
1432 return (0);
1433 }
1434
1435 static void
restore_guest_fpustate(struct vcpu * vcpu)1436 restore_guest_fpustate(struct vcpu *vcpu)
1437 {
1438 /* Save host FPU and restore guest FPU */
1439 fpu_stop_emulating();
1440 hma_fpu_start_guest(vcpu->guestfpu);
1441
1442 /* restore guest XCR0 if XSAVE is enabled in the host */
1443 if (rcr4() & CR4_XSAVE)
1444 load_xcr(0, vcpu->guest_xcr0);
1445
1446 /*
1447 * The FPU is now "dirty" with the guest's state so turn on emulation
1448 * to trap any access to the FPU by the host.
1449 */
1450 fpu_start_emulating();
1451 }
1452
1453 static void
save_guest_fpustate(struct vcpu * vcpu)1454 save_guest_fpustate(struct vcpu *vcpu)
1455 {
1456
1457 if ((rcr0() & CR0_TS) == 0)
1458 panic("fpu emulation not enabled in host!");
1459
1460 /* save guest XCR0 and restore host XCR0 */
1461 if (rcr4() & CR4_XSAVE) {
1462 vcpu->guest_xcr0 = rxcr(0);
1463 load_xcr(0, vmm_get_host_xcr0());
1464 }
1465
1466 /* save guest FPU and restore host FPU */
1467 fpu_stop_emulating();
1468 hma_fpu_stop_guest(vcpu->guestfpu);
1469 /*
1470 * When the host state has been restored, we should not re-enable
1471 * CR0.TS on illumos for eager FPU.
1472 */
1473 }
1474
1475 static int
vcpu_set_state_locked(struct vm * vm,int vcpuid,enum vcpu_state newstate,bool from_idle)1476 vcpu_set_state_locked(struct vm *vm, int vcpuid, enum vcpu_state newstate,
1477 bool from_idle)
1478 {
1479 struct vcpu *vcpu;
1480 int error;
1481
1482 vcpu = &vm->vcpu[vcpuid];
1483 vcpu_assert_locked(vcpu);
1484
1485 /*
1486 * State transitions from the vmmdev_ioctl() must always begin from
1487 * the VCPU_IDLE state. This guarantees that there is only a single
1488 * ioctl() operating on a vcpu at any point.
1489 */
1490 if (from_idle) {
1491 while (vcpu->state != VCPU_IDLE) {
1492 vcpu->reqidle = true;
1493 vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
1494 cv_wait(&vcpu->state_cv, &vcpu->lock);
1495 vcpu->reqidle = false;
1496 }
1497 } else {
1498 KASSERT(vcpu->state != VCPU_IDLE, ("invalid transition from "
1499 "vcpu idle state"));
1500 }
1501
1502 if (vcpu->state == VCPU_RUNNING) {
1503 KASSERT(vcpu->hostcpu == curcpu, ("curcpu %d and hostcpu %d "
1504 "mismatch for running vcpu", curcpu, vcpu->hostcpu));
1505 } else {
1506 KASSERT(vcpu->hostcpu == NOCPU, ("Invalid hostcpu %d for a "
1507 "vcpu that is not running", vcpu->hostcpu));
1508 }
1509
1510 /*
1511 * The following state transitions are allowed:
1512 * IDLE -> FROZEN -> IDLE
1513 * FROZEN -> RUNNING -> FROZEN
1514 * FROZEN -> SLEEPING -> FROZEN
1515 */
1516 switch (vcpu->state) {
1517 case VCPU_IDLE:
1518 case VCPU_RUNNING:
1519 case VCPU_SLEEPING:
1520 error = (newstate != VCPU_FROZEN);
1521 break;
1522 case VCPU_FROZEN:
1523 error = (newstate == VCPU_FROZEN);
1524 break;
1525 default:
1526 error = 1;
1527 break;
1528 }
1529
1530 if (error)
1531 return (EBUSY);
1532
1533 vcpu->state = newstate;
1534 if (newstate == VCPU_RUNNING)
1535 vcpu->hostcpu = curcpu;
1536 else
1537 vcpu->hostcpu = NOCPU;
1538
1539 if (newstate == VCPU_IDLE) {
1540 cv_broadcast(&vcpu->state_cv);
1541 }
1542
1543 return (0);
1544 }
1545
1546 static void
vcpu_require_state(struct vm * vm,int vcpuid,enum vcpu_state newstate)1547 vcpu_require_state(struct vm *vm, int vcpuid, enum vcpu_state newstate)
1548 {
1549 int error;
1550
1551 if ((error = vcpu_set_state(vm, vcpuid, newstate, false)) != 0)
1552 panic("Error %d setting state to %d\n", error, newstate);
1553 }
1554
1555 static void
vcpu_require_state_locked(struct vm * vm,int vcpuid,enum vcpu_state newstate)1556 vcpu_require_state_locked(struct vm *vm, int vcpuid, enum vcpu_state newstate)
1557 {
1558 int error;
1559
1560 if ((error = vcpu_set_state_locked(vm, vcpuid, newstate, false)) != 0)
1561 panic("Error %d setting state to %d", error, newstate);
1562 }
1563
1564 /*
1565 * Emulate a guest 'hlt' by sleeping until the vcpu is ready to run.
1566 */
1567 static int
vm_handle_hlt(struct vm * vm,int vcpuid,bool intr_disabled)1568 vm_handle_hlt(struct vm *vm, int vcpuid, bool intr_disabled)
1569 {
1570 struct vcpu *vcpu;
1571 int vcpu_halted, vm_halted;
1572 bool userspace_exit = false;
1573
1574 KASSERT(!CPU_ISSET(vcpuid, &vm->halted_cpus), ("vcpu already halted"));
1575
1576 vcpu = &vm->vcpu[vcpuid];
1577 vcpu_halted = 0;
1578 vm_halted = 0;
1579
1580 vcpu_lock(vcpu);
1581 while (1) {
1582 /*
1583 * Do a final check for pending interrupts (including NMI and
1584 * INIT) before putting this thread to sleep.
1585 */
1586 if (vm_nmi_pending(vm, vcpuid))
1587 break;
1588 if (vcpu_run_state_pending(vm, vcpuid))
1589 break;
1590 if (!intr_disabled) {
1591 if (vm_extint_pending(vm, vcpuid) ||
1592 vlapic_pending_intr(vcpu->vlapic, NULL)) {
1593 break;
1594 }
1595 }
1596
1597 /*
1598 * Also check for software events which would cause a wake-up.
1599 * This will set the appropriate exitcode directly, rather than
1600 * requiring a trip through VM_RUN().
1601 */
1602 if (vcpu_sleep_bailout_checks(vm, vcpuid)) {
1603 userspace_exit = true;
1604 break;
1605 }
1606
1607 /*
1608 * Some Linux guests implement "halt" by having all vcpus
1609 * execute HLT with interrupts disabled. 'halted_cpus' keeps
1610 * track of the vcpus that have entered this state. When all
1611 * vcpus enter the halted state the virtual machine is halted.
1612 */
1613 if (intr_disabled) {
1614 if (!vcpu_halted && halt_detection_enabled) {
1615 vcpu_halted = 1;
1616 CPU_SET_ATOMIC(vcpuid, &vm->halted_cpus);
1617 }
1618 if (CPU_CMP(&vm->halted_cpus, &vm->active_cpus) == 0) {
1619 vm_halted = 1;
1620 break;
1621 }
1622 }
1623
1624 vcpu_ustate_change(vm, vcpuid, VU_IDLE);
1625 vcpu_require_state_locked(vm, vcpuid, VCPU_SLEEPING);
1626 (void) cv_wait_sig(&vcpu->vcpu_cv, &vcpu->lock);
1627 vcpu_require_state_locked(vm, vcpuid, VCPU_FROZEN);
1628 vcpu_ustate_change(vm, vcpuid, VU_EMU_KERN);
1629 }
1630
1631 if (vcpu_halted)
1632 CPU_CLR_ATOMIC(vcpuid, &vm->halted_cpus);
1633
1634 vcpu_unlock(vcpu);
1635
1636 if (vm_halted) {
1637 (void) vm_suspend(vm, VM_SUSPEND_HALT, -1);
1638 }
1639
1640 return (userspace_exit ? -1 : 0);
1641 }
1642
1643 static int
vm_handle_paging(struct vm * vm,int vcpuid)1644 vm_handle_paging(struct vm *vm, int vcpuid)
1645 {
1646 struct vcpu *vcpu = &vm->vcpu[vcpuid];
1647 vm_client_t *vmc = vcpu->vmclient;
1648 struct vm_exit *vme = &vcpu->exitinfo;
1649 const int ftype = vme->u.paging.fault_type;
1650
1651 ASSERT0(vme->inst_length);
1652 ASSERT(ftype == PROT_READ || ftype == PROT_WRITE || ftype == PROT_EXEC);
1653
1654 if (vmc_fault(vmc, vme->u.paging.gpa, ftype) != 0) {
1655 /*
1656 * If the fault cannot be serviced, kick it out to userspace for
1657 * handling (or more likely, halting the instance).
1658 */
1659 return (-1);
1660 }
1661
1662 return (0);
1663 }
1664
1665 int
vm_service_mmio_read(struct vm * vm,int cpuid,uint64_t gpa,uint64_t * rval,int rsize)1666 vm_service_mmio_read(struct vm *vm, int cpuid, uint64_t gpa, uint64_t *rval,
1667 int rsize)
1668 {
1669 int err = ESRCH;
1670
1671 if (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE) {
1672 struct vlapic *vlapic = vm_lapic(vm, cpuid);
1673
1674 err = vlapic_mmio_read(vlapic, gpa, rval, rsize);
1675 } else if (gpa >= VIOAPIC_BASE && gpa < VIOAPIC_BASE + VIOAPIC_SIZE) {
1676 err = vioapic_mmio_read(vm, cpuid, gpa, rval, rsize);
1677 } else if (gpa >= VHPET_BASE && gpa < VHPET_BASE + VHPET_SIZE) {
1678 err = vhpet_mmio_read(vm, cpuid, gpa, rval, rsize);
1679 }
1680
1681 return (err);
1682 }
1683
1684 int
vm_service_mmio_write(struct vm * vm,int cpuid,uint64_t gpa,uint64_t wval,int wsize)1685 vm_service_mmio_write(struct vm *vm, int cpuid, uint64_t gpa, uint64_t wval,
1686 int wsize)
1687 {
1688 int err = ESRCH;
1689
1690 if (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE) {
1691 struct vlapic *vlapic = vm_lapic(vm, cpuid);
1692
1693 err = vlapic_mmio_write(vlapic, gpa, wval, wsize);
1694 } else if (gpa >= VIOAPIC_BASE && gpa < VIOAPIC_BASE + VIOAPIC_SIZE) {
1695 err = vioapic_mmio_write(vm, cpuid, gpa, wval, wsize);
1696 } else if (gpa >= VHPET_BASE && gpa < VHPET_BASE + VHPET_SIZE) {
1697 err = vhpet_mmio_write(vm, cpuid, gpa, wval, wsize);
1698 }
1699
1700 return (err);
1701 }
1702
1703 static int
vm_handle_mmio_emul(struct vm * vm,int vcpuid)1704 vm_handle_mmio_emul(struct vm *vm, int vcpuid)
1705 {
1706 struct vie *vie;
1707 struct vcpu *vcpu;
1708 struct vm_exit *vme;
1709 uint64_t inst_addr;
1710 int error, fault, cs_d;
1711
1712 vcpu = &vm->vcpu[vcpuid];
1713 vme = &vcpu->exitinfo;
1714 vie = vcpu->vie_ctx;
1715
1716 KASSERT(vme->inst_length == 0, ("%s: invalid inst_length %d",
1717 __func__, vme->inst_length));
1718
1719 inst_addr = vme->rip + vme->u.mmio_emul.cs_base;
1720 cs_d = vme->u.mmio_emul.cs_d;
1721
1722 /* Fetch the faulting instruction */
1723 if (vie_needs_fetch(vie)) {
1724 error = vie_fetch_instruction(vie, vm, vcpuid, inst_addr,
1725 &fault);
1726 if (error != 0) {
1727 return (error);
1728 } else if (fault) {
1729 /*
1730 * If a fault during instruction fetch was encountered,
1731 * it will have asserted that the appropriate exception
1732 * be injected at next entry.
1733 * No further work is required.
1734 */
1735 return (0);
1736 }
1737 }
1738
1739 if (vie_decode_instruction(vie, vm, vcpuid, cs_d) != 0) {
1740 /* Dump (unrecognized) instruction bytes in userspace */
1741 vie_fallback_exitinfo(vie, vme);
1742 return (-1);
1743 }
1744 if (vme->u.mmio_emul.gla != VIE_INVALID_GLA &&
1745 vie_verify_gla(vie, vm, vcpuid, vme->u.mmio_emul.gla) != 0) {
1746 /* Decoded GLA does not match GLA from VM exit state */
1747 vie_fallback_exitinfo(vie, vme);
1748 return (-1);
1749 }
1750
1751 repeat:
1752 error = vie_emulate_mmio(vie, vm, vcpuid);
1753 if (error < 0) {
1754 /*
1755 * MMIO not handled by any of the in-kernel-emulated devices, so
1756 * make a trip out to userspace for it.
1757 */
1758 vie_exitinfo(vie, vme);
1759 } else if (error == EAGAIN) {
1760 /*
1761 * Continue emulating the rep-prefixed instruction, which has
1762 * not completed its iterations.
1763 *
1764 * In case this can be emulated in-kernel and has a high
1765 * repetition count (causing a tight spin), it should be
1766 * deferential to yield conditions.
1767 */
1768 if (!vcpu_should_yield(vm, vcpuid)) {
1769 goto repeat;
1770 } else {
1771 /*
1772 * Defer to the contending load by making a trip to
1773 * userspace with a no-op (BOGUS) exit reason.
1774 */
1775 vie_reset(vie);
1776 vme->exitcode = VM_EXITCODE_BOGUS;
1777 return (-1);
1778 }
1779 } else if (error == 0) {
1780 /* Update %rip now that instruction has been emulated */
1781 vie_advance_pc(vie, &vcpu->nextrip);
1782 }
1783 return (error);
1784 }
1785
1786 static int
vm_handle_inout(struct vm * vm,int vcpuid,struct vm_exit * vme)1787 vm_handle_inout(struct vm *vm, int vcpuid, struct vm_exit *vme)
1788 {
1789 struct vcpu *vcpu;
1790 struct vie *vie;
1791 int err;
1792
1793 vcpu = &vm->vcpu[vcpuid];
1794 vie = vcpu->vie_ctx;
1795
1796 repeat:
1797 err = vie_emulate_inout(vie, vm, vcpuid);
1798
1799 if (err < 0) {
1800 /*
1801 * In/out not handled by any of the in-kernel-emulated devices,
1802 * so make a trip out to userspace for it.
1803 */
1804 vie_exitinfo(vie, vme);
1805 return (err);
1806 } else if (err == EAGAIN) {
1807 /*
1808 * Continue emulating the rep-prefixed ins/outs, which has not
1809 * completed its iterations.
1810 *
1811 * In case this can be emulated in-kernel and has a high
1812 * repetition count (causing a tight spin), it should be
1813 * deferential to yield conditions.
1814 */
1815 if (!vcpu_should_yield(vm, vcpuid)) {
1816 goto repeat;
1817 } else {
1818 /*
1819 * Defer to the contending load by making a trip to
1820 * userspace with a no-op (BOGUS) exit reason.
1821 */
1822 vie_reset(vie);
1823 vme->exitcode = VM_EXITCODE_BOGUS;
1824 return (-1);
1825 }
1826 } else if (err != 0) {
1827 /* Emulation failure. Bail all the way out to userspace. */
1828 vme->exitcode = VM_EXITCODE_INST_EMUL;
1829 bzero(&vme->u.inst_emul, sizeof (vme->u.inst_emul));
1830 return (-1);
1831 }
1832
1833 vie_advance_pc(vie, &vcpu->nextrip);
1834 return (0);
1835 }
1836
1837 static int
vm_handle_inst_emul(struct vm * vm,int vcpuid)1838 vm_handle_inst_emul(struct vm *vm, int vcpuid)
1839 {
1840 struct vie *vie;
1841 struct vcpu *vcpu;
1842 struct vm_exit *vme;
1843 uint64_t cs_base;
1844 int error, fault, cs_d;
1845
1846 vcpu = &vm->vcpu[vcpuid];
1847 vme = &vcpu->exitinfo;
1848 vie = vcpu->vie_ctx;
1849
1850 vie_cs_info(vie, vm, vcpuid, &cs_base, &cs_d);
1851
1852 /* Fetch the faulting instruction */
1853 ASSERT(vie_needs_fetch(vie));
1854 error = vie_fetch_instruction(vie, vm, vcpuid, vme->rip + cs_base,
1855 &fault);
1856 if (error != 0) {
1857 return (error);
1858 } else if (fault) {
1859 /*
1860 * If a fault during instruction fetch was encounted, it will
1861 * have asserted that the appropriate exception be injected at
1862 * next entry. No further work is required.
1863 */
1864 return (0);
1865 }
1866
1867 if (vie_decode_instruction(vie, vm, vcpuid, cs_d) != 0) {
1868 /* Dump (unrecognized) instruction bytes in userspace */
1869 vie_fallback_exitinfo(vie, vme);
1870 return (-1);
1871 }
1872
1873 error = vie_emulate_other(vie, vm, vcpuid);
1874 if (error != 0) {
1875 /*
1876 * Instruction emulation was unable to complete successfully, so
1877 * kick it out to userspace for handling.
1878 */
1879 vie_fallback_exitinfo(vie, vme);
1880 } else {
1881 /* Update %rip now that instruction has been emulated */
1882 vie_advance_pc(vie, &vcpu->nextrip);
1883 }
1884 return (error);
1885 }
1886
1887 static int
vm_handle_run_state(struct vm * vm,int vcpuid)1888 vm_handle_run_state(struct vm *vm, int vcpuid)
1889 {
1890 struct vcpu *vcpu = &vm->vcpu[vcpuid];
1891 bool handled = false;
1892
1893 vcpu_lock(vcpu);
1894 while (1) {
1895 if ((vcpu->run_state & VRS_PEND_INIT) != 0) {
1896 vcpu_unlock(vcpu);
1897 VERIFY0(vcpu_arch_reset(vm, vcpuid, true));
1898 vcpu_lock(vcpu);
1899
1900 vcpu->run_state &= ~(VRS_RUN | VRS_PEND_INIT);
1901 vcpu->run_state |= VRS_INIT;
1902 }
1903
1904 if ((vcpu->run_state & (VRS_INIT | VRS_RUN | VRS_PEND_SIPI)) ==
1905 (VRS_INIT | VRS_PEND_SIPI)) {
1906 const uint8_t vector = vcpu->sipi_vector;
1907
1908 vcpu_unlock(vcpu);
1909 VERIFY0(vcpu_vector_sipi(vm, vcpuid, vector));
1910 vcpu_lock(vcpu);
1911
1912 vcpu->run_state &= ~VRS_PEND_SIPI;
1913 vcpu->run_state |= VRS_RUN;
1914 }
1915
1916 /*
1917 * If the vCPU is now in the running state, there is no need to
1918 * wait for anything prior to re-entry.
1919 */
1920 if ((vcpu->run_state & VRS_RUN) != 0) {
1921 handled = true;
1922 break;
1923 }
1924
1925 /*
1926 * Also check for software events which would cause a wake-up.
1927 * This will set the appropriate exitcode directly, rather than
1928 * requiring a trip through VM_RUN().
1929 */
1930 if (vcpu_sleep_bailout_checks(vm, vcpuid)) {
1931 break;
1932 }
1933
1934 vcpu_ustate_change(vm, vcpuid, VU_IDLE);
1935 vcpu_require_state_locked(vm, vcpuid, VCPU_SLEEPING);
1936 (void) cv_wait_sig(&vcpu->vcpu_cv, &vcpu->lock);
1937 vcpu_require_state_locked(vm, vcpuid, VCPU_FROZEN);
1938 vcpu_ustate_change(vm, vcpuid, VU_EMU_KERN);
1939 }
1940 vcpu_unlock(vcpu);
1941
1942 return (handled ? 0 : -1);
1943 }
1944
1945 static int
vm_rdmtrr(const struct vm_mtrr * mtrr,uint32_t num,uint64_t * val)1946 vm_rdmtrr(const struct vm_mtrr *mtrr, uint32_t num, uint64_t *val)
1947 {
1948 switch (num) {
1949 case MSR_MTRRcap:
1950 *val = MTRR_CAP_WC | MTRR_CAP_FIXED | VMM_MTRR_VAR_MAX;
1951 break;
1952 case MSR_MTRRdefType:
1953 *val = mtrr->def_type;
1954 break;
1955 case MSR_MTRR4kBase ... MSR_MTRR4kBase + 7:
1956 *val = mtrr->fixed4k[num - MSR_MTRR4kBase];
1957 break;
1958 case MSR_MTRR16kBase ... MSR_MTRR16kBase + 1:
1959 *val = mtrr->fixed16k[num - MSR_MTRR16kBase];
1960 break;
1961 case MSR_MTRR64kBase:
1962 *val = mtrr->fixed64k;
1963 break;
1964 case MSR_MTRRVarBase ... MSR_MTRRVarBase + (VMM_MTRR_VAR_MAX * 2) - 1: {
1965 uint_t offset = num - MSR_MTRRVarBase;
1966 if (offset % 2 == 0) {
1967 *val = mtrr->var[offset / 2].base;
1968 } else {
1969 *val = mtrr->var[offset / 2].mask;
1970 }
1971 break;
1972 }
1973 default:
1974 return (EINVAL);
1975 }
1976
1977 return (0);
1978 }
1979
1980 static int
vm_wrmtrr(struct vm_mtrr * mtrr,uint32_t num,uint64_t val)1981 vm_wrmtrr(struct vm_mtrr *mtrr, uint32_t num, uint64_t val)
1982 {
1983 switch (num) {
1984 case MSR_MTRRcap:
1985 /* MTRRCAP is read only */
1986 return (EPERM);
1987 case MSR_MTRRdefType:
1988 if (val & ~VMM_MTRR_DEF_MASK) {
1989 /* generate #GP on writes to reserved fields */
1990 return (EINVAL);
1991 }
1992 mtrr->def_type = val;
1993 break;
1994 case MSR_MTRR4kBase ... MSR_MTRR4kBase + 7:
1995 mtrr->fixed4k[num - MSR_MTRR4kBase] = val;
1996 break;
1997 case MSR_MTRR16kBase ... MSR_MTRR16kBase + 1:
1998 mtrr->fixed16k[num - MSR_MTRR16kBase] = val;
1999 break;
2000 case MSR_MTRR64kBase:
2001 mtrr->fixed64k = val;
2002 break;
2003 case MSR_MTRRVarBase ... MSR_MTRRVarBase + (VMM_MTRR_VAR_MAX * 2) - 1: {
2004 uint_t offset = num - MSR_MTRRVarBase;
2005 if (offset % 2 == 0) {
2006 if (val & ~VMM_MTRR_PHYSBASE_MASK) {
2007 /* generate #GP on writes to reserved fields */
2008 return (EINVAL);
2009 }
2010 mtrr->var[offset / 2].base = val;
2011 } else {
2012 if (val & ~VMM_MTRR_PHYSMASK_MASK) {
2013 /* generate #GP on writes to reserved fields */
2014 return (EINVAL);
2015 }
2016 mtrr->var[offset / 2].mask = val;
2017 }
2018 break;
2019 }
2020 default:
2021 return (EINVAL);
2022 }
2023
2024 return (0);
2025 }
2026
2027 static bool
is_mtrr_msr(uint32_t msr)2028 is_mtrr_msr(uint32_t msr)
2029 {
2030 switch (msr) {
2031 case MSR_MTRRcap:
2032 case MSR_MTRRdefType:
2033 case MSR_MTRR4kBase ... MSR_MTRR4kBase + 7:
2034 case MSR_MTRR16kBase ... MSR_MTRR16kBase + 1:
2035 case MSR_MTRR64kBase:
2036 case MSR_MTRRVarBase ... MSR_MTRRVarBase + (VMM_MTRR_VAR_MAX * 2) - 1:
2037 return (true);
2038 default:
2039 return (false);
2040 }
2041 }
2042
2043 static int
vm_handle_rdmsr(struct vm * vm,int vcpuid,struct vm_exit * vme)2044 vm_handle_rdmsr(struct vm *vm, int vcpuid, struct vm_exit *vme)
2045 {
2046 struct vcpu *vcpu = &vm->vcpu[vcpuid];
2047 const uint32_t code = vme->u.msr.code;
2048 uint64_t val = 0;
2049
2050 switch (code) {
2051 case MSR_MCG_CAP:
2052 case MSR_MCG_STATUS:
2053 val = 0;
2054 break;
2055
2056 case MSR_MTRRcap:
2057 case MSR_MTRRdefType:
2058 case MSR_MTRR4kBase ... MSR_MTRR4kBase + 7:
2059 case MSR_MTRR16kBase ... MSR_MTRR16kBase + 1:
2060 case MSR_MTRR64kBase:
2061 case MSR_MTRRVarBase ... MSR_MTRRVarBase + (VMM_MTRR_VAR_MAX * 2) - 1:
2062 if (vm_rdmtrr(&vcpu->mtrr, code, &val) != 0)
2063 vm_inject_gp(vm, vcpuid);
2064 break;
2065
2066 case MSR_TSC:
2067 /*
2068 * Get the guest TSC, applying necessary vCPU offsets.
2069 *
2070 * In all likelihood, this should always be handled in guest
2071 * context by VMX/SVM rather than taking an exit. (Both VMX and
2072 * SVM pass through read-only access to MSR_TSC to the guest.)
2073 *
2074 * The VM-wide TSC offset and per-vCPU offset are included in
2075 * the calculations of vcpu_tsc_offset(), so this is sufficient
2076 * to use as the offset in our calculations.
2077 *
2078 * No physical offset is requested of vcpu_tsc_offset() since
2079 * rdtsc_offset() takes care of that instead.
2080 */
2081 val = calc_guest_tsc(rdtsc_offset(), vm->freq_multiplier,
2082 vcpu_tsc_offset(vm, vcpuid, false));
2083 break;
2084
2085 default:
2086 /*
2087 * Anything not handled at this point will be kicked out to
2088 * userspace for attempted processing there.
2089 */
2090 return (-1);
2091 }
2092
2093 VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_RAX,
2094 val & 0xffffffff));
2095 VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_RDX,
2096 val >> 32));
2097 return (0);
2098 }
2099
2100 static int
vm_handle_wrmsr(struct vm * vm,int vcpuid,struct vm_exit * vme)2101 vm_handle_wrmsr(struct vm *vm, int vcpuid, struct vm_exit *vme)
2102 {
2103 struct vcpu *vcpu = &vm->vcpu[vcpuid];
2104 const uint32_t code = vme->u.msr.code;
2105 const uint64_t val = vme->u.msr.wval;
2106
2107 switch (code) {
2108 case MSR_MCG_CAP:
2109 case MSR_MCG_STATUS:
2110 /* Ignore writes */
2111 break;
2112
2113 case MSR_MTRRcap:
2114 case MSR_MTRRdefType:
2115 case MSR_MTRR4kBase ... MSR_MTRR4kBase + 7:
2116 case MSR_MTRR16kBase ... MSR_MTRR16kBase + 1:
2117 case MSR_MTRR64kBase:
2118 case MSR_MTRRVarBase ... MSR_MTRRVarBase + (VMM_MTRR_VAR_MAX * 2) - 1:
2119 if (vm_wrmtrr(&vcpu->mtrr, code, val) != 0)
2120 vm_inject_gp(vm, vcpuid);
2121 break;
2122
2123 case MSR_TSC:
2124 /*
2125 * The effect of writing the TSC MSR is that a subsequent read
2126 * of the TSC would report that value written (plus any time
2127 * elapsed between the write and the read).
2128 *
2129 * To calculate that per-vCPU offset, we can work backwards from
2130 * the guest TSC at the time of write:
2131 *
2132 * value = current guest TSC + vCPU offset
2133 *
2134 * so therefore:
2135 *
2136 * value - current guest TSC = vCPU offset
2137 */
2138 vcpu->tsc_offset = val - calc_guest_tsc(rdtsc_offset(),
2139 vm->freq_multiplier, vm->tsc_offset);
2140 break;
2141
2142 default:
2143 /*
2144 * Anything not handled at this point will be kicked out to
2145 * userspace for attempted processing there.
2146 */
2147 return (-1);
2148 }
2149
2150 return (0);
2151 }
2152
2153 /*
2154 * Has a suspend event been asserted on the VM?
2155 *
2156 * The reason and (in the case of a triple-fault) source vcpuid are optionally
2157 * returned if such a state is present.
2158 */
2159 static bool
vm_is_suspended(struct vm * vm,struct vm_exit * vme)2160 vm_is_suspended(struct vm *vm, struct vm_exit *vme)
2161 {
2162 const int val = vm->suspend_how;
2163 if (val == 0) {
2164 return (false);
2165 } else {
2166 if (vme != NULL) {
2167 vme->exitcode = VM_EXITCODE_SUSPENDED;
2168 vme->u.suspended.how = val;
2169 vme->u.suspended.source = vm->suspend_source;
2170 /*
2171 * Normalize suspend event time and, on the off chance
2172 * that it was recorded as occuring prior to VM boot,
2173 * clamp it to a minimum of 0.
2174 */
2175 vme->u.suspended.when = (uint64_t)
2176 MAX(vm_normalize_hrtime(vm, vm->suspend_when), 0);
2177 }
2178 return (true);
2179 }
2180 }
2181
2182 int
vm_suspend(struct vm * vm,enum vm_suspend_how how,int source)2183 vm_suspend(struct vm *vm, enum vm_suspend_how how, int source)
2184 {
2185 if (how <= VM_SUSPEND_NONE || how >= VM_SUSPEND_LAST) {
2186 return (EINVAL);
2187 }
2188
2189 /*
2190 * Although the common case of calling vm_suspend() is via
2191 * ioctl(VM_SUSPEND), where all the vCPUs will be held in the frozen
2192 * state, it can also be called by a running vCPU to indicate a
2193 * triple-fault. In the latter case, there is no exclusion from a
2194 * racing vm_suspend() from a different vCPU, so assertion of the
2195 * suspended state must be performed carefully.
2196 *
2197 * The `suspend_when` is set first via atomic cmpset to pick a "winner"
2198 * of the suspension race, followed by population of 'suspend_source'.
2199 * Only after those are done, and a membar is emitted will 'suspend_how'
2200 * be set, which makes the suspended state visible to any vCPU checking
2201 * for it. That order will prevent an incomplete suspend state (between
2202 * 'how', 'source', and 'when') from being observed.
2203 */
2204 const hrtime_t now = gethrtime();
2205 if (atomic_cmpset_long((ulong_t *)&vm->suspend_when, 0, now) == 0) {
2206 return (EALREADY);
2207 }
2208 vm->suspend_source = source;
2209 membar_producer();
2210 vm->suspend_how = how;
2211
2212 /* Notify all active vcpus that they are now suspended. */
2213 for (uint_t i = 0; i < vm->maxcpus; i++) {
2214 struct vcpu *vcpu = &vm->vcpu[i];
2215
2216 vcpu_lock(vcpu);
2217
2218 if (!CPU_ISSET(i, &vm->active_cpus)) {
2219 /*
2220 * vCPUs not already marked as active can be ignored,
2221 * since they cannot become marked as active unless the
2222 * VM is reinitialized, clearing the suspended state.
2223 */
2224 vcpu_unlock(vcpu);
2225 continue;
2226 }
2227
2228 switch (vcpu->state) {
2229 case VCPU_IDLE:
2230 case VCPU_FROZEN:
2231 /*
2232 * vCPUs not locked by in-kernel activity can be
2233 * immediately marked as suspended: The ustate is moved
2234 * back to VU_INIT, since no further guest work will
2235 * occur while the VM is in this state.
2236 *
2237 * A FROZEN vCPU may still change its ustate on the way
2238 * out of the kernel, but a subsequent check at the end
2239 * of vm_run() should be adequate to fix it up.
2240 */
2241 vcpu_ustate_change(vm, i, VU_INIT);
2242 break;
2243 default:
2244 /*
2245 * Any vCPUs which are running or waiting in-kernel
2246 * (such as in HLT) are notified to pick up the newly
2247 * suspended state.
2248 */
2249 vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
2250 break;
2251 }
2252 vcpu_unlock(vcpu);
2253 }
2254 return (0);
2255 }
2256
2257 void
vm_exit_run_state(struct vm * vm,int vcpuid,uint64_t rip)2258 vm_exit_run_state(struct vm *vm, int vcpuid, uint64_t rip)
2259 {
2260 struct vm_exit *vmexit;
2261
2262 vmexit = vm_exitinfo(vm, vcpuid);
2263 vmexit->rip = rip;
2264 vmexit->inst_length = 0;
2265 vmexit->exitcode = VM_EXITCODE_RUN_STATE;
2266 vmm_stat_incr(vm, vcpuid, VMEXIT_RUN_STATE, 1);
2267 }
2268
2269 /*
2270 * Some vmm resources, such as the lapic, may have CPU-specific resources
2271 * allocated to them which would benefit from migration onto the host CPU which
2272 * is processing the vcpu state.
2273 */
2274 static void
vm_localize_resources(struct vm * vm,struct vcpu * vcpu)2275 vm_localize_resources(struct vm *vm, struct vcpu *vcpu)
2276 {
2277 /*
2278 * Localizing cyclic resources requires acquisition of cpu_lock, and
2279 * doing so with kpreempt disabled is a recipe for deadlock disaster.
2280 */
2281 VERIFY(curthread->t_preempt == 0);
2282
2283 /*
2284 * Do not bother with localization if this vCPU is about to return to
2285 * the host CPU it was last localized to.
2286 */
2287 if (vcpu->lastloccpu == curcpu)
2288 return;
2289
2290 /*
2291 * Localize system-wide resources to the primary boot vCPU. While any
2292 * of the other vCPUs may access them, it keeps the potential interrupt
2293 * footprint constrained to CPUs involved with this instance.
2294 */
2295 if (vcpu == &vm->vcpu[0]) {
2296 vhpet_localize_resources(vm->vhpet);
2297 vrtc_localize_resources(vm->vrtc);
2298 vatpit_localize_resources(vm->vatpit);
2299 }
2300
2301 vlapic_localize_resources(vcpu->vlapic);
2302
2303 vcpu->lastloccpu = curcpu;
2304 }
2305
2306 static void
vmm_savectx(void * arg)2307 vmm_savectx(void *arg)
2308 {
2309 vm_thread_ctx_t *vtc = arg;
2310 struct vm *vm = vtc->vtc_vm;
2311 const int vcpuid = vtc->vtc_vcpuid;
2312
2313 if (ops->vmsavectx != NULL) {
2314 ops->vmsavectx(vm->cookie, vcpuid);
2315 }
2316
2317 /*
2318 * Account for going off-cpu, unless the vCPU is idled, where being
2319 * off-cpu is the explicit point.
2320 */
2321 if (vm->vcpu[vcpuid].ustate != VU_IDLE) {
2322 vtc->vtc_ustate = vm->vcpu[vcpuid].ustate;
2323 vcpu_ustate_change(vm, vcpuid, VU_SCHED);
2324 }
2325
2326 /*
2327 * If the CPU holds the restored guest FPU state, save it and restore
2328 * the host FPU state before this thread goes off-cpu.
2329 */
2330 if ((vtc->vtc_status & VTCS_FPU_RESTORED) != 0) {
2331 struct vcpu *vcpu = &vm->vcpu[vcpuid];
2332
2333 save_guest_fpustate(vcpu);
2334 vtc->vtc_status &= ~VTCS_FPU_RESTORED;
2335 }
2336 }
2337
2338 static void
vmm_restorectx(void * arg)2339 vmm_restorectx(void *arg)
2340 {
2341 vm_thread_ctx_t *vtc = arg;
2342 struct vm *vm = vtc->vtc_vm;
2343 const int vcpuid = vtc->vtc_vcpuid;
2344
2345 /* Complete microstate accounting for vCPU being off-cpu */
2346 if (vm->vcpu[vcpuid].ustate != VU_IDLE) {
2347 vcpu_ustate_change(vm, vcpuid, vtc->vtc_ustate);
2348 }
2349
2350 /*
2351 * When coming back on-cpu, only restore the guest FPU status if the
2352 * thread is in a context marked as requiring it. This should be rare,
2353 * occurring only when a future logic error results in a voluntary
2354 * sleep during the VMRUN critical section.
2355 *
2356 * The common case will result in elision of the guest FPU state
2357 * restoration, deferring that action until it is clearly necessary
2358 * during vm_run.
2359 */
2360 VERIFY((vtc->vtc_status & VTCS_FPU_RESTORED) == 0);
2361 if ((vtc->vtc_status & VTCS_FPU_CTX_CRITICAL) != 0) {
2362 struct vcpu *vcpu = &vm->vcpu[vcpuid];
2363
2364 restore_guest_fpustate(vcpu);
2365 vtc->vtc_status |= VTCS_FPU_RESTORED;
2366 }
2367
2368 if (ops->vmrestorectx != NULL) {
2369 ops->vmrestorectx(vm->cookie, vcpuid);
2370 }
2371
2372 }
2373
2374 /* Convenience defines for parsing vm_entry`cmd values */
2375 #define VEC_MASK_FLAGS (VEC_FLAG_EXIT_CONSISTENT)
2376 #define VEC_MASK_CMD (~VEC_MASK_FLAGS)
2377
2378 static int
vm_entry_actions(struct vm * vm,int vcpuid,const struct vm_entry * entry,struct vm_exit * vme)2379 vm_entry_actions(struct vm *vm, int vcpuid, const struct vm_entry *entry,
2380 struct vm_exit *vme)
2381 {
2382 struct vcpu *vcpu = &vm->vcpu[vcpuid];
2383 struct vie *vie = vcpu->vie_ctx;
2384 int err = 0;
2385
2386 const uint_t cmd = entry->cmd & VEC_MASK_CMD;
2387 const uint_t flags = entry->cmd & VEC_MASK_FLAGS;
2388
2389 switch (cmd) {
2390 case VEC_DEFAULT:
2391 break;
2392 case VEC_DISCARD_INSTR:
2393 vie_reset(vie);
2394 break;
2395 case VEC_FULFILL_MMIO:
2396 err = vie_fulfill_mmio(vie, &entry->u.mmio);
2397 if (err == 0) {
2398 err = vie_emulate_mmio(vie, vm, vcpuid);
2399 if (err == 0) {
2400 vie_advance_pc(vie, &vcpu->nextrip);
2401 } else if (err < 0) {
2402 vie_exitinfo(vie, vme);
2403 } else if (err == EAGAIN) {
2404 /*
2405 * Clear the instruction emulation state in
2406 * order to re-enter VM context and continue
2407 * this 'rep <instruction>'
2408 */
2409 vie_reset(vie);
2410 err = 0;
2411 }
2412 }
2413 break;
2414 case VEC_FULFILL_INOUT:
2415 err = vie_fulfill_inout(vie, &entry->u.inout);
2416 if (err == 0) {
2417 err = vie_emulate_inout(vie, vm, vcpuid);
2418 if (err == 0) {
2419 vie_advance_pc(vie, &vcpu->nextrip);
2420 } else if (err < 0) {
2421 vie_exitinfo(vie, vme);
2422 } else if (err == EAGAIN) {
2423 /*
2424 * Clear the instruction emulation state in
2425 * order to re-enter VM context and continue
2426 * this 'rep ins/outs'
2427 */
2428 vie_reset(vie);
2429 err = 0;
2430 }
2431 }
2432 break;
2433 default:
2434 return (EINVAL);
2435 }
2436
2437 /*
2438 * Pay heed to requests for exit-when-vCPU-is-consistent requests, at
2439 * least when we are not immediately bound for another exit due to
2440 * multi-part instruction emulation or related causes.
2441 */
2442 if ((flags & VEC_FLAG_EXIT_CONSISTENT) != 0 && err == 0) {
2443 vcpu->reqconsist = true;
2444 }
2445
2446 return (err);
2447 }
2448
2449 static int
vm_loop_checks(struct vm * vm,int vcpuid,struct vm_exit * vme)2450 vm_loop_checks(struct vm *vm, int vcpuid, struct vm_exit *vme)
2451 {
2452 struct vie *vie;
2453
2454 vie = vm->vcpu[vcpuid].vie_ctx;
2455
2456 if (vie_pending(vie)) {
2457 /*
2458 * Userspace has not fulfilled the pending needs of the
2459 * instruction emulation, so bail back out.
2460 */
2461 vie_exitinfo(vie, vme);
2462 return (-1);
2463 }
2464
2465 return (0);
2466 }
2467
2468 int
vm_run(struct vm * vm,int vcpuid,const struct vm_entry * entry)2469 vm_run(struct vm *vm, int vcpuid, const struct vm_entry *entry)
2470 {
2471 int error;
2472 struct vcpu *vcpu;
2473 struct vm_exit *vme;
2474 bool intr_disabled;
2475 int affinity_type = CPU_CURRENT;
2476
2477 if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2478 return (EINVAL);
2479 if (!CPU_ISSET(vcpuid, &vm->active_cpus))
2480 return (EINVAL);
2481 if (vm->is_paused) {
2482 return (EBUSY);
2483 }
2484
2485 vcpu = &vm->vcpu[vcpuid];
2486 vme = &vcpu->exitinfo;
2487
2488 vcpu_ustate_change(vm, vcpuid, VU_EMU_KERN);
2489
2490 vcpu->vtc.vtc_status = 0;
2491 ctxop_attach(curthread, vcpu->ctxop);
2492
2493 error = vm_entry_actions(vm, vcpuid, entry, vme);
2494 if (error != 0) {
2495 goto exit;
2496 }
2497
2498 restart:
2499 error = vm_loop_checks(vm, vcpuid, vme);
2500 if (error != 0) {
2501 goto exit;
2502 }
2503
2504 thread_affinity_set(curthread, affinity_type);
2505 /*
2506 * Resource localization should happen after the CPU affinity for the
2507 * thread has been set to ensure that access from restricted contexts,
2508 * such as VMX-accelerated APIC operations, can occur without inducing
2509 * cyclic cross-calls.
2510 *
2511 * This must be done prior to disabling kpreempt via critical_enter().
2512 */
2513 vm_localize_resources(vm, vcpu);
2514 affinity_type = CPU_CURRENT;
2515 critical_enter();
2516
2517 /* Force a trip through update_sregs to reload %fs/%gs and friends */
2518 PCB_SET_UPDATE_SEGS(&ttolwp(curthread)->lwp_pcb);
2519
2520 if ((vcpu->vtc.vtc_status & VTCS_FPU_RESTORED) == 0) {
2521 restore_guest_fpustate(vcpu);
2522 vcpu->vtc.vtc_status |= VTCS_FPU_RESTORED;
2523 }
2524 vcpu->vtc.vtc_status |= VTCS_FPU_CTX_CRITICAL;
2525
2526 vcpu_require_state(vm, vcpuid, VCPU_RUNNING);
2527 error = VMRUN(vm->cookie, vcpuid, vcpu->nextrip);
2528 vcpu_require_state(vm, vcpuid, VCPU_FROZEN);
2529
2530 /*
2531 * Once clear of the delicate contexts comprising the VM_RUN handler,
2532 * thread CPU affinity can be loosened while other processing occurs.
2533 */
2534 vcpu->vtc.vtc_status &= ~VTCS_FPU_CTX_CRITICAL;
2535 thread_affinity_clear(curthread);
2536 critical_exit();
2537
2538 if (error != 0) {
2539 /* Communicate out any error from VMRUN() above */
2540 goto exit;
2541 }
2542
2543 vcpu->nextrip = vme->rip + vme->inst_length;
2544 switch (vme->exitcode) {
2545 case VM_EXITCODE_RUN_STATE:
2546 error = vm_handle_run_state(vm, vcpuid);
2547 break;
2548 case VM_EXITCODE_IOAPIC_EOI:
2549 vioapic_process_eoi(vm, vcpuid,
2550 vme->u.ioapic_eoi.vector);
2551 break;
2552 case VM_EXITCODE_HLT:
2553 intr_disabled = ((vme->u.hlt.rflags & PSL_I) == 0);
2554 error = vm_handle_hlt(vm, vcpuid, intr_disabled);
2555 break;
2556 case VM_EXITCODE_PAGING:
2557 error = vm_handle_paging(vm, vcpuid);
2558 break;
2559 case VM_EXITCODE_MMIO_EMUL:
2560 error = vm_handle_mmio_emul(vm, vcpuid);
2561 break;
2562 case VM_EXITCODE_INOUT:
2563 error = vm_handle_inout(vm, vcpuid, vme);
2564 break;
2565 case VM_EXITCODE_INST_EMUL:
2566 error = vm_handle_inst_emul(vm, vcpuid);
2567 break;
2568 case VM_EXITCODE_MONITOR:
2569 case VM_EXITCODE_MWAIT:
2570 case VM_EXITCODE_VMINSN:
2571 vm_inject_ud(vm, vcpuid);
2572 break;
2573 case VM_EXITCODE_RDMSR:
2574 error = vm_handle_rdmsr(vm, vcpuid, vme);
2575 break;
2576 case VM_EXITCODE_WRMSR:
2577 error = vm_handle_wrmsr(vm, vcpuid, vme);
2578 break;
2579 case VM_EXITCODE_HT:
2580 affinity_type = CPU_BEST;
2581 break;
2582 case VM_EXITCODE_MTRAP:
2583 VERIFY0(vm_suspend_cpu(vm, vcpuid));
2584 error = -1;
2585 break;
2586 default:
2587 /* handled in userland */
2588 error = -1;
2589 break;
2590 }
2591
2592 if (error == 0) {
2593 /* VM exit conditions handled in-kernel, continue running */
2594 goto restart;
2595 }
2596
2597 exit:
2598 kpreempt_disable();
2599 ctxop_detach(curthread, vcpu->ctxop);
2600 /* Make sure all of the needed vCPU context state is saved */
2601 vmm_savectx(&vcpu->vtc);
2602 kpreempt_enable();
2603
2604 /*
2605 * Bill time in userspace against VU_EMU_USER, unless the VM is
2606 * suspended, in which case VU_INIT is the choice.
2607 */
2608 vcpu_ustate_change(vm, vcpuid,
2609 vm_is_suspended(vm, NULL) ? VU_INIT : VU_EMU_USER);
2610
2611 return (error);
2612 }
2613
2614 int
vm_restart_instruction(void * arg,int vcpuid)2615 vm_restart_instruction(void *arg, int vcpuid)
2616 {
2617 struct vm *vm;
2618 struct vcpu *vcpu;
2619 enum vcpu_state state;
2620 uint64_t rip;
2621 int error;
2622
2623 vm = arg;
2624 if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2625 return (EINVAL);
2626
2627 vcpu = &vm->vcpu[vcpuid];
2628 state = vcpu_get_state(vm, vcpuid, NULL);
2629 if (state == VCPU_RUNNING) {
2630 /*
2631 * When a vcpu is "running" the next instruction is determined
2632 * by adding 'rip' and 'inst_length' in the vcpu's 'exitinfo'.
2633 * Thus setting 'inst_length' to zero will cause the current
2634 * instruction to be restarted.
2635 */
2636 vcpu->exitinfo.inst_length = 0;
2637 } else if (state == VCPU_FROZEN) {
2638 /*
2639 * When a vcpu is "frozen" it is outside the critical section
2640 * around VMRUN() and 'nextrip' points to the next instruction.
2641 * Thus instruction restart is achieved by setting 'nextrip'
2642 * to the vcpu's %rip.
2643 */
2644 error = vm_get_register(vm, vcpuid, VM_REG_GUEST_RIP, &rip);
2645 KASSERT(!error, ("%s: error %d getting rip", __func__, error));
2646 vcpu->nextrip = rip;
2647 } else {
2648 panic("%s: invalid state %d", __func__, state);
2649 }
2650 return (0);
2651 }
2652
2653 int
vm_exit_intinfo(struct vm * vm,int vcpuid,uint64_t info)2654 vm_exit_intinfo(struct vm *vm, int vcpuid, uint64_t info)
2655 {
2656 struct vcpu *vcpu;
2657
2658 if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2659 return (EINVAL);
2660
2661 vcpu = &vm->vcpu[vcpuid];
2662
2663 if (VM_INTINFO_PENDING(info)) {
2664 const uint32_t type = VM_INTINFO_TYPE(info);
2665 const uint8_t vector = VM_INTINFO_VECTOR(info);
2666
2667 if (type == VM_INTINFO_NMI && vector != IDT_NMI)
2668 return (EINVAL);
2669 if (type == VM_INTINFO_HWEXCP && vector >= 32)
2670 return (EINVAL);
2671 if (info & VM_INTINFO_MASK_RSVD)
2672 return (EINVAL);
2673 } else {
2674 info = 0;
2675 }
2676 vcpu->exit_intinfo = info;
2677 return (0);
2678 }
2679
2680 enum exc_class {
2681 EXC_BENIGN,
2682 EXC_CONTRIBUTORY,
2683 EXC_PAGEFAULT
2684 };
2685
2686 #define IDT_VE 20 /* Virtualization Exception (Intel specific) */
2687
2688 static enum exc_class
exception_class(uint64_t info)2689 exception_class(uint64_t info)
2690 {
2691 ASSERT(VM_INTINFO_PENDING(info));
2692
2693 /* Table 6-4, "Interrupt and Exception Classes", Intel SDM, Vol 3 */
2694 switch (VM_INTINFO_TYPE(info)) {
2695 case VM_INTINFO_HWINTR:
2696 case VM_INTINFO_SWINTR:
2697 case VM_INTINFO_NMI:
2698 return (EXC_BENIGN);
2699 default:
2700 /*
2701 * Hardware exception.
2702 *
2703 * SVM and VT-x use identical type values to represent NMI,
2704 * hardware interrupt and software interrupt.
2705 *
2706 * SVM uses type '3' for all exceptions. VT-x uses type '3'
2707 * for exceptions except #BP and #OF. #BP and #OF use a type
2708 * value of '5' or '6'. Therefore we don't check for explicit
2709 * values of 'type' to classify 'intinfo' into a hardware
2710 * exception.
2711 */
2712 break;
2713 }
2714
2715 switch (VM_INTINFO_VECTOR(info)) {
2716 case IDT_PF:
2717 case IDT_VE:
2718 return (EXC_PAGEFAULT);
2719 case IDT_DE:
2720 case IDT_TS:
2721 case IDT_NP:
2722 case IDT_SS:
2723 case IDT_GP:
2724 return (EXC_CONTRIBUTORY);
2725 default:
2726 return (EXC_BENIGN);
2727 }
2728 }
2729
2730 /*
2731 * Fetch event pending injection into the guest, if one exists.
2732 *
2733 * Returns true if an event is to be injected (which is placed in `retinfo`).
2734 */
2735 bool
vm_entry_intinfo(struct vm * vm,int vcpuid,uint64_t * retinfo)2736 vm_entry_intinfo(struct vm *vm, int vcpuid, uint64_t *retinfo)
2737 {
2738 struct vcpu *vcpu = &vm->vcpu[vcpuid];
2739 const uint64_t info1 = vcpu->exit_intinfo;
2740 vcpu->exit_intinfo = 0;
2741 const uint64_t info2 = vcpu->exc_pending;
2742 vcpu->exc_pending = 0;
2743
2744 if (VM_INTINFO_PENDING(info1) && VM_INTINFO_PENDING(info2)) {
2745 /*
2746 * If an exception occurs while attempting to call the
2747 * double-fault handler the processor enters shutdown mode
2748 * (aka triple fault).
2749 */
2750 if (VM_INTINFO_TYPE(info1) == VM_INTINFO_HWEXCP &&
2751 VM_INTINFO_VECTOR(info1) == IDT_DF) {
2752 (void) vm_suspend(vm, VM_SUSPEND_TRIPLEFAULT, vcpuid);
2753 *retinfo = 0;
2754 return (false);
2755 }
2756 /*
2757 * "Conditions for Generating a Double Fault"
2758 * Intel SDM, Vol3, Table 6-5
2759 */
2760 const enum exc_class exc1 = exception_class(info1);
2761 const enum exc_class exc2 = exception_class(info2);
2762 if ((exc1 == EXC_CONTRIBUTORY && exc2 == EXC_CONTRIBUTORY) ||
2763 (exc1 == EXC_PAGEFAULT && exc2 != EXC_BENIGN)) {
2764 /* Convert nested fault into a double fault. */
2765 *retinfo =
2766 VM_INTINFO_VALID |
2767 VM_INTINFO_DEL_ERRCODE |
2768 VM_INTINFO_HWEXCP |
2769 IDT_DF;
2770 } else {
2771 /* Handle exceptions serially */
2772 vcpu->exit_intinfo = info1;
2773 *retinfo = info2;
2774 }
2775 return (true);
2776 } else if (VM_INTINFO_PENDING(info1)) {
2777 *retinfo = info1;
2778 return (true);
2779 } else if (VM_INTINFO_PENDING(info2)) {
2780 *retinfo = info2;
2781 return (true);
2782 }
2783
2784 return (false);
2785 }
2786
2787 int
vm_get_intinfo(struct vm * vm,int vcpuid,uint64_t * info1,uint64_t * info2)2788 vm_get_intinfo(struct vm *vm, int vcpuid, uint64_t *info1, uint64_t *info2)
2789 {
2790 struct vcpu *vcpu;
2791
2792 if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2793 return (EINVAL);
2794
2795 vcpu = &vm->vcpu[vcpuid];
2796 *info1 = vcpu->exit_intinfo;
2797 *info2 = vcpu->exc_pending;
2798 return (0);
2799 }
2800
2801 int
vm_inject_exception(struct vm * vm,int vcpuid,uint8_t vector,bool errcode_valid,uint32_t errcode,bool restart_instruction)2802 vm_inject_exception(struct vm *vm, int vcpuid, uint8_t vector,
2803 bool errcode_valid, uint32_t errcode, bool restart_instruction)
2804 {
2805 struct vcpu *vcpu;
2806 uint64_t regval;
2807 int error;
2808
2809 if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2810 return (EINVAL);
2811
2812 if (vector >= 32)
2813 return (EINVAL);
2814
2815 /*
2816 * NMIs are to be injected via their own specialized path using
2817 * vm_inject_nmi().
2818 */
2819 if (vector == IDT_NMI) {
2820 return (EINVAL);
2821 }
2822
2823 /*
2824 * A double fault exception should never be injected directly into
2825 * the guest. It is a derived exception that results from specific
2826 * combinations of nested faults.
2827 */
2828 if (vector == IDT_DF) {
2829 return (EINVAL);
2830 }
2831
2832 vcpu = &vm->vcpu[vcpuid];
2833
2834 if (VM_INTINFO_PENDING(vcpu->exc_pending)) {
2835 /* Unable to inject exception due to one already pending */
2836 return (EBUSY);
2837 }
2838
2839 if (errcode_valid) {
2840 /*
2841 * Exceptions don't deliver an error code in real mode.
2842 */
2843 error = vm_get_register(vm, vcpuid, VM_REG_GUEST_CR0, ®val);
2844 VERIFY0(error);
2845 if ((regval & CR0_PE) == 0) {
2846 errcode_valid = false;
2847 }
2848 }
2849
2850 /*
2851 * From section 26.6.1 "Interruptibility State" in Intel SDM:
2852 *
2853 * Event blocking by "STI" or "MOV SS" is cleared after guest executes
2854 * one instruction or incurs an exception.
2855 */
2856 error = vm_set_register(vm, vcpuid, VM_REG_GUEST_INTR_SHADOW, 0);
2857 VERIFY0(error);
2858
2859 if (restart_instruction) {
2860 VERIFY0(vm_restart_instruction(vm, vcpuid));
2861 }
2862
2863 uint64_t val = VM_INTINFO_VALID | VM_INTINFO_HWEXCP | vector;
2864 if (errcode_valid) {
2865 val |= VM_INTINFO_DEL_ERRCODE;
2866 val |= (uint64_t)errcode << VM_INTINFO_SHIFT_ERRCODE;
2867 }
2868 vcpu->exc_pending = val;
2869 return (0);
2870 }
2871
2872 void
vm_inject_ud(struct vm * vm,int vcpuid)2873 vm_inject_ud(struct vm *vm, int vcpuid)
2874 {
2875 VERIFY0(vm_inject_exception(vm, vcpuid, IDT_UD, false, 0, true));
2876 }
2877
2878 void
vm_inject_gp(struct vm * vm,int vcpuid)2879 vm_inject_gp(struct vm *vm, int vcpuid)
2880 {
2881 VERIFY0(vm_inject_exception(vm, vcpuid, IDT_GP, true, 0, true));
2882 }
2883
2884 void
vm_inject_ac(struct vm * vm,int vcpuid,uint32_t errcode)2885 vm_inject_ac(struct vm *vm, int vcpuid, uint32_t errcode)
2886 {
2887 VERIFY0(vm_inject_exception(vm, vcpuid, IDT_AC, true, errcode, true));
2888 }
2889
2890 void
vm_inject_ss(struct vm * vm,int vcpuid,uint32_t errcode)2891 vm_inject_ss(struct vm *vm, int vcpuid, uint32_t errcode)
2892 {
2893 VERIFY0(vm_inject_exception(vm, vcpuid, IDT_SS, true, errcode, true));
2894 }
2895
2896 void
vm_inject_pf(struct vm * vm,int vcpuid,uint32_t errcode,uint64_t cr2)2897 vm_inject_pf(struct vm *vm, int vcpuid, uint32_t errcode, uint64_t cr2)
2898 {
2899 VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_CR2, cr2));
2900 VERIFY0(vm_inject_exception(vm, vcpuid, IDT_PF, true, errcode, true));
2901 }
2902
2903 static VMM_STAT(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu");
2904
2905 int
vm_inject_nmi(struct vm * vm,int vcpuid)2906 vm_inject_nmi(struct vm *vm, int vcpuid)
2907 {
2908 struct vcpu *vcpu;
2909
2910 if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2911 return (EINVAL);
2912
2913 vcpu = &vm->vcpu[vcpuid];
2914
2915 vcpu->nmi_pending = true;
2916 vcpu_notify_event(vm, vcpuid);
2917 return (0);
2918 }
2919
2920 bool
vm_nmi_pending(struct vm * vm,int vcpuid)2921 vm_nmi_pending(struct vm *vm, int vcpuid)
2922 {
2923 struct vcpu *vcpu = &vm->vcpu[vcpuid];
2924
2925 return (vcpu->nmi_pending);
2926 }
2927
2928 void
vm_nmi_clear(struct vm * vm,int vcpuid)2929 vm_nmi_clear(struct vm *vm, int vcpuid)
2930 {
2931 struct vcpu *vcpu = &vm->vcpu[vcpuid];
2932
2933 ASSERT(vcpu->nmi_pending);
2934
2935 vcpu->nmi_pending = false;
2936 vmm_stat_incr(vm, vcpuid, VCPU_NMI_COUNT, 1);
2937 }
2938
2939 static VMM_STAT(VCPU_EXTINT_COUNT, "number of ExtINTs delivered to vcpu");
2940
2941 int
vm_inject_extint(struct vm * vm,int vcpuid)2942 vm_inject_extint(struct vm *vm, int vcpuid)
2943 {
2944 struct vcpu *vcpu;
2945
2946 if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2947 return (EINVAL);
2948
2949 vcpu = &vm->vcpu[vcpuid];
2950
2951 vcpu->extint_pending = true;
2952 vcpu_notify_event(vm, vcpuid);
2953 return (0);
2954 }
2955
2956 bool
vm_extint_pending(struct vm * vm,int vcpuid)2957 vm_extint_pending(struct vm *vm, int vcpuid)
2958 {
2959 struct vcpu *vcpu = &vm->vcpu[vcpuid];
2960
2961 return (vcpu->extint_pending);
2962 }
2963
2964 void
vm_extint_clear(struct vm * vm,int vcpuid)2965 vm_extint_clear(struct vm *vm, int vcpuid)
2966 {
2967 struct vcpu *vcpu = &vm->vcpu[vcpuid];
2968
2969 ASSERT(vcpu->extint_pending);
2970
2971 vcpu->extint_pending = false;
2972 vmm_stat_incr(vm, vcpuid, VCPU_EXTINT_COUNT, 1);
2973 }
2974
2975 int
vm_inject_init(struct vm * vm,int vcpuid)2976 vm_inject_init(struct vm *vm, int vcpuid)
2977 {
2978 struct vcpu *vcpu;
2979
2980 if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2981 return (EINVAL);
2982
2983 vcpu = &vm->vcpu[vcpuid];
2984 vcpu_lock(vcpu);
2985 vcpu->run_state |= VRS_PEND_INIT;
2986 /*
2987 * As part of queuing the INIT request, clear any pending SIPI. It
2988 * would not otherwise survive across the reset of the vCPU when it
2989 * undergoes the requested INIT. We would not want it to linger when it
2990 * could be mistaken as a subsequent (after the INIT) SIPI request.
2991 */
2992 vcpu->run_state &= ~VRS_PEND_SIPI;
2993 vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
2994
2995 vcpu_unlock(vcpu);
2996 return (0);
2997 }
2998
2999 int
vm_inject_sipi(struct vm * vm,int vcpuid,uint8_t vector)3000 vm_inject_sipi(struct vm *vm, int vcpuid, uint8_t vector)
3001 {
3002 struct vcpu *vcpu;
3003
3004 if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3005 return (EINVAL);
3006
3007 vcpu = &vm->vcpu[vcpuid];
3008 vcpu_lock(vcpu);
3009 vcpu->run_state |= VRS_PEND_SIPI;
3010 vcpu->sipi_vector = vector;
3011 /* SIPI is only actionable if the CPU is waiting in INIT state */
3012 if ((vcpu->run_state & (VRS_INIT | VRS_RUN)) == VRS_INIT) {
3013 vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
3014 }
3015 vcpu_unlock(vcpu);
3016 return (0);
3017 }
3018
3019 bool
vcpu_run_state_pending(struct vm * vm,int vcpuid)3020 vcpu_run_state_pending(struct vm *vm, int vcpuid)
3021 {
3022 struct vcpu *vcpu;
3023
3024 ASSERT(vcpuid >= 0 && vcpuid < vm->maxcpus);
3025 vcpu = &vm->vcpu[vcpuid];
3026
3027 /* Of interest: vCPU not in running state or with pending INIT */
3028 return ((vcpu->run_state & (VRS_RUN | VRS_PEND_INIT)) != VRS_RUN);
3029 }
3030
3031 int
vcpu_arch_reset(struct vm * vm,int vcpuid,bool init_only)3032 vcpu_arch_reset(struct vm *vm, int vcpuid, bool init_only)
3033 {
3034 struct seg_desc desc;
3035 const enum vm_reg_name clear_regs[] = {
3036 VM_REG_GUEST_CR2,
3037 VM_REG_GUEST_CR3,
3038 VM_REG_GUEST_CR4,
3039 VM_REG_GUEST_RAX,
3040 VM_REG_GUEST_RBX,
3041 VM_REG_GUEST_RCX,
3042 VM_REG_GUEST_RSI,
3043 VM_REG_GUEST_RDI,
3044 VM_REG_GUEST_RBP,
3045 VM_REG_GUEST_RSP,
3046 VM_REG_GUEST_R8,
3047 VM_REG_GUEST_R9,
3048 VM_REG_GUEST_R10,
3049 VM_REG_GUEST_R11,
3050 VM_REG_GUEST_R12,
3051 VM_REG_GUEST_R13,
3052 VM_REG_GUEST_R14,
3053 VM_REG_GUEST_R15,
3054 VM_REG_GUEST_DR0,
3055 VM_REG_GUEST_DR1,
3056 VM_REG_GUEST_DR2,
3057 VM_REG_GUEST_DR3,
3058 VM_REG_GUEST_EFER,
3059 };
3060 const enum vm_reg_name data_segs[] = {
3061 VM_REG_GUEST_SS,
3062 VM_REG_GUEST_DS,
3063 VM_REG_GUEST_ES,
3064 VM_REG_GUEST_FS,
3065 VM_REG_GUEST_GS,
3066 };
3067 struct vcpu *vcpu = &vm->vcpu[vcpuid];
3068
3069 if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3070 return (EINVAL);
3071
3072 for (uint_t i = 0; i < nitems(clear_regs); i++) {
3073 VERIFY0(vm_set_register(vm, vcpuid, clear_regs[i], 0));
3074 }
3075
3076 VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_RFLAGS, 2));
3077 VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_RIP, 0xfff0));
3078 VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_CR0, 0x60000010));
3079
3080 /*
3081 * The prescribed contents of %rdx differ slightly between the Intel and
3082 * AMD architectural definitions. The former expects the Extended Model
3083 * in bits 16-19 where the latter expects all the Family, Model, and
3084 * Stepping be there. Common boot ROMs appear to disregard this
3085 * anyways, so we stick with a compromise value similar to what is
3086 * spelled out in the Intel SDM.
3087 */
3088 VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_RDX, 0x600));
3089
3090 VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_DR6, 0xffff0ff0));
3091 VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_DR7, 0x400));
3092
3093 /* CS: Present, R/W, Accessed */
3094 desc.access = 0x0093;
3095 desc.base = 0xffff0000;
3096 desc.limit = 0xffff;
3097 VERIFY0(vm_set_seg_desc(vm, vcpuid, VM_REG_GUEST_CS, &desc));
3098 VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_CS, 0xf000));
3099
3100 /* SS, DS, ES, FS, GS: Present, R/W, Accessed */
3101 desc.access = 0x0093;
3102 desc.base = 0;
3103 desc.limit = 0xffff;
3104 for (uint_t i = 0; i < nitems(data_segs); i++) {
3105 VERIFY0(vm_set_seg_desc(vm, vcpuid, data_segs[i], &desc));
3106 VERIFY0(vm_set_register(vm, vcpuid, data_segs[i], 0));
3107 }
3108
3109 /* GDTR, IDTR */
3110 desc.base = 0;
3111 desc.limit = 0xffff;
3112 VERIFY0(vm_set_seg_desc(vm, vcpuid, VM_REG_GUEST_GDTR, &desc));
3113 VERIFY0(vm_set_seg_desc(vm, vcpuid, VM_REG_GUEST_IDTR, &desc));
3114
3115 /* LDTR: Present, LDT */
3116 desc.access = 0x0082;
3117 desc.base = 0;
3118 desc.limit = 0xffff;
3119 VERIFY0(vm_set_seg_desc(vm, vcpuid, VM_REG_GUEST_LDTR, &desc));
3120 VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_LDTR, 0));
3121
3122 /* TR: Present, 32-bit TSS */
3123 desc.access = 0x008b;
3124 desc.base = 0;
3125 desc.limit = 0xffff;
3126 VERIFY0(vm_set_seg_desc(vm, vcpuid, VM_REG_GUEST_TR, &desc));
3127 VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_TR, 0));
3128
3129 vlapic_reset(vm_lapic(vm, vcpuid));
3130
3131 VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_INTR_SHADOW, 0));
3132
3133 vcpu->exit_intinfo = 0;
3134 vcpu->exc_pending = 0;
3135 vcpu->nmi_pending = false;
3136 vcpu->extint_pending = 0;
3137
3138 /*
3139 * A CPU reset caused by power-on or system reset clears more state than
3140 * one which is trigged from an INIT IPI.
3141 */
3142 if (!init_only) {
3143 vcpu->guest_xcr0 = XFEATURE_ENABLED_X87;
3144 (void) hma_fpu_init(vcpu->guestfpu);
3145
3146 /* XXX: clear MSRs and other pieces */
3147 bzero(&vcpu->mtrr, sizeof (vcpu->mtrr));
3148 }
3149
3150 return (0);
3151 }
3152
3153 static int
vcpu_vector_sipi(struct vm * vm,int vcpuid,uint8_t vector)3154 vcpu_vector_sipi(struct vm *vm, int vcpuid, uint8_t vector)
3155 {
3156 struct seg_desc desc;
3157
3158 if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3159 return (EINVAL);
3160
3161 /* CS: Present, R/W, Accessed */
3162 desc.access = 0x0093;
3163 desc.base = (uint64_t)vector << 12;
3164 desc.limit = 0xffff;
3165 VERIFY0(vm_set_seg_desc(vm, vcpuid, VM_REG_GUEST_CS, &desc));
3166 VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_CS,
3167 (uint64_t)vector << 8));
3168
3169 VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_RIP, 0));
3170
3171 return (0);
3172 }
3173
3174 int
vm_get_capability(struct vm * vm,int vcpu,int type,int * retval)3175 vm_get_capability(struct vm *vm, int vcpu, int type, int *retval)
3176 {
3177 if (vcpu < 0 || vcpu >= vm->maxcpus)
3178 return (EINVAL);
3179
3180 if (type < 0 || type >= VM_CAP_MAX)
3181 return (EINVAL);
3182
3183 return (VMGETCAP(vm->cookie, vcpu, type, retval));
3184 }
3185
3186 int
vm_set_capability(struct vm * vm,int vcpu,int type,int val)3187 vm_set_capability(struct vm *vm, int vcpu, int type, int val)
3188 {
3189 if (vcpu < 0 || vcpu >= vm->maxcpus)
3190 return (EINVAL);
3191
3192 if (type < 0 || type >= VM_CAP_MAX)
3193 return (EINVAL);
3194
3195 return (VMSETCAP(vm->cookie, vcpu, type, val));
3196 }
3197
3198 vcpu_cpuid_config_t *
vm_cpuid_config(struct vm * vm,int vcpuid)3199 vm_cpuid_config(struct vm *vm, int vcpuid)
3200 {
3201 ASSERT3S(vcpuid, >=, 0);
3202 ASSERT3S(vcpuid, <, VM_MAXCPU);
3203
3204 return (&vm->vcpu[vcpuid].cpuid_cfg);
3205 }
3206
3207 struct vlapic *
vm_lapic(struct vm * vm,int cpu)3208 vm_lapic(struct vm *vm, int cpu)
3209 {
3210 ASSERT3S(cpu, >=, 0);
3211 ASSERT3S(cpu, <, VM_MAXCPU);
3212
3213 return (vm->vcpu[cpu].vlapic);
3214 }
3215
3216 struct vioapic *
vm_ioapic(struct vm * vm)3217 vm_ioapic(struct vm *vm)
3218 {
3219
3220 return (vm->vioapic);
3221 }
3222
3223 struct vhpet *
vm_hpet(struct vm * vm)3224 vm_hpet(struct vm *vm)
3225 {
3226
3227 return (vm->vhpet);
3228 }
3229
3230 void *
vm_iommu_domain(struct vm * vm)3231 vm_iommu_domain(struct vm *vm)
3232 {
3233
3234 return (vm->iommu);
3235 }
3236
3237 int
vcpu_set_state(struct vm * vm,int vcpuid,enum vcpu_state newstate,bool from_idle)3238 vcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state newstate,
3239 bool from_idle)
3240 {
3241 int error;
3242 struct vcpu *vcpu;
3243
3244 if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3245 panic("vcpu_set_state: invalid vcpuid %d", vcpuid);
3246
3247 vcpu = &vm->vcpu[vcpuid];
3248
3249 vcpu_lock(vcpu);
3250 error = vcpu_set_state_locked(vm, vcpuid, newstate, from_idle);
3251 vcpu_unlock(vcpu);
3252
3253 return (error);
3254 }
3255
3256 enum vcpu_state
vcpu_get_state(struct vm * vm,int vcpuid,int * hostcpu)3257 vcpu_get_state(struct vm *vm, int vcpuid, int *hostcpu)
3258 {
3259 struct vcpu *vcpu;
3260 enum vcpu_state state;
3261
3262 if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3263 panic("vcpu_get_state: invalid vcpuid %d", vcpuid);
3264
3265 vcpu = &vm->vcpu[vcpuid];
3266
3267 vcpu_lock(vcpu);
3268 state = vcpu->state;
3269 if (hostcpu != NULL)
3270 *hostcpu = vcpu->hostcpu;
3271 vcpu_unlock(vcpu);
3272
3273 return (state);
3274 }
3275
3276 /*
3277 * Calculate the TSC offset for a vCPU, applying physical CPU adjustments if
3278 * requested. The offset calculations include the VM-wide TSC offset.
3279 */
3280 uint64_t
vcpu_tsc_offset(struct vm * vm,int vcpuid,bool phys_adj)3281 vcpu_tsc_offset(struct vm *vm, int vcpuid, bool phys_adj)
3282 {
3283 ASSERT(vcpuid >= 0 && vcpuid < vm->maxcpus);
3284
3285 uint64_t vcpu_off = vm->tsc_offset + vm->vcpu[vcpuid].tsc_offset;
3286
3287 if (phys_adj) {
3288 /* Include any offset for the current physical CPU too */
3289 vcpu_off += vmm_host_tsc_delta();
3290 }
3291
3292 return (vcpu_off);
3293 }
3294
3295 uint64_t
vm_get_freq_multiplier(struct vm * vm)3296 vm_get_freq_multiplier(struct vm *vm)
3297 {
3298 return (vm->freq_multiplier);
3299 }
3300
3301 /* Normalize hrtime against the boot time for a VM */
3302 hrtime_t
vm_normalize_hrtime(struct vm * vm,hrtime_t hrt)3303 vm_normalize_hrtime(struct vm *vm, hrtime_t hrt)
3304 {
3305 /* To avoid underflow/overflow UB, perform math as unsigned */
3306 return ((hrtime_t)((uint64_t)hrt - (uint64_t)vm->boot_hrtime));
3307 }
3308
3309 /* Denormalize hrtime against the boot time for a VM */
3310 hrtime_t
vm_denormalize_hrtime(struct vm * vm,hrtime_t hrt)3311 vm_denormalize_hrtime(struct vm *vm, hrtime_t hrt)
3312 {
3313 /* To avoid underflow/overflow UB, perform math as unsigned */
3314 return ((hrtime_t)((uint64_t)hrt + (uint64_t)vm->boot_hrtime));
3315 }
3316
3317 int
vm_activate_cpu(struct vm * vm,int vcpuid)3318 vm_activate_cpu(struct vm *vm, int vcpuid)
3319 {
3320
3321 if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3322 return (EINVAL);
3323
3324 if (CPU_ISSET(vcpuid, &vm->active_cpus))
3325 return (EBUSY);
3326
3327 if (vm_is_suspended(vm, NULL)) {
3328 return (EBUSY);
3329 }
3330
3331 CPU_SET_ATOMIC(vcpuid, &vm->active_cpus);
3332
3333 /*
3334 * It is possible that this vCPU was undergoing activation at the same
3335 * time that the VM was being suspended.
3336 */
3337 if (vm_is_suspended(vm, NULL)) {
3338 return (EBUSY);
3339 }
3340
3341 return (0);
3342 }
3343
3344 int
vm_suspend_cpu(struct vm * vm,int vcpuid)3345 vm_suspend_cpu(struct vm *vm, int vcpuid)
3346 {
3347 int i;
3348
3349 if (vcpuid < -1 || vcpuid >= vm->maxcpus)
3350 return (EINVAL);
3351
3352 if (vcpuid == -1) {
3353 vm->debug_cpus = vm->active_cpus;
3354 for (i = 0; i < vm->maxcpus; i++) {
3355 if (CPU_ISSET(i, &vm->active_cpus))
3356 vcpu_notify_event(vm, i);
3357 }
3358 } else {
3359 if (!CPU_ISSET(vcpuid, &vm->active_cpus))
3360 return (EINVAL);
3361
3362 CPU_SET_ATOMIC(vcpuid, &vm->debug_cpus);
3363 vcpu_notify_event(vm, vcpuid);
3364 }
3365 return (0);
3366 }
3367
3368 int
vm_resume_cpu(struct vm * vm,int vcpuid)3369 vm_resume_cpu(struct vm *vm, int vcpuid)
3370 {
3371
3372 if (vcpuid < -1 || vcpuid >= vm->maxcpus)
3373 return (EINVAL);
3374
3375 if (vcpuid == -1) {
3376 CPU_ZERO(&vm->debug_cpus);
3377 } else {
3378 if (!CPU_ISSET(vcpuid, &vm->debug_cpus))
3379 return (EINVAL);
3380
3381 CPU_CLR_ATOMIC(vcpuid, &vm->debug_cpus);
3382 }
3383 return (0);
3384 }
3385
3386 static bool
vcpu_bailout_checks(struct vm * vm,int vcpuid)3387 vcpu_bailout_checks(struct vm *vm, int vcpuid)
3388 {
3389 struct vcpu *vcpu = &vm->vcpu[vcpuid];
3390 struct vm_exit *vme = &vcpu->exitinfo;
3391
3392 ASSERT(vcpuid >= 0 && vcpuid < vm->maxcpus);
3393
3394 /*
3395 * Check if VM is suspended, only passing the 'vm_exit *' to be
3396 * populated if this check is being performed as part of entry.
3397 */
3398 if (vm_is_suspended(vm, vme)) {
3399 /* Confirm exit details are as expected */
3400 VERIFY3S(vme->exitcode, ==, VM_EXITCODE_SUSPENDED);
3401 VERIFY(vme->u.suspended.how > VM_SUSPEND_NONE &&
3402 vme->u.suspended.how < VM_SUSPEND_LAST);
3403
3404 return (true);
3405 }
3406 if (vcpu->reqidle) {
3407 /*
3408 * Another thread is trying to lock this vCPU and is waiting for
3409 * it to enter the VCPU_IDLE state. Take a lap with a BOGUS
3410 * exit to allow other thread(s) access to this vCPU.
3411 */
3412 vme->exitcode = VM_EXITCODE_BOGUS;
3413 vmm_stat_incr(vm, vcpuid, VMEXIT_REQIDLE, 1);
3414 return (true);
3415 }
3416 if (vcpu->reqbarrier) {
3417 /*
3418 * Similar to 'reqidle', userspace has requested that this vCPU
3419 * be pushed to a barrier by exiting to userspace. Take that
3420 * lap with BOGUS and clear the flag.
3421 */
3422 vme->exitcode = VM_EXITCODE_BOGUS;
3423 vcpu->reqbarrier = false;
3424 return (true);
3425 }
3426 if (vcpu->reqconsist) {
3427 /*
3428 * We only expect exit-when-consistent requests to be asserted
3429 * during entry, not as an otherwise spontaneous condition. As
3430 * such, we do not count it among the exit statistics, and emit
3431 * the expected BOGUS exitcode, while clearing the request.
3432 */
3433 vme->exitcode = VM_EXITCODE_BOGUS;
3434 vcpu->reqconsist = false;
3435 return (true);
3436 }
3437 if (vcpu_should_yield(vm, vcpuid)) {
3438 vme->exitcode = VM_EXITCODE_BOGUS;
3439 vmm_stat_incr(vm, vcpuid, VMEXIT_ASTPENDING, 1);
3440 return (true);
3441 }
3442 if (CPU_ISSET(vcpuid, &vm->debug_cpus)) {
3443 vme->exitcode = VM_EXITCODE_DEBUG;
3444 return (true);
3445 }
3446
3447 return (false);
3448 }
3449
3450 static bool
vcpu_sleep_bailout_checks(struct vm * vm,int vcpuid)3451 vcpu_sleep_bailout_checks(struct vm *vm, int vcpuid)
3452 {
3453 if (vcpu_bailout_checks(vm, vcpuid)) {
3454 struct vcpu *vcpu = &vm->vcpu[vcpuid];
3455 struct vm_exit *vme = &vcpu->exitinfo;
3456
3457 /*
3458 * Bail-out check done prior to sleeping (in vCPU contexts like
3459 * HLT or wait-for-SIPI) expect that %rip is already populated
3460 * in the vm_exit structure, and we would only modify the
3461 * exitcode and clear the inst_length.
3462 */
3463 vme->inst_length = 0;
3464 return (true);
3465 }
3466 return (false);
3467 }
3468
3469 bool
vcpu_entry_bailout_checks(struct vm * vm,int vcpuid,uint64_t rip)3470 vcpu_entry_bailout_checks(struct vm *vm, int vcpuid, uint64_t rip)
3471 {
3472 if (vcpu_bailout_checks(vm, vcpuid)) {
3473 struct vcpu *vcpu = &vm->vcpu[vcpuid];
3474 struct vm_exit *vme = &vcpu->exitinfo;
3475
3476 /*
3477 * Bail-out checks done as part of VM entry require an updated
3478 * %rip to populate the vm_exit struct if any of the conditions
3479 * of interest are matched in the check.
3480 */
3481 vme->rip = rip;
3482 vme->inst_length = 0;
3483 return (true);
3484 }
3485 return (false);
3486 }
3487
3488 int
vm_vcpu_barrier(struct vm * vm,int vcpuid)3489 vm_vcpu_barrier(struct vm *vm, int vcpuid)
3490 {
3491 if (vcpuid >= 0 && vcpuid < vm->maxcpus) {
3492 struct vcpu *vcpu = &vm->vcpu[vcpuid];
3493
3494 /* Push specified vCPU to barrier */
3495 vcpu_lock(vcpu);
3496 if (CPU_ISSET(vcpuid, &vm->active_cpus)) {
3497 vcpu->reqbarrier = true;
3498 vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
3499 }
3500 vcpu_unlock(vcpu);
3501
3502 return (0);
3503 } else if (vcpuid == -1) {
3504 /* Push all (active) vCPUs to barrier */
3505 for (int i = 0; i < vm->maxcpus; i++) {
3506 struct vcpu *vcpu = &vm->vcpu[i];
3507
3508 vcpu_lock(vcpu);
3509 if (CPU_ISSET(vcpuid, &vm->active_cpus)) {
3510 vcpu->reqbarrier = true;
3511 vcpu_notify_event_locked(vcpu,
3512 VCPU_NOTIFY_EXIT);
3513 }
3514 vcpu_unlock(vcpu);
3515 }
3516
3517 return (0);
3518 } else {
3519 return (EINVAL);
3520 }
3521 }
3522
3523 cpuset_t
vm_active_cpus(struct vm * vm)3524 vm_active_cpus(struct vm *vm)
3525 {
3526 return (vm->active_cpus);
3527 }
3528
3529 cpuset_t
vm_debug_cpus(struct vm * vm)3530 vm_debug_cpus(struct vm *vm)
3531 {
3532 return (vm->debug_cpus);
3533 }
3534
3535 void *
vcpu_stats(struct vm * vm,int vcpuid)3536 vcpu_stats(struct vm *vm, int vcpuid)
3537 {
3538
3539 return (vm->vcpu[vcpuid].stats);
3540 }
3541
3542 int
vm_get_x2apic_state(struct vm * vm,int vcpuid,enum x2apic_state * state)3543 vm_get_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state *state)
3544 {
3545 if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3546 return (EINVAL);
3547
3548 *state = vm->vcpu[vcpuid].x2apic_state;
3549
3550 return (0);
3551 }
3552
3553 int
vm_set_x2apic_state(struct vm * vm,int vcpuid,enum x2apic_state state)3554 vm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
3555 {
3556 if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3557 return (EINVAL);
3558
3559 if (state >= X2APIC_STATE_LAST)
3560 return (EINVAL);
3561
3562 vm->vcpu[vcpuid].x2apic_state = state;
3563
3564 vlapic_set_x2apic_state(vm, vcpuid, state);
3565
3566 return (0);
3567 }
3568
3569 /*
3570 * This function is called to ensure that a vcpu "sees" a pending event
3571 * as soon as possible:
3572 * - If the vcpu thread is sleeping then it is woken up.
3573 * - If the vcpu is running on a different host_cpu then an IPI will be directed
3574 * to the host_cpu to cause the vcpu to trap into the hypervisor.
3575 */
3576 static void
vcpu_notify_event_locked(struct vcpu * vcpu,vcpu_notify_t ntype)3577 vcpu_notify_event_locked(struct vcpu *vcpu, vcpu_notify_t ntype)
3578 {
3579 int hostcpu;
3580
3581 ASSERT(ntype == VCPU_NOTIFY_APIC || VCPU_NOTIFY_EXIT);
3582
3583 hostcpu = vcpu->hostcpu;
3584 if (vcpu->state == VCPU_RUNNING) {
3585 KASSERT(hostcpu != NOCPU, ("vcpu running on invalid hostcpu"));
3586 if (hostcpu != curcpu) {
3587 if (ntype == VCPU_NOTIFY_APIC) {
3588 vlapic_post_intr(vcpu->vlapic, hostcpu);
3589 } else {
3590 poke_cpu(hostcpu);
3591 }
3592 } else {
3593 /*
3594 * If the 'vcpu' is running on 'curcpu' then it must
3595 * be sending a notification to itself (e.g. SELF_IPI).
3596 * The pending event will be picked up when the vcpu
3597 * transitions back to guest context.
3598 */
3599 }
3600 } else {
3601 KASSERT(hostcpu == NOCPU, ("vcpu state %d not consistent "
3602 "with hostcpu %d", vcpu->state, hostcpu));
3603 if (vcpu->state == VCPU_SLEEPING) {
3604 cv_signal(&vcpu->vcpu_cv);
3605 }
3606 }
3607 }
3608
3609 void
vcpu_notify_event(struct vm * vm,int vcpuid)3610 vcpu_notify_event(struct vm *vm, int vcpuid)
3611 {
3612 struct vcpu *vcpu = &vm->vcpu[vcpuid];
3613
3614 vcpu_lock(vcpu);
3615 vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
3616 vcpu_unlock(vcpu);
3617 }
3618
3619 void
vcpu_notify_event_type(struct vm * vm,int vcpuid,vcpu_notify_t ntype)3620 vcpu_notify_event_type(struct vm *vm, int vcpuid, vcpu_notify_t ntype)
3621 {
3622 struct vcpu *vcpu = &vm->vcpu[vcpuid];
3623
3624 if (ntype == VCPU_NOTIFY_NONE) {
3625 return;
3626 }
3627
3628 vcpu_lock(vcpu);
3629 vcpu_notify_event_locked(vcpu, ntype);
3630 vcpu_unlock(vcpu);
3631 }
3632
3633 void
vcpu_ustate_change(struct vm * vm,int vcpuid,enum vcpu_ustate ustate)3634 vcpu_ustate_change(struct vm *vm, int vcpuid, enum vcpu_ustate ustate)
3635 {
3636 struct vcpu *vcpu = &vm->vcpu[vcpuid];
3637 const hrtime_t now = gethrtime();
3638
3639 ASSERT3S(ustate, <, VU_MAX);
3640 ASSERT3S(ustate, >=, VU_INIT);
3641
3642 if (ustate == vcpu->ustate) {
3643 return;
3644 }
3645
3646 const hrtime_t delta = now - vcpu->ustate_when;
3647 vcpu->ustate_total[vcpu->ustate] += delta;
3648
3649 membar_producer();
3650
3651 vcpu->ustate_when = now;
3652 vcpu->ustate = ustate;
3653 }
3654
3655 void *
vm_get_cookie(struct vm * vm)3656 vm_get_cookie(struct vm *vm)
3657 {
3658 return (vm->cookie);
3659 }
3660
3661 struct vmspace *
vm_get_vmspace(struct vm * vm)3662 vm_get_vmspace(struct vm *vm)
3663 {
3664
3665 return (vm->vmspace);
3666 }
3667
3668 struct vm_client *
vm_get_vmclient(struct vm * vm,int vcpuid)3669 vm_get_vmclient(struct vm *vm, int vcpuid)
3670 {
3671 return (vm->vcpu[vcpuid].vmclient);
3672 }
3673
3674 int
vm_apicid2vcpuid(struct vm * vm,int apicid)3675 vm_apicid2vcpuid(struct vm *vm, int apicid)
3676 {
3677 /*
3678 * XXX apic id is assumed to be numerically identical to vcpu id
3679 */
3680 return (apicid);
3681 }
3682
3683 struct vatpic *
vm_atpic(struct vm * vm)3684 vm_atpic(struct vm *vm)
3685 {
3686 return (vm->vatpic);
3687 }
3688
3689 struct vatpit *
vm_atpit(struct vm * vm)3690 vm_atpit(struct vm *vm)
3691 {
3692 return (vm->vatpit);
3693 }
3694
3695 struct vpmtmr *
vm_pmtmr(struct vm * vm)3696 vm_pmtmr(struct vm *vm)
3697 {
3698
3699 return (vm->vpmtmr);
3700 }
3701
3702 struct vrtc *
vm_rtc(struct vm * vm)3703 vm_rtc(struct vm *vm)
3704 {
3705
3706 return (vm->vrtc);
3707 }
3708
3709 enum vm_reg_name
vm_segment_name(int seg)3710 vm_segment_name(int seg)
3711 {
3712 static enum vm_reg_name seg_names[] = {
3713 VM_REG_GUEST_ES,
3714 VM_REG_GUEST_CS,
3715 VM_REG_GUEST_SS,
3716 VM_REG_GUEST_DS,
3717 VM_REG_GUEST_FS,
3718 VM_REG_GUEST_GS
3719 };
3720
3721 KASSERT(seg >= 0 && seg < nitems(seg_names),
3722 ("%s: invalid segment encoding %d", __func__, seg));
3723 return (seg_names[seg]);
3724 }
3725
3726 void
vm_copy_teardown(struct vm * vm,int vcpuid,struct vm_copyinfo * copyinfo,uint_t num_copyinfo)3727 vm_copy_teardown(struct vm *vm, int vcpuid, struct vm_copyinfo *copyinfo,
3728 uint_t num_copyinfo)
3729 {
3730 for (uint_t idx = 0; idx < num_copyinfo; idx++) {
3731 if (copyinfo[idx].cookie != NULL) {
3732 (void) vmp_release((vm_page_t *)copyinfo[idx].cookie);
3733 }
3734 }
3735 bzero(copyinfo, num_copyinfo * sizeof (struct vm_copyinfo));
3736 }
3737
3738 int
vm_copy_setup(struct vm * vm,int vcpuid,struct vm_guest_paging * paging,uint64_t gla,size_t len,int prot,struct vm_copyinfo * copyinfo,uint_t num_copyinfo,int * fault)3739 vm_copy_setup(struct vm *vm, int vcpuid, struct vm_guest_paging *paging,
3740 uint64_t gla, size_t len, int prot, struct vm_copyinfo *copyinfo,
3741 uint_t num_copyinfo, int *fault)
3742 {
3743 uint_t idx, nused;
3744 size_t n, off, remaining;
3745 vm_client_t *vmc = vm_get_vmclient(vm, vcpuid);
3746
3747 bzero(copyinfo, sizeof (struct vm_copyinfo) * num_copyinfo);
3748
3749 nused = 0;
3750 remaining = len;
3751 while (remaining > 0) {
3752 uint64_t gpa;
3753 int error;
3754
3755 if (nused >= num_copyinfo)
3756 return (EFAULT);
3757 error = vm_gla2gpa(vm, vcpuid, paging, gla, prot, &gpa, fault);
3758 if (error || *fault)
3759 return (error);
3760 off = gpa & PAGEOFFSET;
3761 n = min(remaining, PAGESIZE - off);
3762 copyinfo[nused].gpa = gpa;
3763 copyinfo[nused].len = n;
3764 remaining -= n;
3765 gla += n;
3766 nused++;
3767 }
3768
3769 for (idx = 0; idx < nused; idx++) {
3770 vm_page_t *vmp;
3771 caddr_t hva;
3772
3773 vmp = vmc_hold(vmc, copyinfo[idx].gpa & PAGEMASK, prot);
3774 if (vmp == NULL) {
3775 break;
3776 }
3777 if ((prot & PROT_WRITE) != 0) {
3778 hva = (caddr_t)vmp_get_writable(vmp);
3779 } else {
3780 hva = (caddr_t)vmp_get_readable(vmp);
3781 }
3782 copyinfo[idx].hva = hva + (copyinfo[idx].gpa & PAGEOFFSET);
3783 copyinfo[idx].cookie = vmp;
3784 copyinfo[idx].prot = prot;
3785 }
3786
3787 if (idx != nused) {
3788 vm_copy_teardown(vm, vcpuid, copyinfo, num_copyinfo);
3789 return (EFAULT);
3790 } else {
3791 *fault = 0;
3792 return (0);
3793 }
3794 }
3795
3796 void
vm_copyin(struct vm * vm,int vcpuid,struct vm_copyinfo * copyinfo,void * kaddr,size_t len)3797 vm_copyin(struct vm *vm, int vcpuid, struct vm_copyinfo *copyinfo, void *kaddr,
3798 size_t len)
3799 {
3800 char *dst;
3801 int idx;
3802
3803 dst = kaddr;
3804 idx = 0;
3805 while (len > 0) {
3806 ASSERT(copyinfo[idx].prot & PROT_READ);
3807
3808 bcopy(copyinfo[idx].hva, dst, copyinfo[idx].len);
3809 len -= copyinfo[idx].len;
3810 dst += copyinfo[idx].len;
3811 idx++;
3812 }
3813 }
3814
3815 void
vm_copyout(struct vm * vm,int vcpuid,const void * kaddr,struct vm_copyinfo * copyinfo,size_t len)3816 vm_copyout(struct vm *vm, int vcpuid, const void *kaddr,
3817 struct vm_copyinfo *copyinfo, size_t len)
3818 {
3819 const char *src;
3820 int idx;
3821
3822 src = kaddr;
3823 idx = 0;
3824 while (len > 0) {
3825 ASSERT(copyinfo[idx].prot & PROT_WRITE);
3826
3827 bcopy(src, copyinfo[idx].hva, copyinfo[idx].len);
3828 len -= copyinfo[idx].len;
3829 src += copyinfo[idx].len;
3830 idx++;
3831 }
3832 }
3833
3834 /*
3835 * Return the amount of in-use and wired memory for the VM. Since
3836 * these are global stats, only return the values with for vCPU 0
3837 */
3838 VMM_STAT_DECLARE(VMM_MEM_RESIDENT);
3839
3840 static void
vm_get_rescnt(struct vm * vm,int vcpu,struct vmm_stat_type * stat)3841 vm_get_rescnt(struct vm *vm, int vcpu, struct vmm_stat_type *stat)
3842 {
3843 if (vcpu == 0) {
3844 vmm_stat_set(vm, vcpu, VMM_MEM_RESIDENT,
3845 PAGE_SIZE * vmspace_resident_count(vm->vmspace));
3846 }
3847 }
3848
3849 VMM_STAT_FUNC(VMM_MEM_RESIDENT, "Resident memory", vm_get_rescnt);
3850
3851 int
vm_ioport_access(struct vm * vm,int vcpuid,bool in,uint16_t port,uint8_t bytes,uint32_t * val)3852 vm_ioport_access(struct vm *vm, int vcpuid, bool in, uint16_t port,
3853 uint8_t bytes, uint32_t *val)
3854 {
3855 return (vm_inout_access(&vm->ioports, in, port, bytes, val));
3856 }
3857
3858 /*
3859 * bhyve-internal interfaces to attach or detach IO port handlers.
3860 * Must be called with VM write lock held for safety.
3861 */
3862 int
vm_ioport_attach(struct vm * vm,uint16_t port,ioport_handler_t func,void * arg,void ** cookie)3863 vm_ioport_attach(struct vm *vm, uint16_t port, ioport_handler_t func, void *arg,
3864 void **cookie)
3865 {
3866 int err;
3867 err = vm_inout_attach(&vm->ioports, port, IOPF_DEFAULT, func, arg);
3868 if (err == 0) {
3869 *cookie = (void *)IOP_GEN_COOKIE(func, arg, port);
3870 }
3871 return (err);
3872 }
3873 int
vm_ioport_detach(struct vm * vm,void ** cookie,ioport_handler_t * old_func,void ** old_arg)3874 vm_ioport_detach(struct vm *vm, void **cookie, ioport_handler_t *old_func,
3875 void **old_arg)
3876 {
3877 uint16_t port = IOP_PORT_FROM_COOKIE((uintptr_t)*cookie);
3878 int err;
3879
3880 err = vm_inout_detach(&vm->ioports, port, false, old_func, old_arg);
3881 if (err == 0) {
3882 *cookie = NULL;
3883 }
3884 return (err);
3885 }
3886
3887 /*
3888 * External driver interfaces to attach or detach IO port handlers.
3889 * Must be called with VM write lock held for safety.
3890 */
3891 int
vm_ioport_hook(struct vm * vm,uint16_t port,ioport_handler_t func,void * arg,void ** cookie)3892 vm_ioport_hook(struct vm *vm, uint16_t port, ioport_handler_t func,
3893 void *arg, void **cookie)
3894 {
3895 int err;
3896
3897 if (port == 0) {
3898 return (EINVAL);
3899 }
3900
3901 err = vm_inout_attach(&vm->ioports, port, IOPF_DRV_HOOK, func, arg);
3902 if (err == 0) {
3903 *cookie = (void *)IOP_GEN_COOKIE(func, arg, port);
3904 }
3905 return (err);
3906 }
3907 void
vm_ioport_unhook(struct vm * vm,void ** cookie)3908 vm_ioport_unhook(struct vm *vm, void **cookie)
3909 {
3910 uint16_t port = IOP_PORT_FROM_COOKIE((uintptr_t)*cookie);
3911 ioport_handler_t old_func;
3912 void *old_arg;
3913 int err;
3914
3915 err = vm_inout_detach(&vm->ioports, port, true, &old_func, &old_arg);
3916
3917 /* ioport-hook-using drivers are expected to be well-behaved */
3918 VERIFY0(err);
3919 VERIFY(IOP_GEN_COOKIE(old_func, old_arg, port) == (uintptr_t)*cookie);
3920
3921 *cookie = NULL;
3922 }
3923
3924 int
vmm_kstat_update_vcpu(struct kstat * ksp,int rw)3925 vmm_kstat_update_vcpu(struct kstat *ksp, int rw)
3926 {
3927 struct vm *vm = ksp->ks_private;
3928 vmm_vcpu_kstats_t *vvk = ksp->ks_data;
3929 const int vcpuid = vvk->vvk_vcpu.value.ui32;
3930 struct vcpu *vcpu = &vm->vcpu[vcpuid];
3931
3932 ASSERT3U(vcpuid, <, VM_MAXCPU);
3933
3934 vvk->vvk_time_init.value.ui64 = vcpu->ustate_total[VU_INIT];
3935 vvk->vvk_time_run.value.ui64 = vcpu->ustate_total[VU_RUN];
3936 vvk->vvk_time_idle.value.ui64 = vcpu->ustate_total[VU_IDLE];
3937 vvk->vvk_time_emu_kern.value.ui64 = vcpu->ustate_total[VU_EMU_KERN];
3938 vvk->vvk_time_emu_user.value.ui64 = vcpu->ustate_total[VU_EMU_USER];
3939 vvk->vvk_time_sched.value.ui64 = vcpu->ustate_total[VU_SCHED];
3940
3941 return (0);
3942 }
3943
3944 SET_DECLARE(vmm_data_version_entries, const vmm_data_version_entry_t);
3945
3946 static int
vmm_data_find(const vmm_data_req_t * req,const vmm_data_version_entry_t ** resp)3947 vmm_data_find(const vmm_data_req_t *req, const vmm_data_version_entry_t **resp)
3948 {
3949 const vmm_data_version_entry_t **vdpp, *vdp;
3950
3951 ASSERT(resp != NULL);
3952 ASSERT(req->vdr_result_len != NULL);
3953
3954 SET_FOREACH(vdpp, vmm_data_version_entries) {
3955 vdp = *vdpp;
3956 if (vdp->vdve_class != req->vdr_class ||
3957 vdp->vdve_version != req->vdr_version) {
3958 continue;
3959 }
3960
3961 /*
3962 * Enforce any data length expectation expressed by the provider
3963 * for this data.
3964 */
3965 if (vdp->vdve_len_expect != 0 &&
3966 vdp->vdve_len_expect > req->vdr_len) {
3967 *req->vdr_result_len = vdp->vdve_len_expect;
3968 return (ENOSPC);
3969 }
3970
3971 /*
3972 * Make sure that the provided vcpuid is acceptable for the
3973 * backend handler.
3974 */
3975 if (vdp->vdve_readf != NULL || vdp->vdve_writef != NULL) {
3976 /*
3977 * While it is tempting to demand the -1 sentinel value
3978 * in vcpuid here, that expectation was not established
3979 * for early consumers, so it is ignored.
3980 */
3981 } else if (vdp->vdve_vcpu_readf != NULL ||
3982 vdp->vdve_vcpu_writef != NULL) {
3983 /*
3984 * Per-vCPU handlers which permit "wildcard" access will
3985 * accept a vcpuid of -1 (for VM-wide data), while all
3986 * others expect vcpuid [0, VM_MAXCPU).
3987 */
3988 const int llimit = vdp->vdve_vcpu_wildcard ? -1 : 0;
3989 if (req->vdr_vcpuid < llimit ||
3990 req->vdr_vcpuid >= VM_MAXCPU) {
3991 return (EINVAL);
3992 }
3993 } else {
3994 /*
3995 * A provider with neither VM-wide nor per-vCPU handlers
3996 * is completely unexpected. Such a situation should be
3997 * made into a compile-time error. Bail out for now,
3998 * rather than punishing the user with a panic.
3999 */
4000 return (EINVAL);
4001 }
4002
4003
4004 *resp = vdp;
4005 return (0);
4006 }
4007 return (EINVAL);
4008 }
4009
4010 static void *
vmm_data_from_class(const vmm_data_req_t * req,struct vm * vm)4011 vmm_data_from_class(const vmm_data_req_t *req, struct vm *vm)
4012 {
4013 switch (req->vdr_class) {
4014 case VDC_REGISTER:
4015 case VDC_MSR:
4016 case VDC_FPU:
4017 case VDC_LAPIC:
4018 case VDC_VMM_ARCH:
4019 /*
4020 * These have per-CPU handling which is dispatched outside
4021 * vmm_data_version_entries listing.
4022 */
4023 panic("Unexpected per-vcpu class %u", req->vdr_class);
4024 break;
4025
4026 case VDC_IOAPIC:
4027 return (vm->vioapic);
4028 case VDC_ATPIT:
4029 return (vm->vatpit);
4030 case VDC_ATPIC:
4031 return (vm->vatpic);
4032 case VDC_HPET:
4033 return (vm->vhpet);
4034 case VDC_PM_TIMER:
4035 return (vm->vpmtmr);
4036 case VDC_RTC:
4037 return (vm->vrtc);
4038 case VDC_VMM_TIME:
4039 return (vm);
4040 case VDC_VERSION:
4041 /*
4042 * Play along with all of the other classes which need backup
4043 * data, even though version info does not require it.
4044 */
4045 return (vm);
4046
4047 default:
4048 /* The data class will have been validated by now */
4049 panic("Unexpected class %u", req->vdr_class);
4050 }
4051 }
4052
4053 const uint32_t default_msr_iter[] = {
4054 /*
4055 * Although EFER is also available via the get/set-register interface,
4056 * we include it in the default list of emitted MSRs.
4057 */
4058 MSR_EFER,
4059
4060 /*
4061 * While gsbase and fsbase are accessible via the MSR accessors, they
4062 * are not included in MSR iteration since they are covered by the
4063 * segment descriptor interface too.
4064 */
4065 MSR_KGSBASE,
4066
4067 MSR_STAR,
4068 MSR_LSTAR,
4069 MSR_CSTAR,
4070 MSR_SF_MASK,
4071
4072 MSR_SYSENTER_CS_MSR,
4073 MSR_SYSENTER_ESP_MSR,
4074 MSR_SYSENTER_EIP_MSR,
4075
4076 MSR_PAT,
4077
4078 MSR_TSC,
4079
4080 MSR_MTRRcap,
4081 MSR_MTRRdefType,
4082 MSR_MTRR4kBase, MSR_MTRR4kBase + 1, MSR_MTRR4kBase + 2,
4083 MSR_MTRR4kBase + 3, MSR_MTRR4kBase + 4, MSR_MTRR4kBase + 5,
4084 MSR_MTRR4kBase + 6, MSR_MTRR4kBase + 7,
4085 MSR_MTRR16kBase, MSR_MTRR16kBase + 1,
4086 MSR_MTRR64kBase,
4087 };
4088
4089 static int
vmm_data_read_msr(struct vm * vm,int vcpuid,uint32_t msr,uint64_t * value)4090 vmm_data_read_msr(struct vm *vm, int vcpuid, uint32_t msr, uint64_t *value)
4091 {
4092 int err = 0;
4093
4094 switch (msr) {
4095 case MSR_TSC:
4096 /*
4097 * The vmm-data interface for MSRs provides access to the
4098 * per-vCPU offset of the TSC, when reading/writing MSR_TSC.
4099 *
4100 * The VM-wide offset (and scaling) of the guest TSC is accessed
4101 * via the VMM_TIME data class.
4102 */
4103 *value = vm->vcpu[vcpuid].tsc_offset;
4104 return (0);
4105
4106 default:
4107 if (is_mtrr_msr(msr)) {
4108 err = vm_rdmtrr(&vm->vcpu[vcpuid].mtrr, msr, value);
4109 } else {
4110 err = ops->vmgetmsr(vm->cookie, vcpuid, msr, value);
4111 }
4112 break;
4113 }
4114
4115 return (err);
4116 }
4117
4118 static int
vmm_data_write_msr(struct vm * vm,int vcpuid,uint32_t msr,uint64_t value)4119 vmm_data_write_msr(struct vm *vm, int vcpuid, uint32_t msr, uint64_t value)
4120 {
4121 int err = 0;
4122
4123 switch (msr) {
4124 case MSR_TSC:
4125 /* See vmm_data_read_msr() for more detail */
4126 vm->vcpu[vcpuid].tsc_offset = value;
4127 return (0);
4128 case MSR_MTRRcap: {
4129 /*
4130 * MTRRcap is read-only. If the desired value matches the
4131 * existing one, consider it a success.
4132 */
4133 uint64_t comp;
4134 err = vm_rdmtrr(&vm->vcpu[vcpuid].mtrr, msr, &comp);
4135 if (err == 0 && comp != value) {
4136 return (EINVAL);
4137 }
4138 break;
4139 }
4140 default:
4141 if (is_mtrr_msr(msr)) {
4142 /* MTRRcap is already handled above */
4143 ASSERT3U(msr, !=, MSR_MTRRcap);
4144
4145 err = vm_wrmtrr(&vm->vcpu[vcpuid].mtrr, msr, value);
4146 } else {
4147 err = ops->vmsetmsr(vm->cookie, vcpuid, msr, value);
4148 }
4149 break;
4150 }
4151
4152 return (err);
4153 }
4154
4155 static int
vmm_data_read_msrs(struct vm * vm,int vcpuid,const vmm_data_req_t * req)4156 vmm_data_read_msrs(struct vm *vm, int vcpuid, const vmm_data_req_t *req)
4157 {
4158 VERIFY3U(req->vdr_class, ==, VDC_MSR);
4159 VERIFY3U(req->vdr_version, ==, 1);
4160
4161 struct vdi_field_entry_v1 *entryp = req->vdr_data;
4162
4163 /* Specific MSRs requested */
4164 if ((req->vdr_flags & VDX_FLAG_READ_COPYIN) != 0) {
4165 const uint_t count =
4166 req->vdr_len / sizeof (struct vdi_field_entry_v1);
4167
4168 for (uint_t i = 0; i < count; i++, entryp++) {
4169 int err = vmm_data_read_msr(vm, vcpuid,
4170 entryp->vfe_ident, &entryp->vfe_value);
4171
4172 if (err != 0) {
4173 return (err);
4174 }
4175 }
4176
4177 *req->vdr_result_len =
4178 count * sizeof (struct vdi_field_entry_v1);
4179 return (0);
4180 }
4181
4182 /*
4183 * If specific MSRs are not requested, try to provide all those which we
4184 * know about instead.
4185 */
4186 const uint_t num_msrs = nitems(default_msr_iter) +
4187 (VMM_MTRR_VAR_MAX * 2);
4188 const uint32_t output_len =
4189 num_msrs * sizeof (struct vdi_field_entry_v1);
4190
4191 *req->vdr_result_len = output_len;
4192 if (req->vdr_len < output_len) {
4193 return (ENOSPC);
4194 }
4195
4196 /* Output the MSRs in the default list */
4197 for (uint_t i = 0; i < nitems(default_msr_iter); i++, entryp++) {
4198 entryp->vfe_ident = default_msr_iter[i];
4199
4200 /* All of these MSRs are expected to work */
4201 VERIFY0(vmm_data_read_msr(vm, vcpuid, entryp->vfe_ident,
4202 &entryp->vfe_value));
4203 }
4204
4205 /* Output the variable MTRRs */
4206 for (uint_t i = 0; i < (VMM_MTRR_VAR_MAX * 2); i++, entryp++) {
4207 entryp->vfe_ident = MSR_MTRRVarBase + i;
4208
4209 /* All of these MSRs are expected to work */
4210 VERIFY0(vmm_data_read_msr(vm, vcpuid, entryp->vfe_ident,
4211 &entryp->vfe_value));
4212 }
4213 return (0);
4214 }
4215
4216 static int
vmm_data_write_msrs(struct vm * vm,int vcpuid,const vmm_data_req_t * req)4217 vmm_data_write_msrs(struct vm *vm, int vcpuid, const vmm_data_req_t *req)
4218 {
4219 VERIFY3U(req->vdr_class, ==, VDC_MSR);
4220 VERIFY3U(req->vdr_version, ==, 1);
4221
4222 const struct vdi_field_entry_v1 *entryp = req->vdr_data;
4223 const uint_t entry_count =
4224 req->vdr_len / sizeof (struct vdi_field_entry_v1);
4225
4226 /*
4227 * First make sure that all of the MSRs can be manipulated.
4228 * For now, this check is done by going though the getmsr handler
4229 */
4230 for (uint_t i = 0; i < entry_count; i++, entryp++) {
4231 const uint64_t msr = entryp->vfe_ident;
4232 uint64_t val;
4233
4234 if (vmm_data_read_msr(vm, vcpuid, msr, &val) != 0) {
4235 return (EINVAL);
4236 }
4237 }
4238
4239 /*
4240 * Fairly confident that all of the 'set' operations are at least
4241 * targeting valid MSRs, continue on.
4242 */
4243 entryp = req->vdr_data;
4244 for (uint_t i = 0; i < entry_count; i++, entryp++) {
4245 int err = vmm_data_write_msr(vm, vcpuid, entryp->vfe_ident,
4246 entryp->vfe_value);
4247
4248 if (err != 0) {
4249 return (err);
4250 }
4251 }
4252 *req->vdr_result_len = entry_count * sizeof (struct vdi_field_entry_v1);
4253
4254 return (0);
4255 }
4256
4257 static const vmm_data_version_entry_t msr_v1 = {
4258 .vdve_class = VDC_MSR,
4259 .vdve_version = 1,
4260 .vdve_len_per_item = sizeof (struct vdi_field_entry_v1),
4261 .vdve_vcpu_readf = vmm_data_read_msrs,
4262 .vdve_vcpu_writef = vmm_data_write_msrs,
4263 };
4264 VMM_DATA_VERSION(msr_v1);
4265
4266 static const uint32_t vmm_arch_v1_fields[] = {
4267 VAI_VM_IS_PAUSED,
4268 };
4269
4270 static const uint32_t vmm_arch_v1_vcpu_fields[] = {
4271 VAI_PEND_NMI,
4272 VAI_PEND_EXTINT,
4273 VAI_PEND_EXCP,
4274 VAI_PEND_INTINFO,
4275 };
4276
4277 static bool
vmm_read_arch_field(struct vm * vm,int vcpuid,uint32_t ident,uint64_t * valp)4278 vmm_read_arch_field(struct vm *vm, int vcpuid, uint32_t ident, uint64_t *valp)
4279 {
4280 ASSERT(valp != NULL);
4281
4282 if (vcpuid == -1) {
4283 switch (ident) {
4284 case VAI_VM_IS_PAUSED:
4285 *valp = vm->is_paused ? 1 : 0;
4286 return (true);
4287 default:
4288 break;
4289 }
4290 } else {
4291 VERIFY(vcpuid >= 0 && vcpuid <= VM_MAXCPU);
4292
4293 struct vcpu *vcpu = &vm->vcpu[vcpuid];
4294 switch (ident) {
4295 case VAI_PEND_NMI:
4296 *valp = vcpu->nmi_pending != 0 ? 1 : 0;
4297 return (true);
4298 case VAI_PEND_EXTINT:
4299 *valp = vcpu->extint_pending != 0 ? 1 : 0;
4300 return (true);
4301 case VAI_PEND_EXCP:
4302 *valp = vcpu->exc_pending;
4303 return (true);
4304 case VAI_PEND_INTINFO:
4305 *valp = vcpu->exit_intinfo;
4306 return (true);
4307 default:
4308 break;
4309 }
4310 }
4311 return (false);
4312 }
4313
4314 static int
vmm_data_read_varch(struct vm * vm,int vcpuid,const vmm_data_req_t * req)4315 vmm_data_read_varch(struct vm *vm, int vcpuid, const vmm_data_req_t *req)
4316 {
4317 VERIFY3U(req->vdr_class, ==, VDC_VMM_ARCH);
4318 VERIFY3U(req->vdr_version, ==, 1);
4319
4320 /* per-vCPU fields are handled separately from VM-wide ones */
4321 if (vcpuid != -1 && (vcpuid < 0 || vcpuid >= VM_MAXCPU)) {
4322 return (EINVAL);
4323 }
4324
4325 struct vdi_field_entry_v1 *entryp = req->vdr_data;
4326
4327 /* Specific fields requested */
4328 if ((req->vdr_flags & VDX_FLAG_READ_COPYIN) != 0) {
4329 const uint_t count =
4330 req->vdr_len / sizeof (struct vdi_field_entry_v1);
4331
4332 for (uint_t i = 0; i < count; i++, entryp++) {
4333 if (!vmm_read_arch_field(vm, vcpuid, entryp->vfe_ident,
4334 &entryp->vfe_value)) {
4335 return (EINVAL);
4336 }
4337 }
4338 *req->vdr_result_len =
4339 count * sizeof (struct vdi_field_entry_v1);
4340 return (0);
4341 }
4342
4343 /* Emit all of the possible values */
4344 const uint32_t *idents;
4345 uint_t ident_count;
4346
4347 if (vcpuid == -1) {
4348 idents = vmm_arch_v1_fields;
4349 ident_count = nitems(vmm_arch_v1_fields);
4350 } else {
4351 idents = vmm_arch_v1_vcpu_fields;
4352 ident_count = nitems(vmm_arch_v1_vcpu_fields);
4353
4354 }
4355
4356 const uint32_t total_size =
4357 ident_count * sizeof (struct vdi_field_entry_v1);
4358
4359 *req->vdr_result_len = total_size;
4360 if (req->vdr_len < total_size) {
4361 return (ENOSPC);
4362 }
4363 for (uint_t i = 0; i < ident_count; i++, entryp++) {
4364 entryp->vfe_ident = idents[i];
4365 VERIFY(vmm_read_arch_field(vm, vcpuid, entryp->vfe_ident,
4366 &entryp->vfe_value));
4367 }
4368 return (0);
4369 }
4370
4371 static int
vmm_data_write_varch_vcpu(struct vm * vm,int vcpuid,const vmm_data_req_t * req)4372 vmm_data_write_varch_vcpu(struct vm *vm, int vcpuid, const vmm_data_req_t *req)
4373 {
4374 VERIFY3U(req->vdr_class, ==, VDC_VMM_ARCH);
4375 VERIFY3U(req->vdr_version, ==, 1);
4376
4377 if (vcpuid < 0 || vcpuid >= VM_MAXCPU) {
4378 return (EINVAL);
4379 }
4380
4381 const struct vdi_field_entry_v1 *entryp = req->vdr_data;
4382 const uint_t entry_count =
4383 req->vdr_len / sizeof (struct vdi_field_entry_v1);
4384 struct vcpu *vcpu = &vm->vcpu[vcpuid];
4385
4386 for (uint_t i = 0; i < entry_count; i++, entryp++) {
4387 const uint64_t val = entryp->vfe_value;
4388
4389 switch (entryp->vfe_ident) {
4390 case VAI_PEND_NMI:
4391 vcpu->nmi_pending = (val != 0);
4392 break;
4393 case VAI_PEND_EXTINT:
4394 vcpu->extint_pending = (val != 0);
4395 break;
4396 case VAI_PEND_EXCP:
4397 if (!VM_INTINFO_PENDING(val)) {
4398 vcpu->exc_pending = 0;
4399 } else if (VM_INTINFO_TYPE(val) != VM_INTINFO_HWEXCP ||
4400 (val & VM_INTINFO_MASK_RSVD) != 0) {
4401 /* reject improperly-formed hw exception */
4402 return (EINVAL);
4403 } else {
4404 vcpu->exc_pending = val;
4405 }
4406 break;
4407 case VAI_PEND_INTINFO:
4408 if (vm_exit_intinfo(vm, vcpuid, val) != 0) {
4409 return (EINVAL);
4410 }
4411 break;
4412 default:
4413 return (EINVAL);
4414 }
4415 }
4416
4417 *req->vdr_result_len = entry_count * sizeof (struct vdi_field_entry_v1);
4418 return (0);
4419 }
4420
4421 static int
vmm_data_write_varch(struct vm * vm,int vcpuid,const vmm_data_req_t * req)4422 vmm_data_write_varch(struct vm *vm, int vcpuid, const vmm_data_req_t *req)
4423 {
4424 VERIFY3U(req->vdr_class, ==, VDC_VMM_ARCH);
4425 VERIFY3U(req->vdr_version, ==, 1);
4426
4427 /* per-vCPU fields are handled separately from VM-wide ones */
4428 if (vcpuid != -1) {
4429 return (vmm_data_write_varch_vcpu(vm, vcpuid, req));
4430 }
4431
4432 const struct vdi_field_entry_v1 *entryp = req->vdr_data;
4433 const uint_t entry_count =
4434 req->vdr_len / sizeof (struct vdi_field_entry_v1);
4435
4436 if (entry_count > 0) {
4437 if (entryp->vfe_ident == VAI_VM_IS_PAUSED) {
4438 /*
4439 * The VM_PAUSE and VM_RESUME ioctls are the officially
4440 * sanctioned mechanisms for setting the is-paused state
4441 * of the VM.
4442 */
4443 return (EPERM);
4444 } else {
4445 /* no other valid arch entries at this time */
4446 return (EINVAL);
4447 }
4448 }
4449
4450 *req->vdr_result_len = entry_count * sizeof (struct vdi_field_entry_v1);
4451 return (0);
4452 }
4453
4454 static const vmm_data_version_entry_t vmm_arch_v1 = {
4455 .vdve_class = VDC_VMM_ARCH,
4456 .vdve_version = 1,
4457 .vdve_len_per_item = sizeof (struct vdi_field_entry_v1),
4458 .vdve_vcpu_readf = vmm_data_read_varch,
4459 .vdve_vcpu_writef = vmm_data_write_varch,
4460
4461 /*
4462 * Handlers for VMM_ARCH can process VM-wide (vcpuid == -1) entries in
4463 * addition to vCPU specific ones.
4464 */
4465 .vdve_vcpu_wildcard = true,
4466 };
4467 VMM_DATA_VERSION(vmm_arch_v1);
4468
4469
4470 /*
4471 * GUEST TIME SUPPORT
4472 *
4473 * Broadly, there are two categories of functionality related to time passing in
4474 * the guest: the guest's TSC and timers used by emulated devices.
4475 *
4476 * ---------------------------
4477 * GUEST TSC "VIRTUALIZATION"
4478 * ---------------------------
4479 *
4480 * The TSC can be read either via an instruction (rdtsc/rdtscp) or by reading
4481 * the TSC MSR.
4482 *
4483 * When a guest reads the TSC via its MSR, the guest will exit and we emulate
4484 * the rdmsr. More typically, the guest reads the TSC via a rdtsc(p)
4485 * instruction. Both SVM and VMX support virtualizing the guest TSC in hardware
4486 * -- that is, a guest will not generally exit on a rdtsc instruction.
4487 *
4488 * To support hardware-virtualized guest TSC, both SVM and VMX provide two knobs
4489 * for the hypervisor to adjust the guest's view of the TSC:
4490 * - TSC offset
4491 * - TSC frequency multiplier (also called "frequency ratio")
4492 *
4493 * When a guest calls rdtsc(p), the TSC value it sees is the sum of:
4494 * guest_tsc = (host TSC, scaled according to frequency multiplier)
4495 * + (TSC offset, programmed by hypervisor)
4496 *
4497 * See the discussions of the TSC offset and frequency multiplier below for more
4498 * details on each of these.
4499 *
4500 * --------------------
4501 * TSC OFFSET OVERVIEW
4502 * --------------------
4503 *
4504 * The TSC offset is a value added to the host TSC (which may be scaled first)
4505 * to provide the guest TSC. This offset addition is generally done by hardware,
4506 * but may be used in emulating the TSC if necessary.
4507 *
4508 * Recall that general formula for calculating the guest TSC is:
4509 *
4510 * guest_tsc = (host TSC, scaled if needed) + TSC offset
4511 *
4512 * Intuitively, the TSC offset is simply an offset of the host's TSC to make the
4513 * guest's view of the TSC appear correct: The guest TSC should be 0 at boot and
4514 * monotonically increase at a roughly constant frequency. Thus in the simplest
4515 * case, the TSC offset is just the negated value of the host TSC when the guest
4516 * was booted, assuming they have the same frequencies.
4517 *
4518 * In practice, there are several factors that can make calculating the TSC
4519 * offset more complicated, including:
4520 *
4521 * (1) the physical CPU the guest is running on
4522 * (2) whether the guest has written to the TSC of that vCPU
4523 * (3) differing host and guest frequencies, like after a live migration
4524 * (4) a guest running on a different system than where it was booted, like
4525 * after a live migration
4526 *
4527 * We will explore each of these factors individually. See below for a
4528 * summary.
4529 *
4530 *
4531 * (1) Physical CPU offsets
4532 *
4533 * The system maintains a set of per-CPU offsets to the TSC to provide a
4534 * consistent view of the TSC regardless of the CPU a thread is running on.
4535 * These offsets are included automatically as a part of rdtsc_offset().
4536 *
4537 * The per-CPU offset must be included as a part reading the host TSC when
4538 * calculating the offset before running the guest on a given CPU.
4539 *
4540 *
4541 * (2) Guest TSC writes (vCPU offsets)
4542 *
4543 * The TSC is a writable MSR. When a guest writes to the TSC, this operation
4544 * should result in the TSC, when read from that vCPU, shows the value written,
4545 * plus whatever time has elapsed since the read.
4546 *
4547 * To support this, when the guest writes to the TSC, we store an additional
4548 * vCPU offset calculated to make future reads of the TSC map to what the guest
4549 * expects.
4550 *
4551 *
4552 * (3) Differing host and guest frequencies (host TSC scaling)
4553 *
4554 * A guest has the same frequency of its host when it boots, but it may be
4555 * migrated to a machine with a different TSC frequency. Systems expect that
4556 * their TSC frequency does not change. To support this fiction in which a guest
4557 * is running on hardware of a different TSC frequency, the hypervisor can
4558 * program a "frequency multiplier" that represents the ratio of guest/host
4559 * frequency.
4560 *
4561 * Any time a host TSC is used in calculations for the offset, it should be
4562 * "scaled" according to this multiplier, and the hypervisor should program the
4563 * multiplier before running a guest so that the hardware virtualization of the
4564 * TSC functions properly. Similarly, the multiplier should be used in any TSC
4565 * emulation.
4566 *
4567 * See below for more details about the frequency multiplier.
4568 *
4569 *
4570 * (4) Guest running on a system it did not boot on ("base guest TSC")
4571 *
4572 * When a guest boots, its TSC offset is simply the negated host TSC at the time
4573 * it booted. If a guest is migrated from a source host to a target host, the
4574 * TSC offset from the source host is no longer useful for several reasons:
4575 * - the target host TSC has no relationship to the source host TSC
4576 * - the guest did not boot on the target system, so the TSC of the target host
4577 * is not sufficient to describe how long the guest has been running prior to
4578 * migration
4579 * - the target system may have a different TSC frequency than the source system
4580 *
4581 * Ignoring the issue of frequency differences for a moment, let's consider how
4582 * to re-align the guest TSC with the host TSC of the target host. Intuitively,
4583 * for the guest to see the correct TSC, we still want to add some offset to the
4584 * host TSC that offsets how long this guest has been running on
4585 * the system.
4586 *
4587 * An example here might be helpful. Consider a source host and target host,
4588 * both with TSC frequencies of 1GHz. On the source host, the guest and host TSC
4589 * values might look like:
4590 *
4591 * +----------------------------------------------------------------------+
4592 * | Event | source host TSC | guest TSC |
4593 * ------------------------------------------------------------------------
4594 * | guest boot (t=0s) | 5000000000 | 5000000000 + -5000000000 |
4595 * | | | 0 |
4596 * ------------------------------------------------------------------------
4597 * | guest rdtsc (t=10s)) | 15000000000 | 15000000000 + -5000000000 |
4598 * | | | 10000000000 |
4599 * ------------------------------------------------------------------------
4600 * | migration (t=15s) | 20000000000 | 20000000000 + -5000000000 |
4601 * | | | 15000000000 |
4602 * +----------------------------------------------------------------------+
4603 *
4604 * Ignoring the time it takes for a guest to physically migrate machines, on the
4605 * target host, we would expect the TSC to continue functioning as such:
4606 *
4607 * +----------------------------------------------------------------------+
4608 * | Event | target host TSC | guest TSC |
4609 * ------------------------------------------------------------------------
4610 * | guest migrate (t=15s) | 300000000000 | 15000000000 |
4611 * ------------------------------------------------------------------------
4612 * | guest rdtsc (t=20s)) | 305000000000 | 20000000000 |
4613 * ------------------------------------------------------------------------
4614 *
4615 * In order to produce a correct TSC value here, we can calculate a new
4616 * "effective" boot TSC that maps to what the host TSC would've been had it been
4617 * booted on the target. We add that to the guest TSC when it began to run on
4618 * this machine, and negate them both to get a new offset. In this example, the
4619 * effective boot TSC is: -(300000000000 - 15000000000) = -285000000000.
4620 *
4621 * +-------------------------------------------------------------------------+
4622 * | Event | target host TSC | guest TSC |
4623 * ---------------------------------------------------------------------------
4624 * | guest "boot" (t=0s) | 285000000000 | 285000000000 + -285000000000 |
4625 * | | | 0 |
4626 * ---------------------------------------------------------------------------
4627 * | guest migrate (t=15s) | 300000000000 | 300000000000 + -285000000000 |
4628 * | | | 15000000000 |
4629 * ---------------------------------------------------------------------------
4630 * | guest rdtsc (t=20s)) | 305000000000 | 305000000000 + -285000000000 |
4631 * | | | 20000000000 |
4632 * --------------------------------------------------------------------------+
4633 *
4634 * To support the offset calculation following a migration, the VMM data time
4635 * interface allows callers to set a "base guest TSC", which is the TSC value of
4636 * the guest when it began running on the host. The current guest TSC can be
4637 * requested via a read of the time data. See below for details on that
4638 * interface.
4639 *
4640 * Frequency differences between the host and the guest are accounted for when
4641 * scaling the host TSC. See below for details on the frequency multiplier.
4642 *
4643 *
4644 * --------------------
4645 * TSC OFFSET SUMMARY
4646 * --------------------
4647 *
4648 * Factoring in all of the components to the TSC above, the TSC offset that is
4649 * programmed by the hypervisor before running a given vCPU is:
4650 *
4651 * offset = -((base host TSC, scaled if needed) - base_guest_tsc) + vCPU offset
4652 *
4653 * This offset is stored in two pieces. Per-vCPU offsets are stored with the
4654 * given vCPU and added in when programming the offset. The rest of the offset
4655 * is stored as a VM-wide offset, and computed either at boot or when the time
4656 * data is written to.
4657 *
4658 * It is safe to add the vCPU offset and the VM-wide offsets together because
4659 * the vCPU offset is in terms of the guest TSC. The host TSC is scaled before
4660 * using it in calculations, so all TSC values are applicable to the same
4661 * frequency.
4662 *
4663 * Note: Though both the VM-wide offset and per-vCPU offsets may be negative, we
4664 * store them as unsigned values and perform all offsetting math unsigned. This
4665 * is to avoid UB from signed overflow.
4666 *
4667 * -------------------------
4668 * TSC FREQUENCY MULTIPLIER
4669 * -------------------------
4670 *
4671 * In order to account for frequency differences between the host and guest, SVM
4672 * and VMX provide an interface to set a "frequency multiplier" (or "frequency
4673 * ratio") representing guest to host frequency. In a hardware-virtualized read
4674 * of the TSC, the host TSC is scaled using this multiplier prior to adding the
4675 * programmed TSC offset.
4676 *
4677 * Both platforms represent the ratio as a fixed point number, where the lower
4678 * bits are used as a fractional component, and some number of the upper bits
4679 * are used as the integer component.
4680 *
4681 * Some example multipliers, for a platform with FRAC fractional bits in the
4682 * multiplier:
4683 * - guest frequency == host: 1 << FRAC
4684 * - guest frequency is 2x host: 1 << (FRAC + 1)
4685 * - guest frequency is 0.5x host: 1 << (FRAC - 1), as the highest-order
4686 * fractional bit represents 1/2
4687 * - guest frequency is 2.5x host: (1 << FRAC) | (1 << (FRAC - 1))
4688 * and so on.
4689 *
4690 * In general, the frequency multiplier is calculated as follows:
4691 * (guest_hz * (1 << FRAC_SIZE)) / host_hz
4692 *
4693 * The multiplier should be used any time the host TSC value is used in
4694 * calculations with the guest TSC (and their frequencies differ). The function
4695 * `vmm_scale_tsc` is intended to be used for these purposes, as it will scale
4696 * the host TSC only if needed.
4697 *
4698 * The multiplier should also be programmed by the hypervisor before the guest
4699 * is run.
4700 *
4701 *
4702 * ----------------------------
4703 * DEVICE TIMERS (BOOT_HRTIME)
4704 * ----------------------------
4705 *
4706 * Emulated devices use timers to do things such as scheduling periodic events.
4707 * These timers are scheduled relative to the hrtime of the host. When device
4708 * state is exported or imported, we use boot_hrtime to normalize these timers
4709 * against the host hrtime. The boot_hrtime represents the hrtime of the host
4710 * when the guest was booted.
4711 *
4712 * If a guest is migrated to a different machine, boot_hrtime must be adjusted
4713 * to match the hrtime of when the guest was effectively booted on the target
4714 * host. This allows timers to continue functioning when device state is
4715 * imported on the target.
4716 *
4717 *
4718 * ------------------------
4719 * VMM DATA TIME INTERFACE
4720 * ------------------------
4721 *
4722 * In order to facilitate live migrations of guests, we provide an interface,
4723 * via the VMM data read/write ioctls, for userspace to make changes to the
4724 * guest's view of the TSC and device timers, allowing these features to
4725 * continue functioning after a migration.
4726 *
4727 * The interface was designed to expose the minimal amount of data needed for a
4728 * userspace component to make adjustments to the guest's view of time (e.g., to
4729 * account for time passing in a live migration). At a minimum, such a program
4730 * needs:
4731 * - the current guest TSC
4732 * - guest TSC frequency
4733 * - guest's boot_hrtime
4734 * - timestamps of when this data was taken (hrtime for hrtime calculations, and
4735 * wall clock time for computing time deltas between machines)
4736 *
4737 * The wall clock time is provided for consumers to make adjustments to the
4738 * guest TSC and boot_hrtime based on deltas observed during migrations. It may
4739 * be prudent for consumers to use this data only in circumstances where the
4740 * source and target have well-synchronized wall clocks, but nothing in the
4741 * interface depends on this assumption.
4742 *
4743 * On writes, consumers write back:
4744 * - the base guest TSC (used for TSC offset calculations)
4745 * - desired boot_hrtime
4746 * - guest_frequency (cannot change)
4747 * - hrtime of when this data was adjusted
4748 * - (wall clock time on writes is ignored)
4749 *
4750 * The interface will adjust the input guest TSC slightly, based on the input
4751 * hrtime, to account for latency between userspace calculations and application
4752 * of the data on the kernel side. This amounts to adding a small amount of
4753 * additional "uptime" for the guest.
4754 *
4755 * After the adjustments, the interface updates the VM-wide TSC offset and
4756 * boot_hrtime. Per-vCPU offsets are not adjusted, as those are already in terms
4757 * of the guest TSC and can be exported/imported via the MSR VMM data interface.
4758 *
4759 *
4760 * --------------------------------
4761 * SUPPORTED PLATFORMS AND CAVEATS
4762 * --------------------------------
4763 *
4764 * While both VMX and SVM offer TSC scaling as a feature, at this time only SVM
4765 * is supported by bhyve.
4766 *
4767 * The time data interface is designed such that Intel support can be added
4768 * easily, and all other aspects of the time interface should work on Intel.
4769 * (Without frequency control though, in practice, doing live migrations of
4770 * guests on Intel will not work for time-related things, as two machines
4771 * rarely have exactly the same frequency).
4772 *
4773 * Additionally, while on both SVM and VMX the frequency multiplier is a fixed
4774 * point number, each uses a different number of fractional and integer bits for
4775 * the multiplier. As such, calculating the multiplier and fractional bit size
4776 * is requested via the vmm_ops.
4777 *
4778 * Care should be taken to set reasonable limits for ratios based on the
4779 * platform, as the difference in fractional bits can lead to slightly different
4780 * tradeoffs in terms of representable ratios and potentially overflowing
4781 * calculations.
4782 */
4783
4784 /*
4785 * Scales the TSC if needed, based on the input frequency multiplier.
4786 */
4787 static uint64_t
vmm_scale_tsc(uint64_t tsc,uint64_t mult)4788 vmm_scale_tsc(uint64_t tsc, uint64_t mult)
4789 {
4790 const uint32_t frac_size = ops->fr_fracsize;
4791
4792 if (mult != VM_TSCM_NOSCALE) {
4793 VERIFY3U(frac_size, >, 0);
4794 return (scale_tsc(tsc, mult, frac_size));
4795 } else {
4796 return (tsc);
4797 }
4798 }
4799
4800 /*
4801 * Calculate the frequency multiplier, which represents the ratio of
4802 * guest_hz / host_hz. The frequency multiplier is a fixed point number with
4803 * `frac_sz` fractional bits (fractional bits begin at bit 0).
4804 *
4805 * See comment for "calc_freq_multiplier" in "vmm_time_support.S" for more
4806 * information about valid input to this function.
4807 */
4808 uint64_t
vmm_calc_freq_multiplier(uint64_t guest_hz,uint64_t host_hz,uint32_t frac_size)4809 vmm_calc_freq_multiplier(uint64_t guest_hz, uint64_t host_hz,
4810 uint32_t frac_size)
4811 {
4812 VERIFY3U(guest_hz, !=, 0);
4813 VERIFY3U(frac_size, >, 0);
4814 VERIFY3U(frac_size, <, 64);
4815
4816 return (calc_freq_multiplier(guest_hz, host_hz, frac_size));
4817 }
4818
4819 /*
4820 * Calculate the guest VM-wide TSC offset.
4821 *
4822 * offset = - ((base host TSC, scaled if needed) - base_guest_tsc)
4823 *
4824 * The base_host_tsc and the base_guest_tsc are the TSC values of the host
4825 * (read on the system) and the guest (calculated) at the same point in time.
4826 * This allows us to fix the guest TSC at this point in time as a base, either
4827 * following boot (guest TSC = 0), or a change to the guest's time data from
4828 * userspace (such as in the case of a migration).
4829 */
4830 static uint64_t
calc_tsc_offset(uint64_t base_host_tsc,uint64_t base_guest_tsc,uint64_t mult)4831 calc_tsc_offset(uint64_t base_host_tsc, uint64_t base_guest_tsc, uint64_t mult)
4832 {
4833 const uint64_t htsc_scaled = vmm_scale_tsc(base_host_tsc, mult);
4834 if (htsc_scaled > base_guest_tsc) {
4835 return ((uint64_t)(- (int64_t)(htsc_scaled - base_guest_tsc)));
4836 } else {
4837 return (base_guest_tsc - htsc_scaled);
4838 }
4839 }
4840
4841 /*
4842 * Calculate an estimate of the guest TSC.
4843 *
4844 * guest_tsc = (host TSC, scaled if needed) + offset
4845 */
4846 static uint64_t
calc_guest_tsc(uint64_t host_tsc,uint64_t mult,uint64_t offset)4847 calc_guest_tsc(uint64_t host_tsc, uint64_t mult, uint64_t offset)
4848 {
4849 return (vmm_scale_tsc(host_tsc, mult) + offset);
4850 }
4851
4852 /*
4853 * Take a non-atomic "snapshot" of the current:
4854 * - TSC
4855 * - hrtime
4856 * - wall clock time
4857 */
4858 static void
vmm_time_snapshot(uint64_t * tsc,hrtime_t * hrtime,timespec_t * hrestime)4859 vmm_time_snapshot(uint64_t *tsc, hrtime_t *hrtime, timespec_t *hrestime)
4860 {
4861 /*
4862 * Disable interrupts while we take the readings: In the absence of a
4863 * mechanism to convert hrtime to hrestime, we want the time between
4864 * each of these measurements to be as small as possible.
4865 */
4866 ulong_t iflag = intr_clear();
4867
4868 hrtime_t hrt = gethrtimeunscaledf();
4869 *tsc = (uint64_t)hrt;
4870 *hrtime = hrt;
4871 scalehrtime(hrtime);
4872 gethrestime(hrestime);
4873
4874 intr_restore(iflag);
4875 }
4876
4877 /*
4878 * Read VMM Time data
4879 *
4880 * Provides:
4881 * - the current guest TSC and TSC frequency
4882 * - guest boot_hrtime
4883 * - timestamps of the read (hrtime and wall clock time)
4884 */
4885 static int
vmm_data_read_vmm_time(void * arg,const vmm_data_req_t * req)4886 vmm_data_read_vmm_time(void *arg, const vmm_data_req_t *req)
4887 {
4888 VERIFY3U(req->vdr_class, ==, VDC_VMM_TIME);
4889 VERIFY3U(req->vdr_version, ==, 1);
4890 VERIFY3U(req->vdr_len, >=, sizeof (struct vdi_time_info_v1));
4891
4892 struct vm *vm = arg;
4893 struct vdi_time_info_v1 *out = req->vdr_data;
4894
4895 /*
4896 * Since write operations on VMM_TIME data are strict about vcpuid
4897 * (see: vmm_data_write_vmm_time()), read operations should be as well.
4898 */
4899 if (req->vdr_vcpuid != -1) {
4900 return (EINVAL);
4901 }
4902
4903 /* Take a snapshot of this point in time */
4904 uint64_t tsc;
4905 hrtime_t hrtime;
4906 timespec_t hrestime;
4907 vmm_time_snapshot(&tsc, &hrtime, &hrestime);
4908
4909 /* Write the output values */
4910 out->vt_guest_freq = vm->guest_freq;
4911
4912 /*
4913 * Use only the VM-wide TSC offset for calculating the guest TSC,
4914 * ignoring per-vCPU offsets. This value is provided as a "base" guest
4915 * TSC at the time of the read; per-vCPU offsets are factored in as
4916 * needed elsewhere, either when running the vCPU or if the guest reads
4917 * the TSC via rdmsr.
4918 */
4919 out->vt_guest_tsc = calc_guest_tsc(tsc, vm->freq_multiplier,
4920 vm->tsc_offset);
4921 out->vt_boot_hrtime = vm->boot_hrtime;
4922 out->vt_hrtime = hrtime;
4923 out->vt_hres_sec = hrestime.tv_sec;
4924 out->vt_hres_ns = hrestime.tv_nsec;
4925
4926 return (0);
4927 }
4928
4929 /*
4930 * Modify VMM Time data related values
4931 *
4932 * This interface serves to allow guests' TSC and device timers to continue
4933 * functioning across live migrations. On a successful write, the VM-wide TSC
4934 * offset and boot_hrtime of the guest are updated.
4935 *
4936 * The interface requires an hrtime of the system at which the caller wrote
4937 * this data; this allows us to adjust the TSC and boot_hrtime slightly to
4938 * account for time passing between the userspace call and application
4939 * of the data here.
4940 *
4941 * There are several possibilities for invalid input, including:
4942 * - a requested guest frequency of 0, or a frequency otherwise unsupported by
4943 * the underlying platform
4944 * - hrtime or boot_hrtime values that appear to be from the future
4945 * - the requested frequency does not match the host, and this system does not
4946 * have hardware TSC scaling support
4947 */
4948 static int
vmm_data_write_vmm_time(void * arg,const vmm_data_req_t * req)4949 vmm_data_write_vmm_time(void *arg, const vmm_data_req_t *req)
4950 {
4951 VERIFY3U(req->vdr_class, ==, VDC_VMM_TIME);
4952 VERIFY3U(req->vdr_version, ==, 1);
4953 VERIFY3U(req->vdr_len, >=, sizeof (struct vdi_time_info_v1));
4954
4955 struct vm *vm = arg;
4956 const struct vdi_time_info_v1 *src = req->vdr_data;
4957
4958 /*
4959 * While vcpuid values != -1 are tolerated by the vmm_data machinery for
4960 * VM-wide endpoints, the time-related data is more strict: It relies on
4961 * write-locking the VM (implied by the vcpuid -1) to prevent vCPUs or
4962 * other bits from observing inconsistent values while the state is
4963 * being written.
4964 */
4965 if (req->vdr_vcpuid != -1) {
4966 return (EINVAL);
4967 }
4968
4969 /*
4970 * Platform-specific checks will verify the requested frequency against
4971 * the supported range further, but a frequency of 0 is never valid.
4972 */
4973 if (src->vt_guest_freq == 0) {
4974 return (EINVAL);
4975 }
4976
4977 /*
4978 * Check whether the request frequency is supported and get the
4979 * frequency multiplier.
4980 */
4981 uint64_t mult = VM_TSCM_NOSCALE;
4982 freqratio_res_t res = ops->vmfreqratio(src->vt_guest_freq,
4983 vmm_host_freq, &mult);
4984 switch (res) {
4985 case FR_SCALING_NOT_SUPPORTED:
4986 /*
4987 * This system doesn't support TSC scaling, and the guest/host
4988 * frequencies differ
4989 */
4990 return (EPERM);
4991 case FR_OUT_OF_RANGE:
4992 /* Requested frequency ratio is too small/large */
4993 return (EINVAL);
4994 case FR_SCALING_NOT_NEEDED:
4995 /* Host and guest frequencies are the same */
4996 VERIFY3U(mult, ==, VM_TSCM_NOSCALE);
4997 break;
4998 case FR_VALID:
4999 VERIFY3U(mult, !=, VM_TSCM_NOSCALE);
5000 break;
5001 }
5002
5003 /*
5004 * Find (and validate) the hrtime delta between the input request and
5005 * when we received it so that we can bump the TSC to account for time
5006 * passing.
5007 *
5008 * We ignore the hrestime as input, as this is a field that
5009 * exists for reads.
5010 */
5011 uint64_t tsc;
5012 hrtime_t hrtime;
5013 timespec_t hrestime;
5014 vmm_time_snapshot(&tsc, &hrtime, &hrestime);
5015 if ((src->vt_hrtime > hrtime) || (src->vt_boot_hrtime > hrtime)) {
5016 /*
5017 * The caller has passed in an hrtime / boot_hrtime from the
5018 * future.
5019 */
5020 return (EINVAL);
5021 }
5022 hrtime_t hrt_delta = hrtime - src->vt_hrtime;
5023
5024 /* Calculate guest TSC adjustment */
5025 const uint64_t host_ticks = unscalehrtime(hrt_delta);
5026 const uint64_t guest_ticks = vmm_scale_tsc(host_ticks,
5027 vm->freq_multiplier);
5028 const uint64_t base_guest_tsc = src->vt_guest_tsc + guest_ticks;
5029
5030 /* Update guest time data */
5031 vm->freq_multiplier = mult;
5032 vm->guest_freq = src->vt_guest_freq;
5033 vm->boot_hrtime = src->vt_boot_hrtime;
5034 vm->tsc_offset = calc_tsc_offset(tsc, base_guest_tsc,
5035 vm->freq_multiplier);
5036
5037 return (0);
5038 }
5039
5040 static const vmm_data_version_entry_t vmm_time_v1 = {
5041 .vdve_class = VDC_VMM_TIME,
5042 .vdve_version = 1,
5043 .vdve_len_expect = sizeof (struct vdi_time_info_v1),
5044 .vdve_readf = vmm_data_read_vmm_time,
5045 .vdve_writef = vmm_data_write_vmm_time,
5046 };
5047 VMM_DATA_VERSION(vmm_time_v1);
5048
5049
5050 static int
vmm_data_read_versions(void * arg,const vmm_data_req_t * req)5051 vmm_data_read_versions(void *arg, const vmm_data_req_t *req)
5052 {
5053 VERIFY3U(req->vdr_class, ==, VDC_VERSION);
5054 VERIFY3U(req->vdr_version, ==, 1);
5055
5056 const uint32_t total_size = SET_COUNT(vmm_data_version_entries) *
5057 sizeof (struct vdi_version_entry_v1);
5058
5059 /* Make sure there is room for all of the entries */
5060 *req->vdr_result_len = total_size;
5061 if (req->vdr_len < *req->vdr_result_len) {
5062 return (ENOSPC);
5063 }
5064
5065 struct vdi_version_entry_v1 *entryp = req->vdr_data;
5066 const vmm_data_version_entry_t **vdpp;
5067 SET_FOREACH(vdpp, vmm_data_version_entries) {
5068 const vmm_data_version_entry_t *vdp = *vdpp;
5069
5070 entryp->vve_class = vdp->vdve_class;
5071 entryp->vve_version = vdp->vdve_version;
5072 entryp->vve_len_expect = vdp->vdve_len_expect;
5073 entryp->vve_len_per_item = vdp->vdve_len_per_item;
5074 entryp++;
5075 }
5076 return (0);
5077 }
5078
5079 static int
vmm_data_write_versions(void * arg,const vmm_data_req_t * req)5080 vmm_data_write_versions(void *arg, const vmm_data_req_t *req)
5081 {
5082 /* Writing to the version information makes no sense */
5083 return (EPERM);
5084 }
5085
5086 static const vmm_data_version_entry_t versions_v1 = {
5087 .vdve_class = VDC_VERSION,
5088 .vdve_version = 1,
5089 .vdve_len_per_item = sizeof (struct vdi_version_entry_v1),
5090 .vdve_readf = vmm_data_read_versions,
5091 .vdve_writef = vmm_data_write_versions,
5092 };
5093 VMM_DATA_VERSION(versions_v1);
5094
5095 int
vmm_data_read(struct vm * vm,const vmm_data_req_t * req)5096 vmm_data_read(struct vm *vm, const vmm_data_req_t *req)
5097 {
5098 int err = 0;
5099
5100 const vmm_data_version_entry_t *entry = NULL;
5101 err = vmm_data_find(req, &entry);
5102 if (err != 0) {
5103 return (err);
5104 }
5105 ASSERT(entry != NULL);
5106
5107 if (entry->vdve_readf != NULL) {
5108 void *datap = vmm_data_from_class(req, vm);
5109
5110 err = entry->vdve_readf(datap, req);
5111 } else if (entry->vdve_vcpu_readf != NULL) {
5112 err = entry->vdve_vcpu_readf(vm, req->vdr_vcpuid, req);
5113 } else {
5114 err = EINVAL;
5115 }
5116
5117 /*
5118 * Successful reads of fixed-length data should populate the length of
5119 * that result.
5120 */
5121 if (err == 0 && entry->vdve_len_expect != 0) {
5122 *req->vdr_result_len = entry->vdve_len_expect;
5123 }
5124
5125 return (err);
5126 }
5127
5128 int
vmm_data_write(struct vm * vm,const vmm_data_req_t * req)5129 vmm_data_write(struct vm *vm, const vmm_data_req_t *req)
5130 {
5131 int err = 0;
5132
5133 const vmm_data_version_entry_t *entry = NULL;
5134 err = vmm_data_find(req, &entry);
5135 if (err != 0) {
5136 return (err);
5137 }
5138 ASSERT(entry != NULL);
5139
5140 if (entry->vdve_writef != NULL) {
5141 void *datap = vmm_data_from_class(req, vm);
5142
5143 err = entry->vdve_writef(datap, req);
5144 } else if (entry->vdve_vcpu_writef != NULL) {
5145 err = entry->vdve_vcpu_writef(vm, req->vdr_vcpuid, req);
5146 } else {
5147 err = EINVAL;
5148 }
5149
5150 /*
5151 * Successful writes of fixed-length data should populate the length of
5152 * that result.
5153 */
5154 if (err == 0 && entry->vdve_len_expect != 0) {
5155 *req->vdr_result_len = entry->vdve_len_expect;
5156 }
5157
5158 return (err);
5159 }
5160