xref: /freebsd/usr.sbin/bhyve/bhyverun.c (revision 4d3fc8b0570b29fb0d6ee9525f104d52176ff0d4)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2011 NetApp, Inc.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  *
28  * $FreeBSD$
29  */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include <sys/types.h>
35 #ifndef WITHOUT_CAPSICUM
36 #include <sys/capsicum.h>
37 #endif
38 #include <sys/mman.h>
39 #ifdef BHYVE_SNAPSHOT
40 #include <sys/socket.h>
41 #include <sys/stat.h>
42 #endif
43 #include <sys/time.h>
44 #ifdef BHYVE_SNAPSHOT
45 #include <sys/un.h>
46 #endif
47 
48 #include <amd64/vmm/intel/vmcs.h>
49 #include <x86/apicreg.h>
50 
51 #include <machine/atomic.h>
52 #include <machine/segments.h>
53 
54 #ifndef WITHOUT_CAPSICUM
55 #include <capsicum_helpers.h>
56 #endif
57 #include <stdio.h>
58 #include <stdlib.h>
59 #include <string.h>
60 #include <err.h>
61 #include <errno.h>
62 #ifdef BHYVE_SNAPSHOT
63 #include <fcntl.h>
64 #endif
65 #include <libgen.h>
66 #include <unistd.h>
67 #include <assert.h>
68 #include <pthread.h>
69 #include <pthread_np.h>
70 #include <sysexits.h>
71 #include <stdbool.h>
72 #include <stdint.h>
73 #ifdef BHYVE_SNAPSHOT
74 #include <ucl.h>
75 #include <unistd.h>
76 
77 #include <libxo/xo.h>
78 #endif
79 
80 #include <machine/vmm.h>
81 #ifndef WITHOUT_CAPSICUM
82 #include <machine/vmm_dev.h>
83 #endif
84 #include <machine/vmm_instruction_emul.h>
85 #include <vmmapi.h>
86 
87 #include "bhyverun.h"
88 #include "acpi.h"
89 #include "atkbdc.h"
90 #include "bootrom.h"
91 #include "config.h"
92 #include "inout.h"
93 #include "debug.h"
94 #include "fwctl.h"
95 #include "gdb.h"
96 #include "ioapic.h"
97 #include "kernemu_dev.h"
98 #include "mem.h"
99 #include "mevent.h"
100 #include "mptbl.h"
101 #include "pci_emul.h"
102 #include "pci_irq.h"
103 #include "pci_lpc.h"
104 #include "smbiostbl.h"
105 #ifdef BHYVE_SNAPSHOT
106 #include "snapshot.h"
107 #endif
108 #include "xmsr.h"
109 #include "spinup_ap.h"
110 #include "rtc.h"
111 #include "vmgenc.h"
112 
113 #define GUEST_NIO_PORT		0x488	/* guest upcalls via i/o port */
114 
115 #define MB		(1024UL * 1024)
116 #define GB		(1024UL * MB)
117 
118 static const char * const vmx_exit_reason_desc[] = {
119 	[EXIT_REASON_EXCEPTION] = "Exception or non-maskable interrupt (NMI)",
120 	[EXIT_REASON_EXT_INTR] = "External interrupt",
121 	[EXIT_REASON_TRIPLE_FAULT] = "Triple fault",
122 	[EXIT_REASON_INIT] = "INIT signal",
123 	[EXIT_REASON_SIPI] = "Start-up IPI (SIPI)",
124 	[EXIT_REASON_IO_SMI] = "I/O system-management interrupt (SMI)",
125 	[EXIT_REASON_SMI] = "Other SMI",
126 	[EXIT_REASON_INTR_WINDOW] = "Interrupt window",
127 	[EXIT_REASON_NMI_WINDOW] = "NMI window",
128 	[EXIT_REASON_TASK_SWITCH] = "Task switch",
129 	[EXIT_REASON_CPUID] = "CPUID",
130 	[EXIT_REASON_GETSEC] = "GETSEC",
131 	[EXIT_REASON_HLT] = "HLT",
132 	[EXIT_REASON_INVD] = "INVD",
133 	[EXIT_REASON_INVLPG] = "INVLPG",
134 	[EXIT_REASON_RDPMC] = "RDPMC",
135 	[EXIT_REASON_RDTSC] = "RDTSC",
136 	[EXIT_REASON_RSM] = "RSM",
137 	[EXIT_REASON_VMCALL] = "VMCALL",
138 	[EXIT_REASON_VMCLEAR] = "VMCLEAR",
139 	[EXIT_REASON_VMLAUNCH] = "VMLAUNCH",
140 	[EXIT_REASON_VMPTRLD] = "VMPTRLD",
141 	[EXIT_REASON_VMPTRST] = "VMPTRST",
142 	[EXIT_REASON_VMREAD] = "VMREAD",
143 	[EXIT_REASON_VMRESUME] = "VMRESUME",
144 	[EXIT_REASON_VMWRITE] = "VMWRITE",
145 	[EXIT_REASON_VMXOFF] = "VMXOFF",
146 	[EXIT_REASON_VMXON] = "VMXON",
147 	[EXIT_REASON_CR_ACCESS] = "Control-register accesses",
148 	[EXIT_REASON_DR_ACCESS] = "MOV DR",
149 	[EXIT_REASON_INOUT] = "I/O instruction",
150 	[EXIT_REASON_RDMSR] = "RDMSR",
151 	[EXIT_REASON_WRMSR] = "WRMSR",
152 	[EXIT_REASON_INVAL_VMCS] =
153 	    "VM-entry failure due to invalid guest state",
154 	[EXIT_REASON_INVAL_MSR] = "VM-entry failure due to MSR loading",
155 	[EXIT_REASON_MWAIT] = "MWAIT",
156 	[EXIT_REASON_MTF] = "Monitor trap flag",
157 	[EXIT_REASON_MONITOR] = "MONITOR",
158 	[EXIT_REASON_PAUSE] = "PAUSE",
159 	[EXIT_REASON_MCE_DURING_ENTRY] =
160 	    "VM-entry failure due to machine-check event",
161 	[EXIT_REASON_TPR] = "TPR below threshold",
162 	[EXIT_REASON_APIC_ACCESS] = "APIC access",
163 	[EXIT_REASON_VIRTUALIZED_EOI] = "Virtualized EOI",
164 	[EXIT_REASON_GDTR_IDTR] = "Access to GDTR or IDTR",
165 	[EXIT_REASON_LDTR_TR] = "Access to LDTR or TR",
166 	[EXIT_REASON_EPT_FAULT] = "EPT violation",
167 	[EXIT_REASON_EPT_MISCONFIG] = "EPT misconfiguration",
168 	[EXIT_REASON_INVEPT] = "INVEPT",
169 	[EXIT_REASON_RDTSCP] = "RDTSCP",
170 	[EXIT_REASON_VMX_PREEMPT] = "VMX-preemption timer expired",
171 	[EXIT_REASON_INVVPID] = "INVVPID",
172 	[EXIT_REASON_WBINVD] = "WBINVD",
173 	[EXIT_REASON_XSETBV] = "XSETBV",
174 	[EXIT_REASON_APIC_WRITE] = "APIC write",
175 	[EXIT_REASON_RDRAND] = "RDRAND",
176 	[EXIT_REASON_INVPCID] = "INVPCID",
177 	[EXIT_REASON_VMFUNC] = "VMFUNC",
178 	[EXIT_REASON_ENCLS] = "ENCLS",
179 	[EXIT_REASON_RDSEED] = "RDSEED",
180 	[EXIT_REASON_PM_LOG_FULL] = "Page-modification log full",
181 	[EXIT_REASON_XSAVES] = "XSAVES",
182 	[EXIT_REASON_XRSTORS] = "XRSTORS"
183 };
184 
185 typedef int (*vmexit_handler_t)(struct vmctx *, struct vm_exit *, int *vcpu);
186 
187 int guest_ncpus;
188 uint16_t cpu_cores, cpu_sockets, cpu_threads;
189 
190 int raw_stdio = 0;
191 
192 static char *progname;
193 static const int BSP = 0;
194 
195 static cpuset_t cpumask;
196 
197 static void vm_loop(struct vmctx *ctx, int vcpu);
198 
199 static struct bhyvestats {
200 	uint64_t	vmexit_bogus;
201 	uint64_t	vmexit_reqidle;
202 	uint64_t	vmexit_hlt;
203 	uint64_t	vmexit_pause;
204 	uint64_t	vmexit_mtrap;
205 	uint64_t	vmexit_inst_emul;
206 	uint64_t	cpu_switch_rotate;
207 	uint64_t	cpu_switch_direct;
208 } stats;
209 
210 static struct mt_vmm_info {
211 	pthread_t	mt_thr;
212 	struct vmctx	*mt_ctx;
213 	int		mt_vcpu;
214 } *mt_vmm_info;
215 
216 static cpuset_t **vcpumap;
217 
218 static void
219 usage(int code)
220 {
221 
222         fprintf(stderr,
223 		"Usage: %s [-AaCDeHhPSuWwxY]\n"
224 		"       %*s [-c [[cpus=]numcpus][,sockets=n][,cores=n][,threads=n]]\n"
225 		"       %*s [-G port] [-k config_file] [-l lpc] [-m mem] [-o var=value]\n"
226 		"       %*s [-p vcpu:hostcpu] [-r file] [-s pci] [-U uuid] vmname\n"
227 		"       -A: create ACPI tables\n"
228 		"       -a: local apic is in xAPIC mode (deprecated)\n"
229 		"       -C: include guest memory in core file\n"
230 		"       -c: number of CPUs and/or topology specification\n"
231 		"       -D: destroy on power-off\n"
232 		"       -e: exit on unhandled I/O access\n"
233 		"       -G: start a debug server\n"
234 		"       -H: vmexit from the guest on HLT\n"
235 		"       -h: help\n"
236 		"       -k: key=value flat config file\n"
237 		"       -K: PS2 keyboard layout\n"
238 		"       -l: LPC device configuration\n"
239 		"       -m: memory size\n"
240 		"       -o: set config 'var' to 'value'\n"
241 		"       -P: vmexit from the guest on pause\n"
242 		"       -p: pin 'vcpu' to 'hostcpu'\n"
243 #ifdef BHYVE_SNAPSHOT
244 		"       -r: path to checkpoint file\n"
245 #endif
246 		"       -S: guest memory cannot be swapped\n"
247 		"       -s: <slot,driver,configinfo> PCI slot config\n"
248 		"       -U: UUID\n"
249 		"       -u: RTC keeps UTC time\n"
250 		"       -W: force virtio to use single-vector MSI\n"
251 		"       -w: ignore unimplemented MSRs\n"
252 		"       -x: local APIC is in x2APIC mode\n"
253 		"       -Y: disable MPtable generation\n",
254 		progname, (int)strlen(progname), "", (int)strlen(progname), "",
255 		(int)strlen(progname), "");
256 
257 	exit(code);
258 }
259 
260 /*
261  * XXX This parser is known to have the following issues:
262  * 1.  It accepts null key=value tokens ",," as setting "cpus" to an
263  *     empty string.
264  *
265  * The acceptance of a null specification ('-c ""') is by design to match the
266  * manual page syntax specification, this results in a topology of 1 vCPU.
267  */
268 static int
269 topology_parse(const char *opt)
270 {
271 	char *cp, *str, *tofree;
272 
273 	if (*opt == '\0') {
274 		set_config_value("sockets", "1");
275 		set_config_value("cores", "1");
276 		set_config_value("threads", "1");
277 		set_config_value("cpus", "1");
278 		return (0);
279 	}
280 
281 	tofree = str = strdup(opt);
282 	if (str == NULL)
283 		errx(4, "Failed to allocate memory");
284 
285 	while ((cp = strsep(&str, ",")) != NULL) {
286 		if (strncmp(cp, "cpus=", strlen("cpus=")) == 0)
287 			set_config_value("cpus", cp + strlen("cpus="));
288 		else if (strncmp(cp, "sockets=", strlen("sockets=")) == 0)
289 			set_config_value("sockets", cp + strlen("sockets="));
290 		else if (strncmp(cp, "cores=", strlen("cores=")) == 0)
291 			set_config_value("cores", cp + strlen("cores="));
292 		else if (strncmp(cp, "threads=", strlen("threads=")) == 0)
293 			set_config_value("threads", cp + strlen("threads="));
294 #ifdef notyet  /* Do not expose this until vmm.ko implements it */
295 		else if (strncmp(cp, "maxcpus=", strlen("maxcpus=")) == 0)
296 			set_config_value("maxcpus", cp + strlen("maxcpus="));
297 #endif
298 		else if (strchr(cp, '=') != NULL)
299 			goto out;
300 		else
301 			set_config_value("cpus", cp);
302 	}
303 	free(tofree);
304 	return (0);
305 
306 out:
307 	free(tofree);
308 	return (-1);
309 }
310 
311 static int
312 parse_int_value(const char *key, const char *value, int minval, int maxval)
313 {
314 	char *cp;
315 	long lval;
316 
317 	errno = 0;
318 	lval = strtol(value, &cp, 0);
319 	if (errno != 0 || *cp != '\0' || cp == value || lval < minval ||
320 	    lval > maxval)
321 		errx(4, "Invalid value for %s: '%s'", key, value);
322 	return (lval);
323 }
324 
325 /*
326  * Set the sockets, cores, threads, and guest_cpus variables based on
327  * the configured topology.
328  *
329  * The limits of UINT16_MAX are due to the types passed to
330  * vm_set_topology().  vmm.ko may enforce tighter limits.
331  */
332 static void
333 calc_topology(void)
334 {
335 	const char *value;
336 	bool explicit_cpus;
337 	uint64_t ncpus;
338 
339 	value = get_config_value("cpus");
340 	if (value != NULL) {
341 		guest_ncpus = parse_int_value("cpus", value, 1, UINT16_MAX);
342 		explicit_cpus = true;
343 	} else {
344 		guest_ncpus = 1;
345 		explicit_cpus = false;
346 	}
347 	value = get_config_value("cores");
348 	if (value != NULL)
349 		cpu_cores = parse_int_value("cores", value, 1, UINT16_MAX);
350 	else
351 		cpu_cores = 1;
352 	value = get_config_value("threads");
353 	if (value != NULL)
354 		cpu_threads = parse_int_value("threads", value, 1, UINT16_MAX);
355 	else
356 		cpu_threads = 1;
357 	value = get_config_value("sockets");
358 	if (value != NULL)
359 		cpu_sockets = parse_int_value("sockets", value, 1, UINT16_MAX);
360 	else
361 		cpu_sockets = guest_ncpus;
362 
363 	/*
364 	 * Compute sockets * cores * threads avoiding overflow.  The
365 	 * range check above insures these are 16 bit values.
366 	 */
367 	ncpus = (uint64_t)cpu_sockets * cpu_cores * cpu_threads;
368 	if (ncpus > UINT16_MAX)
369 		errx(4, "Computed number of vCPUs too high: %ju",
370 		    (uintmax_t)ncpus);
371 
372 	if (explicit_cpus) {
373 		if (guest_ncpus != (int)ncpus)
374 			errx(4, "Topology (%d sockets, %d cores, %d threads) "
375 			    "does not match %d vCPUs",
376 			    cpu_sockets, cpu_cores, cpu_threads,
377 			    guest_ncpus);
378 	} else
379 		guest_ncpus = ncpus;
380 }
381 
382 static int
383 pincpu_parse(const char *opt)
384 {
385 	const char *value;
386 	char *newval;
387 	char key[16];
388 	int vcpu, pcpu;
389 
390 	if (sscanf(opt, "%d:%d", &vcpu, &pcpu) != 2) {
391 		fprintf(stderr, "invalid format: %s\n", opt);
392 		return (-1);
393 	}
394 
395 	if (vcpu < 0) {
396 		fprintf(stderr, "invalid vcpu '%d'\n", vcpu);
397 		return (-1);
398 	}
399 
400 	if (pcpu < 0 || pcpu >= CPU_SETSIZE) {
401 		fprintf(stderr, "hostcpu '%d' outside valid range from "
402 		    "0 to %d\n", pcpu, CPU_SETSIZE - 1);
403 		return (-1);
404 	}
405 
406 	snprintf(key, sizeof(key), "vcpu.%d.cpuset", vcpu);
407 	value = get_config_value(key);
408 
409 	if (asprintf(&newval, "%s%s%d", value != NULL ? value : "",
410 	    value != NULL ? "," : "", pcpu) == -1) {
411 		perror("failed to build new cpuset string");
412 		return (-1);
413 	}
414 
415 	set_config_value(key, newval);
416 	free(newval);
417 	return (0);
418 }
419 
420 static void
421 parse_cpuset(int vcpu, const char *list, cpuset_t *set)
422 {
423 	char *cp, *token;
424 	int pcpu, start;
425 
426 	CPU_ZERO(set);
427 	start = -1;
428 	token = __DECONST(char *, list);
429 	for (;;) {
430 		pcpu = strtoul(token, &cp, 0);
431 		if (cp == token)
432 			errx(4, "invalid cpuset for vcpu %d: '%s'", vcpu, list);
433 		if (pcpu < 0 || pcpu >= CPU_SETSIZE)
434 			errx(4, "hostcpu '%d' outside valid range from 0 to %d",
435 			    pcpu, CPU_SETSIZE - 1);
436 		switch (*cp) {
437 		case ',':
438 		case '\0':
439 			if (start >= 0) {
440 				if (start > pcpu)
441 					errx(4, "Invalid hostcpu range %d-%d",
442 					    start, pcpu);
443 				while (start < pcpu) {
444 					CPU_SET(start, set);
445 					start++;
446 				}
447 				start = -1;
448 			}
449 			CPU_SET(pcpu, set);
450 			break;
451 		case '-':
452 			if (start >= 0)
453 				errx(4, "invalid cpuset for vcpu %d: '%s'",
454 				    vcpu, list);
455 			start = pcpu;
456 			break;
457 		default:
458 			errx(4, "invalid cpuset for vcpu %d: '%s'", vcpu, list);
459 		}
460 		if (*cp == '\0')
461 			break;
462 		token = cp + 1;
463 	}
464 }
465 
466 static void
467 build_vcpumaps(void)
468 {
469 	char key[16];
470 	const char *value;
471 	int vcpu;
472 
473 	vcpumap = calloc(guest_ncpus, sizeof(*vcpumap));
474 	for (vcpu = 0; vcpu < guest_ncpus; vcpu++) {
475 		snprintf(key, sizeof(key), "vcpu.%d.cpuset", vcpu);
476 		value = get_config_value(key);
477 		if (value == NULL)
478 			continue;
479 		vcpumap[vcpu] = malloc(sizeof(cpuset_t));
480 		if (vcpumap[vcpu] == NULL)
481 			err(4, "Failed to allocate cpuset for vcpu %d", vcpu);
482 		parse_cpuset(vcpu, value, vcpumap[vcpu]);
483 	}
484 }
485 
486 void
487 vm_inject_fault(void *arg, int vcpu, int vector, int errcode_valid,
488     int errcode)
489 {
490 	struct vmctx *ctx;
491 	int error, restart_instruction;
492 
493 	ctx = arg;
494 	restart_instruction = 1;
495 
496 	error = vm_inject_exception(ctx, vcpu, vector, errcode_valid, errcode,
497 	    restart_instruction);
498 	assert(error == 0);
499 }
500 
501 void *
502 paddr_guest2host(struct vmctx *ctx, uintptr_t gaddr, size_t len)
503 {
504 
505 	return (vm_map_gpa(ctx, gaddr, len));
506 }
507 
508 #ifdef BHYVE_SNAPSHOT
509 uintptr_t
510 paddr_host2guest(struct vmctx *ctx, void *addr)
511 {
512 	return (vm_rev_map_gpa(ctx, addr));
513 }
514 #endif
515 
516 int
517 fbsdrun_virtio_msix(void)
518 {
519 
520 	return (get_config_bool_default("virtio_msix", true));
521 }
522 
523 static void *
524 fbsdrun_start_thread(void *param)
525 {
526 	char tname[MAXCOMLEN + 1];
527 	struct mt_vmm_info *mtp;
528 	int error, vcpu;
529 
530 	mtp = param;
531 	vcpu = mtp->mt_vcpu;
532 
533 	snprintf(tname, sizeof(tname), "vcpu %d", vcpu);
534 	pthread_set_name_np(mtp->mt_thr, tname);
535 
536 	if (vcpumap[vcpu] != NULL) {
537 		error = pthread_setaffinity_np(mtp->mt_thr, sizeof(cpuset_t),
538 		    vcpumap[vcpu]);
539 		assert(error == 0);
540 	}
541 
542 #ifdef BHYVE_SNAPSHOT
543 	checkpoint_cpu_add(vcpu);
544 #endif
545 	gdb_cpu_add(vcpu);
546 
547 	vm_loop(mtp->mt_ctx, vcpu);
548 
549 	/* not reached */
550 	exit(1);
551 	return (NULL);
552 }
553 
554 static void
555 fbsdrun_addcpu(struct vmctx *ctx, int newcpu)
556 {
557 	int error;
558 
559 	error = vm_activate_cpu(ctx, newcpu);
560 	if (error != 0)
561 		err(EX_OSERR, "could not activate CPU %d", newcpu);
562 
563 	CPU_SET_ATOMIC(newcpu, &cpumask);
564 
565 	vm_suspend_cpu(ctx, newcpu);
566 
567 	mt_vmm_info[newcpu].mt_ctx = ctx;
568 	mt_vmm_info[newcpu].mt_vcpu = newcpu;
569 
570 	error = pthread_create(&mt_vmm_info[newcpu].mt_thr, NULL,
571 	    fbsdrun_start_thread, &mt_vmm_info[newcpu]);
572 	assert(error == 0);
573 }
574 
575 static int
576 fbsdrun_deletecpu(int vcpu)
577 {
578 
579 	if (!CPU_ISSET(vcpu, &cpumask)) {
580 		fprintf(stderr, "Attempting to delete unknown cpu %d\n", vcpu);
581 		exit(4);
582 	}
583 
584 	CPU_CLR_ATOMIC(vcpu, &cpumask);
585 	return (CPU_EMPTY(&cpumask));
586 }
587 
588 static int
589 vmexit_handle_notify(struct vmctx *ctx __unused, struct vm_exit *vme __unused,
590     int *pvcpu __unused, uint32_t eax __unused)
591 {
592 #if BHYVE_DEBUG
593 	/*
594 	 * put guest-driven debug here
595 	 */
596 #endif
597 	return (VMEXIT_CONTINUE);
598 }
599 
600 static int
601 vmexit_inout(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
602 {
603 	int error;
604 	int bytes, port, in, out;
605 	int vcpu;
606 
607 	vcpu = *pvcpu;
608 
609 	port = vme->u.inout.port;
610 	bytes = vme->u.inout.bytes;
611 	in = vme->u.inout.in;
612 	out = !in;
613 
614         /* Extra-special case of host notifications */
615         if (out && port == GUEST_NIO_PORT) {
616                 error = vmexit_handle_notify(ctx, vme, pvcpu, vme->u.inout.eax);
617 		return (error);
618 	}
619 
620 	error = emulate_inout(ctx, vcpu, vme);
621 	if (error) {
622 		fprintf(stderr, "Unhandled %s%c 0x%04x at 0x%lx\n",
623 		    in ? "in" : "out",
624 		    bytes == 1 ? 'b' : (bytes == 2 ? 'w' : 'l'),
625 		    port, vme->rip);
626 		return (VMEXIT_ABORT);
627 	} else {
628 		return (VMEXIT_CONTINUE);
629 	}
630 }
631 
632 static int
633 vmexit_rdmsr(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
634 {
635 	uint64_t val;
636 	uint32_t eax, edx;
637 	int error;
638 
639 	val = 0;
640 	error = emulate_rdmsr(ctx, *pvcpu, vme->u.msr.code, &val);
641 	if (error != 0) {
642 		fprintf(stderr, "rdmsr to register %#x on vcpu %d\n",
643 		    vme->u.msr.code, *pvcpu);
644 		if (get_config_bool("x86.strictmsr")) {
645 			vm_inject_gp(ctx, *pvcpu);
646 			return (VMEXIT_CONTINUE);
647 		}
648 	}
649 
650 	eax = val;
651 	error = vm_set_register(ctx, *pvcpu, VM_REG_GUEST_RAX, eax);
652 	assert(error == 0);
653 
654 	edx = val >> 32;
655 	error = vm_set_register(ctx, *pvcpu, VM_REG_GUEST_RDX, edx);
656 	assert(error == 0);
657 
658 	return (VMEXIT_CONTINUE);
659 }
660 
661 static int
662 vmexit_wrmsr(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
663 {
664 	int error;
665 
666 	error = emulate_wrmsr(ctx, *pvcpu, vme->u.msr.code, vme->u.msr.wval);
667 	if (error != 0) {
668 		fprintf(stderr, "wrmsr to register %#x(%#lx) on vcpu %d\n",
669 		    vme->u.msr.code, vme->u.msr.wval, *pvcpu);
670 		if (get_config_bool("x86.strictmsr")) {
671 			vm_inject_gp(ctx, *pvcpu);
672 			return (VMEXIT_CONTINUE);
673 		}
674 	}
675 	return (VMEXIT_CONTINUE);
676 }
677 
678 #define	DEBUG_EPT_MISCONFIG
679 #ifdef DEBUG_EPT_MISCONFIG
680 #define	VMCS_GUEST_PHYSICAL_ADDRESS	0x00002400
681 
682 static uint64_t ept_misconfig_gpa, ept_misconfig_pte[4];
683 static int ept_misconfig_ptenum;
684 #endif
685 
686 static const char *
687 vmexit_vmx_desc(uint32_t exit_reason)
688 {
689 
690 	if (exit_reason >= nitems(vmx_exit_reason_desc) ||
691 	    vmx_exit_reason_desc[exit_reason] == NULL)
692 		return ("Unknown");
693 	return (vmx_exit_reason_desc[exit_reason]);
694 }
695 
696 static int
697 vmexit_vmx(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
698 {
699 
700 	fprintf(stderr, "vm exit[%d]\n", *pvcpu);
701 	fprintf(stderr, "\treason\t\tVMX\n");
702 	fprintf(stderr, "\trip\t\t0x%016lx\n", vme->rip);
703 	fprintf(stderr, "\tinst_length\t%d\n", vme->inst_length);
704 	fprintf(stderr, "\tstatus\t\t%d\n", vme->u.vmx.status);
705 	fprintf(stderr, "\texit_reason\t%u (%s)\n", vme->u.vmx.exit_reason,
706 	    vmexit_vmx_desc(vme->u.vmx.exit_reason));
707 	fprintf(stderr, "\tqualification\t0x%016lx\n",
708 	    vme->u.vmx.exit_qualification);
709 	fprintf(stderr, "\tinst_type\t\t%d\n", vme->u.vmx.inst_type);
710 	fprintf(stderr, "\tinst_error\t\t%d\n", vme->u.vmx.inst_error);
711 #ifdef DEBUG_EPT_MISCONFIG
712 	if (vme->u.vmx.exit_reason == EXIT_REASON_EPT_MISCONFIG) {
713 		vm_get_register(ctx, *pvcpu,
714 		    VMCS_IDENT(VMCS_GUEST_PHYSICAL_ADDRESS),
715 		    &ept_misconfig_gpa);
716 		vm_get_gpa_pmap(ctx, ept_misconfig_gpa, ept_misconfig_pte,
717 		    &ept_misconfig_ptenum);
718 		fprintf(stderr, "\tEPT misconfiguration:\n");
719 		fprintf(stderr, "\t\tGPA: %#lx\n", ept_misconfig_gpa);
720 		fprintf(stderr, "\t\tPTE(%d): %#lx %#lx %#lx %#lx\n",
721 		    ept_misconfig_ptenum, ept_misconfig_pte[0],
722 		    ept_misconfig_pte[1], ept_misconfig_pte[2],
723 		    ept_misconfig_pte[3]);
724 	}
725 #endif	/* DEBUG_EPT_MISCONFIG */
726 	return (VMEXIT_ABORT);
727 }
728 
729 static int
730 vmexit_svm(struct vmctx *ctx __unused, struct vm_exit *vme, int *pvcpu)
731 {
732 
733 	fprintf(stderr, "vm exit[%d]\n", *pvcpu);
734 	fprintf(stderr, "\treason\t\tSVM\n");
735 	fprintf(stderr, "\trip\t\t0x%016lx\n", vme->rip);
736 	fprintf(stderr, "\tinst_length\t%d\n", vme->inst_length);
737 	fprintf(stderr, "\texitcode\t%#lx\n", vme->u.svm.exitcode);
738 	fprintf(stderr, "\texitinfo1\t%#lx\n", vme->u.svm.exitinfo1);
739 	fprintf(stderr, "\texitinfo2\t%#lx\n", vme->u.svm.exitinfo2);
740 	return (VMEXIT_ABORT);
741 }
742 
743 static int
744 vmexit_bogus(struct vmctx *ctx __unused, struct vm_exit *vme,
745     int *pvcpu __unused)
746 {
747 
748 	assert(vme->inst_length == 0);
749 
750 	stats.vmexit_bogus++;
751 
752 	return (VMEXIT_CONTINUE);
753 }
754 
755 static int
756 vmexit_reqidle(struct vmctx *ctx __unused, struct vm_exit *vme,
757     int *pvcpu __unused)
758 {
759 
760 	assert(vme->inst_length == 0);
761 
762 	stats.vmexit_reqidle++;
763 
764 	return (VMEXIT_CONTINUE);
765 }
766 
767 static int
768 vmexit_hlt(struct vmctx *ctx __unused, struct vm_exit *vme __unused,
769     int *pvcpu __unused)
770 {
771 
772 	stats.vmexit_hlt++;
773 
774 	/*
775 	 * Just continue execution with the next instruction. We use
776 	 * the HLT VM exit as a way to be friendly with the host
777 	 * scheduler.
778 	 */
779 	return (VMEXIT_CONTINUE);
780 }
781 
782 static int
783 vmexit_pause(struct vmctx *ctx __unused, struct vm_exit *vme __unused,
784     int *pvcpu __unused)
785 {
786 
787 	stats.vmexit_pause++;
788 
789 	return (VMEXIT_CONTINUE);
790 }
791 
792 static int
793 vmexit_mtrap(struct vmctx *ctx __unused, struct vm_exit *vme, int *pvcpu)
794 {
795 
796 	assert(vme->inst_length == 0);
797 
798 	stats.vmexit_mtrap++;
799 
800 #ifdef BHYVE_SNAPSHOT
801 	checkpoint_cpu_suspend(*pvcpu);
802 #endif
803 	gdb_cpu_mtrap(*pvcpu);
804 #ifdef BHYVE_SNAPSHOT
805 	checkpoint_cpu_resume(*pvcpu);
806 #endif
807 
808 	return (VMEXIT_CONTINUE);
809 }
810 
811 static int
812 vmexit_inst_emul(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
813 {
814 	int err, i, cs_d;
815 	struct vie *vie;
816 	enum vm_cpu_mode mode;
817 
818 	stats.vmexit_inst_emul++;
819 
820 	vie = &vme->u.inst_emul.vie;
821 	if (!vie->decoded) {
822 		/*
823 		 * Attempt to decode in userspace as a fallback.  This allows
824 		 * updating instruction decode in bhyve without rebooting the
825 		 * kernel (rapid prototyping), albeit with much slower
826 		 * emulation.
827 		 */
828 		vie_restart(vie);
829 		mode = vme->u.inst_emul.paging.cpu_mode;
830 		cs_d = vme->u.inst_emul.cs_d;
831 		if (vmm_decode_instruction(mode, cs_d, vie) != 0)
832 			goto fail;
833 		if (vm_set_register(ctx, *pvcpu, VM_REG_GUEST_RIP,
834 		    vme->rip + vie->num_processed) != 0)
835 			goto fail;
836 	}
837 
838 	err = emulate_mem(ctx, *pvcpu, vme->u.inst_emul.gpa,
839 	    vie, &vme->u.inst_emul.paging);
840 	if (err) {
841 		if (err == ESRCH) {
842 			EPRINTLN("Unhandled memory access to 0x%lx\n",
843 			    vme->u.inst_emul.gpa);
844 		}
845 		goto fail;
846 	}
847 
848 	return (VMEXIT_CONTINUE);
849 
850 fail:
851 	fprintf(stderr, "Failed to emulate instruction sequence [ ");
852 	for (i = 0; i < vie->num_valid; i++)
853 		fprintf(stderr, "%02x", vie->inst[i]);
854 	FPRINTLN(stderr, " ] at 0x%lx", vme->rip);
855 	return (VMEXIT_ABORT);
856 }
857 
858 static pthread_mutex_t resetcpu_mtx = PTHREAD_MUTEX_INITIALIZER;
859 static pthread_cond_t resetcpu_cond = PTHREAD_COND_INITIALIZER;
860 
861 static int
862 vmexit_suspend(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
863 {
864 	enum vm_suspend_how how;
865 
866 	how = vme->u.suspended.how;
867 
868 	fbsdrun_deletecpu(*pvcpu);
869 
870 	if (*pvcpu != BSP) {
871 		pthread_mutex_lock(&resetcpu_mtx);
872 		pthread_cond_signal(&resetcpu_cond);
873 		pthread_mutex_unlock(&resetcpu_mtx);
874 		pthread_exit(NULL);
875 	}
876 
877 	pthread_mutex_lock(&resetcpu_mtx);
878 	while (!CPU_EMPTY(&cpumask)) {
879 		pthread_cond_wait(&resetcpu_cond, &resetcpu_mtx);
880 	}
881 	pthread_mutex_unlock(&resetcpu_mtx);
882 
883 	switch (how) {
884 	case VM_SUSPEND_RESET:
885 		exit(0);
886 	case VM_SUSPEND_POWEROFF:
887 		if (get_config_bool_default("destroy_on_poweroff", false))
888 			vm_destroy(ctx);
889 		exit(1);
890 	case VM_SUSPEND_HALT:
891 		exit(2);
892 	case VM_SUSPEND_TRIPLEFAULT:
893 		exit(3);
894 	default:
895 		fprintf(stderr, "vmexit_suspend: invalid reason %d\n", how);
896 		exit(100);
897 	}
898 	return (0);	/* NOTREACHED */
899 }
900 
901 static int
902 vmexit_debug(struct vmctx *ctx __unused, struct vm_exit *vme __unused,
903     int *pvcpu)
904 {
905 
906 #ifdef BHYVE_SNAPSHOT
907 	checkpoint_cpu_suspend(*pvcpu);
908 #endif
909 	gdb_cpu_suspend(*pvcpu);
910 #ifdef BHYVE_SNAPSHOT
911 	checkpoint_cpu_resume(*pvcpu);
912 #endif
913 	return (VMEXIT_CONTINUE);
914 }
915 
916 static int
917 vmexit_breakpoint(struct vmctx *ctx __unused, struct vm_exit *vme, int *pvcpu)
918 {
919 
920 	gdb_cpu_breakpoint(*pvcpu, vme);
921 	return (VMEXIT_CONTINUE);
922 }
923 
924 static int
925 vmexit_ipi(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu __unused)
926 {
927 	int error = -1;
928 	int i;
929 	switch (vme->u.ipi.mode) {
930 	case APIC_DELMODE_INIT:
931 		CPU_FOREACH_ISSET(i, &vme->u.ipi.dmask) {
932 			error = vm_suspend_cpu(ctx, i);
933 			if (error) {
934 				warnx("%s: failed to suspend cpu %d\n",
935 				    __func__, i);
936 				break;
937 			}
938 		}
939 		break;
940 	case APIC_DELMODE_STARTUP:
941 		CPU_FOREACH_ISSET(i, &vme->u.ipi.dmask) {
942 			spinup_ap(ctx, i, vme->u.ipi.vector << PAGE_SHIFT);
943 		}
944 		error = 0;
945 		break;
946 	default:
947 		break;
948 	}
949 
950 	return (error);
951 }
952 
953 static vmexit_handler_t handler[VM_EXITCODE_MAX] = {
954 	[VM_EXITCODE_INOUT]  = vmexit_inout,
955 	[VM_EXITCODE_INOUT_STR]  = vmexit_inout,
956 	[VM_EXITCODE_VMX]    = vmexit_vmx,
957 	[VM_EXITCODE_SVM]    = vmexit_svm,
958 	[VM_EXITCODE_BOGUS]  = vmexit_bogus,
959 	[VM_EXITCODE_REQIDLE] = vmexit_reqidle,
960 	[VM_EXITCODE_RDMSR]  = vmexit_rdmsr,
961 	[VM_EXITCODE_WRMSR]  = vmexit_wrmsr,
962 	[VM_EXITCODE_MTRAP]  = vmexit_mtrap,
963 	[VM_EXITCODE_INST_EMUL] = vmexit_inst_emul,
964 	[VM_EXITCODE_SUSPENDED] = vmexit_suspend,
965 	[VM_EXITCODE_TASK_SWITCH] = vmexit_task_switch,
966 	[VM_EXITCODE_DEBUG] = vmexit_debug,
967 	[VM_EXITCODE_BPT] = vmexit_breakpoint,
968 	[VM_EXITCODE_IPI] = vmexit_ipi,
969 };
970 
971 static void
972 vm_loop(struct vmctx *ctx, int vcpu)
973 {
974 	struct vm_exit vme;
975 	int error, rc;
976 	enum vm_exitcode exitcode;
977 	cpuset_t active_cpus;
978 
979 	error = vm_active_cpus(ctx, &active_cpus);
980 	assert(CPU_ISSET(vcpu, &active_cpus));
981 
982 	while (1) {
983 		error = vm_run(ctx, vcpu, &vme);
984 		if (error != 0)
985 			break;
986 
987 		exitcode = vme.exitcode;
988 		if (exitcode >= VM_EXITCODE_MAX || handler[exitcode] == NULL) {
989 			fprintf(stderr, "vm_loop: unexpected exitcode 0x%x\n",
990 			    exitcode);
991 			exit(4);
992 		}
993 
994 		rc = (*handler[exitcode])(ctx, &vme, &vcpu);
995 
996 		switch (rc) {
997 		case VMEXIT_CONTINUE:
998 			break;
999 		case VMEXIT_ABORT:
1000 			abort();
1001 		default:
1002 			exit(4);
1003 		}
1004 	}
1005 	fprintf(stderr, "vm_run error %d, errno %d\n", error, errno);
1006 }
1007 
1008 static int
1009 num_vcpus_allowed(struct vmctx *ctx)
1010 {
1011 	uint16_t sockets, cores, threads, maxcpus;
1012 	int tmp, error;
1013 
1014 	/*
1015 	 * The guest is allowed to spinup more than one processor only if the
1016 	 * UNRESTRICTED_GUEST capability is available.
1017 	 */
1018 	error = vm_get_capability(ctx, BSP, VM_CAP_UNRESTRICTED_GUEST, &tmp);
1019 	if (error != 0)
1020 		return (1);
1021 
1022 	error = vm_get_topology(ctx, &sockets, &cores, &threads, &maxcpus);
1023 	if (error == 0)
1024 		return (maxcpus);
1025 	else
1026 		return (1);
1027 }
1028 
1029 static void
1030 fbsdrun_set_capabilities(struct vmctx *ctx, int cpu)
1031 {
1032 	int err, tmp;
1033 
1034 	if (get_config_bool_default("x86.vmexit_on_hlt", false)) {
1035 		err = vm_get_capability(ctx, cpu, VM_CAP_HALT_EXIT, &tmp);
1036 		if (err < 0) {
1037 			fprintf(stderr, "VM exit on HLT not supported\n");
1038 			exit(4);
1039 		}
1040 		vm_set_capability(ctx, cpu, VM_CAP_HALT_EXIT, 1);
1041 		if (cpu == BSP)
1042 			handler[VM_EXITCODE_HLT] = vmexit_hlt;
1043 	}
1044 
1045 	if (get_config_bool_default("x86.vmexit_on_pause", false)) {
1046 		/*
1047 		 * pause exit support required for this mode
1048 		 */
1049 		err = vm_get_capability(ctx, cpu, VM_CAP_PAUSE_EXIT, &tmp);
1050 		if (err < 0) {
1051 			fprintf(stderr,
1052 			    "SMP mux requested, no pause support\n");
1053 			exit(4);
1054 		}
1055 		vm_set_capability(ctx, cpu, VM_CAP_PAUSE_EXIT, 1);
1056 		if (cpu == BSP)
1057 			handler[VM_EXITCODE_PAUSE] = vmexit_pause;
1058         }
1059 
1060 	if (get_config_bool_default("x86.x2apic", false))
1061 		err = vm_set_x2apic_state(ctx, cpu, X2APIC_ENABLED);
1062 	else
1063 		err = vm_set_x2apic_state(ctx, cpu, X2APIC_DISABLED);
1064 
1065 	if (err) {
1066 		fprintf(stderr, "Unable to set x2apic state (%d)\n", err);
1067 		exit(4);
1068 	}
1069 
1070 	vm_set_capability(ctx, cpu, VM_CAP_ENABLE_INVPCID, 1);
1071 
1072 	err = vm_set_capability(ctx, cpu, VM_CAP_IPI_EXIT, 1);
1073 	assert(err == 0);
1074 }
1075 
1076 static struct vmctx *
1077 do_open(const char *vmname)
1078 {
1079 	struct vmctx *ctx;
1080 	int error;
1081 	bool reinit, romboot;
1082 
1083 	reinit = romboot = false;
1084 
1085 	if (lpc_bootrom())
1086 		romboot = true;
1087 
1088 	error = vm_create(vmname);
1089 	if (error) {
1090 		if (errno == EEXIST) {
1091 			if (romboot) {
1092 				reinit = true;
1093 			} else {
1094 				/*
1095 				 * The virtual machine has been setup by the
1096 				 * userspace bootloader.
1097 				 */
1098 			}
1099 		} else {
1100 			perror("vm_create");
1101 			exit(4);
1102 		}
1103 	} else {
1104 		if (!romboot) {
1105 			/*
1106 			 * If the virtual machine was just created then a
1107 			 * bootrom must be configured to boot it.
1108 			 */
1109 			fprintf(stderr, "virtual machine cannot be booted\n");
1110 			exit(4);
1111 		}
1112 	}
1113 
1114 	ctx = vm_open(vmname);
1115 	if (ctx == NULL) {
1116 		perror("vm_open");
1117 		exit(4);
1118 	}
1119 
1120 #ifndef WITHOUT_CAPSICUM
1121 	if (vm_limit_rights(ctx) != 0)
1122 		err(EX_OSERR, "vm_limit_rights");
1123 #endif
1124 
1125 	if (reinit) {
1126 		error = vm_reinit(ctx);
1127 		if (error) {
1128 			perror("vm_reinit");
1129 			exit(4);
1130 		}
1131 	}
1132 	error = vm_set_topology(ctx, cpu_sockets, cpu_cores, cpu_threads,
1133 	    0 /* maxcpus, unimplemented */);
1134 	if (error)
1135 		errx(EX_OSERR, "vm_set_topology");
1136 	return (ctx);
1137 }
1138 
1139 static void
1140 spinup_vcpu(struct vmctx *ctx, int vcpu)
1141 {
1142 	int error;
1143 
1144 	if (vcpu != BSP) {
1145 		fbsdrun_set_capabilities(ctx, vcpu);
1146 
1147 		/*
1148 		 * Enable the 'unrestricted guest' mode for APs.
1149 		 *
1150 		 * APs startup in power-on 16-bit mode.
1151 		 */
1152 		error = vm_set_capability(ctx, vcpu, VM_CAP_UNRESTRICTED_GUEST, 1);
1153 		assert(error == 0);
1154 	}
1155 
1156 	fbsdrun_addcpu(ctx, vcpu);
1157 }
1158 
1159 static bool
1160 parse_config_option(const char *option)
1161 {
1162 	const char *value;
1163 	char *path;
1164 
1165 	value = strchr(option, '=');
1166 	if (value == NULL || value[1] == '\0')
1167 		return (false);
1168 	path = strndup(option, value - option);
1169 	if (path == NULL)
1170 		err(4, "Failed to allocate memory");
1171 	set_config_value(path, value + 1);
1172 	return (true);
1173 }
1174 
1175 static void
1176 parse_simple_config_file(const char *path)
1177 {
1178 	FILE *fp;
1179 	char *line, *cp;
1180 	size_t linecap;
1181 	unsigned int lineno;
1182 
1183 	fp = fopen(path, "r");
1184 	if (fp == NULL)
1185 		err(4, "Failed to open configuration file %s", path);
1186 	line = NULL;
1187 	linecap = 0;
1188 	lineno = 1;
1189 	for (lineno = 1; getline(&line, &linecap, fp) > 0; lineno++) {
1190 		if (*line == '#' || *line == '\n')
1191 			continue;
1192 		cp = strchr(line, '\n');
1193 		if (cp != NULL)
1194 			*cp = '\0';
1195 		if (!parse_config_option(line))
1196 			errx(4, "%s line %u: invalid config option '%s'", path,
1197 			    lineno, line);
1198 	}
1199 	free(line);
1200 	fclose(fp);
1201 }
1202 
1203 static void
1204 parse_gdb_options(const char *opt)
1205 {
1206 	const char *sport;
1207 	char *colon;
1208 
1209 	if (opt[0] == 'w') {
1210 		set_config_bool("gdb.wait", true);
1211 		opt++;
1212 	}
1213 
1214 	colon = strrchr(opt, ':');
1215 	if (colon == NULL) {
1216 		sport = opt;
1217 	} else {
1218 		*colon = '\0';
1219 		colon++;
1220 		sport = colon;
1221 		set_config_value("gdb.address", opt);
1222 	}
1223 
1224 	set_config_value("gdb.port", sport);
1225 }
1226 
1227 static void
1228 set_defaults(void)
1229 {
1230 
1231 	set_config_bool("acpi_tables", false);
1232 	set_config_value("memory.size", "256M");
1233 	set_config_bool("x86.strictmsr", true);
1234 }
1235 
1236 int
1237 main(int argc, char *argv[])
1238 {
1239 	int c, error;
1240 	int max_vcpus, memflags;
1241 	struct vmctx *ctx;
1242 	uint64_t rip;
1243 	size_t memsize;
1244 	const char *optstr, *value, *vmname;
1245 #ifdef BHYVE_SNAPSHOT
1246 	char *restore_file;
1247 	struct restore_state rstate;
1248 
1249 	restore_file = NULL;
1250 #endif
1251 
1252 	init_config();
1253 	set_defaults();
1254 	progname = basename(argv[0]);
1255 
1256 #ifdef BHYVE_SNAPSHOT
1257 	optstr = "aehuwxACDHIPSWYk:o:p:G:c:s:m:l:K:U:r:";
1258 #else
1259 	optstr = "aehuwxACDHIPSWYk:o:p:G:c:s:m:l:K:U:";
1260 #endif
1261 	while ((c = getopt(argc, argv, optstr)) != -1) {
1262 		switch (c) {
1263 		case 'a':
1264 			set_config_bool("x86.x2apic", false);
1265 			break;
1266 		case 'A':
1267 			set_config_bool("acpi_tables", true);
1268 			break;
1269 		case 'D':
1270 			set_config_bool("destroy_on_poweroff", true);
1271 			break;
1272 		case 'p':
1273                         if (pincpu_parse(optarg) != 0) {
1274                             errx(EX_USAGE, "invalid vcpu pinning "
1275                                  "configuration '%s'", optarg);
1276                         }
1277 			break;
1278                 case 'c':
1279 			if (topology_parse(optarg) != 0) {
1280 			    errx(EX_USAGE, "invalid cpu topology "
1281 				"'%s'", optarg);
1282 			}
1283 			break;
1284 		case 'C':
1285 			set_config_bool("memory.guest_in_core", true);
1286 			break;
1287 		case 'G':
1288 			parse_gdb_options(optarg);
1289 			break;
1290 		case 'k':
1291 			parse_simple_config_file(optarg);
1292 			break;
1293 		case 'K':
1294 			set_config_value("keyboard.layout", optarg);
1295 			break;
1296 		case 'l':
1297 			if (strncmp(optarg, "help", strlen(optarg)) == 0) {
1298 				lpc_print_supported_devices();
1299 				exit(0);
1300 			} else if (lpc_device_parse(optarg) != 0) {
1301 				errx(EX_USAGE, "invalid lpc device "
1302 				    "configuration '%s'", optarg);
1303 			}
1304 			break;
1305 #ifdef BHYVE_SNAPSHOT
1306 		case 'r':
1307 			restore_file = optarg;
1308 			break;
1309 #endif
1310 		case 's':
1311 			if (strncmp(optarg, "help", strlen(optarg)) == 0) {
1312 				pci_print_supported_devices();
1313 				exit(0);
1314 			} else if (pci_parse_slot(optarg) != 0)
1315 				exit(4);
1316 			else
1317 				break;
1318 		case 'S':
1319 			set_config_bool("memory.wired", true);
1320 			break;
1321                 case 'm':
1322 			set_config_value("memory.size", optarg);
1323 			break;
1324 		case 'o':
1325 			if (!parse_config_option(optarg))
1326 				errx(EX_USAGE, "invalid configuration option '%s'", optarg);
1327 			break;
1328 		case 'H':
1329 			set_config_bool("x86.vmexit_on_hlt", true);
1330 			break;
1331 		case 'I':
1332 			/*
1333 			 * The "-I" option was used to add an ioapic to the
1334 			 * virtual machine.
1335 			 *
1336 			 * An ioapic is now provided unconditionally for each
1337 			 * virtual machine and this option is now deprecated.
1338 			 */
1339 			break;
1340 		case 'P':
1341 			set_config_bool("x86.vmexit_on_pause", true);
1342 			break;
1343 		case 'e':
1344 			set_config_bool("x86.strictio", true);
1345 			break;
1346 		case 'u':
1347 			set_config_bool("rtc.use_localtime", false);
1348 			break;
1349 		case 'U':
1350 			set_config_value("uuid", optarg);
1351 			break;
1352 		case 'w':
1353 			set_config_bool("x86.strictmsr", false);
1354 			break;
1355 		case 'W':
1356 			set_config_bool("virtio_msix", false);
1357 			break;
1358 		case 'x':
1359 			set_config_bool("x86.x2apic", true);
1360 			break;
1361 		case 'Y':
1362 			set_config_bool("x86.mptable", false);
1363 			break;
1364 		case 'h':
1365 			usage(0);
1366 		default:
1367 			usage(1);
1368 		}
1369 	}
1370 	argc -= optind;
1371 	argv += optind;
1372 
1373 	if (argc > 1)
1374 		usage(1);
1375 
1376 #ifdef BHYVE_SNAPSHOT
1377 	if (restore_file != NULL) {
1378 		error = load_restore_file(restore_file, &rstate);
1379 		if (error) {
1380 			fprintf(stderr, "Failed to read checkpoint info from "
1381 					"file: '%s'.\n", restore_file);
1382 			exit(1);
1383 		}
1384 		vmname = lookup_vmname(&rstate);
1385 		if (vmname != NULL)
1386 			set_config_value("name", vmname);
1387 	}
1388 #endif
1389 
1390 	if (argc == 1)
1391 		set_config_value("name", argv[0]);
1392 
1393 	vmname = get_config_value("name");
1394 	if (vmname == NULL)
1395 		usage(1);
1396 
1397 	if (get_config_bool_default("config.dump", false)) {
1398 		dump_config();
1399 		exit(1);
1400 	}
1401 
1402 	calc_topology();
1403 	build_vcpumaps();
1404 
1405 	value = get_config_value("memory.size");
1406 	error = vm_parse_memsize(value, &memsize);
1407 	if (error)
1408 		errx(EX_USAGE, "invalid memsize '%s'", value);
1409 
1410 	ctx = do_open(vmname);
1411 
1412 #ifdef BHYVE_SNAPSHOT
1413 	if (restore_file != NULL) {
1414 		guest_ncpus = lookup_guest_ncpus(&rstate);
1415 		memflags = lookup_memflags(&rstate);
1416 		memsize = lookup_memsize(&rstate);
1417 	}
1418 
1419 	if (guest_ncpus < 1) {
1420 		fprintf(stderr, "Invalid guest vCPUs (%d)\n", guest_ncpus);
1421 		exit(1);
1422 	}
1423 #endif
1424 
1425 	max_vcpus = num_vcpus_allowed(ctx);
1426 	if (guest_ncpus > max_vcpus) {
1427 		fprintf(stderr, "%d vCPUs requested but only %d available\n",
1428 			guest_ncpus, max_vcpus);
1429 		exit(4);
1430 	}
1431 
1432 	fbsdrun_set_capabilities(ctx, BSP);
1433 
1434 	memflags = 0;
1435 	if (get_config_bool_default("memory.wired", false))
1436 		memflags |= VM_MEM_F_WIRED;
1437 	if (get_config_bool_default("memory.guest_in_core", false))
1438 		memflags |= VM_MEM_F_INCORE;
1439 	vm_set_memflags(ctx, memflags);
1440 	error = vm_setup_memory(ctx, memsize, VM_MMAP_ALL);
1441 	if (error) {
1442 		fprintf(stderr, "Unable to setup memory (%d)\n", errno);
1443 		exit(4);
1444 	}
1445 
1446 	error = init_msr();
1447 	if (error) {
1448 		fprintf(stderr, "init_msr error %d", error);
1449 		exit(4);
1450 	}
1451 
1452 	init_mem(guest_ncpus);
1453 	init_inout();
1454 	kernemu_dev_init();
1455 	init_bootrom(ctx);
1456 	atkbdc_init(ctx);
1457 	pci_irq_init(ctx);
1458 	ioapic_init(ctx);
1459 
1460 	rtc_init(ctx);
1461 	sci_init(ctx);
1462 
1463 	/*
1464 	 * Exit if a device emulation finds an error in its initilization
1465 	 */
1466 	if (init_pci(ctx) != 0) {
1467 		perror("device emulation initialization error");
1468 		exit(4);
1469 	}
1470 
1471 	/*
1472 	 * Initialize after PCI, to allow a bootrom file to reserve the high
1473 	 * region.
1474 	 */
1475 	if (get_config_bool("acpi_tables"))
1476 		vmgenc_init(ctx);
1477 
1478 	init_gdb(ctx);
1479 
1480 	if (lpc_bootrom()) {
1481 		if (vm_set_capability(ctx, BSP, VM_CAP_UNRESTRICTED_GUEST, 1)) {
1482 			fprintf(stderr, "ROM boot failed: unrestricted guest "
1483 			    "capability not available\n");
1484 			exit(4);
1485 		}
1486 		error = vcpu_reset(ctx, BSP);
1487 		assert(error == 0);
1488 	}
1489 
1490 	/* Allocate per-VCPU resources. */
1491 	mt_vmm_info = calloc(guest_ncpus, sizeof(*mt_vmm_info));
1492 
1493 	/*
1494 	 * Add all vCPUs.
1495 	 */
1496 	for (int vcpu = 0; vcpu < guest_ncpus; vcpu++) {
1497 		spinup_vcpu(ctx, vcpu);
1498 	}
1499 
1500 #ifdef BHYVE_SNAPSHOT
1501 	if (restore_file != NULL) {
1502 		fprintf(stdout, "Pausing pci devs...\r\n");
1503 		if (vm_pause_user_devs() != 0) {
1504 			fprintf(stderr, "Failed to pause PCI device state.\n");
1505 			exit(1);
1506 		}
1507 
1508 		fprintf(stdout, "Restoring vm mem...\r\n");
1509 		if (restore_vm_mem(ctx, &rstate) != 0) {
1510 			fprintf(stderr, "Failed to restore VM memory.\n");
1511 			exit(1);
1512 		}
1513 
1514 		fprintf(stdout, "Restoring pci devs...\r\n");
1515 		if (vm_restore_user_devs(ctx, &rstate) != 0) {
1516 			fprintf(stderr, "Failed to restore PCI device state.\n");
1517 			exit(1);
1518 		}
1519 
1520 		fprintf(stdout, "Restoring kernel structs...\r\n");
1521 		if (vm_restore_kern_structs(ctx, &rstate) != 0) {
1522 			fprintf(stderr, "Failed to restore kernel structs.\n");
1523 			exit(1);
1524 		}
1525 
1526 		fprintf(stdout, "Resuming pci devs...\r\n");
1527 		if (vm_resume_user_devs() != 0) {
1528 			fprintf(stderr, "Failed to resume PCI device state.\n");
1529 			exit(1);
1530 		}
1531 	}
1532 #endif
1533 
1534 	error = vm_get_register(ctx, BSP, VM_REG_GUEST_RIP, &rip);
1535 	assert(error == 0);
1536 
1537 	/*
1538 	 * build the guest tables, MP etc.
1539 	 */
1540 	if (get_config_bool_default("x86.mptable", true)) {
1541 		error = mptable_build(ctx, guest_ncpus);
1542 		if (error) {
1543 			perror("error to build the guest tables");
1544 			exit(4);
1545 		}
1546 	}
1547 
1548 	error = smbios_build(ctx);
1549 	if (error != 0)
1550 		exit(4);
1551 
1552 	if (get_config_bool("acpi_tables")) {
1553 		error = acpi_build(ctx, guest_ncpus);
1554 		assert(error == 0);
1555 	}
1556 
1557 	if (lpc_bootrom())
1558 		fwctl_init();
1559 
1560 	/*
1561 	 * Change the proc title to include the VM name.
1562 	 */
1563 	setproctitle("%s", vmname);
1564 
1565 #ifdef BHYVE_SNAPSHOT
1566 	/* initialize mutex/cond variables */
1567 	init_snapshot();
1568 
1569 	/*
1570 	 * checkpointing thread for communication with bhyvectl
1571 	 */
1572 	if (init_checkpoint_thread(ctx) != 0)
1573 		errx(EX_OSERR, "Failed to start checkpoint thread");
1574 #endif
1575 
1576 #ifndef WITHOUT_CAPSICUM
1577 	caph_cache_catpages();
1578 
1579 	if (caph_limit_stdout() == -1 || caph_limit_stderr() == -1)
1580 		errx(EX_OSERR, "Unable to apply rights for sandbox");
1581 
1582 	if (caph_enter() == -1)
1583 		errx(EX_OSERR, "cap_enter() failed");
1584 #endif
1585 
1586 #ifdef BHYVE_SNAPSHOT
1587 	if (restore_file != NULL) {
1588 		destroy_restore_state(&rstate);
1589 		if (vm_restore_time(ctx) < 0)
1590 			err(EX_OSERR, "Unable to restore time");
1591 
1592 		for (int i = 0; i < guest_ncpus; i++) {
1593 			if (i == BSP)
1594 				continue;
1595 			vm_resume_cpu(ctx, i);
1596 		}
1597 	}
1598 #endif
1599 	vm_resume_cpu(ctx, BSP);
1600 
1601 	/*
1602 	 * Head off to the main event dispatch loop
1603 	 */
1604 	mevent_dispatch();
1605 
1606 	exit(4);
1607 }
1608