1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011 NetApp, Inc.
5 * Copyright (c) 2026 Hans Rosenfeld
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/types.h>
31 #ifndef WITHOUT_CAPSICUM
32 #include <sys/capsicum.h>
33 #endif
34 #include <sys/cpuset.h>
35 #include <sys/domainset.h>
36 #include <sys/mman.h>
37 #ifdef BHYVE_SNAPSHOT
38 #include <sys/socket.h>
39 #include <sys/stat.h>
40 #endif
41 #include <sys/time.h>
42 #ifdef BHYVE_SNAPSHOT
43 #include <sys/un.h>
44 #endif
45 #include <sys/wait.h>
46
47 #include <machine/atomic.h>
48
49 #ifndef WITHOUT_CAPSICUM
50 #include <capsicum_helpers.h>
51 #endif
52 #include <stdio.h>
53 #include <stdlib.h>
54 #include <string.h>
55 #include <err.h>
56 #include <errno.h>
57 #include <fcntl.h>
58 #include <libgen.h>
59 #include <libutil.h>
60 #include <unistd.h>
61 #include <assert.h>
62 #include <pthread.h>
63 #include <pthread_np.h>
64 #include <sysexits.h>
65 #include <stdbool.h>
66 #include <stdint.h>
67 #ifdef BHYVE_SNAPSHOT
68 #include <ucl.h>
69 #include <unistd.h>
70
71 #include <libxo/xo.h>
72 #endif
73
74 #include <dev/vmm/vmm_mem.h>
75 #include <vmmapi.h>
76
77 #include "acpi.h"
78 #include "bhyverun.h"
79 #include "bootrom.h"
80 #include "config.h"
81 #include "debug.h"
82 #ifdef BHYVE_GDB
83 #include "gdb.h"
84 #endif
85 #include "ipc.h"
86 #include "mem.h"
87 #include "mevent.h"
88 #include "pci_emul.h"
89 #ifdef __amd64__
90 #include "amd64/pci_lpc.h"
91 #endif
92 #include "qemu_fwcfg.h"
93 #ifdef BHYVE_SNAPSHOT
94 #include "snapshot.h"
95 #endif
96 #include "tpm_device.h"
97 #include "vmgenc.h"
98 #include "vmexit.h"
99
100 #define MB (1024UL * 1024)
101 #define GB (1024UL * MB)
102
103 int guest_ncpus;
104 uint16_t cpu_cores, cpu_sockets, cpu_threads;
105
106 int raw_stdio = 0;
107
108 #ifdef BHYVE_SNAPSHOT
109 char *restore_file;
110 #endif
111
112 static const int BSP = 0;
113
114 static cpuset_t cpumask;
115
116 static struct vm_mem_domain guest_domains[VM_MAXMEMDOM];
117 static int guest_ndomains = 0;
118
119 static void vm_loop(struct vmctx *ctx, struct vcpu *vcpu);
120
121 static struct vcpu_info {
122 struct vmctx *ctx;
123 struct vcpu *vcpu;
124 int vcpuid;
125 } *vcpu_info;
126
127 static cpuset_t **vcpumap;
128
129 static void
monitor_pipe_handler(int fd __unused,enum ev_type type __unused,void * param __unused)130 monitor_pipe_handler(int fd __unused, enum ev_type type __unused,
131 void *param __unused)
132 {
133
134 exit(BHYVE_EXIT_ERROR);
135 }
136
137 static void
monitor_pipe_init(int fd)138 monitor_pipe_init(int fd)
139 {
140 #ifndef WITHOUT_CAPSICUM
141 cap_rights_t rights;
142
143 cap_rights_init(&rights, CAP_EVENT);
144 if (caph_rights_limit(fd, &rights) == -1)
145 err(BHYVE_EXIT_ERROR, "Unable to apply rights to monitor pipe");
146 #endif
147
148 if (mevent_add(fd, EVF_READ, monitor_pipe_handler, NULL) == NULL)
149 errx(BHYVE_EXIT_ERROR, "Unable to register parent exit event");
150 }
151
152 /*
153 * XXX This parser is known to have the following issues:
154 * 1. It accepts null key=value tokens ",," as setting "cpus" to an
155 * empty string.
156 *
157 * The acceptance of a null specification ('-c ""') is by design to match the
158 * manual page syntax specification, this results in a topology of 1 vCPU.
159 */
160 int
bhyve_topology_parse(const char * opt)161 bhyve_topology_parse(const char *opt)
162 {
163 char *cp, *str, *tofree;
164
165 if (*opt == '\0') {
166 set_config_value("sockets", "1");
167 set_config_value("cores", "1");
168 set_config_value("threads", "1");
169 set_config_value("cpus", "1");
170 return (0);
171 }
172
173 tofree = str = strdup(opt);
174 if (str == NULL)
175 errx(4, "Failed to allocate memory");
176
177 while ((cp = strsep(&str, ",")) != NULL) {
178 if (strncmp(cp, "cpus=", strlen("cpus=")) == 0)
179 set_config_value("cpus", cp + strlen("cpus="));
180 else if (strncmp(cp, "sockets=", strlen("sockets=")) == 0)
181 set_config_value("sockets", cp + strlen("sockets="));
182 else if (strncmp(cp, "cores=", strlen("cores=")) == 0)
183 set_config_value("cores", cp + strlen("cores="));
184 else if (strncmp(cp, "threads=", strlen("threads=")) == 0)
185 set_config_value("threads", cp + strlen("threads="));
186 else if (strchr(cp, '=') != NULL)
187 goto out;
188 else
189 set_config_value("cpus", cp);
190 }
191 free(tofree);
192 return (0);
193
194 out:
195 free(tofree);
196 return (-1);
197 }
198
199 static int
parse_int_value(const char * key,const char * value,int minval,int maxval)200 parse_int_value(const char *key, const char *value, int minval, int maxval)
201 {
202 char *cp;
203 long lval;
204
205 errno = 0;
206 lval = strtol(value, &cp, 0);
207 if (errno != 0 || *cp != '\0' || cp == value || lval < minval ||
208 lval > maxval)
209 errx(4, "Invalid value for %s: '%s'", key, value);
210 return (lval);
211 }
212
213 int
bhyve_numa_parse(const char * opt)214 bhyve_numa_parse(const char *opt)
215 {
216 int id = -1;
217 nvlist_t *nvl;
218 char *cp, *str, *tofree;
219 char pathbuf[64] = { 0 };
220 char *size = NULL, *cpus = NULL, *domain_policy = NULL;
221
222 if (*opt == '\0') {
223 return (-1);
224 }
225
226 tofree = str = strdup(opt);
227 if (str == NULL)
228 errx(4, "Failed to allocate memory");
229
230 while ((cp = strsep(&str, ",")) != NULL) {
231 if (strncmp(cp, "id=", strlen("id=")) == 0)
232 id = parse_int_value("id", cp + strlen("id="), 0,
233 UINT8_MAX);
234 else if (strncmp(cp, "size=", strlen("size=")) == 0)
235 size = cp + strlen("size=");
236 else if (strncmp(cp,
237 "domain_policy=", strlen("domain_policy=")) == 0)
238 domain_policy = cp + strlen("domain_policy=");
239 else if (strncmp(cp, "cpus=", strlen("cpus=")) == 0)
240 cpus = cp + strlen("cpus=");
241 }
242
243 if (id == -1) {
244 EPRINTLN("Missing NUMA domain ID in '%s'", opt);
245 goto out;
246 }
247
248 snprintf(pathbuf, sizeof(pathbuf), "domains.%d", id);
249 nvl = find_config_node(pathbuf);
250 if (nvl == NULL)
251 nvl = create_config_node(pathbuf);
252 if (size != NULL)
253 set_config_value_node(nvl, "size", size);
254 if (domain_policy != NULL)
255 set_config_value_node(nvl, "domain_policy", domain_policy);
256 if (cpus != NULL)
257 set_config_value_node(nvl, "cpus", cpus);
258
259 free(tofree);
260 return (0);
261
262 out:
263 free(tofree);
264 return (-1);
265 }
266
267 void
bhyve_cfg_warn(const char * old,const char * new)268 bhyve_cfg_warn(const char *old, const char *new)
269 {
270 if (get_config_value(old) != NULL &&
271 get_config_value(new) == NULL) {
272 warnx("'%s' is deprecated, use '%s' instead", old, new);
273 set_config_value(new, get_config_value(old));
274 }
275 }
276
277 static void
calc_mem_affinity(size_t vm_memsize)278 calc_mem_affinity(size_t vm_memsize)
279 {
280 int i;
281 nvlist_t *nvl;
282 bool need_recalc;
283 const char *value;
284 struct vm_mem_domain *dom;
285 char pathbuf[64] = { 0 };
286
287 need_recalc = false;
288 for (i = 0; i < VM_MAXMEMDOM; i++) {
289 dom = &guest_domains[i];
290 snprintf(pathbuf, sizeof(pathbuf), "domains.%d", i);
291 nvl = find_config_node(pathbuf);
292 if (nvl == NULL) {
293 break;
294 }
295
296 value = get_config_value_node(nvl, "size");
297 need_recalc |= value == NULL;
298 if (value != NULL && vm_parse_memsize(value, &dom->size)) {
299 errx(EX_USAGE, "invalid memsize for domain %d: '%s'", i,
300 value);
301 }
302
303 dom->ds_mask = calloc(1, sizeof(domainset_t));
304 if (dom->ds_mask == NULL) {
305 errx(EX_OSERR, "Failed to allocate domainset mask");
306 }
307 dom->ds_size = sizeof(domainset_t);
308 value = get_config_value_node(nvl, "domain_policy");
309 if (value == NULL) {
310 dom->ds_policy = DOMAINSET_POLICY_INVALID;
311 DOMAINSET_ZERO(dom->ds_mask);
312 } else if (domainset_parselist(value, dom->ds_mask, &dom->ds_policy) !=
313 CPUSET_PARSE_OK) {
314 errx(EX_USAGE, "failed to parse domain policy '%s'", value);
315 }
316 }
317
318 guest_ndomains = i;
319 if (guest_ndomains == 0) {
320 /*
321 * No domains were specified - create domain
322 * 0 holding all CPUs and memory.
323 */
324 guest_ndomains = 1;
325 guest_domains[0].size = vm_memsize;
326 } else if (need_recalc) {
327 warnx("At least one domain memory size was not specified, distributing"
328 " total VM memory size across all domains");
329 for (i = 0; i < guest_ndomains; i++) {
330 guest_domains[i].size = vm_memsize / guest_ndomains;
331 }
332 }
333 }
334
335 /*
336 * Set the sockets, cores, threads, and guest_cpus variables based on
337 * the configured topology.
338 *
339 * The limits of UINT16_MAX are due to the types passed to
340 * vm_set_topology(). vmm.ko may enforce tighter limits.
341 */
342 static void
calc_topology(void)343 calc_topology(void)
344 {
345 const char *value;
346 bool explicit_cpus;
347 uint64_t ncpus;
348
349 value = get_config_value("cpus");
350 if (value != NULL) {
351 guest_ncpus = parse_int_value("cpus", value, 1, UINT16_MAX);
352 explicit_cpus = true;
353 } else {
354 guest_ncpus = 1;
355 explicit_cpus = false;
356 }
357 value = get_config_value("cores");
358 if (value != NULL)
359 cpu_cores = parse_int_value("cores", value, 1, UINT16_MAX);
360 else
361 cpu_cores = 1;
362 value = get_config_value("threads");
363 if (value != NULL)
364 cpu_threads = parse_int_value("threads", value, 1, UINT16_MAX);
365 else
366 cpu_threads = 1;
367 value = get_config_value("sockets");
368 if (value != NULL)
369 cpu_sockets = parse_int_value("sockets", value, 1, UINT16_MAX);
370 else
371 cpu_sockets = guest_ncpus;
372
373 /*
374 * Compute sockets * cores * threads avoiding overflow. The
375 * range check above insures these are 16 bit values.
376 */
377 ncpus = (uint64_t)cpu_sockets * cpu_cores * cpu_threads;
378 if (ncpus > UINT16_MAX)
379 errx(4, "Computed number of vCPUs too high: %ju",
380 (uintmax_t)ncpus);
381
382 if (explicit_cpus) {
383 if (guest_ncpus != (int)ncpus)
384 errx(4, "Topology (%d sockets, %d cores, %d threads) "
385 "does not match %d vCPUs",
386 cpu_sockets, cpu_cores, cpu_threads,
387 guest_ncpus);
388 } else
389 guest_ncpus = ncpus;
390 }
391
392 static int
bhyve_pincpu(int vcpu,int pcpu)393 bhyve_pincpu(int vcpu, int pcpu)
394 {
395 const char *value;
396 char *newval;
397 char key[16];
398
399 if (vcpu < 0) {
400 fprintf(stderr, "invalid vcpu '%d'\n", vcpu);
401 return (-1);
402 }
403
404 if (pcpu < 0 || pcpu >= CPU_SETSIZE) {
405 fprintf(stderr, "hostcpu '%d' outside valid range from "
406 "0 to %d\n", pcpu, CPU_SETSIZE - 1);
407 return (-1);
408 }
409
410 snprintf(key, sizeof(key), "vcpu.%d.cpuset", vcpu);
411 value = get_config_value(key);
412
413 if (asprintf(&newval, "%s%s%d", value != NULL ? value : "",
414 value != NULL ? "," : "", pcpu) == -1) {
415 perror("failed to build new cpuset string");
416 return (-1);
417 }
418
419 set_config_value(key, newval);
420 free(newval);
421 return (0);
422 }
423
424 int
bhyve_pincpu_parse(const char * opt)425 bhyve_pincpu_parse(const char *opt)
426 {
427 int vcpu_first, vcpu_last, pcpu_first, pcpu_last;
428 int vcpu, pcpu;
429
430 if (sscanf(opt, "%d-%d:%d-%d", &vcpu_first, &vcpu_last, &pcpu_first, &pcpu_last) == 4) {
431 if (vcpu_first > vcpu_last || pcpu_first > pcpu_last) {
432 fprintf(stderr, "invalid range (must be ascending): %s\n", opt);
433 return (-1);
434 }
435 if ((vcpu_last - vcpu_first) != (pcpu_last - pcpu_first)) {
436 fprintf(stderr, "range sizes do not match: %s\n", opt);
437 return (-1);
438 }
439 for (vcpu = vcpu_first, pcpu = pcpu_first; vcpu <= vcpu_last; vcpu++, pcpu++)
440 if (bhyve_pincpu(vcpu, pcpu) != 0)
441 return (-1);
442 return (0);
443 }
444
445 if (sscanf(opt, "%d:%d", &vcpu, &pcpu) == 2)
446 return (bhyve_pincpu(vcpu, pcpu));
447
448 fprintf(stderr, "invalid format: %s\n", opt);
449 return (-1);
450 }
451
452 static void
parse_cpuset(int vcpu,const char * list,cpuset_t * set)453 parse_cpuset(int vcpu, const char *list, cpuset_t *set)
454 {
455 char *cp, *token;
456 int pcpu, start;
457
458 CPU_ZERO(set);
459 start = -1;
460 token = __DECONST(char *, list);
461 for (;;) {
462 pcpu = strtoul(token, &cp, 0);
463 if (cp == token)
464 errx(4, "invalid cpuset for vcpu %d: '%s'", vcpu, list);
465 if (pcpu < 0 || pcpu >= CPU_SETSIZE)
466 errx(4, "hostcpu '%d' outside valid range from 0 to %d",
467 pcpu, CPU_SETSIZE - 1);
468 switch (*cp) {
469 case ',':
470 case '\0':
471 if (start >= 0) {
472 if (start > pcpu)
473 errx(4, "Invalid hostcpu range %d-%d",
474 start, pcpu);
475 while (start < pcpu) {
476 CPU_SET(start, set);
477 start++;
478 }
479 start = -1;
480 }
481 CPU_SET(pcpu, set);
482 break;
483 case '-':
484 if (start >= 0)
485 errx(4, "invalid cpuset for vcpu %d: '%s'",
486 vcpu, list);
487 start = pcpu;
488 break;
489 default:
490 errx(4, "invalid cpuset for vcpu %d: '%s'", vcpu, list);
491 }
492 if (*cp == '\0')
493 break;
494 token = cp + 1;
495 }
496 }
497
498 static void
build_vcpumaps(void)499 build_vcpumaps(void)
500 {
501 char key[16];
502 const char *value;
503 int vcpu;
504
505 vcpumap = calloc(guest_ncpus, sizeof(*vcpumap));
506 for (vcpu = 0; vcpu < guest_ncpus; vcpu++) {
507 snprintf(key, sizeof(key), "vcpu.%d.cpuset", vcpu);
508 value = get_config_value(key);
509 if (value == NULL)
510 continue;
511 vcpumap[vcpu] = malloc(sizeof(cpuset_t));
512 if (vcpumap[vcpu] == NULL)
513 err(4, "Failed to allocate cpuset for vcpu %d", vcpu);
514 parse_cpuset(vcpu, value, vcpumap[vcpu]);
515 }
516 }
517
518 static void
set_vcpu_affinities(void)519 set_vcpu_affinities(void)
520 {
521 int cpu, error;
522 nvlist_t *nvl = NULL;
523 cpuset_t cpus;
524 const char *value;
525 char pathbuf[64] = { 0 };
526
527 for (int dom = 0; dom < guest_ndomains; dom++) {
528 snprintf(pathbuf, sizeof(pathbuf), "domains.%d", dom);
529 nvl = find_config_node(pathbuf);
530 if (nvl == NULL)
531 break;
532
533 value = get_config_value_node(nvl, "cpus");
534 if (value == NULL) {
535 EPRINTLN("Missing CPU set for domain %d", dom);
536 exit(BHYVE_EXIT_ERROR);
537 }
538
539 parse_cpuset(dom, value, &cpus);
540 CPU_FOREACH_ISSET(cpu, &cpus) {
541 error = acpi_add_vcpu_affinity(cpu, dom);
542 if (error) {
543 EPRINTLN(
544 "Unable to set vCPU %d affinity for domain %d: %s",
545 cpu, dom, strerror(errno));
546 exit(BHYVE_EXIT_ERROR);
547 }
548 }
549 }
550 if (guest_ndomains > 1 || nvl != NULL)
551 return;
552
553 /*
554 * If we're dealing with one domain and no cpuset was provided, create a
555 * default one holding all cpus.
556 */
557 for (cpu = 0; cpu < guest_ncpus; cpu++) {
558 error = acpi_add_vcpu_affinity(cpu, 0);
559 if (error) {
560 EPRINTLN(
561 "Unable to set vCPU %d affinity for domain %d: %s",
562 cpu, 0, strerror(errno));
563 exit(BHYVE_EXIT_ERROR);
564 }
565 }
566 }
567
568 void *
paddr_guest2host(struct vmctx * ctx,uintptr_t gaddr,size_t len)569 paddr_guest2host(struct vmctx *ctx, uintptr_t gaddr, size_t len)
570 {
571
572 return (vm_map_gpa(ctx, gaddr, len));
573 }
574
575 #ifdef BHYVE_SNAPSHOT
576 uintptr_t
paddr_host2guest(struct vmctx * ctx,void * addr)577 paddr_host2guest(struct vmctx *ctx, void *addr)
578 {
579 return (vm_rev_map_gpa(ctx, addr));
580 }
581 #endif
582
583 int
fbsdrun_virtio_msix(void)584 fbsdrun_virtio_msix(void)
585 {
586
587 return (get_config_bool_default("virtio.msix", true));
588 }
589
590 struct vcpu *
fbsdrun_vcpu(int vcpuid)591 fbsdrun_vcpu(int vcpuid)
592 {
593 return (vcpu_info[vcpuid].vcpu);
594 }
595
596 static void *
fbsdrun_start_thread(void * param)597 fbsdrun_start_thread(void *param)
598 {
599 char tname[MAXCOMLEN + 1];
600 struct vcpu_info *vi = param;
601
602 snprintf(tname, sizeof(tname), "vcpu %d", vi->vcpuid);
603 pthread_set_name_np(pthread_self(), tname);
604
605 if (vcpumap[vi->vcpuid] != NULL) {
606 if (pthread_setaffinity_np(pthread_self(),
607 sizeof(cpuset_t), vcpumap[vi->vcpuid]) != 0) {
608 int i;
609 warn("Error pinning vcpu %d to host cpuset@%p", vi->vcpuid,
610 vcpumap[vi->vcpuid]);
611 CPU_FOREACH_ISSET(i, vcpumap[vi->vcpuid]) {
612 warnx("cpu %d enabled\n", i);
613 }
614 exit(BHYVE_EXIT_ERROR);
615 }
616 }
617
618 #ifdef BHYVE_SNAPSHOT
619 checkpoint_cpu_add(vi->vcpuid);
620 #endif
621 #ifdef BHYVE_GDB
622 gdb_cpu_add(vi->vcpu);
623 #endif
624
625 vm_loop(vi->ctx, vi->vcpu);
626 /* We get here if the VM was destroyed asynchronously. */
627 exit(BHYVE_EXIT_ERROR);
628 }
629
630 void
fbsdrun_addcpu(int vcpuid)631 fbsdrun_addcpu(int vcpuid)
632 {
633 struct vcpu_info *vi;
634 pthread_t thr;
635 int error;
636
637 vi = &vcpu_info[vcpuid];
638
639 error = vm_activate_cpu(vi->vcpu);
640 if (error != 0)
641 err(EX_OSERR, "could not activate CPU %d", vi->vcpuid);
642
643 CPU_SET_ATOMIC(vcpuid, &cpumask);
644
645 error = vm_suspend_cpu(vi->vcpu);
646 assert(error == 0);
647
648 error = pthread_create(&thr, NULL, fbsdrun_start_thread, vi);
649 assert(error == 0);
650 }
651
652 void
fbsdrun_deletecpu(int vcpu)653 fbsdrun_deletecpu(int vcpu)
654 {
655 static pthread_mutex_t resetcpu_mtx = PTHREAD_MUTEX_INITIALIZER;
656 static pthread_cond_t resetcpu_cond = PTHREAD_COND_INITIALIZER;
657
658 pthread_mutex_lock(&resetcpu_mtx);
659 if (!CPU_ISSET(vcpu, &cpumask)) {
660 EPRINTLN("Attempting to delete unknown cpu %d", vcpu);
661 exit(BHYVE_EXIT_ERROR);
662 }
663
664 CPU_CLR(vcpu, &cpumask);
665
666 if (vcpu != BSP) {
667 pthread_cond_signal(&resetcpu_cond);
668 pthread_mutex_unlock(&resetcpu_mtx);
669 pthread_exit(NULL);
670 /* NOTREACHED */
671 }
672
673 while (!CPU_EMPTY(&cpumask)) {
674 pthread_cond_wait(&resetcpu_cond, &resetcpu_mtx);
675 }
676 pthread_mutex_unlock(&resetcpu_mtx);
677 }
678
679 int
fbsdrun_suspendcpu(int vcpuid)680 fbsdrun_suspendcpu(int vcpuid)
681 {
682 return (vm_suspend_cpu(vcpu_info[vcpuid].vcpu));
683 }
684
685 static void
vm_loop(struct vmctx * ctx,struct vcpu * vcpu)686 vm_loop(struct vmctx *ctx, struct vcpu *vcpu)
687 {
688 struct vm_exit vme;
689 struct vm_run vmrun;
690 int error, rc;
691 enum vm_exitcode exitcode;
692 cpuset_t active_cpus, dmask;
693
694 error = vm_active_cpus(ctx, &active_cpus);
695 assert(CPU_ISSET(vcpu_id(vcpu), &active_cpus));
696
697 vmrun.vm_exit = &vme;
698 vmrun.cpuset = &dmask;
699 vmrun.cpusetsize = sizeof(dmask);
700
701 while (1) {
702 error = vm_run(vcpu, &vmrun);
703 if (error != 0)
704 break;
705
706 exitcode = vme.exitcode;
707 if (exitcode >= VM_EXITCODE_MAX ||
708 vmexit_handlers[exitcode] == NULL) {
709 warnx("vm_loop: unexpected exitcode 0x%x", exitcode);
710 exit(BHYVE_EXIT_ERROR);
711 }
712
713 rc = (*vmexit_handlers[exitcode])(ctx, vcpu, &vmrun);
714
715 switch (rc) {
716 case VMEXIT_CONTINUE:
717 break;
718 case VMEXIT_ABORT:
719 abort();
720 default:
721 exit(BHYVE_EXIT_ERROR);
722 }
723 }
724 EPRINTLN("vm_run error %d, errno %d", error, errno);
725 }
726
727 static int
num_vcpus_allowed(struct vmctx * ctx,struct vcpu * vcpu)728 num_vcpus_allowed(struct vmctx *ctx, struct vcpu *vcpu)
729 {
730 uint16_t sockets, cores, threads, maxcpus;
731 int tmp, error;
732
733 /*
734 * The guest is allowed to spinup more than one processor only if the
735 * UNRESTRICTED_GUEST capability is available.
736 */
737 error = vm_get_capability(vcpu, VM_CAP_UNRESTRICTED_GUEST, &tmp);
738 if (error != 0)
739 return (1);
740
741 error = vm_get_topology(ctx, &sockets, &cores, &threads, &maxcpus);
742 if (error == 0)
743 return (maxcpus);
744 else
745 return (1);
746 }
747
748 static struct vmctx *
do_open(const char * vmname)749 do_open(const char *vmname)
750 {
751 struct vmctx *ctx;
752 int error, flags;
753 bool romboot, monitor;
754
755 monitor = get_config_bool_default("monitor", false);
756 romboot = bootrom_boot();
757
758 /*
759 * If we don't have a boot ROM, the guest context must have been
760 * initialized by bhyveload(8) or equivalent.
761 */
762 ctx = vm_openf(vmname, romboot ? VMMAPI_OPEN_REINIT : 0);
763 if (ctx == NULL) {
764 if (errno != ENOENT)
765 err(4, "vm_openf");
766 if (!romboot)
767 errx(4, "no bootrom was configured");
768 flags = VMMAPI_OPEN_CREATE;
769 if (monitor)
770 flags |= VMMAPI_OPEN_CREATE_DESTROY_ON_CLOSE;
771 ctx = vm_openf(vmname, flags);
772 if (ctx == NULL)
773 err(4, "vm_openf");
774 }
775
776 #ifndef WITHOUT_CAPSICUM
777 if (vm_limit_rights(ctx) != 0)
778 err(EX_OSERR, "vm_limit_rights");
779 #endif
780
781 error = vm_set_topology(ctx, cpu_sockets, cpu_cores, cpu_threads, 0);
782 if (error)
783 errx(EX_OSERR, "vm_set_topology");
784 return (ctx);
785 }
786
787 bool
bhyve_parse_config_option(const char * option)788 bhyve_parse_config_option(const char *option)
789 {
790 const char *value;
791 char *path;
792
793 value = strchr(option, '=');
794 if (value == NULL || value[1] == '\0')
795 return (false);
796 path = strndup(option, value - option);
797 if (path == NULL)
798 err(4, "Failed to allocate memory");
799 set_config_value(path, value + 1);
800 free(path);
801 return (true);
802 }
803
804 void
bhyve_parse_simple_config_file(const char * path)805 bhyve_parse_simple_config_file(const char *path)
806 {
807 FILE *fp;
808 char *line, *cp;
809 size_t linecap;
810 unsigned int lineno;
811
812 fp = fopen(path, "r");
813 if (fp == NULL)
814 err(4, "Failed to open configuration file %s", path);
815 line = NULL;
816 linecap = 0;
817 lineno = 1;
818 for (lineno = 1; getline(&line, &linecap, fp) > 0; lineno++) {
819 if (*line == '#' || *line == '\n')
820 continue;
821 cp = strchr(line, '\n');
822 if (cp != NULL)
823 *cp = '\0';
824 if (!bhyve_parse_config_option(line))
825 errx(4, "%s line %u: invalid config option '%s'", path,
826 lineno, line);
827 }
828 free(line);
829 fclose(fp);
830 }
831
832 #ifdef BHYVE_GDB
833 void
bhyve_parse_gdb_options(const char * opt)834 bhyve_parse_gdb_options(const char *opt)
835 {
836 const char *sport;
837 char *colon;
838
839 if (opt[0] == 'w') {
840 set_config_bool("gdb.wait", true);
841 opt++;
842 }
843
844 colon = strrchr(opt, ':');
845 if (colon == NULL) {
846 sport = opt;
847 } else {
848 *colon = '\0';
849 colon++;
850 sport = colon;
851 set_config_value("gdb.address", opt);
852 }
853
854 set_config_value("gdb.port", sport);
855 }
856 #endif
857
858 int
main(int argc,char * argv[])859 main(int argc, char *argv[])
860 {
861 int error, status;
862 int max_vcpus, memflags;
863 struct vcpu *bsp;
864 struct vmctx *ctx;
865 size_t memsize;
866 const char *value, *vmname;
867 #ifdef BHYVE_SNAPSHOT
868 struct restore_state rstate;
869 #endif
870
871 bhyve_init_config();
872 bhyve_optparse(argc, argv);
873 argc -= optind;
874 argv += optind;
875
876 if (argc > 1)
877 bhyve_usage(1);
878
879 #ifdef BHYVE_SNAPSHOT
880 if (restore_file != NULL) {
881 error = load_restore_file(restore_file, &rstate);
882 if (error) {
883 fprintf(stderr, "Failed to read checkpoint info from "
884 "file: '%s'.\n", restore_file);
885 exit(BHYVE_EXIT_ERROR);
886 }
887 vmname = lookup_vmname(&rstate);
888 if (vmname != NULL)
889 set_config_value("name", vmname);
890 }
891 #endif
892
893 if (argc == 1)
894 set_config_value("name", argv[0]);
895
896 vmname = get_config_value("name");
897 if (vmname == NULL)
898 bhyve_usage(1);
899
900 if (get_config_bool_default("config.dump", false)) {
901 dump_config();
902 exit(BHYVE_EXIT_POWEROFF);
903 }
904
905 calc_topology();
906 build_vcpumaps();
907
908 value = get_config_value("memory.size");
909 error = vm_parse_memsize(value, &memsize);
910 if (error)
911 errx(EX_USAGE, "invalid memsize '%s'", value);
912
913 ctx = do_open(vmname);
914
915 #ifdef BHYVE_SNAPSHOT
916 if (restore_file != NULL) {
917 guest_ncpus = lookup_guest_ncpus(&rstate);
918 memflags = lookup_memflags(&rstate);
919 memsize = lookup_memsize(&rstate);
920 }
921
922 if (guest_ncpus < 1) {
923 fprintf(stderr, "Invalid guest vCPUs (%d)\n", guest_ncpus);
924 exit(BHYVE_EXIT_ERROR);
925 }
926 #endif
927
928 calc_mem_affinity(memsize);
929 memflags = 0;
930 if (get_config_bool_default("memory.wired", false))
931 memflags |= VM_MEM_F_WIRED;
932 if (get_config_bool_default("memory.guest_in_core", false))
933 memflags |= VM_MEM_F_INCORE;
934 vm_set_memflags(ctx, memflags);
935 error = vm_setup_memory_domains(ctx, VM_MMAP_ALL, guest_domains,
936 guest_ndomains);
937 if (error) {
938 fprintf(stderr, "Unable to setup memory (%d)\n", errno);
939 exit(BHYVE_EXIT_ERROR);
940 }
941
942 set_vcpu_affinities();
943 init_mem(guest_ncpus);
944 init_bootrom(ctx);
945
946 if (get_config_bool_default("monitor", false)) {
947 int monitor_pipe[2];
948 pid_t child;
949
950 while (1) {
951 if (pipe2(monitor_pipe, O_CLOEXEC) == -1)
952 err(BHYVE_EXIT_ERROR, "pipe2");
953
954 child = fork();
955 if (child == -1) {
956 EPRINTLN("Monitor mode fork failed: %s",
957 strerror(errno));
958 exit(BHYVE_EXIT_ERROR);
959 }
960 if (child == 0) {
961 close(monitor_pipe[1]);
962 monitor_pipe_init(monitor_pipe[0]);
963 break;
964 }
965
966 close(monitor_pipe[0]);
967 while ((error = waitpid(child, &status, 0)) == -1 && errno == EINTR)
968 ;
969 close(monitor_pipe[1]);
970 if (error == -1) {
971 EPRINTLN("Monitor mode wait failed: %s",
972 strerror(errno));
973 exit(BHYVE_EXIT_ERROR);
974 }
975 if (WIFSIGNALED(status)) {
976 EPRINTLN("Child process was killed by signal %d",
977 WTERMSIG(status));
978 exit(BHYVE_EXIT_ERROR);
979 } else {
980 status = WEXITSTATUS(status);
981 if (status != BHYVE_EXIT_RESET)
982 exit(status);
983 }
984 if (vm_reinit(ctx) != 0) {
985 EPRINTLN("Monitor mode reinit failed: %s",
986 strerror(errno));
987 exit(BHYVE_EXIT_ERROR);
988 };
989 }
990 }
991
992 bsp = vm_vcpu_open(ctx, BSP);
993 if (bsp == NULL) {
994 fprintf(stderr, "Unable to open boot VCPU: %s",
995 strerror(errno));
996 exit(BHYVE_EXIT_ERROR);
997 }
998
999 max_vcpus = num_vcpus_allowed(ctx, bsp);
1000 if (guest_ncpus > max_vcpus) {
1001 fprintf(stderr, "%d vCPUs requested but only %d available\n",
1002 guest_ncpus, max_vcpus);
1003 exit(BHYVE_EXIT_ERROR);
1004 }
1005
1006 bhyve_init_vcpu(bsp);
1007
1008 /* Allocate per-VCPU resources. */
1009 vcpu_info = calloc(guest_ncpus, sizeof(*vcpu_info));
1010 for (int vcpuid = 0; vcpuid < guest_ncpus; vcpuid++) {
1011 vcpu_info[vcpuid].ctx = ctx;
1012 vcpu_info[vcpuid].vcpuid = vcpuid;
1013 if (vcpuid == BSP)
1014 vcpu_info[vcpuid].vcpu = bsp;
1015 else
1016 vcpu_info[vcpuid].vcpu = vm_vcpu_open(ctx, vcpuid);
1017
1018 if (vcpu_info[vcpuid].vcpu == NULL) {
1019 fprintf(stderr, "Unable to open VCPU %d: %s", vcpuid,
1020 strerror(errno));
1021 exit(BHYVE_EXIT_ERROR);
1022 }
1023 }
1024
1025 if (bhyve_init_platform(ctx, bsp) != 0)
1026 exit(BHYVE_EXIT_ERROR);
1027
1028 if (qemu_fwcfg_init(ctx) != 0) {
1029 fprintf(stderr, "qemu fwcfg initialization error\n");
1030 exit(BHYVE_EXIT_ERROR);
1031 }
1032
1033 if (qemu_fwcfg_add_file("opt/bhyve/hw.ncpu", sizeof(guest_ncpus),
1034 &guest_ncpus) != 0) {
1035 fprintf(stderr, "Could not add qemu fwcfg opt/bhyve/hw.ncpu\n");
1036 exit(BHYVE_EXIT_ERROR);
1037 }
1038
1039 /*
1040 * Exit if a device emulation finds an error in its initialization
1041 */
1042 if (init_pci(ctx) != 0) {
1043 EPRINTLN("Device emulation initialization error: %s",
1044 strerror(errno));
1045 exit(BHYVE_EXIT_ERROR);
1046 }
1047 if (init_tpm(ctx) != 0) {
1048 EPRINTLN("Failed to init TPM device");
1049 exit(BHYVE_EXIT_ERROR);
1050 }
1051
1052 /*
1053 * Initialize after PCI, to allow a bootrom file to reserve the high
1054 * region.
1055 */
1056 if (get_config_bool("acpi_tables"))
1057 vmgenc_init(ctx);
1058
1059 #ifdef BHYVE_GDB
1060 init_gdb(ctx);
1061 #endif
1062
1063 /*
1064 * Add all vCPUs.
1065 */
1066 for (int vcpuid = 0; vcpuid < guest_ncpus; vcpuid++)
1067 bhyve_start_vcpu(vcpu_info[vcpuid].vcpu, vcpuid == BSP);
1068
1069 #ifdef BHYVE_SNAPSHOT
1070 if (restore_file != NULL) {
1071 FPRINTLN(stdout, "Pausing pci devs...");
1072 if (vm_pause_devices() != 0) {
1073 EPRINTLN("Failed to pause PCI device state.");
1074 exit(BHYVE_EXIT_ERROR);
1075 }
1076
1077 FPRINTLN(stdout, "Restoring vm mem...");
1078 if (restore_vm_mem(ctx, &rstate) != 0) {
1079 EPRINTLN("Failed to restore VM memory.");
1080 exit(BHYVE_EXIT_ERROR);
1081 }
1082
1083 FPRINTLN(stdout, "Restoring pci devs...");
1084 if (vm_restore_devices(&rstate) != 0) {
1085 EPRINTLN("Failed to restore PCI device state.");
1086 exit(BHYVE_EXIT_ERROR);
1087 }
1088
1089 FPRINTLN(stdout, "Restoring kernel structs...");
1090 if (vm_restore_kern_structs(ctx, &rstate) != 0) {
1091 EPRINTLN("Failed to restore kernel structs.");
1092 exit(BHYVE_EXIT_ERROR);
1093 }
1094
1095 FPRINTLN(stdout, "Resuming pci devs...");
1096 if (vm_resume_devices() != 0) {
1097 EPRINTLN("Failed to resume PCI device state.");
1098 exit(BHYVE_EXIT_ERROR);
1099 }
1100 }
1101 #endif
1102
1103 if (bhyve_init_platform_late(ctx, bsp) != 0)
1104 exit(BHYVE_EXIT_ERROR);
1105
1106 /*
1107 * Change the proc title to include the VM name.
1108 */
1109 setproctitle("%s", vmname);
1110
1111 /*
1112 * Thread for handling bhyvectl commands.
1113 */
1114 if (init_ipc_thread(ctx, get_config_value("rundir")) != 0)
1115 errx(EX_OSERR, "Failed to start IPC thread");
1116
1117 #ifndef WITHOUT_CAPSICUM
1118 caph_cache_catpages();
1119
1120 if (caph_limit_stdout() == -1 || caph_limit_stderr() == -1)
1121 errx(EX_OSERR, "Unable to apply rights for sandbox");
1122
1123 if (caph_enter() == -1)
1124 errx(EX_OSERR, "cap_enter() failed");
1125 #endif
1126
1127 #ifdef BHYVE_SNAPSHOT
1128 if (restore_file != NULL) {
1129 destroy_restore_state(&rstate);
1130 if (vm_restore_time(ctx) < 0)
1131 err(EX_OSERR, "Unable to restore time");
1132
1133 for (int vcpuid = 0; vcpuid < guest_ncpus; vcpuid++)
1134 vm_resume_cpu(vcpu_info[vcpuid].vcpu);
1135 } else
1136 #endif
1137 vm_resume_cpu(bsp);
1138
1139 /*
1140 * Head off to the main event dispatch loop
1141 */
1142 mevent_dispatch();
1143
1144 exit(BHYVE_EXIT_ERROR);
1145 }
1146