1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011 NetApp, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include <sys/types.h> 35 #ifndef WITHOUT_CAPSICUM 36 #include <sys/capsicum.h> 37 #endif 38 #include <sys/mman.h> 39 #ifdef BHYVE_SNAPSHOT 40 #include <sys/socket.h> 41 #include <sys/stat.h> 42 #endif 43 #include <sys/time.h> 44 #ifdef BHYVE_SNAPSHOT 45 #include <sys/un.h> 46 #endif 47 48 #include <amd64/vmm/intel/vmcs.h> 49 50 #include <machine/atomic.h> 51 #include <machine/segments.h> 52 53 #ifndef WITHOUT_CAPSICUM 54 #include <capsicum_helpers.h> 55 #endif 56 #include <stdio.h> 57 #include <stdlib.h> 58 #include <string.h> 59 #include <err.h> 60 #include <errno.h> 61 #ifdef BHYVE_SNAPSHOT 62 #include <fcntl.h> 63 #endif 64 #include <libgen.h> 65 #include <unistd.h> 66 #include <assert.h> 67 #include <pthread.h> 68 #include <pthread_np.h> 69 #include <sysexits.h> 70 #include <stdbool.h> 71 #include <stdint.h> 72 #ifdef BHYVE_SNAPSHOT 73 #include <ucl.h> 74 #include <unistd.h> 75 76 #include <libxo/xo.h> 77 #endif 78 79 #include <machine/vmm.h> 80 #ifndef WITHOUT_CAPSICUM 81 #include <machine/vmm_dev.h> 82 #endif 83 #include <machine/vmm_instruction_emul.h> 84 #include <vmmapi.h> 85 86 #include "bhyverun.h" 87 #include "acpi.h" 88 #include "atkbdc.h" 89 #include "bootrom.h" 90 #include "inout.h" 91 #include "debug.h" 92 #include "fwctl.h" 93 #include "gdb.h" 94 #include "ioapic.h" 95 #include "kernemu_dev.h" 96 #include "mem.h" 97 #include "mevent.h" 98 #include "mptbl.h" 99 #include "pci_emul.h" 100 #include "pci_irq.h" 101 #include "pci_lpc.h" 102 #include "smbiostbl.h" 103 #ifdef BHYVE_SNAPSHOT 104 #include "snapshot.h" 105 #endif 106 #include "xmsr.h" 107 #include "spinup_ap.h" 108 #include "rtc.h" 109 #include "vmgenc.h" 110 111 #define GUEST_NIO_PORT 0x488 /* guest upcalls via i/o port */ 112 113 #define MB (1024UL * 1024) 114 #define GB (1024UL * MB) 115 116 static const char * const vmx_exit_reason_desc[] = { 117 [EXIT_REASON_EXCEPTION] = "Exception or non-maskable interrupt (NMI)", 118 [EXIT_REASON_EXT_INTR] = "External interrupt", 119 [EXIT_REASON_TRIPLE_FAULT] = "Triple fault", 120 [EXIT_REASON_INIT] = "INIT signal", 121 [EXIT_REASON_SIPI] = "Start-up IPI (SIPI)", 122 [EXIT_REASON_IO_SMI] = "I/O system-management interrupt (SMI)", 123 [EXIT_REASON_SMI] = "Other SMI", 124 [EXIT_REASON_INTR_WINDOW] = "Interrupt window", 125 [EXIT_REASON_NMI_WINDOW] = "NMI window", 126 [EXIT_REASON_TASK_SWITCH] = "Task switch", 127 [EXIT_REASON_CPUID] = "CPUID", 128 [EXIT_REASON_GETSEC] = "GETSEC", 129 [EXIT_REASON_HLT] = "HLT", 130 [EXIT_REASON_INVD] = "INVD", 131 [EXIT_REASON_INVLPG] = "INVLPG", 132 [EXIT_REASON_RDPMC] = "RDPMC", 133 [EXIT_REASON_RDTSC] = "RDTSC", 134 [EXIT_REASON_RSM] = "RSM", 135 [EXIT_REASON_VMCALL] = "VMCALL", 136 [EXIT_REASON_VMCLEAR] = "VMCLEAR", 137 [EXIT_REASON_VMLAUNCH] = "VMLAUNCH", 138 [EXIT_REASON_VMPTRLD] = "VMPTRLD", 139 [EXIT_REASON_VMPTRST] = "VMPTRST", 140 [EXIT_REASON_VMREAD] = "VMREAD", 141 [EXIT_REASON_VMRESUME] = "VMRESUME", 142 [EXIT_REASON_VMWRITE] = "VMWRITE", 143 [EXIT_REASON_VMXOFF] = "VMXOFF", 144 [EXIT_REASON_VMXON] = "VMXON", 145 [EXIT_REASON_CR_ACCESS] = "Control-register accesses", 146 [EXIT_REASON_DR_ACCESS] = "MOV DR", 147 [EXIT_REASON_INOUT] = "I/O instruction", 148 [EXIT_REASON_RDMSR] = "RDMSR", 149 [EXIT_REASON_WRMSR] = "WRMSR", 150 [EXIT_REASON_INVAL_VMCS] = 151 "VM-entry failure due to invalid guest state", 152 [EXIT_REASON_INVAL_MSR] = "VM-entry failure due to MSR loading", 153 [EXIT_REASON_MWAIT] = "MWAIT", 154 [EXIT_REASON_MTF] = "Monitor trap flag", 155 [EXIT_REASON_MONITOR] = "MONITOR", 156 [EXIT_REASON_PAUSE] = "PAUSE", 157 [EXIT_REASON_MCE_DURING_ENTRY] = 158 "VM-entry failure due to machine-check event", 159 [EXIT_REASON_TPR] = "TPR below threshold", 160 [EXIT_REASON_APIC_ACCESS] = "APIC access", 161 [EXIT_REASON_VIRTUALIZED_EOI] = "Virtualized EOI", 162 [EXIT_REASON_GDTR_IDTR] = "Access to GDTR or IDTR", 163 [EXIT_REASON_LDTR_TR] = "Access to LDTR or TR", 164 [EXIT_REASON_EPT_FAULT] = "EPT violation", 165 [EXIT_REASON_EPT_MISCONFIG] = "EPT misconfiguration", 166 [EXIT_REASON_INVEPT] = "INVEPT", 167 [EXIT_REASON_RDTSCP] = "RDTSCP", 168 [EXIT_REASON_VMX_PREEMPT] = "VMX-preemption timer expired", 169 [EXIT_REASON_INVVPID] = "INVVPID", 170 [EXIT_REASON_WBINVD] = "WBINVD", 171 [EXIT_REASON_XSETBV] = "XSETBV", 172 [EXIT_REASON_APIC_WRITE] = "APIC write", 173 [EXIT_REASON_RDRAND] = "RDRAND", 174 [EXIT_REASON_INVPCID] = "INVPCID", 175 [EXIT_REASON_VMFUNC] = "VMFUNC", 176 [EXIT_REASON_ENCLS] = "ENCLS", 177 [EXIT_REASON_RDSEED] = "RDSEED", 178 [EXIT_REASON_PM_LOG_FULL] = "Page-modification log full", 179 [EXIT_REASON_XSAVES] = "XSAVES", 180 [EXIT_REASON_XRSTORS] = "XRSTORS" 181 }; 182 183 typedef int (*vmexit_handler_t)(struct vmctx *, struct vm_exit *, int *vcpu); 184 extern int vmexit_task_switch(struct vmctx *, struct vm_exit *, int *vcpu); 185 186 const char *vmname; 187 188 int guest_ncpus; 189 uint16_t cores, maxcpus, sockets, threads; 190 191 char *guest_uuid_str; 192 193 int raw_stdio = 0; 194 195 static int gdb_port = 0; 196 static int guest_vmexit_on_hlt, guest_vmexit_on_pause; 197 static int virtio_msix = 1; 198 static int x2apic_mode = 0; /* default is xAPIC */ 199 static int destroy_on_poweroff = 0; 200 201 static int strictio; 202 static int strictmsr = 1; 203 204 static int acpi; 205 206 static char *progname; 207 static const int BSP = 0; 208 209 static cpuset_t cpumask; 210 211 static void vm_loop(struct vmctx *ctx, int vcpu, uint64_t rip); 212 213 static struct vm_exit vmexit[VM_MAXCPU]; 214 215 struct bhyvestats { 216 uint64_t vmexit_bogus; 217 uint64_t vmexit_reqidle; 218 uint64_t vmexit_hlt; 219 uint64_t vmexit_pause; 220 uint64_t vmexit_mtrap; 221 uint64_t vmexit_inst_emul; 222 uint64_t cpu_switch_rotate; 223 uint64_t cpu_switch_direct; 224 } stats; 225 226 struct mt_vmm_info { 227 pthread_t mt_thr; 228 struct vmctx *mt_ctx; 229 int mt_vcpu; 230 } mt_vmm_info[VM_MAXCPU]; 231 232 static cpuset_t *vcpumap[VM_MAXCPU] = { NULL }; 233 234 static void 235 usage(int code) 236 { 237 238 fprintf(stderr, 239 "Usage: %s [-aehuwxACDHPSWY]\n" 240 " %*s [-c [[cpus=]numcpus][,sockets=n][,cores=n][,threads=n]]\n" 241 " %*s [-l <lpc>]\n" 242 " %*s [-m mem] [-p vcpu:hostcpu] [-s <pci>] [-U uuid] <vm>\n" 243 " -a: local apic is in xAPIC mode (deprecated)\n" 244 " -A: create ACPI tables\n" 245 " -c: number of cpus and/or topology specification\n" 246 " -C: include guest memory in core file\n" 247 " -D: destroy on power-off\n" 248 " -e: exit on unhandled I/O access\n" 249 " -h: help\n" 250 " -H: vmexit from the guest on hlt\n" 251 " -l: LPC device configuration\n" 252 " -m: memory size in MB\n" 253 #ifdef BHYVE_SNAPSHOT 254 " -r: path to checkpoint file\n" 255 #endif 256 " -p: pin 'vcpu' to 'hostcpu'\n" 257 " -P: vmexit from the guest on pause\n" 258 " -s: <slot,driver,configinfo> PCI slot config\n" 259 " -S: guest memory cannot be swapped\n" 260 " -u: RTC keeps UTC time\n" 261 " -U: uuid\n" 262 " -w: ignore unimplemented MSRs\n" 263 " -W: force virtio to use single-vector MSI\n" 264 " -x: local apic is in x2APIC mode\n" 265 " -Y: disable MPtable generation\n", 266 progname, (int)strlen(progname), "", (int)strlen(progname), "", 267 (int)strlen(progname), ""); 268 269 exit(code); 270 } 271 272 /* 273 * XXX This parser is known to have the following issues: 274 * 1. It accepts null key=value tokens ",,". 275 * 2. It accepts whitespace after = and before value. 276 * 3. Values out of range of INT are silently wrapped. 277 * 4. It doesn't check non-final values. 278 * 5. The apparently bogus limits of UINT16_MAX are for future expansion. 279 * 280 * The acceptance of a null specification ('-c ""') is by design to match the 281 * manual page syntax specification, this results in a topology of 1 vCPU. 282 */ 283 static int 284 topology_parse(const char *opt) 285 { 286 uint64_t ncpus; 287 int c, chk, n, s, t, tmp; 288 char *cp, *str; 289 bool ns, scts; 290 291 c = 1, n = 1, s = 1, t = 1; 292 ns = false, scts = false; 293 str = strdup(opt); 294 if (str == NULL) 295 goto out; 296 297 while ((cp = strsep(&str, ",")) != NULL) { 298 if (sscanf(cp, "%i%n", &tmp, &chk) == 1) { 299 n = tmp; 300 ns = true; 301 } else if (sscanf(cp, "cpus=%i%n", &tmp, &chk) == 1) { 302 n = tmp; 303 ns = true; 304 } else if (sscanf(cp, "sockets=%i%n", &tmp, &chk) == 1) { 305 s = tmp; 306 scts = true; 307 } else if (sscanf(cp, "cores=%i%n", &tmp, &chk) == 1) { 308 c = tmp; 309 scts = true; 310 } else if (sscanf(cp, "threads=%i%n", &tmp, &chk) == 1) { 311 t = tmp; 312 scts = true; 313 #ifdef notyet /* Do not expose this until vmm.ko implements it */ 314 } else if (sscanf(cp, "maxcpus=%i%n", &tmp, &chk) == 1) { 315 m = tmp; 316 #endif 317 /* Skip the empty argument case from -c "" */ 318 } else if (cp[0] == '\0') 319 continue; 320 else 321 goto out; 322 /* Any trailing garbage causes an error */ 323 if (cp[chk] != '\0') 324 goto out; 325 } 326 free(str); 327 str = NULL; 328 329 /* 330 * Range check 1 <= n <= UINT16_MAX all values 331 */ 332 if (n < 1 || s < 1 || c < 1 || t < 1 || 333 n > UINT16_MAX || s > UINT16_MAX || c > UINT16_MAX || 334 t > UINT16_MAX) 335 return (-1); 336 337 /* If only the cpus was specified, use that as sockets */ 338 if (!scts) 339 s = n; 340 /* 341 * Compute sockets * cores * threads avoiding overflow 342 * The range check above insures these are 16 bit values 343 * If n was specified check it against computed ncpus 344 */ 345 ncpus = (uint64_t)s * c * t; 346 if (ncpus > UINT16_MAX || (ns && n != ncpus)) 347 return (-1); 348 349 guest_ncpus = ncpus; 350 sockets = s; 351 cores = c; 352 threads = t; 353 return(0); 354 355 out: 356 free(str); 357 return (-1); 358 } 359 360 static int 361 pincpu_parse(const char *opt) 362 { 363 int vcpu, pcpu; 364 365 if (sscanf(opt, "%d:%d", &vcpu, &pcpu) != 2) { 366 fprintf(stderr, "invalid format: %s\n", opt); 367 return (-1); 368 } 369 370 if (vcpu < 0 || vcpu >= VM_MAXCPU) { 371 fprintf(stderr, "vcpu '%d' outside valid range from 0 to %d\n", 372 vcpu, VM_MAXCPU - 1); 373 return (-1); 374 } 375 376 if (pcpu < 0 || pcpu >= CPU_SETSIZE) { 377 fprintf(stderr, "hostcpu '%d' outside valid range from " 378 "0 to %d\n", pcpu, CPU_SETSIZE - 1); 379 return (-1); 380 } 381 382 if (vcpumap[vcpu] == NULL) { 383 if ((vcpumap[vcpu] = malloc(sizeof(cpuset_t))) == NULL) { 384 perror("malloc"); 385 return (-1); 386 } 387 CPU_ZERO(vcpumap[vcpu]); 388 } 389 CPU_SET(pcpu, vcpumap[vcpu]); 390 return (0); 391 } 392 393 void 394 vm_inject_fault(void *arg, int vcpu, int vector, int errcode_valid, 395 int errcode) 396 { 397 struct vmctx *ctx; 398 int error, restart_instruction; 399 400 ctx = arg; 401 restart_instruction = 1; 402 403 error = vm_inject_exception(ctx, vcpu, vector, errcode_valid, errcode, 404 restart_instruction); 405 assert(error == 0); 406 } 407 408 void * 409 paddr_guest2host(struct vmctx *ctx, uintptr_t gaddr, size_t len) 410 { 411 412 return (vm_map_gpa(ctx, gaddr, len)); 413 } 414 415 #ifdef BHYVE_SNAPSHOT 416 uintptr_t 417 paddr_host2guest(struct vmctx *ctx, void *addr) 418 { 419 return (vm_rev_map_gpa(ctx, addr)); 420 } 421 #endif 422 423 int 424 fbsdrun_vmexit_on_pause(void) 425 { 426 427 return (guest_vmexit_on_pause); 428 } 429 430 int 431 fbsdrun_vmexit_on_hlt(void) 432 { 433 434 return (guest_vmexit_on_hlt); 435 } 436 437 int 438 fbsdrun_virtio_msix(void) 439 { 440 441 return (virtio_msix); 442 } 443 444 static void * 445 fbsdrun_start_thread(void *param) 446 { 447 char tname[MAXCOMLEN + 1]; 448 struct mt_vmm_info *mtp; 449 int vcpu; 450 451 mtp = param; 452 vcpu = mtp->mt_vcpu; 453 454 snprintf(tname, sizeof(tname), "vcpu %d", vcpu); 455 pthread_set_name_np(mtp->mt_thr, tname); 456 457 #ifdef BHYVE_SNAPSHOT 458 checkpoint_cpu_add(vcpu); 459 #endif 460 if (gdb_port != 0) 461 gdb_cpu_add(vcpu); 462 463 vm_loop(mtp->mt_ctx, vcpu, vmexit[vcpu].rip); 464 465 /* not reached */ 466 exit(1); 467 return (NULL); 468 } 469 470 void 471 fbsdrun_addcpu(struct vmctx *ctx, int fromcpu, int newcpu, uint64_t rip) 472 { 473 int error; 474 475 assert(fromcpu == BSP); 476 477 /* 478 * The 'newcpu' must be activated in the context of 'fromcpu'. If 479 * vm_activate_cpu() is delayed until newcpu's pthread starts running 480 * then vmm.ko is out-of-sync with bhyve and this can create a race 481 * with vm_suspend(). 482 */ 483 error = vm_activate_cpu(ctx, newcpu); 484 if (error != 0) 485 err(EX_OSERR, "could not activate CPU %d", newcpu); 486 487 CPU_SET_ATOMIC(newcpu, &cpumask); 488 489 /* 490 * Set up the vmexit struct to allow execution to start 491 * at the given RIP 492 */ 493 vmexit[newcpu].rip = rip; 494 vmexit[newcpu].inst_length = 0; 495 496 mt_vmm_info[newcpu].mt_ctx = ctx; 497 mt_vmm_info[newcpu].mt_vcpu = newcpu; 498 499 error = pthread_create(&mt_vmm_info[newcpu].mt_thr, NULL, 500 fbsdrun_start_thread, &mt_vmm_info[newcpu]); 501 assert(error == 0); 502 } 503 504 static int 505 fbsdrun_deletecpu(struct vmctx *ctx, int vcpu) 506 { 507 508 if (!CPU_ISSET(vcpu, &cpumask)) { 509 fprintf(stderr, "Attempting to delete unknown cpu %d\n", vcpu); 510 exit(4); 511 } 512 513 CPU_CLR_ATOMIC(vcpu, &cpumask); 514 return (CPU_EMPTY(&cpumask)); 515 } 516 517 static int 518 vmexit_handle_notify(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu, 519 uint32_t eax) 520 { 521 #if BHYVE_DEBUG 522 /* 523 * put guest-driven debug here 524 */ 525 #endif 526 return (VMEXIT_CONTINUE); 527 } 528 529 static int 530 vmexit_inout(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu) 531 { 532 int error; 533 int bytes, port, in, out; 534 int vcpu; 535 536 vcpu = *pvcpu; 537 538 port = vme->u.inout.port; 539 bytes = vme->u.inout.bytes; 540 in = vme->u.inout.in; 541 out = !in; 542 543 /* Extra-special case of host notifications */ 544 if (out && port == GUEST_NIO_PORT) { 545 error = vmexit_handle_notify(ctx, vme, pvcpu, vme->u.inout.eax); 546 return (error); 547 } 548 549 error = emulate_inout(ctx, vcpu, vme, strictio); 550 if (error) { 551 fprintf(stderr, "Unhandled %s%c 0x%04x at 0x%lx\n", 552 in ? "in" : "out", 553 bytes == 1 ? 'b' : (bytes == 2 ? 'w' : 'l'), 554 port, vmexit->rip); 555 return (VMEXIT_ABORT); 556 } else { 557 return (VMEXIT_CONTINUE); 558 } 559 } 560 561 static int 562 vmexit_rdmsr(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu) 563 { 564 uint64_t val; 565 uint32_t eax, edx; 566 int error; 567 568 val = 0; 569 error = emulate_rdmsr(ctx, *pvcpu, vme->u.msr.code, &val); 570 if (error != 0) { 571 fprintf(stderr, "rdmsr to register %#x on vcpu %d\n", 572 vme->u.msr.code, *pvcpu); 573 if (strictmsr) { 574 vm_inject_gp(ctx, *pvcpu); 575 return (VMEXIT_CONTINUE); 576 } 577 } 578 579 eax = val; 580 error = vm_set_register(ctx, *pvcpu, VM_REG_GUEST_RAX, eax); 581 assert(error == 0); 582 583 edx = val >> 32; 584 error = vm_set_register(ctx, *pvcpu, VM_REG_GUEST_RDX, edx); 585 assert(error == 0); 586 587 return (VMEXIT_CONTINUE); 588 } 589 590 static int 591 vmexit_wrmsr(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu) 592 { 593 int error; 594 595 error = emulate_wrmsr(ctx, *pvcpu, vme->u.msr.code, vme->u.msr.wval); 596 if (error != 0) { 597 fprintf(stderr, "wrmsr to register %#x(%#lx) on vcpu %d\n", 598 vme->u.msr.code, vme->u.msr.wval, *pvcpu); 599 if (strictmsr) { 600 vm_inject_gp(ctx, *pvcpu); 601 return (VMEXIT_CONTINUE); 602 } 603 } 604 return (VMEXIT_CONTINUE); 605 } 606 607 static int 608 vmexit_spinup_ap(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu) 609 { 610 611 (void)spinup_ap(ctx, *pvcpu, 612 vme->u.spinup_ap.vcpu, vme->u.spinup_ap.rip); 613 614 return (VMEXIT_CONTINUE); 615 } 616 617 #define DEBUG_EPT_MISCONFIG 618 #ifdef DEBUG_EPT_MISCONFIG 619 #define VMCS_GUEST_PHYSICAL_ADDRESS 0x00002400 620 621 static uint64_t ept_misconfig_gpa, ept_misconfig_pte[4]; 622 static int ept_misconfig_ptenum; 623 #endif 624 625 static const char * 626 vmexit_vmx_desc(uint32_t exit_reason) 627 { 628 629 if (exit_reason >= nitems(vmx_exit_reason_desc) || 630 vmx_exit_reason_desc[exit_reason] == NULL) 631 return ("Unknown"); 632 return (vmx_exit_reason_desc[exit_reason]); 633 } 634 635 static int 636 vmexit_vmx(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu) 637 { 638 639 fprintf(stderr, "vm exit[%d]\n", *pvcpu); 640 fprintf(stderr, "\treason\t\tVMX\n"); 641 fprintf(stderr, "\trip\t\t0x%016lx\n", vmexit->rip); 642 fprintf(stderr, "\tinst_length\t%d\n", vmexit->inst_length); 643 fprintf(stderr, "\tstatus\t\t%d\n", vmexit->u.vmx.status); 644 fprintf(stderr, "\texit_reason\t%u (%s)\n", vmexit->u.vmx.exit_reason, 645 vmexit_vmx_desc(vmexit->u.vmx.exit_reason)); 646 fprintf(stderr, "\tqualification\t0x%016lx\n", 647 vmexit->u.vmx.exit_qualification); 648 fprintf(stderr, "\tinst_type\t\t%d\n", vmexit->u.vmx.inst_type); 649 fprintf(stderr, "\tinst_error\t\t%d\n", vmexit->u.vmx.inst_error); 650 #ifdef DEBUG_EPT_MISCONFIG 651 if (vmexit->u.vmx.exit_reason == EXIT_REASON_EPT_MISCONFIG) { 652 vm_get_register(ctx, *pvcpu, 653 VMCS_IDENT(VMCS_GUEST_PHYSICAL_ADDRESS), 654 &ept_misconfig_gpa); 655 vm_get_gpa_pmap(ctx, ept_misconfig_gpa, ept_misconfig_pte, 656 &ept_misconfig_ptenum); 657 fprintf(stderr, "\tEPT misconfiguration:\n"); 658 fprintf(stderr, "\t\tGPA: %#lx\n", ept_misconfig_gpa); 659 fprintf(stderr, "\t\tPTE(%d): %#lx %#lx %#lx %#lx\n", 660 ept_misconfig_ptenum, ept_misconfig_pte[0], 661 ept_misconfig_pte[1], ept_misconfig_pte[2], 662 ept_misconfig_pte[3]); 663 } 664 #endif /* DEBUG_EPT_MISCONFIG */ 665 return (VMEXIT_ABORT); 666 } 667 668 static int 669 vmexit_svm(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu) 670 { 671 672 fprintf(stderr, "vm exit[%d]\n", *pvcpu); 673 fprintf(stderr, "\treason\t\tSVM\n"); 674 fprintf(stderr, "\trip\t\t0x%016lx\n", vmexit->rip); 675 fprintf(stderr, "\tinst_length\t%d\n", vmexit->inst_length); 676 fprintf(stderr, "\texitcode\t%#lx\n", vmexit->u.svm.exitcode); 677 fprintf(stderr, "\texitinfo1\t%#lx\n", vmexit->u.svm.exitinfo1); 678 fprintf(stderr, "\texitinfo2\t%#lx\n", vmexit->u.svm.exitinfo2); 679 return (VMEXIT_ABORT); 680 } 681 682 static int 683 vmexit_bogus(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu) 684 { 685 686 assert(vmexit->inst_length == 0); 687 688 stats.vmexit_bogus++; 689 690 return (VMEXIT_CONTINUE); 691 } 692 693 static int 694 vmexit_reqidle(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu) 695 { 696 697 assert(vmexit->inst_length == 0); 698 699 stats.vmexit_reqidle++; 700 701 return (VMEXIT_CONTINUE); 702 } 703 704 static int 705 vmexit_hlt(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu) 706 { 707 708 stats.vmexit_hlt++; 709 710 /* 711 * Just continue execution with the next instruction. We use 712 * the HLT VM exit as a way to be friendly with the host 713 * scheduler. 714 */ 715 return (VMEXIT_CONTINUE); 716 } 717 718 static int 719 vmexit_pause(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu) 720 { 721 722 stats.vmexit_pause++; 723 724 return (VMEXIT_CONTINUE); 725 } 726 727 static int 728 vmexit_mtrap(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu) 729 { 730 731 assert(vmexit->inst_length == 0); 732 733 stats.vmexit_mtrap++; 734 735 #ifdef BHYVE_SNAPSHOT 736 checkpoint_cpu_suspend(*pvcpu); 737 #endif 738 if (gdb_port != 0) 739 gdb_cpu_mtrap(*pvcpu); 740 #ifdef BHYVE_SNAPSHOT 741 checkpoint_cpu_resume(*pvcpu); 742 #endif 743 744 return (VMEXIT_CONTINUE); 745 } 746 747 static int 748 vmexit_inst_emul(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu) 749 { 750 int err, i, cs_d; 751 struct vie *vie; 752 enum vm_cpu_mode mode; 753 754 stats.vmexit_inst_emul++; 755 756 vie = &vmexit->u.inst_emul.vie; 757 if (!vie->decoded) { 758 /* 759 * Attempt to decode in userspace as a fallback. This allows 760 * updating instruction decode in bhyve without rebooting the 761 * kernel (rapid prototyping), albeit with much slower 762 * emulation. 763 */ 764 vie_restart(vie); 765 mode = vmexit->u.inst_emul.paging.cpu_mode; 766 cs_d = vmexit->u.inst_emul.cs_d; 767 if (vmm_decode_instruction(mode, cs_d, vie) != 0) 768 goto fail; 769 if (vm_set_register(ctx, *pvcpu, VM_REG_GUEST_RIP, 770 vmexit->rip + vie->num_processed) != 0) 771 goto fail; 772 } 773 774 err = emulate_mem(ctx, *pvcpu, vmexit->u.inst_emul.gpa, 775 vie, &vmexit->u.inst_emul.paging); 776 777 if (err) { 778 if (err == ESRCH) { 779 EPRINTLN("Unhandled memory access to 0x%lx\n", 780 vmexit->u.inst_emul.gpa); 781 } 782 goto fail; 783 } 784 785 return (VMEXIT_CONTINUE); 786 787 fail: 788 fprintf(stderr, "Failed to emulate instruction sequence [ "); 789 for (i = 0; i < vie->num_valid; i++) 790 fprintf(stderr, "%02x", vie->inst[i]); 791 FPRINTLN(stderr, " ] at 0x%lx", vmexit->rip); 792 return (VMEXIT_ABORT); 793 } 794 795 static pthread_mutex_t resetcpu_mtx = PTHREAD_MUTEX_INITIALIZER; 796 static pthread_cond_t resetcpu_cond = PTHREAD_COND_INITIALIZER; 797 798 static int 799 vmexit_suspend(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu) 800 { 801 enum vm_suspend_how how; 802 803 how = vmexit->u.suspended.how; 804 805 fbsdrun_deletecpu(ctx, *pvcpu); 806 807 if (*pvcpu != BSP) { 808 pthread_mutex_lock(&resetcpu_mtx); 809 pthread_cond_signal(&resetcpu_cond); 810 pthread_mutex_unlock(&resetcpu_mtx); 811 pthread_exit(NULL); 812 } 813 814 pthread_mutex_lock(&resetcpu_mtx); 815 while (!CPU_EMPTY(&cpumask)) { 816 pthread_cond_wait(&resetcpu_cond, &resetcpu_mtx); 817 } 818 pthread_mutex_unlock(&resetcpu_mtx); 819 820 switch (how) { 821 case VM_SUSPEND_RESET: 822 exit(0); 823 case VM_SUSPEND_POWEROFF: 824 if (destroy_on_poweroff) 825 vm_destroy(ctx); 826 exit(1); 827 case VM_SUSPEND_HALT: 828 exit(2); 829 case VM_SUSPEND_TRIPLEFAULT: 830 exit(3); 831 default: 832 fprintf(stderr, "vmexit_suspend: invalid reason %d\n", how); 833 exit(100); 834 } 835 return (0); /* NOTREACHED */ 836 } 837 838 static int 839 vmexit_debug(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu) 840 { 841 842 #ifdef BHYVE_SNAPSHOT 843 checkpoint_cpu_suspend(*pvcpu); 844 #endif 845 if (gdb_port != 0) 846 gdb_cpu_suspend(*pvcpu); 847 #ifdef BHYVE_SNAPSHOT 848 checkpoint_cpu_resume(*pvcpu); 849 #endif 850 return (VMEXIT_CONTINUE); 851 } 852 853 static int 854 vmexit_breakpoint(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu) 855 { 856 857 if (gdb_port == 0) { 858 fprintf(stderr, "vm_loop: unexpected VMEXIT_DEBUG\n"); 859 exit(4); 860 } 861 gdb_cpu_breakpoint(*pvcpu, vmexit); 862 return (VMEXIT_CONTINUE); 863 } 864 865 static vmexit_handler_t handler[VM_EXITCODE_MAX] = { 866 [VM_EXITCODE_INOUT] = vmexit_inout, 867 [VM_EXITCODE_INOUT_STR] = vmexit_inout, 868 [VM_EXITCODE_VMX] = vmexit_vmx, 869 [VM_EXITCODE_SVM] = vmexit_svm, 870 [VM_EXITCODE_BOGUS] = vmexit_bogus, 871 [VM_EXITCODE_REQIDLE] = vmexit_reqidle, 872 [VM_EXITCODE_RDMSR] = vmexit_rdmsr, 873 [VM_EXITCODE_WRMSR] = vmexit_wrmsr, 874 [VM_EXITCODE_MTRAP] = vmexit_mtrap, 875 [VM_EXITCODE_INST_EMUL] = vmexit_inst_emul, 876 [VM_EXITCODE_SPINUP_AP] = vmexit_spinup_ap, 877 [VM_EXITCODE_SUSPENDED] = vmexit_suspend, 878 [VM_EXITCODE_TASK_SWITCH] = vmexit_task_switch, 879 [VM_EXITCODE_DEBUG] = vmexit_debug, 880 [VM_EXITCODE_BPT] = vmexit_breakpoint, 881 }; 882 883 static void 884 vm_loop(struct vmctx *ctx, int vcpu, uint64_t startrip) 885 { 886 int error, rc; 887 enum vm_exitcode exitcode; 888 cpuset_t active_cpus; 889 890 if (vcpumap[vcpu] != NULL) { 891 error = pthread_setaffinity_np(pthread_self(), 892 sizeof(cpuset_t), vcpumap[vcpu]); 893 assert(error == 0); 894 } 895 896 error = vm_active_cpus(ctx, &active_cpus); 897 assert(CPU_ISSET(vcpu, &active_cpus)); 898 899 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_RIP, startrip); 900 assert(error == 0); 901 902 while (1) { 903 error = vm_run(ctx, vcpu, &vmexit[vcpu]); 904 if (error != 0) 905 break; 906 907 exitcode = vmexit[vcpu].exitcode; 908 if (exitcode >= VM_EXITCODE_MAX || handler[exitcode] == NULL) { 909 fprintf(stderr, "vm_loop: unexpected exitcode 0x%x\n", 910 exitcode); 911 exit(4); 912 } 913 914 rc = (*handler[exitcode])(ctx, &vmexit[vcpu], &vcpu); 915 916 switch (rc) { 917 case VMEXIT_CONTINUE: 918 break; 919 case VMEXIT_ABORT: 920 abort(); 921 default: 922 exit(4); 923 } 924 } 925 fprintf(stderr, "vm_run error %d, errno %d\n", error, errno); 926 } 927 928 static int 929 num_vcpus_allowed(struct vmctx *ctx) 930 { 931 int tmp, error; 932 933 error = vm_get_capability(ctx, BSP, VM_CAP_UNRESTRICTED_GUEST, &tmp); 934 935 /* 936 * The guest is allowed to spinup more than one processor only if the 937 * UNRESTRICTED_GUEST capability is available. 938 */ 939 if (error == 0) 940 return (VM_MAXCPU); 941 else 942 return (1); 943 } 944 945 void 946 fbsdrun_set_capabilities(struct vmctx *ctx, int cpu) 947 { 948 int err, tmp; 949 950 if (fbsdrun_vmexit_on_hlt()) { 951 err = vm_get_capability(ctx, cpu, VM_CAP_HALT_EXIT, &tmp); 952 if (err < 0) { 953 fprintf(stderr, "VM exit on HLT not supported\n"); 954 exit(4); 955 } 956 vm_set_capability(ctx, cpu, VM_CAP_HALT_EXIT, 1); 957 if (cpu == BSP) 958 handler[VM_EXITCODE_HLT] = vmexit_hlt; 959 } 960 961 if (fbsdrun_vmexit_on_pause()) { 962 /* 963 * pause exit support required for this mode 964 */ 965 err = vm_get_capability(ctx, cpu, VM_CAP_PAUSE_EXIT, &tmp); 966 if (err < 0) { 967 fprintf(stderr, 968 "SMP mux requested, no pause support\n"); 969 exit(4); 970 } 971 vm_set_capability(ctx, cpu, VM_CAP_PAUSE_EXIT, 1); 972 if (cpu == BSP) 973 handler[VM_EXITCODE_PAUSE] = vmexit_pause; 974 } 975 976 if (x2apic_mode) 977 err = vm_set_x2apic_state(ctx, cpu, X2APIC_ENABLED); 978 else 979 err = vm_set_x2apic_state(ctx, cpu, X2APIC_DISABLED); 980 981 if (err) { 982 fprintf(stderr, "Unable to set x2apic state (%d)\n", err); 983 exit(4); 984 } 985 986 vm_set_capability(ctx, cpu, VM_CAP_ENABLE_INVPCID, 1); 987 } 988 989 static struct vmctx * 990 do_open(const char *vmname) 991 { 992 struct vmctx *ctx; 993 int error; 994 bool reinit, romboot; 995 #ifndef WITHOUT_CAPSICUM 996 cap_rights_t rights; 997 const cap_ioctl_t *cmds; 998 size_t ncmds; 999 #endif 1000 1001 reinit = romboot = false; 1002 1003 if (lpc_bootrom()) 1004 romboot = true; 1005 1006 error = vm_create(vmname); 1007 if (error) { 1008 if (errno == EEXIST) { 1009 if (romboot) { 1010 reinit = true; 1011 } else { 1012 /* 1013 * The virtual machine has been setup by the 1014 * userspace bootloader. 1015 */ 1016 } 1017 } else { 1018 perror("vm_create"); 1019 exit(4); 1020 } 1021 } else { 1022 if (!romboot) { 1023 /* 1024 * If the virtual machine was just created then a 1025 * bootrom must be configured to boot it. 1026 */ 1027 fprintf(stderr, "virtual machine cannot be booted\n"); 1028 exit(4); 1029 } 1030 } 1031 1032 ctx = vm_open(vmname); 1033 if (ctx == NULL) { 1034 perror("vm_open"); 1035 exit(4); 1036 } 1037 1038 #ifndef WITHOUT_CAPSICUM 1039 cap_rights_init(&rights, CAP_IOCTL, CAP_MMAP_RW); 1040 if (caph_rights_limit(vm_get_device_fd(ctx), &rights) == -1) 1041 errx(EX_OSERR, "Unable to apply rights for sandbox"); 1042 vm_get_ioctls(&ncmds); 1043 cmds = vm_get_ioctls(NULL); 1044 if (cmds == NULL) 1045 errx(EX_OSERR, "out of memory"); 1046 if (caph_ioctls_limit(vm_get_device_fd(ctx), cmds, ncmds) == -1) 1047 errx(EX_OSERR, "Unable to apply rights for sandbox"); 1048 free((cap_ioctl_t *)cmds); 1049 #endif 1050 1051 if (reinit) { 1052 error = vm_reinit(ctx); 1053 if (error) { 1054 perror("vm_reinit"); 1055 exit(4); 1056 } 1057 } 1058 error = vm_set_topology(ctx, sockets, cores, threads, maxcpus); 1059 if (error) 1060 errx(EX_OSERR, "vm_set_topology"); 1061 return (ctx); 1062 } 1063 1064 void 1065 spinup_vcpu(struct vmctx *ctx, int vcpu) 1066 { 1067 int error; 1068 uint64_t rip; 1069 1070 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_RIP, &rip); 1071 assert(error == 0); 1072 1073 fbsdrun_set_capabilities(ctx, vcpu); 1074 error = vm_set_capability(ctx, vcpu, VM_CAP_UNRESTRICTED_GUEST, 1); 1075 assert(error == 0); 1076 1077 fbsdrun_addcpu(ctx, BSP, vcpu, rip); 1078 } 1079 1080 int 1081 main(int argc, char *argv[]) 1082 { 1083 int c, error, err; 1084 int max_vcpus, mptgen, memflags; 1085 int rtc_localtime; 1086 bool gdb_stop; 1087 struct vmctx *ctx; 1088 uint64_t rip; 1089 size_t memsize; 1090 char *optstr; 1091 #ifdef BHYVE_SNAPSHOT 1092 char *restore_file; 1093 struct restore_state rstate; 1094 int vcpu; 1095 1096 restore_file = NULL; 1097 #endif 1098 1099 progname = basename(argv[0]); 1100 gdb_stop = false; 1101 guest_ncpus = 1; 1102 sockets = cores = threads = 1; 1103 maxcpus = 0; 1104 memsize = 256 * MB; 1105 mptgen = 1; 1106 rtc_localtime = 1; 1107 memflags = 0; 1108 1109 #ifdef BHYVE_SNAPSHOT 1110 optstr = "aehuwxACDHIPSWYp:G:c:s:m:l:U:r:"; 1111 #else 1112 optstr = "aehuwxACDHIPSWYp:G:c:s:m:l:U:"; 1113 #endif 1114 while ((c = getopt(argc, argv, optstr)) != -1) { 1115 switch (c) { 1116 case 'a': 1117 x2apic_mode = 0; 1118 break; 1119 case 'A': 1120 acpi = 1; 1121 break; 1122 case 'D': 1123 destroy_on_poweroff = 1; 1124 break; 1125 case 'p': 1126 if (pincpu_parse(optarg) != 0) { 1127 errx(EX_USAGE, "invalid vcpu pinning " 1128 "configuration '%s'", optarg); 1129 } 1130 break; 1131 case 'c': 1132 if (topology_parse(optarg) != 0) { 1133 errx(EX_USAGE, "invalid cpu topology " 1134 "'%s'", optarg); 1135 } 1136 break; 1137 case 'C': 1138 memflags |= VM_MEM_F_INCORE; 1139 break; 1140 case 'G': 1141 if (optarg[0] == 'w') { 1142 gdb_stop = true; 1143 optarg++; 1144 } 1145 gdb_port = atoi(optarg); 1146 break; 1147 case 'l': 1148 if (strncmp(optarg, "help", strlen(optarg)) == 0) { 1149 lpc_print_supported_devices(); 1150 exit(0); 1151 } else if (lpc_device_parse(optarg) != 0) { 1152 errx(EX_USAGE, "invalid lpc device " 1153 "configuration '%s'", optarg); 1154 } 1155 break; 1156 #ifdef BHYVE_SNAPSHOT 1157 case 'r': 1158 restore_file = optarg; 1159 break; 1160 #endif 1161 case 's': 1162 if (strncmp(optarg, "help", strlen(optarg)) == 0) { 1163 pci_print_supported_devices(); 1164 exit(0); 1165 } else if (pci_parse_slot(optarg) != 0) 1166 exit(4); 1167 else 1168 break; 1169 case 'S': 1170 memflags |= VM_MEM_F_WIRED; 1171 break; 1172 case 'm': 1173 error = vm_parse_memsize(optarg, &memsize); 1174 if (error) 1175 errx(EX_USAGE, "invalid memsize '%s'", optarg); 1176 break; 1177 case 'H': 1178 guest_vmexit_on_hlt = 1; 1179 break; 1180 case 'I': 1181 /* 1182 * The "-I" option was used to add an ioapic to the 1183 * virtual machine. 1184 * 1185 * An ioapic is now provided unconditionally for each 1186 * virtual machine and this option is now deprecated. 1187 */ 1188 break; 1189 case 'P': 1190 guest_vmexit_on_pause = 1; 1191 break; 1192 case 'e': 1193 strictio = 1; 1194 break; 1195 case 'u': 1196 rtc_localtime = 0; 1197 break; 1198 case 'U': 1199 guest_uuid_str = optarg; 1200 break; 1201 case 'w': 1202 strictmsr = 0; 1203 break; 1204 case 'W': 1205 virtio_msix = 0; 1206 break; 1207 case 'x': 1208 x2apic_mode = 1; 1209 break; 1210 case 'Y': 1211 mptgen = 0; 1212 break; 1213 case 'h': 1214 usage(0); 1215 default: 1216 usage(1); 1217 } 1218 } 1219 argc -= optind; 1220 argv += optind; 1221 1222 #ifdef BHYVE_SNAPSHOT 1223 if (argc > 1 || (argc == 0 && restore_file == NULL)) 1224 usage(1); 1225 1226 if (restore_file != NULL) { 1227 error = load_restore_file(restore_file, &rstate); 1228 if (error) { 1229 fprintf(stderr, "Failed to read checkpoint info from " 1230 "file: '%s'.\n", restore_file); 1231 exit(1); 1232 } 1233 } 1234 1235 if (argc == 1) { 1236 vmname = argv[0]; 1237 } else { 1238 vmname = lookup_vmname(&rstate); 1239 if (vmname == NULL) { 1240 fprintf(stderr, "Cannot find VM name in restore file. " 1241 "Please specify one.\n"); 1242 exit(1); 1243 } 1244 } 1245 #else 1246 if (argc != 1) 1247 usage(1); 1248 1249 vmname = argv[0]; 1250 #endif 1251 ctx = do_open(vmname); 1252 1253 #ifdef BHYVE_SNAPSHOT 1254 if (restore_file != NULL) { 1255 guest_ncpus = lookup_guest_ncpus(&rstate); 1256 memflags = lookup_memflags(&rstate); 1257 memsize = lookup_memsize(&rstate); 1258 } 1259 1260 if (guest_ncpus < 1) { 1261 fprintf(stderr, "Invalid guest vCPUs (%d)\n", guest_ncpus); 1262 exit(1); 1263 } 1264 #endif 1265 1266 max_vcpus = num_vcpus_allowed(ctx); 1267 if (guest_ncpus > max_vcpus) { 1268 fprintf(stderr, "%d vCPUs requested but only %d available\n", 1269 guest_ncpus, max_vcpus); 1270 exit(4); 1271 } 1272 1273 fbsdrun_set_capabilities(ctx, BSP); 1274 1275 vm_set_memflags(ctx, memflags); 1276 err = vm_setup_memory(ctx, memsize, VM_MMAP_ALL); 1277 if (err) { 1278 fprintf(stderr, "Unable to setup memory (%d)\n", errno); 1279 exit(4); 1280 } 1281 1282 error = init_msr(); 1283 if (error) { 1284 fprintf(stderr, "init_msr error %d", error); 1285 exit(4); 1286 } 1287 1288 init_mem(); 1289 init_inout(); 1290 kernemu_dev_init(); 1291 init_bootrom(ctx); 1292 atkbdc_init(ctx); 1293 pci_irq_init(ctx); 1294 ioapic_init(ctx); 1295 1296 rtc_init(ctx, rtc_localtime); 1297 sci_init(ctx); 1298 1299 /* 1300 * Exit if a device emulation finds an error in its initilization 1301 */ 1302 if (init_pci(ctx) != 0) { 1303 perror("device emulation initialization error"); 1304 exit(4); 1305 } 1306 1307 /* 1308 * Initialize after PCI, to allow a bootrom file to reserve the high 1309 * region. 1310 */ 1311 if (acpi) 1312 vmgenc_init(ctx); 1313 1314 if (gdb_port != 0) 1315 init_gdb(ctx, gdb_port, gdb_stop); 1316 1317 if (lpc_bootrom()) { 1318 if (vm_set_capability(ctx, BSP, VM_CAP_UNRESTRICTED_GUEST, 1)) { 1319 fprintf(stderr, "ROM boot failed: unrestricted guest " 1320 "capability not available\n"); 1321 exit(4); 1322 } 1323 error = vcpu_reset(ctx, BSP); 1324 assert(error == 0); 1325 } 1326 1327 #ifdef BHYVE_SNAPSHOT 1328 if (restore_file != NULL) { 1329 fprintf(stdout, "Pausing pci devs...\r\n"); 1330 if (vm_pause_user_devs(ctx) != 0) { 1331 fprintf(stderr, "Failed to pause PCI device state.\n"); 1332 exit(1); 1333 } 1334 1335 fprintf(stdout, "Restoring vm mem...\r\n"); 1336 if (restore_vm_mem(ctx, &rstate) != 0) { 1337 fprintf(stderr, "Failed to restore VM memory.\n"); 1338 exit(1); 1339 } 1340 1341 fprintf(stdout, "Restoring pci devs...\r\n"); 1342 if (vm_restore_user_devs(ctx, &rstate) != 0) { 1343 fprintf(stderr, "Failed to restore PCI device state.\n"); 1344 exit(1); 1345 } 1346 1347 fprintf(stdout, "Restoring kernel structs...\r\n"); 1348 if (vm_restore_kern_structs(ctx, &rstate) != 0) { 1349 fprintf(stderr, "Failed to restore kernel structs.\n"); 1350 exit(1); 1351 } 1352 1353 fprintf(stdout, "Resuming pci devs...\r\n"); 1354 if (vm_resume_user_devs(ctx) != 0) { 1355 fprintf(stderr, "Failed to resume PCI device state.\n"); 1356 exit(1); 1357 } 1358 } 1359 #endif 1360 1361 error = vm_get_register(ctx, BSP, VM_REG_GUEST_RIP, &rip); 1362 assert(error == 0); 1363 1364 /* 1365 * build the guest tables, MP etc. 1366 */ 1367 if (mptgen) { 1368 error = mptable_build(ctx, guest_ncpus); 1369 if (error) { 1370 perror("error to build the guest tables"); 1371 exit(4); 1372 } 1373 } 1374 1375 error = smbios_build(ctx); 1376 assert(error == 0); 1377 1378 if (acpi) { 1379 error = acpi_build(ctx, guest_ncpus); 1380 assert(error == 0); 1381 } 1382 1383 if (lpc_bootrom()) 1384 fwctl_init(); 1385 1386 /* 1387 * Change the proc title to include the VM name. 1388 */ 1389 setproctitle("%s", vmname); 1390 1391 #ifndef WITHOUT_CAPSICUM 1392 caph_cache_catpages(); 1393 1394 if (caph_limit_stdout() == -1 || caph_limit_stderr() == -1) 1395 errx(EX_OSERR, "Unable to apply rights for sandbox"); 1396 1397 if (caph_enter() == -1) 1398 errx(EX_OSERR, "cap_enter() failed"); 1399 #endif 1400 1401 #ifdef BHYVE_SNAPSHOT 1402 if (restore_file != NULL) 1403 destroy_restore_state(&rstate); 1404 1405 /* 1406 * checkpointing thread for communication with bhyvectl 1407 */ 1408 if (init_checkpoint_thread(ctx) < 0) 1409 printf("Failed to start checkpoint thread!\r\n"); 1410 1411 if (restore_file != NULL) 1412 vm_restore_time(ctx); 1413 #endif 1414 1415 /* 1416 * Add CPU 0 1417 */ 1418 fbsdrun_addcpu(ctx, BSP, BSP, rip); 1419 1420 #ifdef BHYVE_SNAPSHOT 1421 /* 1422 * If we restore a VM, start all vCPUs now (including APs), otherwise, 1423 * let the guest OS to spin them up later via vmexits. 1424 */ 1425 if (restore_file != NULL) { 1426 for (vcpu = 0; vcpu < guest_ncpus; vcpu++) { 1427 if (vcpu == BSP) 1428 continue; 1429 1430 fprintf(stdout, "spinning up vcpu no %d...\r\n", vcpu); 1431 spinup_vcpu(ctx, vcpu); 1432 } 1433 } 1434 #endif 1435 1436 /* 1437 * Head off to the main event dispatch loop 1438 */ 1439 mevent_dispatch(); 1440 1441 exit(4); 1442 } 1443