1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2011 NetApp, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include "opt_bhyve_snapshot.h" 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/sysctl.h> 35 #include <sys/malloc.h> 36 #include <sys/pcpu.h> 37 #include <sys/lock.h> 38 #include <sys/mutex.h> 39 #include <sys/proc.h> 40 #include <sys/rwlock.h> 41 #include <sys/sched.h> 42 #include <sys/smp.h> 43 #include <sys/sx.h> 44 #include <sys/vnode.h> 45 46 #include <vm/vm.h> 47 #include <vm/vm_param.h> 48 #include <vm/vm_extern.h> 49 #include <vm/vm_object.h> 50 #include <vm/vm_page.h> 51 #include <vm/pmap.h> 52 #include <vm/vm_map.h> 53 #include <vm/vm_pager.h> 54 #include <vm/vm_kern.h> 55 #include <vm/vnode_pager.h> 56 #include <vm/swap_pager.h> 57 #include <vm/uma.h> 58 59 #include <x86/apicreg.h> 60 #include <machine/cpu.h> 61 #include <machine/ifunc.h> 62 #include <machine/md_var.h> 63 #include <machine/pcb.h> 64 #include <x86/psl.h> 65 #include <machine/smp.h> 66 67 #include <machine/vmm.h> 68 #include <machine/vmm_instruction_emul.h> 69 #include <machine/vmm_snapshot.h> 70 71 #include <dev/vmm/vmm_dev.h> 72 #include <dev/vmm/vmm_ktr.h> 73 #include <dev/vmm/vmm_mem.h> 74 #include <dev/vmm/vmm_vm.h> 75 76 #include "vmm_ioport.h" 77 #include "vmm_host.h" 78 #include "vmm_mem.h" 79 #include "vmm_util.h" 80 #include "vatpic.h" 81 #include "vatpit.h" 82 #include "vhpet.h" 83 #include "vioapic.h" 84 #include "vlapic.h" 85 #include "vpmtmr.h" 86 #include "vrtc.h" 87 #include "vmm_stat.h" 88 #include "vmm_lapic.h" 89 90 #include "io/ppt.h" 91 #include "io/iommu.h" 92 93 struct vlapic; 94 95 #define VMM_CTR0(vcpu, format) \ 96 VCPU_CTR0((vcpu)->vm, (vcpu)->vcpuid, format) 97 98 #define VMM_CTR1(vcpu, format, p1) \ 99 VCPU_CTR1((vcpu)->vm, (vcpu)->vcpuid, format, p1) 100 101 #define VMM_CTR2(vcpu, format, p1, p2) \ 102 VCPU_CTR2((vcpu)->vm, (vcpu)->vcpuid, format, p1, p2) 103 104 #define VMM_CTR3(vcpu, format, p1, p2, p3) \ 105 VCPU_CTR3((vcpu)->vm, (vcpu)->vcpuid, format, p1, p2, p3) 106 107 #define VMM_CTR4(vcpu, format, p1, p2, p3, p4) \ 108 VCPU_CTR4((vcpu)->vm, (vcpu)->vcpuid, format, p1, p2, p3, p4) 109 110 static void vmmops_panic(void); 111 112 static void 113 vmmops_panic(void) 114 { 115 panic("vmm_ops func called when !vmm_is_intel() && !vmm_is_svm()"); 116 } 117 118 #define DEFINE_VMMOPS_IFUNC(ret_type, opname, args) \ 119 DEFINE_IFUNC(, ret_type, vmmops_##opname, args) \ 120 { \ 121 if (vmm_is_intel()) \ 122 return (vmm_ops_intel.opname); \ 123 else if (vmm_is_svm()) \ 124 return (vmm_ops_amd.opname); \ 125 else \ 126 return ((ret_type (*)args)vmmops_panic); \ 127 } 128 129 DEFINE_VMMOPS_IFUNC(int, modinit, (int ipinum)) 130 DEFINE_VMMOPS_IFUNC(int, modcleanup, (void)) 131 DEFINE_VMMOPS_IFUNC(void, modsuspend, (void)) 132 DEFINE_VMMOPS_IFUNC(void, modresume, (void)) 133 DEFINE_VMMOPS_IFUNC(void *, init, (struct vm *vm, struct pmap *pmap)) 134 DEFINE_VMMOPS_IFUNC(int, run, (void *vcpui, register_t rip, struct pmap *pmap, 135 struct vm_eventinfo *info)) 136 DEFINE_VMMOPS_IFUNC(void, cleanup, (void *vmi)) 137 DEFINE_VMMOPS_IFUNC(void *, vcpu_init, (void *vmi, struct vcpu *vcpu, 138 int vcpu_id)) 139 DEFINE_VMMOPS_IFUNC(void, vcpu_cleanup, (void *vcpui)) 140 DEFINE_VMMOPS_IFUNC(int, getreg, (void *vcpui, int num, uint64_t *retval)) 141 DEFINE_VMMOPS_IFUNC(int, setreg, (void *vcpui, int num, uint64_t val)) 142 DEFINE_VMMOPS_IFUNC(int, getdesc, (void *vcpui, int num, struct seg_desc *desc)) 143 DEFINE_VMMOPS_IFUNC(int, setdesc, (void *vcpui, int num, struct seg_desc *desc)) 144 DEFINE_VMMOPS_IFUNC(int, getcap, (void *vcpui, int num, int *retval)) 145 DEFINE_VMMOPS_IFUNC(int, setcap, (void *vcpui, int num, int val)) 146 DEFINE_VMMOPS_IFUNC(struct vmspace *, vmspace_alloc, (vm_offset_t min, 147 vm_offset_t max)) 148 DEFINE_VMMOPS_IFUNC(void, vmspace_free, (struct vmspace *vmspace)) 149 DEFINE_VMMOPS_IFUNC(struct vlapic *, vlapic_init, (void *vcpui)) 150 DEFINE_VMMOPS_IFUNC(void, vlapic_cleanup, (struct vlapic *vlapic)) 151 #ifdef BHYVE_SNAPSHOT 152 DEFINE_VMMOPS_IFUNC(int, vcpu_snapshot, (void *vcpui, 153 struct vm_snapshot_meta *meta)) 154 DEFINE_VMMOPS_IFUNC(int, restore_tsc, (void *vcpui, uint64_t now)) 155 #endif 156 157 SDT_PROVIDER_DEFINE(vmm); 158 159 static MALLOC_DEFINE(M_VM, "vm", "vm"); 160 161 /* statistics */ 162 static VMM_STAT(VCPU_TOTAL_RUNTIME, "vcpu total runtime"); 163 164 SYSCTL_DECL(_hw_vmm); 165 166 /* 167 * Halt the guest if all vcpus are executing a HLT instruction with 168 * interrupts disabled. 169 */ 170 static int halt_detection_enabled = 1; 171 SYSCTL_INT(_hw_vmm, OID_AUTO, halt_detection, CTLFLAG_RDTUN, 172 &halt_detection_enabled, 0, 173 "Halt VM if all vcpus execute HLT with interrupts disabled"); 174 175 static int trace_guest_exceptions; 176 SYSCTL_INT(_hw_vmm, OID_AUTO, trace_guest_exceptions, CTLFLAG_RDTUN, 177 &trace_guest_exceptions, 0, 178 "Trap into hypervisor on all guest exceptions and reflect them back"); 179 180 static int trap_wbinvd; 181 SYSCTL_INT(_hw_vmm, OID_AUTO, trap_wbinvd, CTLFLAG_RDTUN, &trap_wbinvd, 0, 182 "WBINVD triggers a VM-exit"); 183 184 /* global statistics */ 185 VMM_STAT(VCPU_MIGRATIONS, "vcpu migration across host cpus"); 186 VMM_STAT(VMEXIT_COUNT, "total number of vm exits"); 187 VMM_STAT(VMEXIT_EXTINT, "vm exits due to external interrupt"); 188 VMM_STAT(VMEXIT_HLT, "number of times hlt was intercepted"); 189 VMM_STAT(VMEXIT_CR_ACCESS, "number of times %cr access was intercepted"); 190 VMM_STAT(VMEXIT_RDMSR, "number of times rdmsr was intercepted"); 191 VMM_STAT(VMEXIT_WRMSR, "number of times wrmsr was intercepted"); 192 VMM_STAT(VMEXIT_MTRAP, "number of monitor trap exits"); 193 VMM_STAT(VMEXIT_PAUSE, "number of times pause was intercepted"); 194 VMM_STAT(VMEXIT_INTR_WINDOW, "vm exits due to interrupt window opening"); 195 VMM_STAT(VMEXIT_NMI_WINDOW, "vm exits due to nmi window opening"); 196 VMM_STAT(VMEXIT_INOUT, "number of times in/out was intercepted"); 197 VMM_STAT(VMEXIT_CPUID, "number of times cpuid was intercepted"); 198 VMM_STAT(VMEXIT_NESTED_FAULT, "vm exits due to nested page fault"); 199 VMM_STAT(VMEXIT_INST_EMUL, "vm exits for instruction emulation"); 200 VMM_STAT(VMEXIT_UNKNOWN, "number of vm exits for unknown reason"); 201 VMM_STAT(VMEXIT_ASTPENDING, "number of times astpending at exit"); 202 VMM_STAT(VMEXIT_REQIDLE, "number of times idle requested at exit"); 203 VMM_STAT(VMEXIT_USERSPACE, "number of vm exits handled in userspace"); 204 VMM_STAT(VMEXIT_RENDEZVOUS, "number of times rendezvous pending at exit"); 205 VMM_STAT(VMEXIT_EXCEPTION, "number of vm exits due to exceptions"); 206 207 static void 208 vcpu_cleanup(struct vcpu *vcpu, bool destroy) 209 { 210 vmmops_vlapic_cleanup(vcpu->vlapic); 211 vmmops_vcpu_cleanup(vcpu->cookie); 212 vcpu->cookie = NULL; 213 if (destroy) { 214 vmm_stat_free(vcpu->stats); 215 fpu_save_area_free(vcpu->guestfpu); 216 vcpu_lock_destroy(vcpu); 217 free(vcpu, M_VM); 218 } 219 } 220 221 static struct vcpu * 222 vcpu_alloc(struct vm *vm, int vcpu_id) 223 { 224 struct vcpu *vcpu; 225 226 KASSERT(vcpu_id >= 0 && vcpu_id < vm->maxcpus, 227 ("vcpu_init: invalid vcpu %d", vcpu_id)); 228 229 vcpu = malloc(sizeof(*vcpu), M_VM, M_WAITOK | M_ZERO); 230 vcpu_lock_init(vcpu); 231 vcpu->state = VCPU_IDLE; 232 vcpu->hostcpu = NOCPU; 233 vcpu->vcpuid = vcpu_id; 234 vcpu->vm = vm; 235 vcpu->guestfpu = fpu_save_area_alloc(); 236 vcpu->stats = vmm_stat_alloc(); 237 vcpu->tsc_offset = 0; 238 return (vcpu); 239 } 240 241 static void 242 vcpu_init(struct vcpu *vcpu) 243 { 244 vcpu->cookie = vmmops_vcpu_init(vcpu->vm->cookie, vcpu, vcpu->vcpuid); 245 vcpu->vlapic = vmmops_vlapic_init(vcpu->cookie); 246 vm_set_x2apic_state(vcpu, X2APIC_DISABLED); 247 vcpu->reqidle = 0; 248 vcpu->exitintinfo = 0; 249 vcpu->nmi_pending = 0; 250 vcpu->extint_pending = 0; 251 vcpu->exception_pending = 0; 252 vcpu->guest_xcr0 = XFEATURE_ENABLED_X87; 253 fpu_save_area_reset(vcpu->guestfpu); 254 vmm_stat_init(vcpu->stats); 255 } 256 257 int 258 vcpu_trace_exceptions(struct vcpu *vcpu) 259 { 260 return (trace_guest_exceptions); 261 } 262 263 int 264 vcpu_trap_wbinvd(struct vcpu *vcpu) 265 { 266 return (trap_wbinvd); 267 } 268 269 struct vm_exit * 270 vm_exitinfo(struct vcpu *vcpu) 271 { 272 return (&vcpu->exitinfo); 273 } 274 275 cpuset_t * 276 vm_exitinfo_cpuset(struct vcpu *vcpu) 277 { 278 return (&vcpu->exitinfo_cpuset); 279 } 280 281 int 282 vmm_modinit(void) 283 { 284 if (!vmm_is_hw_supported()) 285 return (ENXIO); 286 287 vmm_host_state_init(); 288 289 vmm_ipinum = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) : 290 &IDTVEC(justreturn), vmm_justreturn); 291 if (vmm_ipinum < 0) 292 vmm_ipinum = IPI_AST; 293 294 vmm_suspend_p = vmmops_modsuspend; 295 vmm_resume_p = vmmops_modresume; 296 297 return (vmmops_modinit(vmm_ipinum)); 298 } 299 300 int 301 vmm_modcleanup(void) 302 { 303 vmm_suspend_p = NULL; 304 vmm_resume_p = NULL; 305 iommu_cleanup(); 306 if (vmm_ipinum != IPI_AST) 307 lapic_ipi_free(vmm_ipinum); 308 return (vmmops_modcleanup()); 309 } 310 311 static void 312 vm_init(struct vm *vm, bool create) 313 { 314 vm->cookie = vmmops_init(vm, vmspace_pmap(vm_vmspace(vm))); 315 vm->iommu = NULL; 316 vm->vioapic = vioapic_init(vm); 317 vm->vhpet = vhpet_init(vm); 318 vm->vatpic = vatpic_init(vm); 319 vm->vatpit = vatpit_init(vm); 320 vm->vpmtmr = vpmtmr_init(vm); 321 if (create) 322 vm->vrtc = vrtc_init(vm); 323 324 CPU_ZERO(&vm->active_cpus); 325 CPU_ZERO(&vm->debug_cpus); 326 CPU_ZERO(&vm->startup_cpus); 327 328 vm->suspend = 0; 329 CPU_ZERO(&vm->suspended_cpus); 330 331 if (!create) { 332 for (int i = 0; i < vm->maxcpus; i++) { 333 if (vm->vcpu[i] != NULL) 334 vcpu_init(vm->vcpu[i]); 335 } 336 } 337 } 338 339 struct vcpu * 340 vm_alloc_vcpu(struct vm *vm, int vcpuid) 341 { 342 struct vcpu *vcpu; 343 344 if (vcpuid < 0 || vcpuid >= vm_get_maxcpus(vm)) 345 return (NULL); 346 347 vcpu = (struct vcpu *) 348 atomic_load_acq_ptr((uintptr_t *)&vm->vcpu[vcpuid]); 349 if (__predict_true(vcpu != NULL)) 350 return (vcpu); 351 352 sx_xlock(&vm->vcpus_init_lock); 353 vcpu = vm->vcpu[vcpuid]; 354 if (vcpu == NULL && !vm->dying) { 355 vcpu = vcpu_alloc(vm, vcpuid); 356 vcpu_init(vcpu); 357 358 /* 359 * Ensure vCPU is fully created before updating pointer 360 * to permit unlocked reads above. 361 */ 362 atomic_store_rel_ptr((uintptr_t *)&vm->vcpu[vcpuid], 363 (uintptr_t)vcpu); 364 } 365 sx_xunlock(&vm->vcpus_init_lock); 366 return (vcpu); 367 } 368 369 int 370 vm_create(const char *name, struct vm **retvm) 371 { 372 struct vm *vm; 373 int error; 374 375 vm = malloc(sizeof(struct vm), M_VM, M_WAITOK | M_ZERO); 376 error = vm_mem_init(&vm->mem, 0, VM_MAXUSER_ADDRESS_LA48); 377 if (error != 0) { 378 free(vm, M_VM); 379 return (error); 380 } 381 strcpy(vm->name, name); 382 mtx_init(&vm->rendezvous_mtx, "vm rendezvous lock", 0, MTX_DEF); 383 sx_init(&vm->vcpus_init_lock, "vm vcpus"); 384 vm->vcpu = malloc(sizeof(*vm->vcpu) * vm_maxcpu, M_VM, M_WAITOK | 385 M_ZERO); 386 387 vm->sockets = 1; 388 vm->cores = 1; /* XXX backwards compatibility */ 389 vm->threads = 1; /* XXX backwards compatibility */ 390 vm->maxcpus = vm_maxcpu; 391 392 vm_init(vm, true); 393 394 *retvm = vm; 395 return (0); 396 } 397 398 static void 399 vm_cleanup(struct vm *vm, bool destroy) 400 { 401 if (destroy) 402 vm_xlock_memsegs(vm); 403 else 404 vm_assert_memseg_xlocked(vm); 405 406 ppt_unassign_all(vm); 407 408 if (vm->iommu != NULL) 409 iommu_destroy_domain(vm->iommu); 410 411 if (destroy) 412 vrtc_cleanup(vm->vrtc); 413 else 414 vrtc_reset(vm->vrtc); 415 vpmtmr_cleanup(vm->vpmtmr); 416 vatpit_cleanup(vm->vatpit); 417 vhpet_cleanup(vm->vhpet); 418 vatpic_cleanup(vm->vatpic); 419 vioapic_cleanup(vm->vioapic); 420 421 for (int i = 0; i < vm->maxcpus; i++) { 422 if (vm->vcpu[i] != NULL) 423 vcpu_cleanup(vm->vcpu[i], destroy); 424 } 425 426 vmmops_cleanup(vm->cookie); 427 428 vm_mem_cleanup(vm); 429 430 if (destroy) { 431 vm_mem_destroy(vm); 432 433 free(vm->vcpu, M_VM); 434 sx_destroy(&vm->vcpus_init_lock); 435 mtx_destroy(&vm->rendezvous_mtx); 436 } 437 } 438 439 void 440 vm_destroy(struct vm *vm) 441 { 442 vm_cleanup(vm, true); 443 free(vm, M_VM); 444 } 445 446 void 447 vm_reset(struct vm *vm) 448 { 449 vm_cleanup(vm, false); 450 vm_init(vm, false); 451 } 452 453 int 454 vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa) 455 { 456 return (vmm_mmio_alloc(vm_vmspace(vm), gpa, len, hpa)); 457 } 458 459 int 460 vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len) 461 { 462 463 vmm_mmio_free(vm_vmspace(vm), gpa, len); 464 return (0); 465 } 466 467 static int 468 vm_iommu_map(struct vm *vm) 469 { 470 pmap_t pmap; 471 vm_paddr_t gpa, hpa; 472 struct vm_mem_map *mm; 473 int error, i; 474 475 sx_assert(&vm->mem.mem_segs_lock, SX_LOCKED); 476 477 pmap = vmspace_pmap(vm_vmspace(vm)); 478 for (i = 0; i < VM_MAX_MEMMAPS; i++) { 479 mm = &vm->mem.mem_maps[i]; 480 if (!vm_memseg_sysmem(vm, mm->segid)) 481 continue; 482 483 KASSERT((mm->flags & VM_MEMMAP_F_IOMMU) == 0, 484 ("iommu map found invalid memmap %#lx/%#lx/%#x", 485 mm->gpa, mm->len, mm->flags)); 486 if ((mm->flags & VM_MEMMAP_F_WIRED) == 0) 487 continue; 488 mm->flags |= VM_MEMMAP_F_IOMMU; 489 490 for (gpa = mm->gpa; gpa < mm->gpa + mm->len; gpa += PAGE_SIZE) { 491 hpa = pmap_extract(pmap, gpa); 492 493 /* 494 * All mappings in the vmm vmspace must be 495 * present since they are managed by vmm in this way. 496 * Because we are in pass-through mode, the 497 * mappings must also be wired. This implies 498 * that all pages must be mapped and wired, 499 * allowing to use pmap_extract() and avoiding the 500 * need to use vm_gpa_hold_global(). 501 * 502 * This could change if/when we start 503 * supporting page faults on IOMMU maps. 504 */ 505 KASSERT(vm_page_wired(PHYS_TO_VM_PAGE(hpa)), 506 ("vm_iommu_map: vm %p gpa %jx hpa %jx not wired", 507 vm, (uintmax_t)gpa, (uintmax_t)hpa)); 508 509 iommu_create_mapping(vm->iommu, gpa, hpa, PAGE_SIZE); 510 } 511 } 512 513 error = iommu_invalidate_tlb(iommu_host_domain()); 514 return (error); 515 } 516 517 static int 518 vm_iommu_unmap(struct vm *vm) 519 { 520 vm_paddr_t gpa; 521 struct vm_mem_map *mm; 522 int error, i; 523 524 sx_assert(&vm->mem.mem_segs_lock, SX_LOCKED); 525 526 for (i = 0; i < VM_MAX_MEMMAPS; i++) { 527 mm = &vm->mem.mem_maps[i]; 528 if (!vm_memseg_sysmem(vm, mm->segid)) 529 continue; 530 531 if ((mm->flags & VM_MEMMAP_F_IOMMU) == 0) 532 continue; 533 mm->flags &= ~VM_MEMMAP_F_IOMMU; 534 KASSERT((mm->flags & VM_MEMMAP_F_WIRED) != 0, 535 ("iommu unmap found invalid memmap %#lx/%#lx/%#x", 536 mm->gpa, mm->len, mm->flags)); 537 538 for (gpa = mm->gpa; gpa < mm->gpa + mm->len; gpa += PAGE_SIZE) { 539 KASSERT(vm_page_wired(PHYS_TO_VM_PAGE(pmap_extract( 540 vmspace_pmap(vm_vmspace(vm)), gpa))), 541 ("vm_iommu_unmap: vm %p gpa %jx not wired", 542 vm, (uintmax_t)gpa)); 543 iommu_remove_mapping(vm->iommu, gpa, PAGE_SIZE); 544 } 545 } 546 547 /* 548 * Invalidate the cached translations associated with the domain 549 * from which pages were removed. 550 */ 551 error = iommu_invalidate_tlb(vm->iommu); 552 return (error); 553 } 554 555 int 556 vm_unassign_pptdev(struct vm *vm, int bus, int slot, int func) 557 { 558 int error; 559 560 error = ppt_unassign_device(vm, bus, slot, func); 561 if (error) 562 return (error); 563 564 if (ppt_assigned_devices(vm) == 0) 565 error = vm_iommu_unmap(vm); 566 567 return (error); 568 } 569 570 int 571 vm_assign_pptdev(struct vm *vm, int bus, int slot, int func) 572 { 573 int error; 574 vm_paddr_t maxaddr; 575 bool map = false; 576 577 /* Set up the IOMMU to do the 'gpa' to 'hpa' translation */ 578 if (ppt_assigned_devices(vm) == 0) { 579 KASSERT(vm->iommu == NULL, 580 ("vm_assign_pptdev: iommu must be NULL")); 581 maxaddr = vmm_sysmem_maxaddr(vm); 582 vm->iommu = iommu_create_domain(maxaddr); 583 if (vm->iommu == NULL) 584 return (ENXIO); 585 map = true; 586 } 587 588 error = ppt_assign_device(vm, bus, slot, func); 589 if (error == 0 && map) 590 error = vm_iommu_map(vm); 591 return (error); 592 } 593 594 int 595 vm_reset_pptdev(struct vm *vm, int bus, int slot, int func) 596 { 597 598 return (ppt_reset_device(vm, bus, slot, func)); 599 } 600 601 int 602 vm_get_register(struct vcpu *vcpu, int reg, uint64_t *retval) 603 { 604 /* Negative values represent VM control structure fields. */ 605 if (reg >= VM_REG_LAST) 606 return (EINVAL); 607 608 return (vmmops_getreg(vcpu->cookie, reg, retval)); 609 } 610 611 int 612 vm_set_register(struct vcpu *vcpu, int reg, uint64_t val) 613 { 614 int error; 615 616 /* Negative values represent VM control structure fields. */ 617 if (reg >= VM_REG_LAST) 618 return (EINVAL); 619 620 error = vmmops_setreg(vcpu->cookie, reg, val); 621 if (error || reg != VM_REG_GUEST_RIP) 622 return (error); 623 624 /* Set 'nextrip' to match the value of %rip */ 625 VMM_CTR1(vcpu, "Setting nextrip to %#lx", val); 626 vcpu->nextrip = val; 627 return (0); 628 } 629 630 static bool 631 is_descriptor_table(int reg) 632 { 633 634 switch (reg) { 635 case VM_REG_GUEST_IDTR: 636 case VM_REG_GUEST_GDTR: 637 return (true); 638 default: 639 return (false); 640 } 641 } 642 643 static bool 644 is_segment_register(int reg) 645 { 646 647 switch (reg) { 648 case VM_REG_GUEST_ES: 649 case VM_REG_GUEST_CS: 650 case VM_REG_GUEST_SS: 651 case VM_REG_GUEST_DS: 652 case VM_REG_GUEST_FS: 653 case VM_REG_GUEST_GS: 654 case VM_REG_GUEST_TR: 655 case VM_REG_GUEST_LDTR: 656 return (true); 657 default: 658 return (false); 659 } 660 } 661 662 int 663 vm_get_seg_desc(struct vcpu *vcpu, int reg, struct seg_desc *desc) 664 { 665 666 if (!is_segment_register(reg) && !is_descriptor_table(reg)) 667 return (EINVAL); 668 669 return (vmmops_getdesc(vcpu->cookie, reg, desc)); 670 } 671 672 int 673 vm_set_seg_desc(struct vcpu *vcpu, int reg, struct seg_desc *desc) 674 { 675 676 if (!is_segment_register(reg) && !is_descriptor_table(reg)) 677 return (EINVAL); 678 679 return (vmmops_setdesc(vcpu->cookie, reg, desc)); 680 } 681 682 static void 683 restore_guest_fpustate(struct vcpu *vcpu) 684 { 685 686 /* flush host state to the pcb */ 687 fpuexit(curthread); 688 689 /* restore guest FPU state */ 690 fpu_enable(); 691 fpurestore(vcpu->guestfpu); 692 693 /* restore guest XCR0 if XSAVE is enabled in the host */ 694 if (rcr4() & CR4_XSAVE) 695 load_xcr(0, vcpu->guest_xcr0); 696 697 /* 698 * The FPU is now "dirty" with the guest's state so disable 699 * the FPU to trap any access by the host. 700 */ 701 fpu_disable(); 702 } 703 704 static void 705 save_guest_fpustate(struct vcpu *vcpu) 706 { 707 708 if ((rcr0() & CR0_TS) == 0) 709 panic("fpu emulation not enabled in host!"); 710 711 /* save guest XCR0 and restore host XCR0 */ 712 if (rcr4() & CR4_XSAVE) { 713 vcpu->guest_xcr0 = rxcr(0); 714 load_xcr(0, vmm_get_host_xcr0()); 715 } 716 717 /* save guest FPU state */ 718 fpu_enable(); 719 fpusave(vcpu->guestfpu); 720 fpu_disable(); 721 } 722 723 static VMM_STAT(VCPU_IDLE_TICKS, "number of ticks vcpu was idle"); 724 725 static void 726 vcpu_require_state(struct vcpu *vcpu, enum vcpu_state newstate) 727 { 728 int error; 729 730 if ((error = vcpu_set_state(vcpu, newstate, false)) != 0) 731 panic("Error %d setting state to %d\n", error, newstate); 732 } 733 734 static void 735 vcpu_require_state_locked(struct vcpu *vcpu, enum vcpu_state newstate) 736 { 737 int error; 738 739 if ((error = vcpu_set_state_locked(vcpu, newstate, false)) != 0) 740 panic("Error %d setting state to %d", error, newstate); 741 } 742 743 /* 744 * Emulate a guest 'hlt' by sleeping until the vcpu is ready to run. 745 */ 746 static int 747 vm_handle_hlt(struct vcpu *vcpu, bool intr_disabled, bool *retu) 748 { 749 struct vm *vm = vcpu->vm; 750 const char *wmesg; 751 struct thread *td; 752 int error, t, vcpuid, vcpu_halted, vm_halted; 753 754 vcpuid = vcpu->vcpuid; 755 vcpu_halted = 0; 756 vm_halted = 0; 757 error = 0; 758 td = curthread; 759 760 KASSERT(!CPU_ISSET(vcpuid, &vm->halted_cpus), ("vcpu already halted")); 761 762 vcpu_lock(vcpu); 763 while (1) { 764 /* 765 * Do a final check for pending NMI or interrupts before 766 * really putting this thread to sleep. Also check for 767 * software events that would cause this vcpu to wakeup. 768 * 769 * These interrupts/events could have happened after the 770 * vcpu returned from vmmops_run() and before it acquired the 771 * vcpu lock above. 772 */ 773 if (vm->rendezvous_func != NULL || vm->suspend || vcpu->reqidle) 774 break; 775 if (vm_nmi_pending(vcpu)) 776 break; 777 if (!intr_disabled) { 778 if (vm_extint_pending(vcpu) || 779 vlapic_pending_intr(vcpu->vlapic, NULL)) { 780 break; 781 } 782 } 783 784 /* Don't go to sleep if the vcpu thread needs to yield */ 785 if (vcpu_should_yield(vcpu)) 786 break; 787 788 if (vcpu_debugged(vcpu)) 789 break; 790 791 /* 792 * Some Linux guests implement "halt" by having all vcpus 793 * execute HLT with interrupts disabled. 'halted_cpus' keeps 794 * track of the vcpus that have entered this state. When all 795 * vcpus enter the halted state the virtual machine is halted. 796 */ 797 if (intr_disabled) { 798 wmesg = "vmhalt"; 799 VMM_CTR0(vcpu, "Halted"); 800 if (!vcpu_halted && halt_detection_enabled) { 801 vcpu_halted = 1; 802 CPU_SET_ATOMIC(vcpuid, &vm->halted_cpus); 803 } 804 if (CPU_CMP(&vm->halted_cpus, &vm->active_cpus) == 0) { 805 vm_halted = 1; 806 break; 807 } 808 } else { 809 wmesg = "vmidle"; 810 } 811 812 t = ticks; 813 vcpu_require_state_locked(vcpu, VCPU_SLEEPING); 814 /* 815 * XXX msleep_spin() cannot be interrupted by signals so 816 * wake up periodically to check pending signals. 817 */ 818 msleep_spin(vcpu, &vcpu->mtx, wmesg, hz); 819 vcpu_require_state_locked(vcpu, VCPU_FROZEN); 820 vmm_stat_incr(vcpu, VCPU_IDLE_TICKS, ticks - t); 821 if (td_ast_pending(td, TDA_SUSPEND)) { 822 vcpu_unlock(vcpu); 823 error = thread_check_susp(td, false); 824 if (error != 0) { 825 if (vcpu_halted) { 826 CPU_CLR_ATOMIC(vcpuid, 827 &vm->halted_cpus); 828 } 829 return (error); 830 } 831 vcpu_lock(vcpu); 832 } 833 } 834 835 if (vcpu_halted) 836 CPU_CLR_ATOMIC(vcpuid, &vm->halted_cpus); 837 838 vcpu_unlock(vcpu); 839 840 if (vm_halted) 841 vm_suspend(vm, VM_SUSPEND_HALT); 842 843 return (0); 844 } 845 846 static int 847 vm_handle_paging(struct vcpu *vcpu, bool *retu) 848 { 849 struct vm *vm = vcpu->vm; 850 int rv, ftype; 851 struct vm_map *map; 852 struct vm_exit *vme; 853 854 vme = &vcpu->exitinfo; 855 856 KASSERT(vme->inst_length == 0, ("%s: invalid inst_length %d", 857 __func__, vme->inst_length)); 858 859 ftype = vme->u.paging.fault_type; 860 KASSERT(ftype == VM_PROT_READ || 861 ftype == VM_PROT_WRITE || ftype == VM_PROT_EXECUTE, 862 ("vm_handle_paging: invalid fault_type %d", ftype)); 863 864 if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) { 865 rv = pmap_emulate_accessed_dirty(vmspace_pmap(vm_vmspace(vm)), 866 vme->u.paging.gpa, ftype); 867 if (rv == 0) { 868 VMM_CTR2(vcpu, "%s bit emulation for gpa %#lx", 869 ftype == VM_PROT_READ ? "accessed" : "dirty", 870 vme->u.paging.gpa); 871 goto done; 872 } 873 } 874 875 map = &vm_vmspace(vm)->vm_map; 876 rv = vm_fault(map, vme->u.paging.gpa, ftype, VM_FAULT_NORMAL, NULL); 877 878 VMM_CTR3(vcpu, "vm_handle_paging rv = %d, gpa = %#lx, " 879 "ftype = %d", rv, vme->u.paging.gpa, ftype); 880 881 if (rv != KERN_SUCCESS) 882 return (EFAULT); 883 done: 884 return (0); 885 } 886 887 static int 888 vm_handle_inst_emul(struct vcpu *vcpu, bool *retu) 889 { 890 struct vie *vie; 891 struct vm_exit *vme; 892 uint64_t gla, gpa, cs_base; 893 struct vm_guest_paging *paging; 894 mem_region_read_t mread; 895 mem_region_write_t mwrite; 896 enum vm_cpu_mode cpu_mode; 897 int cs_d, error, fault; 898 899 vme = &vcpu->exitinfo; 900 901 KASSERT(vme->inst_length == 0, ("%s: invalid inst_length %d", 902 __func__, vme->inst_length)); 903 904 gla = vme->u.inst_emul.gla; 905 gpa = vme->u.inst_emul.gpa; 906 cs_base = vme->u.inst_emul.cs_base; 907 cs_d = vme->u.inst_emul.cs_d; 908 vie = &vme->u.inst_emul.vie; 909 paging = &vme->u.inst_emul.paging; 910 cpu_mode = paging->cpu_mode; 911 912 VMM_CTR1(vcpu, "inst_emul fault accessing gpa %#lx", gpa); 913 914 /* Fetch, decode and emulate the faulting instruction */ 915 if (vie->num_valid == 0) { 916 error = vmm_fetch_instruction(vcpu, paging, vme->rip + cs_base, 917 VIE_INST_SIZE, vie, &fault); 918 } else { 919 /* 920 * The instruction bytes have already been copied into 'vie' 921 */ 922 error = fault = 0; 923 } 924 if (error || fault) 925 return (error); 926 927 if (vmm_decode_instruction(vcpu, gla, cpu_mode, cs_d, vie) != 0) { 928 VMM_CTR1(vcpu, "Error decoding instruction at %#lx", 929 vme->rip + cs_base); 930 *retu = true; /* dump instruction bytes in userspace */ 931 return (0); 932 } 933 934 /* 935 * Update 'nextrip' based on the length of the emulated instruction. 936 */ 937 vme->inst_length = vie->num_processed; 938 vcpu->nextrip += vie->num_processed; 939 VMM_CTR1(vcpu, "nextrip updated to %#lx after instruction decoding", 940 vcpu->nextrip); 941 942 /* return to userland unless this is an in-kernel emulated device */ 943 if (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE) { 944 mread = lapic_mmio_read; 945 mwrite = lapic_mmio_write; 946 } else if (gpa >= VIOAPIC_BASE && gpa < VIOAPIC_BASE + VIOAPIC_SIZE) { 947 mread = vioapic_mmio_read; 948 mwrite = vioapic_mmio_write; 949 } else if (gpa >= VHPET_BASE && gpa < VHPET_BASE + VHPET_SIZE) { 950 mread = vhpet_mmio_read; 951 mwrite = vhpet_mmio_write; 952 } else { 953 *retu = true; 954 return (0); 955 } 956 957 error = vmm_emulate_instruction(vcpu, gpa, vie, paging, mread, mwrite, 958 retu); 959 960 return (error); 961 } 962 963 static int 964 vm_handle_suspend(struct vcpu *vcpu, bool *retu) 965 { 966 struct vm *vm = vcpu->vm; 967 int error, i; 968 struct thread *td; 969 970 error = 0; 971 td = curthread; 972 973 CPU_SET_ATOMIC(vcpu->vcpuid, &vm->suspended_cpus); 974 975 /* 976 * Wait until all 'active_cpus' have suspended themselves. 977 * 978 * Since a VM may be suspended at any time including when one or 979 * more vcpus are doing a rendezvous we need to call the rendezvous 980 * handler while we are waiting to prevent a deadlock. 981 */ 982 vcpu_lock(vcpu); 983 while (error == 0) { 984 if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) { 985 VMM_CTR0(vcpu, "All vcpus suspended"); 986 break; 987 } 988 989 if (vm->rendezvous_func == NULL) { 990 VMM_CTR0(vcpu, "Sleeping during suspend"); 991 vcpu_require_state_locked(vcpu, VCPU_SLEEPING); 992 msleep_spin(vcpu, &vcpu->mtx, "vmsusp", hz); 993 vcpu_require_state_locked(vcpu, VCPU_FROZEN); 994 if (td_ast_pending(td, TDA_SUSPEND)) { 995 vcpu_unlock(vcpu); 996 error = thread_check_susp(td, false); 997 vcpu_lock(vcpu); 998 } 999 } else { 1000 VMM_CTR0(vcpu, "Rendezvous during suspend"); 1001 vcpu_unlock(vcpu); 1002 error = vm_handle_rendezvous(vcpu); 1003 vcpu_lock(vcpu); 1004 } 1005 } 1006 vcpu_unlock(vcpu); 1007 1008 /* 1009 * Wakeup the other sleeping vcpus and return to userspace. 1010 */ 1011 for (i = 0; i < vm->maxcpus; i++) { 1012 if (CPU_ISSET(i, &vm->suspended_cpus)) { 1013 vcpu_notify_event(vm_vcpu(vm, i)); 1014 } 1015 } 1016 1017 *retu = true; 1018 return (error); 1019 } 1020 1021 static int 1022 vm_handle_reqidle(struct vcpu *vcpu, bool *retu) 1023 { 1024 vcpu_lock(vcpu); 1025 KASSERT(vcpu->reqidle, ("invalid vcpu reqidle %d", vcpu->reqidle)); 1026 vcpu->reqidle = 0; 1027 vcpu_unlock(vcpu); 1028 *retu = true; 1029 return (0); 1030 } 1031 1032 static int 1033 vm_handle_db(struct vcpu *vcpu, struct vm_exit *vme, bool *retu) 1034 { 1035 int error, fault; 1036 uint64_t rsp; 1037 uint64_t rflags; 1038 struct vm_copyinfo copyinfo[2]; 1039 1040 *retu = true; 1041 if (!vme->u.dbg.pushf_intercept || vme->u.dbg.tf_shadow_val != 0) { 1042 return (0); 1043 } 1044 1045 vm_get_register(vcpu, VM_REG_GUEST_RSP, &rsp); 1046 error = vm_copy_setup(vcpu, &vme->u.dbg.paging, rsp, sizeof(uint64_t), 1047 VM_PROT_RW, copyinfo, nitems(copyinfo), &fault); 1048 if (error != 0 || fault != 0) { 1049 *retu = false; 1050 return (EINVAL); 1051 } 1052 1053 /* Read pushed rflags value from top of stack. */ 1054 vm_copyin(copyinfo, &rflags, sizeof(uint64_t)); 1055 1056 /* Clear TF bit. */ 1057 rflags &= ~(PSL_T); 1058 1059 /* Write updated value back to memory. */ 1060 vm_copyout(&rflags, copyinfo, sizeof(uint64_t)); 1061 vm_copy_teardown(copyinfo, nitems(copyinfo)); 1062 1063 return (0); 1064 } 1065 1066 void 1067 vm_exit_suspended(struct vcpu *vcpu, uint64_t rip) 1068 { 1069 struct vm *vm = vcpu->vm; 1070 struct vm_exit *vmexit; 1071 1072 KASSERT(vm->suspend > VM_SUSPEND_NONE && vm->suspend < VM_SUSPEND_LAST, 1073 ("vm_exit_suspended: invalid suspend type %d", vm->suspend)); 1074 1075 vmexit = vm_exitinfo(vcpu); 1076 vmexit->rip = rip; 1077 vmexit->inst_length = 0; 1078 vmexit->exitcode = VM_EXITCODE_SUSPENDED; 1079 vmexit->u.suspended.how = vm->suspend; 1080 } 1081 1082 void 1083 vm_exit_debug(struct vcpu *vcpu, uint64_t rip) 1084 { 1085 struct vm_exit *vmexit; 1086 1087 vmexit = vm_exitinfo(vcpu); 1088 vmexit->rip = rip; 1089 vmexit->inst_length = 0; 1090 vmexit->exitcode = VM_EXITCODE_DEBUG; 1091 } 1092 1093 void 1094 vm_exit_rendezvous(struct vcpu *vcpu, uint64_t rip) 1095 { 1096 struct vm_exit *vmexit; 1097 1098 vmexit = vm_exitinfo(vcpu); 1099 vmexit->rip = rip; 1100 vmexit->inst_length = 0; 1101 vmexit->exitcode = VM_EXITCODE_RENDEZVOUS; 1102 vmm_stat_incr(vcpu, VMEXIT_RENDEZVOUS, 1); 1103 } 1104 1105 void 1106 vm_exit_reqidle(struct vcpu *vcpu, uint64_t rip) 1107 { 1108 struct vm_exit *vmexit; 1109 1110 vmexit = vm_exitinfo(vcpu); 1111 vmexit->rip = rip; 1112 vmexit->inst_length = 0; 1113 vmexit->exitcode = VM_EXITCODE_REQIDLE; 1114 vmm_stat_incr(vcpu, VMEXIT_REQIDLE, 1); 1115 } 1116 1117 void 1118 vm_exit_astpending(struct vcpu *vcpu, uint64_t rip) 1119 { 1120 struct vm_exit *vmexit; 1121 1122 vmexit = vm_exitinfo(vcpu); 1123 vmexit->rip = rip; 1124 vmexit->inst_length = 0; 1125 vmexit->exitcode = VM_EXITCODE_BOGUS; 1126 vmm_stat_incr(vcpu, VMEXIT_ASTPENDING, 1); 1127 } 1128 1129 int 1130 vm_run(struct vcpu *vcpu) 1131 { 1132 struct vm *vm = vcpu->vm; 1133 struct vm_eventinfo evinfo; 1134 int error, vcpuid; 1135 struct pcb *pcb; 1136 uint64_t tscval; 1137 struct vm_exit *vme; 1138 bool retu, intr_disabled; 1139 pmap_t pmap; 1140 1141 vcpuid = vcpu->vcpuid; 1142 1143 if (!CPU_ISSET(vcpuid, &vm->active_cpus)) 1144 return (EINVAL); 1145 1146 if (CPU_ISSET(vcpuid, &vm->suspended_cpus)) 1147 return (EINVAL); 1148 1149 pmap = vmspace_pmap(vm_vmspace(vm)); 1150 vme = &vcpu->exitinfo; 1151 evinfo.rptr = &vm->rendezvous_req_cpus; 1152 evinfo.sptr = &vm->suspend; 1153 evinfo.iptr = &vcpu->reqidle; 1154 restart: 1155 critical_enter(); 1156 1157 KASSERT(!CPU_ISSET(curcpu, &pmap->pm_active), 1158 ("vm_run: absurd pm_active")); 1159 1160 tscval = rdtsc(); 1161 1162 pcb = PCPU_GET(curpcb); 1163 set_pcb_flags(pcb, PCB_FULL_IRET); 1164 1165 restore_guest_fpustate(vcpu); 1166 1167 vcpu_require_state(vcpu, VCPU_RUNNING); 1168 error = vmmops_run(vcpu->cookie, vcpu->nextrip, pmap, &evinfo); 1169 vcpu_require_state(vcpu, VCPU_FROZEN); 1170 1171 save_guest_fpustate(vcpu); 1172 1173 vmm_stat_incr(vcpu, VCPU_TOTAL_RUNTIME, rdtsc() - tscval); 1174 1175 critical_exit(); 1176 1177 if (error == 0) { 1178 retu = false; 1179 vcpu->nextrip = vme->rip + vme->inst_length; 1180 switch (vme->exitcode) { 1181 case VM_EXITCODE_REQIDLE: 1182 error = vm_handle_reqidle(vcpu, &retu); 1183 break; 1184 case VM_EXITCODE_SUSPENDED: 1185 error = vm_handle_suspend(vcpu, &retu); 1186 break; 1187 case VM_EXITCODE_IOAPIC_EOI: 1188 vioapic_process_eoi(vm, vme->u.ioapic_eoi.vector); 1189 break; 1190 case VM_EXITCODE_RENDEZVOUS: 1191 error = vm_handle_rendezvous(vcpu); 1192 break; 1193 case VM_EXITCODE_HLT: 1194 intr_disabled = ((vme->u.hlt.rflags & PSL_I) == 0); 1195 error = vm_handle_hlt(vcpu, intr_disabled, &retu); 1196 break; 1197 case VM_EXITCODE_PAGING: 1198 error = vm_handle_paging(vcpu, &retu); 1199 break; 1200 case VM_EXITCODE_INST_EMUL: 1201 error = vm_handle_inst_emul(vcpu, &retu); 1202 break; 1203 case VM_EXITCODE_INOUT: 1204 case VM_EXITCODE_INOUT_STR: 1205 error = vm_handle_inout(vcpu, vme, &retu); 1206 break; 1207 case VM_EXITCODE_DB: 1208 error = vm_handle_db(vcpu, vme, &retu); 1209 break; 1210 case VM_EXITCODE_MONITOR: 1211 case VM_EXITCODE_MWAIT: 1212 case VM_EXITCODE_VMINSN: 1213 vm_inject_ud(vcpu); 1214 break; 1215 default: 1216 retu = true; /* handled in userland */ 1217 break; 1218 } 1219 } 1220 1221 /* 1222 * VM_EXITCODE_INST_EMUL could access the apic which could transform the 1223 * exit code into VM_EXITCODE_IPI. 1224 */ 1225 if (error == 0 && vme->exitcode == VM_EXITCODE_IPI) 1226 error = vm_handle_ipi(vcpu, vme, &retu); 1227 1228 if (error == 0 && retu == false) 1229 goto restart; 1230 1231 vmm_stat_incr(vcpu, VMEXIT_USERSPACE, 1); 1232 VMM_CTR2(vcpu, "retu %d/%d", error, vme->exitcode); 1233 1234 return (error); 1235 } 1236 1237 int 1238 vm_restart_instruction(struct vcpu *vcpu) 1239 { 1240 enum vcpu_state state; 1241 uint64_t rip; 1242 int error __diagused; 1243 1244 state = vcpu_get_state(vcpu, NULL); 1245 if (state == VCPU_RUNNING) { 1246 /* 1247 * When a vcpu is "running" the next instruction is determined 1248 * by adding 'rip' and 'inst_length' in the vcpu's 'exitinfo'. 1249 * Thus setting 'inst_length' to zero will cause the current 1250 * instruction to be restarted. 1251 */ 1252 vcpu->exitinfo.inst_length = 0; 1253 VMM_CTR1(vcpu, "restarting instruction at %#lx by " 1254 "setting inst_length to zero", vcpu->exitinfo.rip); 1255 } else if (state == VCPU_FROZEN) { 1256 /* 1257 * When a vcpu is "frozen" it is outside the critical section 1258 * around vmmops_run() and 'nextrip' points to the next 1259 * instruction. Thus instruction restart is achieved by setting 1260 * 'nextrip' to the vcpu's %rip. 1261 */ 1262 error = vm_get_register(vcpu, VM_REG_GUEST_RIP, &rip); 1263 KASSERT(!error, ("%s: error %d getting rip", __func__, error)); 1264 VMM_CTR2(vcpu, "restarting instruction by updating " 1265 "nextrip from %#lx to %#lx", vcpu->nextrip, rip); 1266 vcpu->nextrip = rip; 1267 } else { 1268 panic("%s: invalid state %d", __func__, state); 1269 } 1270 return (0); 1271 } 1272 1273 int 1274 vm_exit_intinfo(struct vcpu *vcpu, uint64_t info) 1275 { 1276 int type, vector; 1277 1278 if (info & VM_INTINFO_VALID) { 1279 type = info & VM_INTINFO_TYPE; 1280 vector = info & 0xff; 1281 if (type == VM_INTINFO_NMI && vector != IDT_NMI) 1282 return (EINVAL); 1283 if (type == VM_INTINFO_HWEXCEPTION && vector >= 32) 1284 return (EINVAL); 1285 if (info & VM_INTINFO_RSVD) 1286 return (EINVAL); 1287 } else { 1288 info = 0; 1289 } 1290 VMM_CTR2(vcpu, "%s: info1(%#lx)", __func__, info); 1291 vcpu->exitintinfo = info; 1292 return (0); 1293 } 1294 1295 enum exc_class { 1296 EXC_BENIGN, 1297 EXC_CONTRIBUTORY, 1298 EXC_PAGEFAULT 1299 }; 1300 1301 #define IDT_VE 20 /* Virtualization Exception (Intel specific) */ 1302 1303 static enum exc_class 1304 exception_class(uint64_t info) 1305 { 1306 int type, vector; 1307 1308 KASSERT(info & VM_INTINFO_VALID, ("intinfo must be valid: %#lx", info)); 1309 type = info & VM_INTINFO_TYPE; 1310 vector = info & 0xff; 1311 1312 /* Table 6-4, "Interrupt and Exception Classes", Intel SDM, Vol 3 */ 1313 switch (type) { 1314 case VM_INTINFO_HWINTR: 1315 case VM_INTINFO_SWINTR: 1316 case VM_INTINFO_NMI: 1317 return (EXC_BENIGN); 1318 default: 1319 /* 1320 * Hardware exception. 1321 * 1322 * SVM and VT-x use identical type values to represent NMI, 1323 * hardware interrupt and software interrupt. 1324 * 1325 * SVM uses type '3' for all exceptions. VT-x uses type '3' 1326 * for exceptions except #BP and #OF. #BP and #OF use a type 1327 * value of '5' or '6'. Therefore we don't check for explicit 1328 * values of 'type' to classify 'intinfo' into a hardware 1329 * exception. 1330 */ 1331 break; 1332 } 1333 1334 switch (vector) { 1335 case IDT_PF: 1336 case IDT_VE: 1337 return (EXC_PAGEFAULT); 1338 case IDT_DE: 1339 case IDT_TS: 1340 case IDT_NP: 1341 case IDT_SS: 1342 case IDT_GP: 1343 return (EXC_CONTRIBUTORY); 1344 default: 1345 return (EXC_BENIGN); 1346 } 1347 } 1348 1349 static int 1350 nested_fault(struct vcpu *vcpu, uint64_t info1, uint64_t info2, 1351 uint64_t *retinfo) 1352 { 1353 enum exc_class exc1, exc2; 1354 int type1, vector1; 1355 1356 KASSERT(info1 & VM_INTINFO_VALID, ("info1 %#lx is not valid", info1)); 1357 KASSERT(info2 & VM_INTINFO_VALID, ("info2 %#lx is not valid", info2)); 1358 1359 /* 1360 * If an exception occurs while attempting to call the double-fault 1361 * handler the processor enters shutdown mode (aka triple fault). 1362 */ 1363 type1 = info1 & VM_INTINFO_TYPE; 1364 vector1 = info1 & 0xff; 1365 if (type1 == VM_INTINFO_HWEXCEPTION && vector1 == IDT_DF) { 1366 VMM_CTR2(vcpu, "triple fault: info1(%#lx), info2(%#lx)", 1367 info1, info2); 1368 vm_suspend(vcpu->vm, VM_SUSPEND_TRIPLEFAULT); 1369 *retinfo = 0; 1370 return (0); 1371 } 1372 1373 /* 1374 * Table 6-5 "Conditions for Generating a Double Fault", Intel SDM, Vol3 1375 */ 1376 exc1 = exception_class(info1); 1377 exc2 = exception_class(info2); 1378 if ((exc1 == EXC_CONTRIBUTORY && exc2 == EXC_CONTRIBUTORY) || 1379 (exc1 == EXC_PAGEFAULT && exc2 != EXC_BENIGN)) { 1380 /* Convert nested fault into a double fault. */ 1381 *retinfo = IDT_DF; 1382 *retinfo |= VM_INTINFO_VALID | VM_INTINFO_HWEXCEPTION; 1383 *retinfo |= VM_INTINFO_DEL_ERRCODE; 1384 } else { 1385 /* Handle exceptions serially */ 1386 *retinfo = info2; 1387 } 1388 return (1); 1389 } 1390 1391 static uint64_t 1392 vcpu_exception_intinfo(struct vcpu *vcpu) 1393 { 1394 uint64_t info = 0; 1395 1396 if (vcpu->exception_pending) { 1397 info = vcpu->exc_vector & 0xff; 1398 info |= VM_INTINFO_VALID | VM_INTINFO_HWEXCEPTION; 1399 if (vcpu->exc_errcode_valid) { 1400 info |= VM_INTINFO_DEL_ERRCODE; 1401 info |= (uint64_t)vcpu->exc_errcode << 32; 1402 } 1403 } 1404 return (info); 1405 } 1406 1407 int 1408 vm_entry_intinfo(struct vcpu *vcpu, uint64_t *retinfo) 1409 { 1410 uint64_t info1, info2; 1411 int valid; 1412 1413 info1 = vcpu->exitintinfo; 1414 vcpu->exitintinfo = 0; 1415 1416 info2 = 0; 1417 if (vcpu->exception_pending) { 1418 info2 = vcpu_exception_intinfo(vcpu); 1419 vcpu->exception_pending = 0; 1420 VMM_CTR2(vcpu, "Exception %d delivered: %#lx", 1421 vcpu->exc_vector, info2); 1422 } 1423 1424 if ((info1 & VM_INTINFO_VALID) && (info2 & VM_INTINFO_VALID)) { 1425 valid = nested_fault(vcpu, info1, info2, retinfo); 1426 } else if (info1 & VM_INTINFO_VALID) { 1427 *retinfo = info1; 1428 valid = 1; 1429 } else if (info2 & VM_INTINFO_VALID) { 1430 *retinfo = info2; 1431 valid = 1; 1432 } else { 1433 valid = 0; 1434 } 1435 1436 if (valid) { 1437 VMM_CTR4(vcpu, "%s: info1(%#lx), info2(%#lx), " 1438 "retinfo(%#lx)", __func__, info1, info2, *retinfo); 1439 } 1440 1441 return (valid); 1442 } 1443 1444 int 1445 vm_get_intinfo(struct vcpu *vcpu, uint64_t *info1, uint64_t *info2) 1446 { 1447 *info1 = vcpu->exitintinfo; 1448 *info2 = vcpu_exception_intinfo(vcpu); 1449 return (0); 1450 } 1451 1452 int 1453 vm_inject_exception(struct vcpu *vcpu, int vector, int errcode_valid, 1454 uint32_t errcode, int restart_instruction) 1455 { 1456 uint64_t regval; 1457 int error __diagused; 1458 1459 if (vector < 0 || vector >= 32) 1460 return (EINVAL); 1461 1462 /* 1463 * A double fault exception should never be injected directly into 1464 * the guest. It is a derived exception that results from specific 1465 * combinations of nested faults. 1466 */ 1467 if (vector == IDT_DF) 1468 return (EINVAL); 1469 1470 if (vcpu->exception_pending) { 1471 VMM_CTR2(vcpu, "Unable to inject exception %d due to " 1472 "pending exception %d", vector, vcpu->exc_vector); 1473 return (EBUSY); 1474 } 1475 1476 if (errcode_valid) { 1477 /* 1478 * Exceptions don't deliver an error code in real mode. 1479 */ 1480 error = vm_get_register(vcpu, VM_REG_GUEST_CR0, ®val); 1481 KASSERT(!error, ("%s: error %d getting CR0", __func__, error)); 1482 if (!(regval & CR0_PE)) 1483 errcode_valid = 0; 1484 } 1485 1486 /* 1487 * From section 26.6.1 "Interruptibility State" in Intel SDM: 1488 * 1489 * Event blocking by "STI" or "MOV SS" is cleared after guest executes 1490 * one instruction or incurs an exception. 1491 */ 1492 error = vm_set_register(vcpu, VM_REG_GUEST_INTR_SHADOW, 0); 1493 KASSERT(error == 0, ("%s: error %d clearing interrupt shadow", 1494 __func__, error)); 1495 1496 if (restart_instruction) 1497 vm_restart_instruction(vcpu); 1498 1499 vcpu->exception_pending = 1; 1500 vcpu->exc_vector = vector; 1501 vcpu->exc_errcode = errcode; 1502 vcpu->exc_errcode_valid = errcode_valid; 1503 VMM_CTR1(vcpu, "Exception %d pending", vector); 1504 return (0); 1505 } 1506 1507 void 1508 vm_inject_fault(struct vcpu *vcpu, int vector, int errcode_valid, int errcode) 1509 { 1510 int error __diagused, restart_instruction; 1511 1512 restart_instruction = 1; 1513 1514 error = vm_inject_exception(vcpu, vector, errcode_valid, 1515 errcode, restart_instruction); 1516 KASSERT(error == 0, ("vm_inject_exception error %d", error)); 1517 } 1518 1519 void 1520 vm_inject_pf(struct vcpu *vcpu, int error_code, uint64_t cr2) 1521 { 1522 int error __diagused; 1523 1524 VMM_CTR2(vcpu, "Injecting page fault: error_code %#x, cr2 %#lx", 1525 error_code, cr2); 1526 1527 error = vm_set_register(vcpu, VM_REG_GUEST_CR2, cr2); 1528 KASSERT(error == 0, ("vm_set_register(cr2) error %d", error)); 1529 1530 vm_inject_fault(vcpu, IDT_PF, 1, error_code); 1531 } 1532 1533 static VMM_STAT(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu"); 1534 1535 int 1536 vm_inject_nmi(struct vcpu *vcpu) 1537 { 1538 1539 vcpu->nmi_pending = 1; 1540 vcpu_notify_event(vcpu); 1541 return (0); 1542 } 1543 1544 int 1545 vm_nmi_pending(struct vcpu *vcpu) 1546 { 1547 return (vcpu->nmi_pending); 1548 } 1549 1550 void 1551 vm_nmi_clear(struct vcpu *vcpu) 1552 { 1553 if (vcpu->nmi_pending == 0) 1554 panic("vm_nmi_clear: inconsistent nmi_pending state"); 1555 1556 vcpu->nmi_pending = 0; 1557 vmm_stat_incr(vcpu, VCPU_NMI_COUNT, 1); 1558 } 1559 1560 static VMM_STAT(VCPU_EXTINT_COUNT, "number of ExtINTs delivered to vcpu"); 1561 1562 int 1563 vm_inject_extint(struct vcpu *vcpu) 1564 { 1565 1566 vcpu->extint_pending = 1; 1567 vcpu_notify_event(vcpu); 1568 return (0); 1569 } 1570 1571 int 1572 vm_extint_pending(struct vcpu *vcpu) 1573 { 1574 return (vcpu->extint_pending); 1575 } 1576 1577 void 1578 vm_extint_clear(struct vcpu *vcpu) 1579 { 1580 if (vcpu->extint_pending == 0) 1581 panic("vm_extint_clear: inconsistent extint_pending state"); 1582 1583 vcpu->extint_pending = 0; 1584 vmm_stat_incr(vcpu, VCPU_EXTINT_COUNT, 1); 1585 } 1586 1587 int 1588 vm_get_capability(struct vcpu *vcpu, int type, int *retval) 1589 { 1590 if (type < 0 || type >= VM_CAP_MAX) 1591 return (EINVAL); 1592 1593 return (vmmops_getcap(vcpu->cookie, type, retval)); 1594 } 1595 1596 int 1597 vm_set_capability(struct vcpu *vcpu, int type, int val) 1598 { 1599 if (type < 0 || type >= VM_CAP_MAX) 1600 return (EINVAL); 1601 1602 return (vmmops_setcap(vcpu->cookie, type, val)); 1603 } 1604 1605 struct vlapic * 1606 vm_lapic(struct vcpu *vcpu) 1607 { 1608 return (vcpu->vlapic); 1609 } 1610 1611 struct vioapic * 1612 vm_ioapic(struct vm *vm) 1613 { 1614 1615 return (vm->vioapic); 1616 } 1617 1618 struct vhpet * 1619 vm_hpet(struct vm *vm) 1620 { 1621 1622 return (vm->vhpet); 1623 } 1624 1625 bool 1626 vmm_is_pptdev(int bus, int slot, int func) 1627 { 1628 int b, f, i, n, s; 1629 char *val, *cp, *cp2; 1630 bool found; 1631 1632 /* 1633 * XXX 1634 * The length of an environment variable is limited to 128 bytes which 1635 * puts an upper limit on the number of passthru devices that may be 1636 * specified using a single environment variable. 1637 * 1638 * Work around this by scanning multiple environment variable 1639 * names instead of a single one - yuck! 1640 */ 1641 const char *names[] = { "pptdevs", "pptdevs2", "pptdevs3", NULL }; 1642 1643 /* set pptdevs="1/2/3 4/5/6 7/8/9 10/11/12" */ 1644 found = false; 1645 for (i = 0; names[i] != NULL && !found; i++) { 1646 cp = val = kern_getenv(names[i]); 1647 while (cp != NULL && *cp != '\0') { 1648 if ((cp2 = strchr(cp, ' ')) != NULL) 1649 *cp2 = '\0'; 1650 1651 n = sscanf(cp, "%d/%d/%d", &b, &s, &f); 1652 if (n == 3 && bus == b && slot == s && func == f) { 1653 found = true; 1654 break; 1655 } 1656 1657 if (cp2 != NULL) 1658 *cp2++ = ' '; 1659 1660 cp = cp2; 1661 } 1662 freeenv(val); 1663 } 1664 return (found); 1665 } 1666 1667 void * 1668 vm_iommu_domain(struct vm *vm) 1669 { 1670 1671 return (vm->iommu); 1672 } 1673 1674 /* 1675 * Returns the subset of vCPUs in tostart that are awaiting startup. 1676 * These vCPUs are also marked as no longer awaiting startup. 1677 */ 1678 cpuset_t 1679 vm_start_cpus(struct vm *vm, const cpuset_t *tostart) 1680 { 1681 cpuset_t set; 1682 1683 mtx_lock(&vm->rendezvous_mtx); 1684 CPU_AND(&set, &vm->startup_cpus, tostart); 1685 CPU_ANDNOT(&vm->startup_cpus, &vm->startup_cpus, &set); 1686 mtx_unlock(&vm->rendezvous_mtx); 1687 return (set); 1688 } 1689 1690 void 1691 vm_await_start(struct vm *vm, const cpuset_t *waiting) 1692 { 1693 mtx_lock(&vm->rendezvous_mtx); 1694 CPU_OR(&vm->startup_cpus, &vm->startup_cpus, waiting); 1695 mtx_unlock(&vm->rendezvous_mtx); 1696 } 1697 1698 int 1699 vm_get_x2apic_state(struct vcpu *vcpu, enum x2apic_state *state) 1700 { 1701 *state = vcpu->x2apic_state; 1702 1703 return (0); 1704 } 1705 1706 int 1707 vm_set_x2apic_state(struct vcpu *vcpu, enum x2apic_state state) 1708 { 1709 if (state >= X2APIC_STATE_LAST) 1710 return (EINVAL); 1711 1712 vcpu->x2apic_state = state; 1713 1714 vlapic_set_x2apic_state(vcpu, state); 1715 1716 return (0); 1717 } 1718 1719 void 1720 vcpu_notify_lapic(struct vcpu *vcpu) 1721 { 1722 vcpu_lock(vcpu); 1723 if (vcpu->state == VCPU_RUNNING && vcpu->hostcpu != curcpu) 1724 vlapic_post_intr(vcpu->vlapic, vcpu->hostcpu, vmm_ipinum); 1725 else 1726 vcpu_notify_event_locked(vcpu); 1727 vcpu_unlock(vcpu); 1728 } 1729 1730 int 1731 vm_apicid2vcpuid(struct vm *vm, int apicid) 1732 { 1733 /* 1734 * XXX apic id is assumed to be numerically identical to vcpu id 1735 */ 1736 return (apicid); 1737 } 1738 1739 int 1740 vm_smp_rendezvous(struct vcpu *vcpu, cpuset_t dest, 1741 vm_rendezvous_func_t func, void *arg) 1742 { 1743 struct vm *vm = vcpu->vm; 1744 int error, i; 1745 1746 /* 1747 * Enforce that this function is called without any locks 1748 */ 1749 WITNESS_WARN(WARN_PANIC, NULL, "vm_smp_rendezvous"); 1750 1751 restart: 1752 mtx_lock(&vm->rendezvous_mtx); 1753 if (vm->rendezvous_func != NULL) { 1754 /* 1755 * If a rendezvous is already in progress then we need to 1756 * call the rendezvous handler in case this 'vcpu' is one 1757 * of the targets of the rendezvous. 1758 */ 1759 VMM_CTR0(vcpu, "Rendezvous already in progress"); 1760 mtx_unlock(&vm->rendezvous_mtx); 1761 error = vm_handle_rendezvous(vcpu); 1762 if (error != 0) 1763 return (error); 1764 goto restart; 1765 } 1766 KASSERT(vm->rendezvous_func == NULL, ("vm_smp_rendezvous: previous " 1767 "rendezvous is still in progress")); 1768 1769 VMM_CTR0(vcpu, "Initiating rendezvous"); 1770 vm->rendezvous_req_cpus = dest; 1771 CPU_ZERO(&vm->rendezvous_done_cpus); 1772 vm->rendezvous_arg = arg; 1773 vm->rendezvous_func = func; 1774 mtx_unlock(&vm->rendezvous_mtx); 1775 1776 /* 1777 * Wake up any sleeping vcpus and trigger a VM-exit in any running 1778 * vcpus so they handle the rendezvous as soon as possible. 1779 */ 1780 for (i = 0; i < vm->maxcpus; i++) { 1781 if (CPU_ISSET(i, &dest)) 1782 vcpu_notify_event(vm_vcpu(vm, i)); 1783 } 1784 1785 return (vm_handle_rendezvous(vcpu)); 1786 } 1787 1788 struct vatpic * 1789 vm_atpic(struct vm *vm) 1790 { 1791 return (vm->vatpic); 1792 } 1793 1794 struct vatpit * 1795 vm_atpit(struct vm *vm) 1796 { 1797 return (vm->vatpit); 1798 } 1799 1800 struct vpmtmr * 1801 vm_pmtmr(struct vm *vm) 1802 { 1803 1804 return (vm->vpmtmr); 1805 } 1806 1807 struct vrtc * 1808 vm_rtc(struct vm *vm) 1809 { 1810 1811 return (vm->vrtc); 1812 } 1813 1814 enum vm_reg_name 1815 vm_segment_name(int seg) 1816 { 1817 static enum vm_reg_name seg_names[] = { 1818 VM_REG_GUEST_ES, 1819 VM_REG_GUEST_CS, 1820 VM_REG_GUEST_SS, 1821 VM_REG_GUEST_DS, 1822 VM_REG_GUEST_FS, 1823 VM_REG_GUEST_GS 1824 }; 1825 1826 KASSERT(seg >= 0 && seg < nitems(seg_names), 1827 ("%s: invalid segment encoding %d", __func__, seg)); 1828 return (seg_names[seg]); 1829 } 1830 1831 void 1832 vm_copy_teardown(struct vm_copyinfo *copyinfo, int num_copyinfo) 1833 { 1834 int idx; 1835 1836 for (idx = 0; idx < num_copyinfo; idx++) { 1837 if (copyinfo[idx].cookie != NULL) 1838 vm_gpa_release(copyinfo[idx].cookie); 1839 } 1840 bzero(copyinfo, num_copyinfo * sizeof(struct vm_copyinfo)); 1841 } 1842 1843 int 1844 vm_copy_setup(struct vcpu *vcpu, struct vm_guest_paging *paging, 1845 uint64_t gla, size_t len, int prot, struct vm_copyinfo *copyinfo, 1846 int num_copyinfo, int *fault) 1847 { 1848 int error, idx, nused; 1849 size_t n, off, remaining; 1850 void *hva, *cookie; 1851 uint64_t gpa; 1852 1853 bzero(copyinfo, sizeof(struct vm_copyinfo) * num_copyinfo); 1854 1855 nused = 0; 1856 remaining = len; 1857 while (remaining > 0) { 1858 if (nused >= num_copyinfo) 1859 return (EFAULT); 1860 error = vm_gla2gpa(vcpu, paging, gla, prot, &gpa, fault); 1861 if (error || *fault) 1862 return (error); 1863 off = gpa & PAGE_MASK; 1864 n = min(remaining, PAGE_SIZE - off); 1865 copyinfo[nused].gpa = gpa; 1866 copyinfo[nused].len = n; 1867 remaining -= n; 1868 gla += n; 1869 nused++; 1870 } 1871 1872 for (idx = 0; idx < nused; idx++) { 1873 hva = vm_gpa_hold(vcpu, copyinfo[idx].gpa, 1874 copyinfo[idx].len, prot, &cookie); 1875 if (hva == NULL) 1876 break; 1877 copyinfo[idx].hva = hva; 1878 copyinfo[idx].cookie = cookie; 1879 } 1880 1881 if (idx != nused) { 1882 vm_copy_teardown(copyinfo, num_copyinfo); 1883 return (EFAULT); 1884 } else { 1885 *fault = 0; 1886 return (0); 1887 } 1888 } 1889 1890 void 1891 vm_copyin(struct vm_copyinfo *copyinfo, void *kaddr, size_t len) 1892 { 1893 char *dst; 1894 int idx; 1895 1896 dst = kaddr; 1897 idx = 0; 1898 while (len > 0) { 1899 bcopy(copyinfo[idx].hva, dst, copyinfo[idx].len); 1900 len -= copyinfo[idx].len; 1901 dst += copyinfo[idx].len; 1902 idx++; 1903 } 1904 } 1905 1906 void 1907 vm_copyout(const void *kaddr, struct vm_copyinfo *copyinfo, size_t len) 1908 { 1909 const char *src; 1910 int idx; 1911 1912 src = kaddr; 1913 idx = 0; 1914 while (len > 0) { 1915 bcopy(src, copyinfo[idx].hva, copyinfo[idx].len); 1916 len -= copyinfo[idx].len; 1917 src += copyinfo[idx].len; 1918 idx++; 1919 } 1920 } 1921 1922 /* 1923 * Return the amount of in-use and wired memory for the VM. Since 1924 * these are global stats, only return the values with for vCPU 0 1925 */ 1926 VMM_STAT_DECLARE(VMM_MEM_RESIDENT); 1927 VMM_STAT_DECLARE(VMM_MEM_WIRED); 1928 1929 static void 1930 vm_get_rescnt(struct vcpu *vcpu, struct vmm_stat_type *stat) 1931 { 1932 1933 if (vcpu->vcpuid == 0) { 1934 vmm_stat_set(vcpu, VMM_MEM_RESIDENT, PAGE_SIZE * 1935 vmspace_resident_count(vm_vmspace(vcpu->vm))); 1936 } 1937 } 1938 1939 static void 1940 vm_get_wiredcnt(struct vcpu *vcpu, struct vmm_stat_type *stat) 1941 { 1942 1943 if (vcpu->vcpuid == 0) { 1944 vmm_stat_set(vcpu, VMM_MEM_WIRED, PAGE_SIZE * 1945 pmap_wired_count(vmspace_pmap(vm_vmspace(vcpu->vm)))); 1946 } 1947 } 1948 1949 VMM_STAT_FUNC(VMM_MEM_RESIDENT, "Resident memory", vm_get_rescnt); 1950 VMM_STAT_FUNC(VMM_MEM_WIRED, "Wired memory", vm_get_wiredcnt); 1951 1952 #ifdef BHYVE_SNAPSHOT 1953 static int 1954 vm_snapshot_vcpus(struct vm *vm, struct vm_snapshot_meta *meta) 1955 { 1956 uint64_t tsc, now; 1957 int ret; 1958 struct vcpu *vcpu; 1959 uint16_t i, maxcpus; 1960 1961 now = rdtsc(); 1962 maxcpus = vm_get_maxcpus(vm); 1963 for (i = 0; i < maxcpus; i++) { 1964 vcpu = vm->vcpu[i]; 1965 if (vcpu == NULL) 1966 continue; 1967 1968 SNAPSHOT_VAR_OR_LEAVE(vcpu->x2apic_state, meta, ret, done); 1969 SNAPSHOT_VAR_OR_LEAVE(vcpu->exitintinfo, meta, ret, done); 1970 SNAPSHOT_VAR_OR_LEAVE(vcpu->exc_vector, meta, ret, done); 1971 SNAPSHOT_VAR_OR_LEAVE(vcpu->exc_errcode_valid, meta, ret, done); 1972 SNAPSHOT_VAR_OR_LEAVE(vcpu->exc_errcode, meta, ret, done); 1973 SNAPSHOT_VAR_OR_LEAVE(vcpu->guest_xcr0, meta, ret, done); 1974 SNAPSHOT_VAR_OR_LEAVE(vcpu->exitinfo, meta, ret, done); 1975 SNAPSHOT_VAR_OR_LEAVE(vcpu->nextrip, meta, ret, done); 1976 1977 /* 1978 * Save the absolute TSC value by adding now to tsc_offset. 1979 * 1980 * It will be turned turned back into an actual offset when the 1981 * TSC restore function is called 1982 */ 1983 tsc = now + vcpu->tsc_offset; 1984 SNAPSHOT_VAR_OR_LEAVE(tsc, meta, ret, done); 1985 if (meta->op == VM_SNAPSHOT_RESTORE) 1986 vcpu->tsc_offset = tsc; 1987 } 1988 1989 done: 1990 return (ret); 1991 } 1992 1993 static int 1994 vm_snapshot_vm(struct vm *vm, struct vm_snapshot_meta *meta) 1995 { 1996 int ret; 1997 1998 ret = vm_snapshot_vcpus(vm, meta); 1999 if (ret != 0) 2000 goto done; 2001 2002 SNAPSHOT_VAR_OR_LEAVE(vm->startup_cpus, meta, ret, done); 2003 done: 2004 return (ret); 2005 } 2006 2007 static int 2008 vm_snapshot_vcpu(struct vm *vm, struct vm_snapshot_meta *meta) 2009 { 2010 int error; 2011 struct vcpu *vcpu; 2012 uint16_t i, maxcpus; 2013 2014 error = 0; 2015 2016 maxcpus = vm_get_maxcpus(vm); 2017 for (i = 0; i < maxcpus; i++) { 2018 vcpu = vm->vcpu[i]; 2019 if (vcpu == NULL) 2020 continue; 2021 2022 error = vmmops_vcpu_snapshot(vcpu->cookie, meta); 2023 if (error != 0) { 2024 printf("%s: failed to snapshot vmcs/vmcb data for " 2025 "vCPU: %d; error: %d\n", __func__, i, error); 2026 goto done; 2027 } 2028 } 2029 2030 done: 2031 return (error); 2032 } 2033 2034 /* 2035 * Save kernel-side structures to user-space for snapshotting. 2036 */ 2037 int 2038 vm_snapshot_req(struct vm *vm, struct vm_snapshot_meta *meta) 2039 { 2040 int ret = 0; 2041 2042 switch (meta->dev_req) { 2043 case STRUCT_VMCX: 2044 ret = vm_snapshot_vcpu(vm, meta); 2045 break; 2046 case STRUCT_VM: 2047 ret = vm_snapshot_vm(vm, meta); 2048 break; 2049 case STRUCT_VIOAPIC: 2050 ret = vioapic_snapshot(vm_ioapic(vm), meta); 2051 break; 2052 case STRUCT_VLAPIC: 2053 ret = vlapic_snapshot(vm, meta); 2054 break; 2055 case STRUCT_VHPET: 2056 ret = vhpet_snapshot(vm_hpet(vm), meta); 2057 break; 2058 case STRUCT_VATPIC: 2059 ret = vatpic_snapshot(vm_atpic(vm), meta); 2060 break; 2061 case STRUCT_VATPIT: 2062 ret = vatpit_snapshot(vm_atpit(vm), meta); 2063 break; 2064 case STRUCT_VPMTMR: 2065 ret = vpmtmr_snapshot(vm_pmtmr(vm), meta); 2066 break; 2067 case STRUCT_VRTC: 2068 ret = vrtc_snapshot(vm_rtc(vm), meta); 2069 break; 2070 default: 2071 printf("%s: failed to find the requested type %#x\n", 2072 __func__, meta->dev_req); 2073 ret = (EINVAL); 2074 } 2075 return (ret); 2076 } 2077 2078 void 2079 vm_set_tsc_offset(struct vcpu *vcpu, uint64_t offset) 2080 { 2081 vcpu->tsc_offset = offset; 2082 } 2083 2084 int 2085 vm_restore_time(struct vm *vm) 2086 { 2087 int error; 2088 uint64_t now; 2089 struct vcpu *vcpu; 2090 uint16_t i, maxcpus; 2091 2092 now = rdtsc(); 2093 2094 error = vhpet_restore_time(vm_hpet(vm)); 2095 if (error) 2096 return (error); 2097 2098 maxcpus = vm_get_maxcpus(vm); 2099 for (i = 0; i < maxcpus; i++) { 2100 vcpu = vm->vcpu[i]; 2101 if (vcpu == NULL) 2102 continue; 2103 2104 error = vmmops_restore_tsc(vcpu->cookie, 2105 vcpu->tsc_offset - now); 2106 if (error) 2107 return (error); 2108 } 2109 2110 return (0); 2111 } 2112 #endif 2113