Lines Matching +full:gpa +full:- +full:0
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
91 void *cookie; /* (i) cpu-specific data */
95 #define vcpu_lock_initialized(v) mtx_initialized(&((v)->mtx))
96 #define vcpu_lock_init(v) mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN)
97 #define vcpu_lock_destroy(v) mtx_destroy(&((v)->mtx))
98 #define vcpu_lock(v) mtx_lock_spin(&((v)->mtx))
99 #define vcpu_unlock(v) mtx_unlock_spin(&((v)->mtx))
100 #define vcpu_assert_locked(v) mtx_assert(&((v)->mtx), MA_OWNED)
117 void *cookie; /* (i) cpu-specific data */
148 SYSCTL_INT(_hw_vmm, OID_AUTO, ipinum, CTLFLAG_RD, &vmm_ipinum, 0,
153 &vm_maxcpu, 0, "Maximum number of vCPUs");
166 #define VM_MAXCPU MIN(0xffff - 1, CPU_SETSIZE)
171 vmmops_vcpu_cleanup(vcpu->cookie); in vcpu_cleanup()
172 vcpu->cookie = NULL; in vcpu_cleanup()
174 vmm_stat_free(vcpu->stats); in vcpu_cleanup()
175 fpu_save_area_free(vcpu->guestfpu); in vcpu_cleanup()
185 KASSERT(vcpu_id >= 0 && vcpu_id < vm->maxcpus, in vcpu_alloc()
190 vcpu->state = VCPU_IDLE; in vcpu_alloc()
191 vcpu->hostcpu = NOCPU; in vcpu_alloc()
192 vcpu->vcpuid = vcpu_id; in vcpu_alloc()
193 vcpu->vm = vm; in vcpu_alloc()
194 vcpu->guestfpu = fpu_save_area_alloc(); in vcpu_alloc()
195 vcpu->stats = vmm_stat_alloc(); in vcpu_alloc()
202 vcpu->cookie = vmmops_vcpu_init(vcpu->vm->cookie, vcpu, vcpu->vcpuid); in vcpu_init()
203 MPASS(vcpu->cookie != NULL); in vcpu_init()
204 fpu_save_area_reset(vcpu->guestfpu); in vcpu_init()
205 vmm_stat_init(vcpu->stats); in vcpu_init()
211 return (&vcpu->exitinfo); in vm_exitinfo()
227 if (vm_maxcpu == 0) in vmm_init()
241 if (error != 0) in vmm_handler()
244 if (error == 0) in vmm_handler()
251 if (error == 0 && vmm_initialized) { in vmm_handler()
255 * Something bad happened - prevent new in vmm_handler()
263 error = 0; in vmm_handler()
278 * - vmm device initialization requires an initialized devfs.
288 vm->cookie = vmmops_init(vm, vmspace_pmap(vm->vmspace)); in vm_init()
289 MPASS(vm->cookie != NULL); in vm_init()
291 CPU_ZERO(&vm->active_cpus); in vm_init()
292 CPU_ZERO(&vm->debug_cpus); in vm_init()
294 vm->suspend = 0; in vm_init()
295 CPU_ZERO(&vm->suspended_cpus); in vm_init()
297 memset(vm->mmio_region, 0, sizeof(vm->mmio_region)); in vm_init()
300 for (i = 0; i < vm->maxcpus; i++) { in vm_init()
301 if (vm->vcpu[i] != NULL) in vm_init()
302 vcpu_init(vm->vcpu[i]); in vm_init()
310 sx_xlock(&vm->vcpus_init_lock); in vm_disable_vcpu_creation()
311 vm->dying = true; in vm_disable_vcpu_creation()
312 sx_xunlock(&vm->vcpus_init_lock); in vm_disable_vcpu_creation()
320 if (vcpuid < 0 || vcpuid >= vm_get_maxcpus(vm)) in vm_alloc_vcpu()
324 if (vcpuid >= aplic_max_cpu_count(vm->cookie)) in vm_alloc_vcpu()
328 atomic_load_acq_ptr((uintptr_t *)&vm->vcpu[vcpuid]); in vm_alloc_vcpu()
332 sx_xlock(&vm->vcpus_init_lock); in vm_alloc_vcpu()
333 vcpu = vm->vcpu[vcpuid]; in vm_alloc_vcpu()
334 if (vcpu == NULL && !vm->dying) { in vm_alloc_vcpu()
342 atomic_store_rel_ptr((uintptr_t *)&vm->vcpu[vcpuid], in vm_alloc_vcpu()
345 sx_xunlock(&vm->vcpus_init_lock); in vm_alloc_vcpu()
352 sx_slock(&vm->vcpus_init_lock); in vm_slock_vcpus()
358 sx_unlock(&vm->vcpus_init_lock); in vm_unlock_vcpus()
377 vmspace = vmmops_vmspace_alloc(0, 1ul << 39); in vm_create()
382 strcpy(vm->name, name); in vm_create()
383 vm->vmspace = vmspace; in vm_create()
384 vm_mem_init(&vm->mem); in vm_create()
385 sx_init(&vm->vcpus_init_lock, "vm vcpus"); in vm_create()
387 vm->sockets = 1; in vm_create()
388 vm->cores = 1; /* XXX backwards compatibility */ in vm_create()
389 vm->threads = 1; /* XXX backwards compatibility */ in vm_create()
390 vm->maxcpus = vm_maxcpu; in vm_create()
392 vm->vcpu = malloc(sizeof(*vm->vcpu) * vm->maxcpus, M_VMM, in vm_create()
398 return (0); in vm_create()
405 *sockets = vm->sockets; in vm_get_topology()
406 *cores = vm->cores; in vm_get_topology()
407 *threads = vm->threads; in vm_get_topology()
408 *maxcpus = vm->maxcpus; in vm_get_topology()
414 return (vm->maxcpus); in vm_get_maxcpus()
422 if ((sockets * cores * threads) > vm->maxcpus) in vm_set_topology()
424 vm->sockets = sockets; in vm_set_topology()
425 vm->cores = cores; in vm_set_topology()
426 vm->threads = threads; in vm_set_topology()
427 return(0); in vm_set_topology()
440 aplic_detach_from_vm(vm->cookie); in vm_cleanup()
442 for (i = 0; i < vm->maxcpus; i++) { in vm_cleanup()
443 if (vm->vcpu[i] != NULL) in vm_cleanup()
444 vcpu_cleanup(vm->vcpu[i], destroy); in vm_cleanup()
447 vmmops_cleanup(vm->cookie); in vm_cleanup()
453 vmmops_vmspace_free(vm->vmspace); in vm_cleanup()
454 vm->vmspace = NULL; in vm_cleanup()
456 for (i = 0; i < vm->maxcpus; i++) in vm_cleanup()
457 free(vm->vcpu[i], M_VMM); in vm_cleanup()
458 free(vm->vcpu, M_VMM); in vm_cleanup()
459 sx_destroy(&vm->vcpus_init_lock); in vm_cleanup()
480 if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) { in vm_reinit()
483 error = 0; in vm_reinit()
494 return (vm->name); in vm_name()
499 uint64_t gla, int prot, uint64_t *gpa, int *is_fault) in vm_gla2gpa_nofault() argument
501 return (vmmops_gla2gpa(vcpu->cookie, paging, gla, prot, gpa, is_fault)); in vm_gla2gpa_nofault()
510 for (i = 0; i < nitems(vm->mmio_region); i++) { in vm_register_inst_handler()
511 if (vm->mmio_region[i].start == 0 && in vm_register_inst_handler()
512 vm->mmio_region[i].end == 0) { in vm_register_inst_handler()
513 vm->mmio_region[i].start = start; in vm_register_inst_handler()
514 vm->mmio_region[i].end = start + size; in vm_register_inst_handler()
515 vm->mmio_region[i].read = mmio_read; in vm_register_inst_handler()
516 vm->mmio_region[i].write = mmio_write; in vm_register_inst_handler()
529 for (i = 0; i < nitems(vm->mmio_region); i++) { in vm_deregister_inst_handler()
530 if (vm->mmio_region[i].start == start && in vm_deregister_inst_handler()
531 vm->mmio_region[i].end == start + size) { in vm_deregister_inst_handler()
532 memset(&vm->mmio_region[i], 0, in vm_deregister_inst_handler()
533 sizeof(vm->mmio_region[i])); in vm_deregister_inst_handler()
538 panic("%s: Invalid MMIO region: %lx - %lx", __func__, start, in vm_deregister_inst_handler()
554 vm = vcpu->vm; in vm_handle_inst_emul()
555 hyp = vm->cookie; in vm_handle_inst_emul()
556 if (!hyp->aplic_attached) in vm_handle_inst_emul()
559 vme = &vcpu->exitinfo; in vm_handle_inst_emul()
560 vie = &vme->u.inst_emul.vie; in vm_handle_inst_emul()
561 paging = &vme->u.inst_emul.paging; in vm_handle_inst_emul()
563 fault_ipa = vme->u.inst_emul.gpa; in vm_handle_inst_emul()
566 for (i = 0; i < nitems(vm->mmio_region); i++) { in vm_handle_inst_emul()
567 if (vm->mmio_region[i].start <= fault_ipa && in vm_handle_inst_emul()
568 vm->mmio_region[i].end > fault_ipa) { in vm_handle_inst_emul()
569 vmr = &vm->mmio_region[i]; in vm_handle_inst_emul()
577 vmr->read, vmr->write, retu); in vm_handle_inst_emul()
582 return (0); in vm_handle_inst_emul()
593 if (atomic_cmpset_int(&vm->suspend, 0, how) == 0) { in vm_suspend()
595 vm->suspend, how); in vm_suspend()
604 for (i = 0; i < vm->maxcpus; i++) { in vm_suspend()
605 if (CPU_ISSET(i, &vm->active_cpus)) in vm_suspend()
609 return (0); in vm_suspend()
615 struct vm *vm = vcpu->vm; in vm_exit_suspended()
618 KASSERT(vm->suspend > VM_SUSPEND_NONE && vm->suspend < VM_SUSPEND_LAST, in vm_exit_suspended()
619 ("vm_exit_suspended: invalid suspend type %d", vm->suspend)); in vm_exit_suspended()
622 vmexit->pc = pc; in vm_exit_suspended()
623 vmexit->inst_length = 4; in vm_exit_suspended()
624 vmexit->exitcode = VM_EXITCODE_SUSPENDED; in vm_exit_suspended()
625 vmexit->u.suspended.how = vm->suspend; in vm_exit_suspended()
634 vmexit->pc = pc; in vm_exit_debug()
635 vmexit->inst_length = 4; in vm_exit_debug()
636 vmexit->exitcode = VM_EXITCODE_DEBUG; in vm_exit_debug()
642 struct vm *vm = vcpu->vm; in vm_activate_cpu()
644 if (CPU_ISSET(vcpu->vcpuid, &vm->active_cpus)) in vm_activate_cpu()
647 CPU_SET_ATOMIC(vcpu->vcpuid, &vm->active_cpus); in vm_activate_cpu()
648 return (0); in vm_activate_cpu()
656 vm->debug_cpus = vm->active_cpus; in vm_suspend_cpu()
657 for (int i = 0; i < vm->maxcpus; i++) { in vm_suspend_cpu()
658 if (CPU_ISSET(i, &vm->active_cpus)) in vm_suspend_cpu()
662 if (!CPU_ISSET(vcpu->vcpuid, &vm->active_cpus)) in vm_suspend_cpu()
665 CPU_SET_ATOMIC(vcpu->vcpuid, &vm->debug_cpus); in vm_suspend_cpu()
668 return (0); in vm_suspend_cpu()
676 CPU_ZERO(&vm->debug_cpus); in vm_resume_cpu()
678 if (!CPU_ISSET(vcpu->vcpuid, &vm->debug_cpus)) in vm_resume_cpu()
681 CPU_CLR_ATOMIC(vcpu->vcpuid, &vm->debug_cpus); in vm_resume_cpu()
683 return (0); in vm_resume_cpu()
690 return (CPU_ISSET(vcpu->vcpuid, &vcpu->vm->debug_cpus)); in vcpu_debugged()
697 return (vm->active_cpus); in vm_active_cpus()
704 return (vm->debug_cpus); in vm_debug_cpus()
711 return (vm->suspended_cpus); in vm_suspended_cpus()
719 return (vcpu->stats); in vcpu_stats()
725 * - If the vcpu thread is sleeping then it is woken up.
726 * - If the vcpu is running on a different host_cpu then an IPI will be directed
734 hostcpu = vcpu->hostcpu; in vcpu_notify_event_locked()
735 if (vcpu->state == VCPU_RUNNING) { in vcpu_notify_event_locked()
749 "with hostcpu %d", vcpu->state, hostcpu)); in vcpu_notify_event_locked()
750 if (vcpu->state == VCPU_SLEEPING) in vcpu_notify_event_locked()
766 return (vm->vmspace); in vm_vmspace()
772 return (&vm->mem); in vm_mem()
782 /* Ensure the VFP state will be re-loaded when exiting the guest. */ in restore_guest_fpustate()
787 fpe_restore(vcpu->guestfpu); in restore_guest_fpustate()
802 fpe_store(vcpu->guestfpu); in save_guest_fpustate()
823 while (vcpu->state != VCPU_IDLE) { in vcpu_set_state_locked()
825 msleep_spin(&vcpu->state, &vcpu->mtx, "vmstat", hz); in vcpu_set_state_locked()
828 KASSERT(vcpu->state != VCPU_IDLE, ("invalid transition from " in vcpu_set_state_locked()
832 if (vcpu->state == VCPU_RUNNING) { in vcpu_set_state_locked()
833 KASSERT(vcpu->hostcpu == curcpu, ("curcpu %d and hostcpu %d " in vcpu_set_state_locked()
834 "mismatch for running vcpu", curcpu, vcpu->hostcpu)); in vcpu_set_state_locked()
836 KASSERT(vcpu->hostcpu == NOCPU, ("Invalid hostcpu %d for a " in vcpu_set_state_locked()
837 "vcpu that is not running", vcpu->hostcpu)); in vcpu_set_state_locked()
842 * IDLE -> FROZEN -> IDLE in vcpu_set_state_locked()
843 * FROZEN -> RUNNING -> FROZEN in vcpu_set_state_locked()
844 * FROZEN -> SLEEPING -> FROZEN in vcpu_set_state_locked()
846 switch (vcpu->state) { in vcpu_set_state_locked()
863 vcpu->state = newstate; in vcpu_set_state_locked()
865 vcpu->hostcpu = curcpu; in vcpu_set_state_locked()
867 vcpu->hostcpu = NOCPU; in vcpu_set_state_locked()
870 wakeup(&vcpu->state); in vcpu_set_state_locked()
872 return (0); in vcpu_set_state_locked()
880 if ((error = vcpu_set_state(vcpu, newstate, false)) != 0) in vcpu_require_state()
889 if ((error = vcpu_set_state_locked(vcpu, newstate, false)) != 0) in vcpu_require_state_locked()
897 if (type < 0 || type >= VM_CAP_MAX) in vm_get_capability()
900 return (vmmops_getcap(vcpu->cookie, type, retval)); in vm_get_capability()
907 if (type < 0 || type >= VM_CAP_MAX) in vm_set_capability()
910 return (vmmops_setcap(vcpu->cookie, type, val)); in vm_set_capability()
917 return (vcpu->vm); in vcpu_vm()
924 return (vcpu->vcpuid); in vcpu_vcpuid()
931 return (vcpu->cookie); in vcpu_get_cookie()
938 return (vm->vcpu[vcpuid]); in vm_vcpu()
959 state = vcpu->state; in vcpu_get_state()
961 *hostcpu = vcpu->hostcpu; in vcpu_get_state()
974 return (vmmops_getreg(vcpu->cookie, reg, retval)); in vm_get_register()
984 error = vmmops_setreg(vcpu->cookie, reg, val); in vm_set_register()
988 vcpu->nextpc = val; in vm_set_register()
990 return (0); in vm_set_register()
997 return (vm->cookie); in vm_get_cookie()
1004 return (vmmops_exception(vcpu->cookie, scause)); in vm_inject_exception()
1011 return (aplic_attach_to_vm(vm->cookie, descr)); in vm_attach_aplic()
1018 return (aplic_inject_irq(vm->cookie, -1, irq, true)); in vm_assert_irq()
1025 return (aplic_inject_irq(vm->cookie, -1, irq, false)); in vm_deassert_irq()
1033 return (aplic_inject_msi(vm->cookie, msg, addr)); in vm_raise_msi()
1043 if (aplic_check_pending(vcpu->cookie)) in vm_handle_wfi()
1046 if (riscv_check_ipi(vcpu->cookie, false)) in vm_handle_wfi()
1049 if (riscv_check_interrupts_pending(vcpu->cookie)) in vm_handle_wfi()
1060 msleep_spin(vcpu, &vcpu->mtx, "vmidle", hz); in vm_handle_wfi()
1067 return (0); in vm_handle_wfi()
1080 vm = vcpu->vm; in vm_handle_paging()
1081 vme = &vcpu->exitinfo; in vm_handle_paging()
1083 pmap = vmspace_pmap(vm->vmspace); in vm_handle_paging()
1084 addr = (vme->htval << 2) & ~(PAGE_SIZE - 1); in vm_handle_paging()
1088 switch (vme->scause) { in vm_handle_paging()
1099 panic("unknown page trap: %lu", vme->scause); in vm_handle_paging()
1104 return (0); in vm_handle_paging()
1106 map = &vm->vmspace->vm_map; in vm_handle_paging()
1114 return (0); in vm_handle_paging()
1120 struct vm *vm = vcpu->vm; in vm_handle_suspend()
1124 error = 0; in vm_handle_suspend()
1127 CPU_SET_ATOMIC(vcpu->vcpuid, &vm->suspended_cpus); in vm_handle_suspend()
1137 while (error == 0) { in vm_handle_suspend()
1138 if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) in vm_handle_suspend()
1142 msleep_spin(vcpu, &vcpu->mtx, "vmsusp", hz); in vm_handle_suspend()
1155 for (i = 0; i < vm->maxcpus; i++) { in vm_handle_suspend()
1156 if (CPU_ISSET(i, &vm->suspended_cpus)) { in vm_handle_suspend()
1176 vm = vcpu->vm; in vm_run()
1180 vcpuid = vcpu->vcpuid; in vm_run()
1182 if (!CPU_ISSET(vcpuid, &vm->active_cpus)) in vm_run()
1185 if (CPU_ISSET(vcpuid, &vm->suspended_cpus)) in vm_run()
1188 pmap = vmspace_pmap(vm->vmspace); in vm_run()
1189 vme = &vcpu->exitinfo; in vm_run()
1191 evinfo.sptr = &vm->suspend; in vm_run()
1199 error = vmmops_run(vcpu->cookie, vcpu->nextpc, pmap, &evinfo); in vm_run()
1206 if (error == 0) { in vm_run()
1208 switch (vme->exitcode) { in vm_run()
1210 vcpu->nextpc = vme->pc + vme->inst_length; in vm_run()
1214 vcpu->nextpc = vme->pc + vme->inst_length; in vm_run()
1219 vcpu->nextpc = vme->pc + vme->inst_length; in vm_run()
1223 vcpu->nextpc = vme->pc; in vm_run()
1227 vcpu->nextpc = vme->pc; in vm_run()
1229 error = 0; in vm_run()
1232 vcpu->nextpc = vme->pc; in vm_run()
1237 vcpu->nextpc = vme->pc; in vm_run()
1243 if (error == 0 && retu == false) in vm_run()