Lines Matching full:vcpu
76 struct vcpu { struct
80 int hostcpu; /* host cpuid this vcpu last ran on */ argument
91 #define vcpu_lock_init(v) mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN) argument
151 struct vcpu **vcpu; /* (i) guest vcpus */ member
166 static int vm_handle_wfi(struct vcpu *vcpu,
172 static VMM_STAT(VCPU_TOTAL_RUNTIME, "vcpu total runtime");
178 "IPI vector used for vcpu notifications");
240 static void vcpu_notify_event_locked(struct vcpu *vcpu);
290 vcpu_cleanup(struct vcpu *vcpu, bool destroy) in vcpu_cleanup() argument
292 vmmops_vcpu_cleanup(vcpu->cookie); in vcpu_cleanup()
293 vcpu->cookie = NULL; in vcpu_cleanup()
295 vmm_stat_free(vcpu->stats); in vcpu_cleanup()
296 fpu_save_area_free(vcpu->guestfpu); in vcpu_cleanup()
297 vcpu_lock_destroy(vcpu); in vcpu_cleanup()
301 static struct vcpu *
304 struct vcpu *vcpu; in vcpu_alloc() local
307 ("vcpu_alloc: invalid vcpu %d", vcpu_id)); in vcpu_alloc()
309 vcpu = malloc(sizeof(*vcpu), M_VMM, M_WAITOK | M_ZERO); in vcpu_alloc()
310 vcpu_lock_init(vcpu); in vcpu_alloc()
311 vcpu->state = VCPU_IDLE; in vcpu_alloc()
312 vcpu->hostcpu = NOCPU; in vcpu_alloc()
313 vcpu->vcpuid = vcpu_id; in vcpu_alloc()
314 vcpu->vm = vm; in vcpu_alloc()
315 vcpu->guestfpu = fpu_save_area_alloc(); in vcpu_alloc()
316 vcpu->stats = vmm_stat_alloc(); in vcpu_alloc()
317 return (vcpu); in vcpu_alloc()
321 vcpu_init(struct vcpu *vcpu) in vcpu_init() argument
323 vcpu->cookie = vmmops_vcpu_init(vcpu->vm->cookie, vcpu, vcpu->vcpuid); in vcpu_init()
324 MPASS(vcpu->cookie != NULL); in vcpu_init()
325 fpu_save_area_reset(vcpu->guestfpu); in vcpu_init()
326 vmm_stat_init(vcpu->stats); in vcpu_init()
330 vm_exitinfo(struct vcpu *vcpu) in vm_exitinfo() argument
332 return (&vcpu->exitinfo); in vm_exitinfo()
423 if (vm->vcpu[i] != NULL) in vm_init()
424 vcpu_init(vm->vcpu[i]); in vm_init()
437 struct vcpu *
440 struct vcpu *vcpu; in vm_alloc_vcpu() local
449 vcpu = (struct vcpu *) in vm_alloc_vcpu()
450 atomic_load_acq_ptr((uintptr_t *)&vm->vcpu[vcpuid]); in vm_alloc_vcpu()
451 if (__predict_true(vcpu != NULL)) in vm_alloc_vcpu()
452 return (vcpu); in vm_alloc_vcpu()
455 vcpu = vm->vcpu[vcpuid]; in vm_alloc_vcpu()
456 if (vcpu == NULL && !vm->dying) { in vm_alloc_vcpu()
457 vcpu = vcpu_alloc(vm, vcpuid); in vm_alloc_vcpu()
458 vcpu_init(vcpu); in vm_alloc_vcpu()
461 * Ensure vCPU is fully created before updating pointer in vm_alloc_vcpu()
464 atomic_store_rel_ptr((uintptr_t *)&vm->vcpu[vcpuid], in vm_alloc_vcpu()
465 (uintptr_t)vcpu); in vm_alloc_vcpu()
468 return (vcpu); in vm_alloc_vcpu()
514 vm->vcpu = malloc(sizeof(*vm->vcpu) * vm->maxcpus, M_VMM, in vm_create()
572 if (vm->vcpu[i] != NULL) in vm_cleanup()
573 vcpu_cleanup(vm->vcpu[i], destroy); in vm_cleanup()
602 free(vm->vcpu[i], M_VMM); in vm_cleanup()
603 free(vm->vcpu, M_VMM); in vm_cleanup()
662 * This function is called in the context of a running vcpu which acts as
666 vm_mem_allocated(struct vcpu *vcpu, vm_paddr_t gpa) in vm_mem_allocated() argument
668 struct vm *vm = vcpu->vm; in vm_mem_allocated()
674 state = vcpu_get_state(vcpu, &hostcpu); in vm_mem_allocated()
676 ("%s: invalid vcpu state %d/%d", __func__, state, hostcpu)); in vm_mem_allocated()
919 vm_gla2gpa_nofault(struct vcpu *vcpu, struct vm_guest_paging *paging, in vm_gla2gpa_nofault() argument
923 vmmops_gla2gpa(vcpu->cookie, paging, gla, prot, gpa, is_fault); in vm_gla2gpa_nofault()
928 vmm_reg_raz(struct vcpu *vcpu, uint64_t *rval, void *arg) in vmm_reg_raz() argument
935 vmm_reg_read_arg(struct vcpu *vcpu, uint64_t *rval, void *arg) in vmm_reg_read_arg() argument
942 vmm_reg_wi(struct vcpu *vcpu, uint64_t wval, void *arg) in vmm_reg_wi() argument
1046 vm_handle_reg_emul(struct vcpu *vcpu, bool *retu) in vm_handle_reg_emul() argument
1053 vm = vcpu->vm; in vm_handle_reg_emul()
1054 vme = &vcpu->exitinfo; in vm_handle_reg_emul()
1064 rv = vmm_emulate_register(vcpu, vre, in vm_handle_reg_emul()
1077 rv = vmm_emulate_register(vcpu, vre, in vm_handle_reg_emul()
1132 vm_handle_inst_emul(struct vcpu *vcpu, bool *retu) in vm_handle_inst_emul() argument
1143 vm = vcpu->vm; in vm_handle_inst_emul()
1148 vme = &vcpu->exitinfo; in vm_handle_inst_emul()
1165 error = vmm_emulate_instruction(vcpu, fault_ipa, vie, paging, in vm_handle_inst_emul()
1202 vm_exit_suspended(struct vcpu *vcpu, uint64_t pc) in vm_exit_suspended() argument
1204 struct vm *vm = vcpu->vm; in vm_exit_suspended()
1210 vmexit = vm_exitinfo(vcpu); in vm_exit_suspended()
1218 vm_exit_debug(struct vcpu *vcpu, uint64_t pc) in vm_exit_debug() argument
1222 vmexit = vm_exitinfo(vcpu); in vm_exit_debug()
1229 vm_activate_cpu(struct vcpu *vcpu) in vm_activate_cpu() argument
1231 struct vm *vm = vcpu->vm; in vm_activate_cpu()
1233 if (CPU_ISSET(vcpu->vcpuid, &vm->active_cpus)) in vm_activate_cpu()
1236 CPU_SET_ATOMIC(vcpu->vcpuid, &vm->active_cpus); in vm_activate_cpu()
1242 vm_suspend_cpu(struct vm *vm, struct vcpu *vcpu) in vm_suspend_cpu() argument
1244 if (vcpu == NULL) { in vm_suspend_cpu()
1251 if (!CPU_ISSET(vcpu->vcpuid, &vm->active_cpus)) in vm_suspend_cpu()
1254 CPU_SET_ATOMIC(vcpu->vcpuid, &vm->debug_cpus); in vm_suspend_cpu()
1255 vcpu_notify_event(vcpu); in vm_suspend_cpu()
1261 vm_resume_cpu(struct vm *vm, struct vcpu *vcpu) in vm_resume_cpu() argument
1264 if (vcpu == NULL) { in vm_resume_cpu()
1267 if (!CPU_ISSET(vcpu->vcpuid, &vm->debug_cpus)) in vm_resume_cpu()
1270 CPU_CLR_ATOMIC(vcpu->vcpuid, &vm->debug_cpus); in vm_resume_cpu()
1276 vcpu_debugged(struct vcpu *vcpu) in vcpu_debugged() argument
1279 return (CPU_ISSET(vcpu->vcpuid, &vcpu->vm->debug_cpus)); in vcpu_debugged()
1305 vcpu_stats(struct vcpu *vcpu) in vcpu_stats() argument
1308 return (vcpu->stats); in vcpu_stats()
1312 * This function is called to ensure that a vcpu "sees" a pending event
1314 * - If the vcpu thread is sleeping then it is woken up.
1315 * - If the vcpu is running on a different host_cpu then an IPI will be directed
1316 * to the host_cpu to cause the vcpu to trap into the hypervisor.
1319 vcpu_notify_event_locked(struct vcpu *vcpu) in vcpu_notify_event_locked() argument
1323 hostcpu = vcpu->hostcpu; in vcpu_notify_event_locked()
1324 if (vcpu->state == VCPU_RUNNING) { in vcpu_notify_event_locked()
1325 KASSERT(hostcpu != NOCPU, ("vcpu running on invalid hostcpu")); in vcpu_notify_event_locked()
1330 * If the 'vcpu' is running on 'curcpu' then it must in vcpu_notify_event_locked()
1332 * The pending event will be picked up when the vcpu in vcpu_notify_event_locked()
1337 KASSERT(hostcpu == NOCPU, ("vcpu state %d not consistent " in vcpu_notify_event_locked()
1338 "with hostcpu %d", vcpu->state, hostcpu)); in vcpu_notify_event_locked()
1339 if (vcpu->state == VCPU_SLEEPING) in vcpu_notify_event_locked()
1340 wakeup_one(vcpu); in vcpu_notify_event_locked()
1345 vcpu_notify_event(struct vcpu *vcpu) in vcpu_notify_event() argument
1347 vcpu_lock(vcpu); in vcpu_notify_event()
1348 vcpu_notify_event_locked(vcpu); in vcpu_notify_event()
1349 vcpu_unlock(vcpu); in vcpu_notify_event()
1353 restore_guest_fpustate(struct vcpu *vcpu) in restore_guest_fpustate() argument
1363 vfp_restore(vcpu->guestfpu); in restore_guest_fpustate()
1373 save_guest_fpustate(struct vcpu *vcpu) in save_guest_fpustate() argument
1381 vfp_store(vcpu->guestfpu); in save_guest_fpustate()
1388 vcpu_set_state_locked(struct vcpu *vcpu, enum vcpu_state newstate, in vcpu_set_state_locked() argument
1393 vcpu_assert_locked(vcpu); in vcpu_set_state_locked()
1398 * ioctl() operating on a vcpu at any point. in vcpu_set_state_locked()
1401 while (vcpu->state != VCPU_IDLE) { in vcpu_set_state_locked()
1402 vcpu_notify_event_locked(vcpu); in vcpu_set_state_locked()
1403 msleep_spin(&vcpu->state, &vcpu->mtx, "vmstat", hz); in vcpu_set_state_locked()
1406 KASSERT(vcpu->state != VCPU_IDLE, ("invalid transition from " in vcpu_set_state_locked()
1407 "vcpu idle state")); in vcpu_set_state_locked()
1410 if (vcpu->state == VCPU_RUNNING) { in vcpu_set_state_locked()
1411 KASSERT(vcpu->hostcpu == curcpu, ("curcpu %d and hostcpu %d " in vcpu_set_state_locked()
1412 "mismatch for running vcpu", curcpu, vcpu->hostcpu)); in vcpu_set_state_locked()
1414 KASSERT(vcpu->hostcpu == NOCPU, ("Invalid hostcpu %d for a " in vcpu_set_state_locked()
1415 "vcpu that is not running", vcpu->hostcpu)); in vcpu_set_state_locked()
1424 switch (vcpu->state) { in vcpu_set_state_locked()
1441 vcpu->state = newstate; in vcpu_set_state_locked()
1443 vcpu->hostcpu = curcpu; in vcpu_set_state_locked()
1445 vcpu->hostcpu = NOCPU; in vcpu_set_state_locked()
1448 wakeup(&vcpu->state); in vcpu_set_state_locked()
1454 vcpu_require_state(struct vcpu *vcpu, enum vcpu_state newstate) in vcpu_require_state() argument
1458 if ((error = vcpu_set_state(vcpu, newstate, false)) != 0) in vcpu_require_state()
1463 vcpu_require_state_locked(struct vcpu *vcpu, enum vcpu_state newstate) in vcpu_require_state_locked() argument
1467 if ((error = vcpu_set_state_locked(vcpu, newstate, false)) != 0) in vcpu_require_state_locked()
1472 vm_get_capability(struct vcpu *vcpu, int type, int *retval) in vm_get_capability() argument
1477 return (vmmops_getcap(vcpu->cookie, type, retval)); in vm_get_capability()
1481 vm_set_capability(struct vcpu *vcpu, int type, int val) in vm_set_capability() argument
1486 return (vmmops_setcap(vcpu->cookie, type, val)); in vm_set_capability()
1490 vcpu_vm(struct vcpu *vcpu) in vcpu_vm() argument
1492 return (vcpu->vm); in vcpu_vm()
1496 vcpu_vcpuid(struct vcpu *vcpu) in vcpu_vcpuid() argument
1498 return (vcpu->vcpuid); in vcpu_vcpuid()
1502 vcpu_get_cookie(struct vcpu *vcpu) in vcpu_get_cookie() argument
1504 return (vcpu->cookie); in vcpu_get_cookie()
1507 struct vcpu *
1510 return (vm->vcpu[vcpuid]); in vm_vcpu()
1514 vcpu_set_state(struct vcpu *vcpu, enum vcpu_state newstate, bool from_idle) in vcpu_set_state() argument
1518 vcpu_lock(vcpu); in vcpu_set_state()
1519 error = vcpu_set_state_locked(vcpu, newstate, from_idle); in vcpu_set_state()
1520 vcpu_unlock(vcpu); in vcpu_set_state()
1526 vcpu_get_state(struct vcpu *vcpu, int *hostcpu) in vcpu_get_state() argument
1530 vcpu_lock(vcpu); in vcpu_get_state()
1531 state = vcpu->state; in vcpu_get_state()
1533 *hostcpu = vcpu->hostcpu; in vcpu_get_state()
1534 vcpu_unlock(vcpu); in vcpu_get_state()
1572 vm_gpa_hold(struct vcpu *vcpu, vm_paddr_t gpa, size_t len, int reqprot, in vm_gpa_hold() argument
1577 * The current vcpu should be frozen to ensure 'vm_memmap[]' in vm_gpa_hold()
1580 int state = vcpu_get_state(vcpu, NULL); in vm_gpa_hold()
1581 KASSERT(state == VCPU_FROZEN, ("%s: invalid vcpu state %d", in vm_gpa_hold()
1584 return (_vm_gpa_hold(vcpu->vm, gpa, len, reqprot, cookie)); in vm_gpa_hold()
1604 vm_get_register(struct vcpu *vcpu, int reg, uint64_t *retval) in vm_get_register() argument
1610 return (vmmops_getreg(vcpu->cookie, reg, retval)); in vm_get_register()
1614 vm_set_register(struct vcpu *vcpu, int reg, uint64_t val) in vm_set_register() argument
1620 error = vmmops_setreg(vcpu->cookie, reg, val); in vm_set_register()
1624 vcpu->nextpc = val; in vm_set_register()
1636 vm_inject_exception(struct vcpu *vcpu, uint64_t esr, uint64_t far) in vm_inject_exception() argument
1638 return (vmmops_exception(vcpu->cookie, esr, far)); in vm_inject_exception()
1668 vm_handle_smccc_call(struct vcpu *vcpu, struct vm_exit *vme, bool *retu) in vm_handle_smccc_call() argument
1673 hypctx = vcpu_get_cookie(vcpu); in vm_handle_smccc_call()
1688 vm_handle_wfi(struct vcpu *vcpu, struct vm_exit *vme, bool *retu) in vm_handle_wfi() argument
1690 vcpu_lock(vcpu); in vm_handle_wfi()
1692 if (vgic_has_pending_irq(vcpu->cookie)) in vm_handle_wfi()
1695 if (vcpu_should_yield(vcpu)) in vm_handle_wfi()
1698 vcpu_require_state_locked(vcpu, VCPU_SLEEPING); in vm_handle_wfi()
1703 msleep_spin(vcpu, &vcpu->mtx, "vmidle", hz); in vm_handle_wfi()
1704 vcpu_require_state_locked(vcpu, VCPU_FROZEN); in vm_handle_wfi()
1706 vcpu_unlock(vcpu); in vm_handle_wfi()
1713 vm_handle_paging(struct vcpu *vcpu, bool *retu) in vm_handle_paging() argument
1715 struct vm *vm = vcpu->vm; in vm_handle_paging()
1722 vme = &vcpu->exitinfo; in vm_handle_paging()
1724 pmap = vmspace_pmap(vcpu->vm->vmspace); in vm_handle_paging()
1750 vm_handle_suspend(struct vcpu *vcpu, bool *retu) in vm_handle_suspend() argument
1752 struct vm *vm = vcpu->vm; in vm_handle_suspend()
1759 CPU_SET_ATOMIC(vcpu->vcpuid, &vm->suspended_cpus); in vm_handle_suspend()
1768 vcpu_lock(vcpu); in vm_handle_suspend()
1773 vcpu_require_state_locked(vcpu, VCPU_SLEEPING); in vm_handle_suspend()
1774 msleep_spin(vcpu, &vcpu->mtx, "vmsusp", hz); in vm_handle_suspend()
1775 vcpu_require_state_locked(vcpu, VCPU_FROZEN); in vm_handle_suspend()
1777 vcpu_unlock(vcpu); in vm_handle_suspend()
1779 vcpu_lock(vcpu); in vm_handle_suspend()
1782 vcpu_unlock(vcpu); in vm_handle_suspend()
1798 vm_run(struct vcpu *vcpu) in vm_run() argument
1800 struct vm *vm = vcpu->vm; in vm_run()
1807 vcpuid = vcpu->vcpuid; in vm_run()
1816 vme = &vcpu->exitinfo; in vm_run()
1823 restore_guest_fpustate(vcpu); in vm_run()
1825 vcpu_require_state(vcpu, VCPU_RUNNING); in vm_run()
1826 error = vmmops_run(vcpu->cookie, vcpu->nextpc, pmap, &evinfo); in vm_run()
1827 vcpu_require_state(vcpu, VCPU_FROZEN); in vm_run()
1829 save_guest_fpustate(vcpu); in vm_run()
1837 vcpu->nextpc = vme->pc + vme->inst_length; in vm_run()
1838 error = vm_handle_inst_emul(vcpu, &retu); in vm_run()
1842 vcpu->nextpc = vme->pc + vme->inst_length; in vm_run()
1843 error = vm_handle_reg_emul(vcpu, &retu); in vm_run()
1851 vcpu->nextpc = vme->pc; in vm_run()
1856 error = vm_handle_smccc_call(vcpu, vme, &retu); in vm_run()
1860 vcpu->nextpc = vme->pc + vme->inst_length; in vm_run()
1861 error = vm_handle_wfi(vcpu, vme, &retu); in vm_run()
1865 vcpu->nextpc = vme->pc; in vm_run()
1866 error = vm_handle_paging(vcpu, &retu); in vm_run()
1870 vcpu->nextpc = vme->pc; in vm_run()
1871 error = vm_handle_suspend(vcpu, &retu); in vm_run()
1876 vcpu->nextpc = vme->pc; in vm_run()