86 	struct vm	*vm;		/* (o) */  member
121 struct vm {  struct
281 vcpu_alloc(struct vm *vm, int vcpu_id)  in vcpu_alloc()  argument
285 	KASSERT(vcpu_id >= 0 && vcpu_id < vm->maxcpus,  in vcpu_alloc()
293 	vcpu->vm = vm;  in vcpu_alloc()
302 	vcpu->cookie = vmmops_vcpu_init(vcpu->vm->cookie, vcpu, vcpu->vcpuid);  in vcpu_init()
406 vm_init(struct vm *vm, bool create)  in vm_init()  argument
410 	vm->cookie = vmmops_init(vm, vmspace_pmap(vm->vmspace));  in vm_init()
411 	MPASS(vm->cookie != NULL);  in vm_init()
413 	CPU_ZERO(&vm->active_cpus);  in vm_init()
414 	CPU_ZERO(&vm->debug_cpus);  in vm_init()
416 	vm->suspend = 0;  in vm_init()
417 	CPU_ZERO(&vm->suspended_cpus);  in vm_init()
419 	memset(vm->mmio_region, 0, sizeof(vm->mmio_region));  in vm_init()
420 	memset(vm->special_reg, 0, sizeof(vm->special_reg));  in vm_init()
423 		for (i = 0; i < vm->maxcpus; i++) {  in vm_init()
424 			if (vm->vcpu[i] != NULL)  in vm_init()
425 				vcpu_init(vm->vcpu[i]);  in vm_init()
431 vm_disable_vcpu_creation(struct vm *vm)  in vm_disable_vcpu_creation()  argument
433 	sx_xlock(&vm->vcpus_init_lock);  in vm_disable_vcpu_creation()
434 	vm->dying = true;  in vm_disable_vcpu_creation()
435 	sx_xunlock(&vm->vcpus_init_lock);  in vm_disable_vcpu_creation()
439 vm_alloc_vcpu(struct vm *vm, int vcpuid)  in vm_alloc_vcpu()  argument
443 	if (vcpuid < 0 || vcpuid >= vm_get_maxcpus(vm))  in vm_alloc_vcpu()
447 	if (vcpuid >= vgic_max_cpu_count(vm->cookie))  in vm_alloc_vcpu()
451 	    atomic_load_acq_ptr((uintptr_t *)&vm->vcpu[vcpuid]);  in vm_alloc_vcpu()
455 	sx_xlock(&vm->vcpus_init_lock);  in vm_alloc_vcpu()
456 	vcpu = vm->vcpu[vcpuid];  in vm_alloc_vcpu()
457 	if (vcpu == NULL && !vm->dying) {  in vm_alloc_vcpu()
458 		vcpu = vcpu_alloc(vm, vcpuid);  in vm_alloc_vcpu()
465 		atomic_store_rel_ptr((uintptr_t *)&vm->vcpu[vcpuid],  in vm_alloc_vcpu()
468 	sx_xunlock(&vm->vcpus_init_lock);  in vm_alloc_vcpu()
473 vm_slock_vcpus(struct vm *vm)  in vm_slock_vcpus()  argument
475 	sx_slock(&vm->vcpus_init_lock);  in vm_slock_vcpus()
479 vm_unlock_vcpus(struct vm *vm)  in vm_unlock_vcpus()  argument
481 	sx_unlock(&vm->vcpus_init_lock);  in vm_unlock_vcpus()
485 vm_create(const char *name, struct vm **retvm)  in vm_create()
487 	struct vm *vm;  in vm_create()  local
504 	vm = malloc(sizeof(struct vm), M_VMM, M_WAITOK | M_ZERO);  in vm_create()
505 	strcpy(vm->name, name);  in vm_create()
506 	vm->vmspace = vmspace;  in vm_create()
507 	vm_mem_init(&vm->mem);  in vm_create()
508 	sx_init(&vm->vcpus_init_lock, "vm vcpus");  in vm_create()
510 	vm->sockets = 1;  in vm_create()
511 	vm->cores = 1;			/* XXX backwards compatibility */  in vm_create()
512 	vm->threads = 1;		/* XXX backwards compatibility */  in vm_create()
513 	vm->maxcpus = vm_maxcpu;  in vm_create()
515 	vm->vcpu = malloc(sizeof(*vm->vcpu) * vm->maxcpus, M_VMM,  in vm_create()
518 	vm_init(vm, true);  in vm_create()
520 	*retvm = vm;  in vm_create()
525 vm_get_topology(struct vm *vm, uint16_t *sockets, uint16_t *cores,  in vm_get_topology()  argument
528 	*sockets = vm->sockets;  in vm_get_topology()
529 	*cores = vm->cores;  in vm_get_topology()
530 	*threads = vm->threads;  in vm_get_topology()
531 	*maxcpus = vm->maxcpus;  in vm_get_topology()
535 vm_get_maxcpus(struct vm *vm)  in vm_get_maxcpus()  argument
537 	return (vm->maxcpus);  in vm_get_maxcpus()
541 vm_set_topology(struct vm *vm, uint16_t sockets, uint16_t cores,  in vm_set_topology()  argument
545 	if ((sockets * cores * threads) > vm->maxcpus)  in vm_set_topology()
547 	vm->sockets = sockets;  in vm_set_topology()
548 	vm->cores = cores;  in vm_set_topology()
549 	vm->threads = threads;  in vm_set_topology()
554 vm_cleanup(struct vm *vm, bool destroy)  in vm_cleanup()  argument
560 		vm_xlock_memsegs(vm);  in vm_cleanup()
561 		pmap = vmspace_pmap(vm->vmspace);  in vm_cleanup()
569 		vm_assert_memseg_xlocked(vm);  in vm_cleanup()
572 	vgic_detach_from_vm(vm->cookie);  in vm_cleanup()
574 	for (i = 0; i < vm->maxcpus; i++) {  in vm_cleanup()
575 		if (vm->vcpu[i] != NULL)  in vm_cleanup()
576 			vcpu_cleanup(vm->vcpu[i], destroy);  in vm_cleanup()
579 	vmmops_cleanup(vm->cookie);  in vm_cleanup()
581 	vm_mem_cleanup(vm);  in vm_cleanup()
583 		vm_mem_destroy(vm);  in vm_cleanup()
585 		vmmops_vmspace_free(vm->vmspace);  in vm_cleanup()
586 		vm->vmspace = NULL;  in vm_cleanup()
588 		for (i = 0; i < vm->maxcpus; i++)  in vm_cleanup()
589 			free(vm->vcpu[i], M_VMM);  in vm_cleanup()
590 		free(vm->vcpu, M_VMM);  in vm_cleanup()
591 		sx_destroy(&vm->vcpus_init_lock);  in vm_cleanup()
596 vm_destroy(struct vm *vm)  in vm_destroy()  argument
598 	vm_cleanup(vm, true);  in vm_destroy()
599 	free(vm, M_VMM);  in vm_destroy()
603 vm_reinit(struct vm *vm)  in vm_reinit()  argument
610 	if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) {  in vm_reinit()
611 		vm_cleanup(vm, false);  in vm_reinit()
612 		vm_init(vm, false);  in vm_reinit()
622 vm_name(struct vm *vm)  in vm_name()  argument
624 	return (vm->name);  in vm_name()
714 vm_register_reg_handler(struct vm *vm, uint64_t iss, uint64_t mask,  in vm_register_reg_handler()  argument
719 	for (i = 0; i < nitems(vm->special_reg); i++) {  in vm_register_reg_handler()
720 		if (vm->special_reg[i].esr_iss == 0 &&  in vm_register_reg_handler()
721 		    vm->special_reg[i].esr_mask == 0) {  in vm_register_reg_handler()
722 			vm->special_reg[i].esr_iss = iss;  in vm_register_reg_handler()
723 			vm->special_reg[i].esr_mask = mask;  in vm_register_reg_handler()
724 			vm->special_reg[i].reg_read = reg_read;  in vm_register_reg_handler()
725 			vm->special_reg[i].reg_write = reg_write;  in vm_register_reg_handler()
726 			vm->special_reg[i].arg = arg;  in vm_register_reg_handler()
735 vm_deregister_reg_handler(struct vm *vm, uint64_t iss, uint64_t mask)  in vm_deregister_reg_handler()  argument
739 	for (i = 0; i < nitems(vm->special_reg); i++) {  in vm_deregister_reg_handler()
740 		if (vm->special_reg[i].esr_iss == iss &&  in vm_deregister_reg_handler()
741 		    vm->special_reg[i].esr_mask == mask) {  in vm_deregister_reg_handler()
742 			memset(&vm->special_reg[i], 0,  in vm_deregister_reg_handler()
743 			    sizeof(vm->special_reg[i]));  in vm_deregister_reg_handler()
755 	struct vm *vm;  in vm_handle_reg_emul()  local
760 	vm = vcpu->vm;  in vm_handle_reg_emul()
764 	for (i = 0; i < nitems(vm->special_reg); i++) {  in vm_handle_reg_emul()
765 		if (vm->special_reg[i].esr_iss == 0 &&  in vm_handle_reg_emul()
766 		    vm->special_reg[i].esr_mask == 0)  in vm_handle_reg_emul()
769 		if ((vre->inst_syndrome & vm->special_reg[i].esr_mask) ==  in vm_handle_reg_emul()
770 		    vm->special_reg[i].esr_iss) {  in vm_handle_reg_emul()
772 			    vm->special_reg[i].reg_read,  in vm_handle_reg_emul()
773 			    vm->special_reg[i].reg_write,  in vm_handle_reg_emul()
774 			    vm->special_reg[i].arg);  in vm_handle_reg_emul()
801 vm_register_inst_handler(struct vm *vm, uint64_t start, uint64_t size,  in vm_register_inst_handler()  argument
806 	for (i = 0; i < nitems(vm->mmio_region); i++) {  in vm_register_inst_handler()
807 		if (vm->mmio_region[i].start == 0 &&  in vm_register_inst_handler()
808 		    vm->mmio_region[i].end == 0) {  in vm_register_inst_handler()
809 			vm->mmio_region[i].start = start;  in vm_register_inst_handler()
810 			vm->mmio_region[i].end = start + size;  in vm_register_inst_handler()
811 			vm->mmio_region[i].read = mmio_read;  in vm_register_inst_handler()
812 			vm->mmio_region[i].write = mmio_write;  in vm_register_inst_handler()
821 vm_deregister_inst_handler(struct vm *vm, uint64_t start, uint64_t size)  in vm_deregister_inst_handler()  argument
825 	for (i = 0; i < nitems(vm->mmio_region); i++) {  in vm_deregister_inst_handler()
826 		if (vm->mmio_region[i].start == start &&  in vm_deregister_inst_handler()
827 		    vm->mmio_region[i].end == start + size) {  in vm_deregister_inst_handler()
828 			memset(&vm->mmio_region[i], 0,  in vm_deregister_inst_handler()
829 			    sizeof(vm->mmio_region[i]));  in vm_deregister_inst_handler()
841 	struct vm *vm;  in vm_handle_inst_emul()  local
850 	vm = vcpu->vm;  in vm_handle_inst_emul()
851 	hyp = vm->cookie;  in vm_handle_inst_emul()
862 	for (i = 0; i < nitems(vm->mmio_region); i++) {  in vm_handle_inst_emul()
863 		if (vm->mmio_region[i].start <= fault_ipa &&  in vm_handle_inst_emul()
864 		    vm->mmio_region[i].end > fault_ipa) {  in vm_handle_inst_emul()
865 			vmr = &vm->mmio_region[i];  in vm_handle_inst_emul()
882 vm_suspend(struct vm *vm, enum vm_suspend_how how)  in vm_suspend()  argument
889 	if (atomic_cmpset_int(&vm->suspend, 0, how) == 0) {  in vm_suspend()
890 		VM_CTR2(vm, "virtual machine already suspended %d/%d",  in vm_suspend()
891 		    vm->suspend, how);  in vm_suspend()
895 	VM_CTR1(vm, "virtual machine successfully suspended %d", how);  in vm_suspend()
900 	for (i = 0; i < vm->maxcpus; i++) {  in vm_suspend()
901 		if (CPU_ISSET(i, &vm->active_cpus))  in vm_suspend()
902 			vcpu_notify_event(vm_vcpu(vm, i));  in vm_suspend()
911 	struct vm *vm = vcpu->vm;  in vm_exit_suspended()  local
914 	KASSERT(vm->suspend > VM_SUSPEND_NONE && vm->suspend < VM_SUSPEND_LAST,  in vm_exit_suspended()
915 	    ("vm_exit_suspended: invalid suspend type %d", vm->suspend));  in vm_exit_suspended()
921 	vmexit->u.suspended.how = vm->suspend;  in vm_exit_suspended()
938 	struct vm *vm = vcpu->vm;  in vm_activate_cpu()  local
940 	if (CPU_ISSET(vcpu->vcpuid, &vm->active_cpus))  in vm_activate_cpu()
943 	CPU_SET_ATOMIC(vcpu->vcpuid, &vm->active_cpus);  in vm_activate_cpu()
949 vm_suspend_cpu(struct vm *vm, struct vcpu *vcpu)  in vm_suspend_cpu()  argument
952 		vm->debug_cpus = vm->active_cpus;  in vm_suspend_cpu()
953 		for (int i = 0; i < vm->maxcpus; i++) {  in vm_suspend_cpu()
954 			if (CPU_ISSET(i, &vm->active_cpus))  in vm_suspend_cpu()
955 				vcpu_notify_event(vm_vcpu(vm, i));  in vm_suspend_cpu()
958 		if (!CPU_ISSET(vcpu->vcpuid, &vm->active_cpus))  in vm_suspend_cpu()
961 		CPU_SET_ATOMIC(vcpu->vcpuid, &vm->debug_cpus);  in vm_suspend_cpu()
968 vm_resume_cpu(struct vm *vm, struct vcpu *vcpu)  in vm_resume_cpu()  argument
972 		CPU_ZERO(&vm->debug_cpus);  in vm_resume_cpu()
974 		if (!CPU_ISSET(vcpu->vcpuid, &vm->debug_cpus))  in vm_resume_cpu()
977 		CPU_CLR_ATOMIC(vcpu->vcpuid, &vm->debug_cpus);  in vm_resume_cpu()
986 	return (CPU_ISSET(vcpu->vcpuid, &vcpu->vm->debug_cpus));  in vcpu_debugged()
990 vm_active_cpus(struct vm *vm)  in vm_active_cpus()  argument
993 	return (vm->active_cpus);  in vm_active_cpus()
997 vm_debug_cpus(struct vm *vm)  in vm_debug_cpus()  argument
1000 	return (vm->debug_cpus);  in vm_debug_cpus()
1004 vm_suspended_cpus(struct vm *vm)  in vm_suspended_cpus()  argument
1007 	return (vm->suspended_cpus);  in vm_suspended_cpus()
1060 vm_vmspace(struct vm *vm)  in vm_vmspace()  argument
1062 	return (vm->vmspace);  in vm_vmspace()
1066 vm_mem(struct vm *vm)  in vm_mem()  argument
1068 	return (&vm->mem);  in vm_mem()
1208 struct vm *
1211 	return (vcpu->vm);  in vcpu_vm()
1227 vm_vcpu(struct vm *vm, int vcpuid)  in vm_vcpu()  argument
1229 	return (vm->vcpu[vcpuid]);  in vm_vcpu()
1285 vm_get_cookie(struct vm *vm)  in vm_get_cookie()  argument
1287 	return (vm->cookie);  in vm_get_cookie()
1297 vm_attach_vgic(struct vm *vm, struct vm_vgic_descr *descr)  in vm_attach_vgic()  argument
1299 	return (vgic_attach_to_vm(vm->cookie, descr));  in vm_attach_vgic()
1303 vm_assert_irq(struct vm *vm, uint32_t irq)  in vm_assert_irq()  argument
1305 	return (vgic_inject_irq(vm->cookie, -1, irq, true));  in vm_assert_irq()
1309 vm_deassert_irq(struct vm *vm, uint32_t irq)  in vm_deassert_irq()  argument
1311 	return (vgic_inject_irq(vm->cookie, -1, irq, false));  in vm_deassert_irq()
1315 vm_raise_msi(struct vm *vm, uint64_t msg, uint64_t addr, int bus, int slot,  in vm_raise_msi()  argument
1319 	return (vgic_inject_msi(vm->cookie, msg, addr));  in vm_raise_msi()
1370 	struct vm *vm = vcpu->vm;  in vm_handle_paging()  local
1379 	pmap = vmspace_pmap(vcpu->vm->vmspace);  in vm_handle_paging()
1396 	map = &vm->vmspace->vm_map;  in vm_handle_paging()
1407 	struct vm *vm = vcpu->vm;  in vm_handle_suspend()  local
1414 	CPU_SET_ATOMIC(vcpu->vcpuid, &vm->suspended_cpus);  in vm_handle_suspend()
1425 		if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0)  in vm_handle_suspend()
1442 	for (i = 0; i < vm->maxcpus; i++) {  in vm_handle_suspend()
1443 		if (CPU_ISSET(i, &vm->suspended_cpus)) {  in vm_handle_suspend()
1444 			vcpu_notify_event(vm_vcpu(vm, i));  in vm_handle_suspend()
1455 	struct vm *vm = vcpu->vm;  in vm_run()  local
1464 	if (!CPU_ISSET(vcpuid, &vm->active_cpus))  in vm_run()
1467 	if (CPU_ISSET(vcpuid, &vm->suspended_cpus))  in vm_run()
1470 	pmap = vmspace_pmap(vm->vmspace);  in vm_run()
1473 	evinfo.sptr = &vm->suspend;  in vm_run()