50 #include <vm/vm.h>
51 #include <vm/vm_object.h>
52 #include <vm/vm_page.h>
53 #include <vm/pmap.h>
54 #include <vm/vm_map.h>
55 #include <vm/vm_extern.h>
56 #include <vm/vm_param.h>
64 #include <machine/vm.h>
89 	struct vm	*vm;		/* (o) */  member
110  * (o) initialized the first time the VM is created
111  * (i) initialized when VM is created and when it is reinitialized
114 struct vm {  struct
118 	int		suspend;		/* (i) stop VM execution */
127 	/* The following describe the vm cpu topology */  argument
149 VMM_STAT(VMEXIT_COUNT, "total number of vm exits");  argument
167 vcpu_alloc(struct vm *vm, int vcpu_id)  in vcpu_alloc()  argument
171 	KASSERT(vcpu_id >= 0 && vcpu_id < vm->maxcpus,  in vcpu_alloc()
179 	vcpu->vm = vm;  in vcpu_alloc()
188 	vcpu->cookie = vmmops_vcpu_init(vcpu->vm->cookie, vcpu, vcpu->vcpuid);  in vcpu_init()
213 vm_init(struct vm *vm, bool create)  in vm_init()  argument
217 	vm->cookie = vmmops_init(vm, vmspace_pmap(vm_vmspace(vm)));  in vm_init()
218 	MPASS(vm->cookie != NULL);  in vm_init()
220 	CPU_ZERO(&vm->active_cpus);  in vm_init()
221 	CPU_ZERO(&vm->debug_cpus);  in vm_init()
223 	vm->suspend = 0;  in vm_init()
224 	CPU_ZERO(&vm->suspended_cpus);  in vm_init()
226 	memset(vm->mmio_region, 0, sizeof(vm->mmio_region));  in vm_init()
229 		for (i = 0; i < vm->maxcpus; i++) {  in vm_init()
230 			if (vm->vcpu[i] != NULL)  in vm_init()
231 				vcpu_init(vm->vcpu[i]);  in vm_init()
237 vm_disable_vcpu_creation(struct vm *vm)  in vm_disable_vcpu_creation()  argument
239 	sx_xlock(&vm->vcpus_init_lock);  in vm_disable_vcpu_creation()
240 	vm->dying = true;  in vm_disable_vcpu_creation()
241 	sx_xunlock(&vm->vcpus_init_lock);  in vm_disable_vcpu_creation()
245 vm_alloc_vcpu(struct vm *vm, int vcpuid)  in vm_alloc_vcpu()  argument
249 	if (vcpuid < 0 || vcpuid >= vm_get_maxcpus(vm))  in vm_alloc_vcpu()
253 	    atomic_load_acq_ptr((uintptr_t *)&vm->vcpu[vcpuid]);  in vm_alloc_vcpu()
257 	sx_xlock(&vm->vcpus_init_lock);  in vm_alloc_vcpu()
258 	vcpu = vm->vcpu[vcpuid];  in vm_alloc_vcpu()
259 	if (vcpu == NULL && !vm->dying) {  in vm_alloc_vcpu()
260 		vcpu = vcpu_alloc(vm, vcpuid);  in vm_alloc_vcpu()
267 		atomic_store_rel_ptr((uintptr_t *)&vm->vcpu[vcpuid],  in vm_alloc_vcpu()
270 	sx_xunlock(&vm->vcpus_init_lock);  in vm_alloc_vcpu()
275 vm_lock_vcpus(struct vm *vm)  in vm_lock_vcpus()  argument
277 	sx_xlock(&vm->vcpus_init_lock);  in vm_lock_vcpus()
281 vm_unlock_vcpus(struct vm *vm)  in vm_unlock_vcpus()  argument
283 	sx_unlock(&vm->vcpus_init_lock);  in vm_unlock_vcpus()
287 vm_create(const char *name, struct vm **retvm)  in vm_create()
289 	struct vm *vm;  in vm_create()  local
292 	vm = malloc(sizeof(struct vm), M_VMM, M_WAITOK | M_ZERO);  in vm_create()
293 	error = vm_mem_init(&vm->mem, 0, 1ul << 39);  in vm_create()
295 		free(vm, M_VMM);  in vm_create()
298 	strcpy(vm->name, name);  in vm_create()
299 	sx_init(&vm->vcpus_init_lock, "vm vcpus");  in vm_create()
301 	vm->sockets = 1;  in vm_create()
302 	vm->cores = 1;			/* XXX backwards compatibility */  in vm_create()
303 	vm->threads = 1;		/* XXX backwards compatibility */  in vm_create()
304 	vm->maxcpus = vm_maxcpu;  in vm_create()
306 	vm->vcpu = malloc(sizeof(*vm->vcpu) * vm->maxcpus, M_VMM,  in vm_create()
309 	vm_init(vm, true);  in vm_create()
311 	*retvm = vm;  in vm_create()
316 vm_get_topology(struct vm *vm, uint16_t *sockets, uint16_t *cores,  in vm_get_topology()  argument
319 	*sockets = vm->sockets;  in vm_get_topology()
320 	*cores = vm->cores;  in vm_get_topology()
321 	*threads = vm->threads;  in vm_get_topology()
322 	*maxcpus = vm->maxcpus;  in vm_get_topology()
326 vm_get_maxcpus(struct vm *vm)  in vm_get_maxcpus()  argument
328 	return (vm->maxcpus);  in vm_get_maxcpus()
332 vm_set_topology(struct vm *vm, uint16_t sockets, uint16_t cores,  in vm_set_topology()  argument
336 	if ((sockets * cores * threads) > vm->maxcpus)  in vm_set_topology()
338 	vm->sockets = sockets;  in vm_set_topology()
339 	vm->cores = cores;  in vm_set_topology()
340 	vm->threads = threads;  in vm_set_topology()
345 vm_cleanup(struct vm *vm, bool destroy)  in vm_cleanup()  argument
350 		vm_xlock_memsegs(vm);  in vm_cleanup()
352 		vm_assert_memseg_xlocked(vm);  in vm_cleanup()
354 	aplic_detach_from_vm(vm->cookie);  in vm_cleanup()
356 	for (i = 0; i < vm->maxcpus; i++) {  in vm_cleanup()
357 		if (vm->vcpu[i] != NULL)  in vm_cleanup()
358 			vcpu_cleanup(vm->vcpu[i], destroy);  in vm_cleanup()
361 	vmmops_cleanup(vm->cookie);  in vm_cleanup()
363 	vm_mem_cleanup(vm);  in vm_cleanup()
365 		vm_mem_destroy(vm);  in vm_cleanup()
367 		free(vm->vcpu, M_VMM);  in vm_cleanup()
368 		sx_destroy(&vm->vcpus_init_lock);  in vm_cleanup()
373 vm_destroy(struct vm *vm)  in vm_destroy()  argument
376 	vm_cleanup(vm, true);  in vm_destroy()
378 	free(vm, M_VMM);  in vm_destroy()
382 vm_reinit(struct vm *vm)  in vm_reinit()  argument
389 	if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) {  in vm_reinit()
390 		vm_cleanup(vm, false);  in vm_reinit()
391 		vm_init(vm, false);  in vm_reinit()
401 vm_name(struct vm *vm)  in vm_name()  argument
403 	return (vm->name);  in vm_name()
414 vm_register_inst_handler(struct vm *vm, uint64_t start, uint64_t size,  in vm_register_inst_handler()  argument
419 	for (i = 0; i < nitems(vm->mmio_region); i++) {  in vm_register_inst_handler()
420 		if (vm->mmio_region[i].start == 0 &&  in vm_register_inst_handler()
421 		    vm->mmio_region[i].end == 0) {  in vm_register_inst_handler()
422 			vm->mmio_region[i].start = start;  in vm_register_inst_handler()
423 			vm->mmio_region[i].end = start + size;  in vm_register_inst_handler()
424 			vm->mmio_region[i].read = mmio_read;  in vm_register_inst_handler()
425 			vm->mmio_region[i].write = mmio_write;  in vm_register_inst_handler()
434 vm_deregister_inst_handler(struct vm *vm, uint64_t start, uint64_t size)  in vm_deregister_inst_handler()  argument
438 	for (i = 0; i < nitems(vm->mmio_region); i++) {  in vm_deregister_inst_handler()
439 		if (vm->mmio_region[i].start == start &&  in vm_deregister_inst_handler()
440 		    vm->mmio_region[i].end == start + size) {  in vm_deregister_inst_handler()
441 			memset(&vm->mmio_region[i], 0,  in vm_deregister_inst_handler()
442 			    sizeof(vm->mmio_region[i]));  in vm_deregister_inst_handler()
454 	struct vm *vm;  in vm_handle_inst_emul()  local
463 	vm = vcpu->vm;  in vm_handle_inst_emul()
464 	hyp = vm->cookie;  in vm_handle_inst_emul()
475 	for (i = 0; i < nitems(vm->mmio_region); i++) {  in vm_handle_inst_emul()
476 		if (vm->mmio_region[i].start <= fault_ipa &&  in vm_handle_inst_emul()
477 		    vm->mmio_region[i].end > fault_ipa) {  in vm_handle_inst_emul()
478 			vmr = &vm->mmio_region[i];  in vm_handle_inst_emul()
495 vm_suspend(struct vm *vm, enum vm_suspend_how how)  in vm_suspend()  argument
502 	if (atomic_cmpset_int(&vm->suspend, 0, how) == 0) {  in vm_suspend()
503 		VM_CTR2(vm, "virtual machine already suspended %d/%d",  in vm_suspend()
504 		    vm->suspend, how);  in vm_suspend()
508 	VM_CTR1(vm, "virtual machine successfully suspended %d", how);  in vm_suspend()
513 	for (i = 0; i < vm->maxcpus; i++) {  in vm_suspend()
514 		if (CPU_ISSET(i, &vm->active_cpus))  in vm_suspend()
515 			vcpu_notify_event(vm_vcpu(vm, i));  in vm_suspend()
524 	struct vm *vm = vcpu->vm;  in vm_exit_suspended()  local
527 	KASSERT(vm->suspend > VM_SUSPEND_NONE && vm->suspend < VM_SUSPEND_LAST,  in vm_exit_suspended()
528 	    ("vm_exit_suspended: invalid suspend type %d", vm->suspend));  in vm_exit_suspended()
534 	vmexit->u.suspended.how = vm->suspend;  in vm_exit_suspended()
551 	struct vm *vm = vcpu->vm;  in vm_activate_cpu()  local
553 	if (CPU_ISSET(vcpu->vcpuid, &vm->active_cpus))  in vm_activate_cpu()
556 	CPU_SET_ATOMIC(vcpu->vcpuid, &vm->active_cpus);  in vm_activate_cpu()
562 vm_suspend_cpu(struct vm *vm, struct vcpu *vcpu)  in vm_suspend_cpu()  argument
565 		vm->debug_cpus = vm->active_cpus;  in vm_suspend_cpu()
566 		for (int i = 0; i < vm->maxcpus; i++) {  in vm_suspend_cpu()
567 			if (CPU_ISSET(i, &vm->active_cpus))  in vm_suspend_cpu()
568 				vcpu_notify_event(vm_vcpu(vm, i));  in vm_suspend_cpu()
571 		if (!CPU_ISSET(vcpu->vcpuid, &vm->active_cpus))  in vm_suspend_cpu()
574 		CPU_SET_ATOMIC(vcpu->vcpuid, &vm->debug_cpus);  in vm_suspend_cpu()
581 vm_resume_cpu(struct vm *vm, struct vcpu *vcpu)  in vm_resume_cpu()  argument
585 		CPU_ZERO(&vm->debug_cpus);  in vm_resume_cpu()
587 		if (!CPU_ISSET(vcpu->vcpuid, &vm->debug_cpus))  in vm_resume_cpu()
590 		CPU_CLR_ATOMIC(vcpu->vcpuid, &vm->debug_cpus);  in vm_resume_cpu()
599 	return (CPU_ISSET(vcpu->vcpuid, &vcpu->vm->debug_cpus));  in vcpu_debugged()
603 vm_active_cpus(struct vm *vm)  in vm_active_cpus()  argument
606 	return (vm->active_cpus);  in vm_active_cpus()
610 vm_debug_cpus(struct vm *vm)  in vm_debug_cpus()  argument
613 	return (vm->debug_cpus);  in vm_debug_cpus()
617 vm_suspended_cpus(struct vm *vm)  in vm_suspended_cpus()  argument
620 	return (vm->suspended_cpus);  in vm_suspended_cpus()
673 vm_mem(struct vm *vm)  in vm_mem()  argument
675 	return (&vm->mem);  in vm_mem()
816 struct vm *
820 	return (vcpu->vm);  in vcpu_vm()
838 vm_vcpu(struct vm *vm, int vcpuid)  in vm_vcpu()  argument
841 	return (vm->vcpu[vcpuid]);  in vm_vcpu()
896 vm_get_cookie(struct vm *vm)  in vm_get_cookie()  argument
899 	return (vm->cookie);  in vm_get_cookie()
910 vm_attach_aplic(struct vm *vm, struct vm_aplic_descr *descr)  in vm_attach_aplic()  argument
913 	return (aplic_attach_to_vm(vm->cookie, descr));  in vm_attach_aplic()
917 vm_assert_irq(struct vm *vm, uint32_t irq)  in vm_assert_irq()  argument
920 	return (aplic_inject_irq(vm->cookie, -1, irq, true));  in vm_assert_irq()
924 vm_deassert_irq(struct vm *vm, uint32_t irq)  in vm_deassert_irq()  argument
927 	return (aplic_inject_irq(vm->cookie, -1, irq, false));  in vm_deassert_irq()
931 vm_raise_msi(struct vm *vm, uint64_t msg, uint64_t addr, int bus, int slot,  in vm_raise_msi()  argument
935 	return (aplic_inject_msi(vm->cookie, msg, addr));  in vm_raise_msi()
941 	struct vm *vm;  in vm_handle_wfi()  local
943 	vm = vcpu->vm;  in vm_handle_wfi()
946 		if (vm->suspend)  in vm_handle_wfi()
979 	struct vm *vm;  in vm_handle_paging()  local
986 	vm = vcpu->vm;  in vm_handle_paging()
989 	pmap = vmspace_pmap(vm_vmspace(vm));  in vm_handle_paging()
1012 	map = &vm_vmspace(vm)->vm_map;  in vm_handle_paging()
1026 	struct vm *vm = vcpu->vm;  in vm_handle_suspend()  local
1033 	CPU_SET_ATOMIC(vcpu->vcpuid, &vm->suspended_cpus);  in vm_handle_suspend()
1038 	 * Since a VM may be suspended at any time including when one or  in vm_handle_suspend()
1044 		if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0)  in vm_handle_suspend()
1061 	for (i = 0; i < vm->maxcpus; i++) {  in vm_handle_suspend()
1062 		if (CPU_ISSET(i, &vm->suspended_cpus)) {  in vm_handle_suspend()
1063 			vcpu_notify_event(vm_vcpu(vm, i));  in vm_handle_suspend()
1076 	struct vm *vm;  in vm_run()  local
1082 	vm = vcpu->vm;  in vm_run()
1088 	if (!CPU_ISSET(vcpuid, &vm->active_cpus))  in vm_run()
1091 	if (CPU_ISSET(vcpuid, &vm->suspended_cpus))  in vm_run()
1094 	pmap = vmspace_pmap(vm_vmspace(vm));  in vm_run()
1097 	evinfo.sptr = &vm->suspend;  in vm_run()