233 	vmspace_t *vms;  in vmspace_alloc()  local
243 	vms = kmem_zalloc(sizeof (*vms), KM_SLEEP);  in vmspace_alloc()
244 	vms->vms_size = size;  in vmspace_alloc()
245 	list_create(&vms->vms_maplist, sizeof (vmspace_mapping_t),  in vmspace_alloc()
247 	list_create(&vms->vms_clients, sizeof (vm_client_t),  in vmspace_alloc()
250 	vms->vms_gpt = vmm_gpt_alloc(pte_ops);  in vmspace_alloc()
251 	vms->vms_pt_gen = 1;  in vmspace_alloc()
252 	vms->vms_track_dirty = track_dirty;  in vmspace_alloc()
254 	return (vms);  in vmspace_alloc()
262 vmspace_destroy(vmspace_t *vms)  in vmspace_destroy()  argument
264 	mutex_enter(&vms->vms_lock);  in vmspace_destroy()
265 	VERIFY(list_is_empty(&vms->vms_maplist));  in vmspace_destroy()
267 	if (!list_is_empty(&vms->vms_clients)) {  in vmspace_destroy()
268 		vm_client_t *vmc = list_head(&vms->vms_clients);  in vmspace_destroy()
270 			vmc = vmc_space_orphan(vmc, vms);  in vmspace_destroy()
276 		while (!list_is_empty(&vms->vms_clients)) {  in vmspace_destroy()
277 			cv_wait(&vms->vms_cv, &vms->vms_lock);  in vmspace_destroy()
280 	VERIFY(list_is_empty(&vms->vms_clients));  in vmspace_destroy()
282 	vmm_gpt_free(vms->vms_gpt);  in vmspace_destroy()
283 	mutex_exit(&vms->vms_lock);  in vmspace_destroy()
285 	mutex_destroy(&vms->vms_lock);  in vmspace_destroy()
286 	cv_destroy(&vms->vms_cv);  in vmspace_destroy()
287 	list_destroy(&vms->vms_maplist);  in vmspace_destroy()
288 	list_destroy(&vms->vms_clients);  in vmspace_destroy()
290 	kmem_free(vms, sizeof (*vms));  in vmspace_destroy()
297 vmspace_resident_count(vmspace_t *vms)  in vmspace_resident_count()  argument
299 	return (vms->vms_pages_mapped);  in vmspace_resident_count()
312 vmspace_bits_operate(vmspace_t *vms, uint64_t gpa, size_t len,  in vmspace_bits_operate()  argument
319 	vmm_gpt_t *gpt = vms->vms_gpt;  in vmspace_bits_operate()
363 				int err = vmspace_ensure_mapped(vms, gpa,  in vmspace_bits_operate()
406 	vmspace_hold_enter(vms);  in vmspace_bits_operate()
407 	vms->vms_pt_gen++;  in vmspace_bits_operate()
408 	vmspace_clients_invalidate(vms, gpa, len);  in vmspace_bits_operate()
409 	vmspace_hold_exit(vms, true);  in vmspace_bits_operate()
416 vmspace_get_tracking(vmspace_t *vms)  in vmspace_get_tracking()  argument
418 	mutex_enter(&vms->vms_lock);  in vmspace_get_tracking()
419 	const bool val = vms->vms_track_dirty;  in vmspace_get_tracking()
420 	mutex_exit(&vms->vms_lock);  in vmspace_get_tracking()
428 vmspace_set_tracking(vmspace_t *vms, bool enable_dirty_tracking)  in vmspace_set_tracking()  argument
430 	if (enable_dirty_tracking && !vmm_gpt_can_track_dirty(vms->vms_gpt)) {  in vmspace_set_tracking()
435 	vmspace_hold_enter(vms);  in vmspace_set_tracking()
436 	if (vms->vms_track_dirty == enable_dirty_tracking) {  in vmspace_set_tracking()
438 		vmspace_hold_exit(vms, false);  in vmspace_set_tracking()
442 	vms->vms_track_dirty = enable_dirty_tracking;  in vmspace_set_tracking()
445 	for (vm_client_t *vmc = list_head(&vms->vms_clients);  in vmspace_set_tracking()
447 	    vmc = list_next(&vms->vms_clients, vmc)) {  in vmspace_set_tracking()
457 	vms->vms_pt_gen++;  in vmspace_set_tracking()
458 	vmspace_clients_invalidate(vms, 0, vms->vms_size);  in vmspace_set_tracking()
460 	vmspace_hold_exit(vms, true);  in vmspace_set_tracking()
631 vm_mapping_find(vmspace_t *vms, uintptr_t addr, size_t size)  in vm_mapping_find()  argument
634 	list_t *ml = &vms->vms_maplist;  in vm_mapping_find()
639 	if (addr >= vms->vms_size) {  in vm_mapping_find()
661 vm_mapping_gap(vmspace_t *vms, uintptr_t addr, size_t size)  in vm_mapping_gap()  argument
664 	list_t *ml = &vms->vms_maplist;  in vm_mapping_gap()
667 	ASSERT(MUTEX_HELD(&vms->vms_lock));  in vm_mapping_gap()
685 vm_mapping_remove(vmspace_t *vms, vmspace_mapping_t *vmsm)  in vm_mapping_remove()  argument
687 	list_t *ml = &vms->vms_maplist;  in vm_mapping_remove()
689 	ASSERT(MUTEX_HELD(&vms->vms_lock));  in vm_mapping_remove()
690 	ASSERT(vms->vms_held);  in vm_mapping_remove()
706 vmspace_hold_enter(vmspace_t *vms)  in vmspace_hold_enter()  argument
708 	mutex_enter(&vms->vms_lock);  in vmspace_hold_enter()
709 	VERIFY(!vms->vms_held);  in vmspace_hold_enter()
711 	vm_client_t *vmc = list_head(&vms->vms_clients);  in vmspace_hold_enter()
712 	for (; vmc != NULL; vmc = list_next(&vms->vms_clients, vmc)) {  in vmspace_hold_enter()
715 	vms->vms_held = true;  in vmspace_hold_enter()
729 vmspace_hold_exit(vmspace_t *vms, bool kick_on_cpu)  in vmspace_hold_exit()  argument
731 	ASSERT(MUTEX_HELD(&vms->vms_lock));  in vmspace_hold_exit()
732 	VERIFY(vms->vms_held);  in vmspace_hold_exit()
734 	vm_client_t *vmc = list_head(&vms->vms_clients);  in vmspace_hold_exit()
735 	for (; vmc != NULL; vmc = list_next(&vms->vms_clients, vmc)) {  in vmspace_hold_exit()
738 	vms->vms_held = false;  in vmspace_hold_exit()
739 	mutex_exit(&vms->vms_lock);  in vmspace_hold_exit()
743 vmspace_clients_invalidate(vmspace_t *vms, uintptr_t gpa, size_t len)  in vmspace_clients_invalidate()  argument
745 	ASSERT(MUTEX_HELD(&vms->vms_lock));  in vmspace_clients_invalidate()
746 	VERIFY(vms->vms_held);  in vmspace_clients_invalidate()
748 	for (vm_client_t *vmc = list_head(&vms->vms_clients);  in vmspace_clients_invalidate()
750 	    vmc = list_next(&vms->vms_clients, vmc)) {  in vmspace_clients_invalidate()
751 		vmc_space_invalidate(vmc, gpa, len, vms->vms_pt_gen);  in vmspace_clients_invalidate()
767 vmspace_map(vmspace_t *vms, vm_object_t *vmo, uintptr_t obj_off, uintptr_t addr,  in vmspace_map()  argument
777 	if ((addr + len) >= vms->vms_size) {  in vmspace_map()
783 	vmspace_hold_enter(vms);  in vmspace_map()
784 	if (!vm_mapping_gap(vms, addr, len)) {  in vmspace_map()
793 		list_insert_tail(&vms->vms_maplist, vmsm);  in vmspace_map()
799 		vmm_gpt_populate_region(vms->vms_gpt, addr, len);  in vmspace_map()
801 	vmspace_hold_exit(vms, false);  in vmspace_map()
812 vmspace_unmap(vmspace_t *vms, uintptr_t addr, uintptr_t len)  in vmspace_unmap()  argument
821 	vmspace_hold_enter(vms);  in vmspace_unmap()
823 	if ((vmsm = vm_mapping_find(vms, addr, len)) == NULL ||  in vmspace_unmap()
825 		vmspace_hold_exit(vms, false);  in vmspace_unmap()
830 	for (vmc = list_head(&vms->vms_clients); vmc != NULL;  in vmspace_unmap()
831 	    vmc = list_next(&vms->vms_clients, vmc)) {  in vmspace_unmap()
836 	if (vmm_gpt_unmap_region(vms->vms_gpt, addr, len) != 0) {  in vmspace_unmap()
837 		vms->vms_pt_gen++;  in vmspace_unmap()
838 		gen = vms->vms_pt_gen;  in vmspace_unmap()
841 	vmm_gpt_vacate_region(vms->vms_gpt, addr, len);  in vmspace_unmap()
848 		vmspace_clients_invalidate(vms, addr, len);  in vmspace_unmap()
851 	vm_mapping_remove(vms, vmsm);  in vmspace_unmap()
852 	vmspace_hold_exit(vms, true);  in vmspace_unmap()
861 vmspace_ensure_mapped(vmspace_t *vms, uintptr_t gpa, int req_prot, pfn_t *pfnp,  in vmspace_ensure_mapped()  argument
871 	vmsm = vm_mapping_find(vms, gpa, PAGESIZE);  in vmspace_ensure_mapped()
883 	if (vmm_gpt_map_at(vms->vms_gpt, leaf_pte, pfn, vmsm->vmsm_prot,  in vmspace_ensure_mapped()
885 		atomic_inc_64(&vms->vms_pages_mapped);  in vmspace_ensure_mapped()
898 vmspace_lookup_map(vmspace_t *vms, uintptr_t gpa, int req_prot, pfn_t *pfnp,  in vmspace_lookup_map()  argument
901 	vmm_gpt_t *gpt = vms->vms_gpt;  in vmspace_lookup_map()
925 		int err = vmspace_ensure_mapped(vms, gpa, req_prot, &pfn, leaf);  in vmspace_lookup_map()
948 vmspace_populate(vmspace_t *vms, uintptr_t addr, uintptr_t len)  in vmspace_populate()  argument
951 	mutex_enter(&vms->vms_lock);  in vmspace_populate()
954 	if ((vmsm = vm_mapping_find(vms, addr, len)) == NULL) {  in vmspace_populate()
955 		mutex_exit(&vms->vms_lock);  in vmspace_populate()
968 		if (vmm_gpt_map(vms->vms_gpt, gpa, pfn, prot, attr)) {  in vmspace_populate()
972 	atomic_add_64(&vms->vms_pages_mapped, populated);  in vmspace_populate()
974 	mutex_exit(&vms->vms_lock);  in vmspace_populate()
982 vmspace_client_alloc(vmspace_t *vms)  in vmspace_client_alloc()  argument
987 	vmc->vmc_space = vms;  in vmspace_client_alloc()
994 	vmc->vmc_track_dirty = vms->vms_track_dirty;  in vmspace_client_alloc()
996 	mutex_enter(&vms->vms_lock);  in vmspace_client_alloc()
997 	list_insert_tail(&vms->vms_clients, vmc);  in vmspace_client_alloc()
998 	mutex_exit(&vms->vms_lock);  in vmspace_client_alloc()
1007 vmspace_table_root(vmspace_t *vms)  in vmspace_table_root()  argument
1009 	return (vmm_gpt_get_pmtp(vms->vms_gpt, vms->vms_track_dirty));  in vmspace_table_root()
1016 vmspace_table_gen(vmspace_t *vms)  in vmspace_table_gen()  argument
1018 	return (vms->vms_pt_gen);  in vmspace_table_gen()
1068 	vmspace_t *vms = vmc->vmc_space;  in vmc_table_enter()  local
1078 	gen = vms->vms_pt_gen;  in vmc_table_enter()
1229 vmc_space_orphan(vm_client_t *vmc, vmspace_t *vms)  in vmc_space_orphan()  argument
1233 	ASSERT(MUTEX_HELD(&vms->vms_lock));  in vmc_space_orphan()
1236 	VERIFY3P(vmc->vmc_space, ==, vms);  in vmc_space_orphan()
1244 		next = list_next(&vms->vms_clients, vmc);  in vmc_space_orphan()
1265 		next = list_next(&vms->vms_clients, vmc);  in vmc_space_orphan()
1266 		list_remove(&vms->vms_clients, vmc);  in vmc_space_orphan()
1279 	vmspace_t *vms = vmc->vmc_space;  in vmc_hold_ext()  local
1295 	if (vmspace_lookup_map(vms, gpa, prot, &pfn, &ptep) != 0) {  in vmc_hold_ext()
1328 	vmspace_t *vms = vmc->vmc_space;  in vmc_fault()  local
1333 		err = vmspace_lookup_map(vms, gpa & PAGEMASK, prot, NULL, NULL);  in vmc_fault()
1348 	vmspace_t *vms = vmc->vmc_space;  in vmc_clone()  local
1350 	return (vmspace_client_alloc(vms));  in vmc_clone()
1388 		vmspace_t *vms;  in vmc_destroy()  local
1400 		vms = vmc->vmc_space;  in vmc_destroy()
1403 		mutex_enter(&vms->vms_lock);  in vmc_destroy()
1405 		list_remove(&vms->vms_clients, vmc);  in vmc_destroy()
1411 		cv_signal(&vms->vms_cv);  in vmc_destroy()
1413 		mutex_exit(&vms->vms_lock);  in vmc_destroy()