268 		vm_client_t *vmc = list_head(&vms->vms_clients);  in vmspace_destroy()  local
269 		while (vmc != NULL) {  in vmspace_destroy()
270 			vmc = vmc_space_orphan(vmc, vms);  in vmspace_destroy()
445 	for (vm_client_t *vmc = list_head(&vms->vms_clients);  in vmspace_set_tracking()  local
446 	    vmc != NULL;  in vmspace_set_tracking()
447 	    vmc = list_next(&vms->vms_clients, vmc)) {  in vmspace_set_tracking()
448 		mutex_enter(&vmc->vmc_lock);  in vmspace_set_tracking()
449 		vmc->vmc_track_dirty = enable_dirty_tracking;  in vmspace_set_tracking()
450 		mutex_exit(&vmc->vmc_lock);  in vmspace_set_tracking()
711 	vm_client_t *vmc = list_head(&vms->vms_clients);  in vmspace_hold_enter()  local
712 	for (; vmc != NULL; vmc = list_next(&vms->vms_clients, vmc)) {  in vmspace_hold_enter()
713 		vmc_space_hold(vmc);  in vmspace_hold_enter()
734 	vm_client_t *vmc = list_head(&vms->vms_clients);  in vmspace_hold_exit()  local
735 	for (; vmc != NULL; vmc = list_next(&vms->vms_clients, vmc)) {  in vmspace_hold_exit()
736 		vmc_space_release(vmc, kick_on_cpu);  in vmspace_hold_exit()
748 	for (vm_client_t *vmc = list_head(&vms->vms_clients);  in vmspace_clients_invalidate()  local
749 	    vmc != NULL;  in vmspace_clients_invalidate()
750 	    vmc = list_next(&vms->vms_clients, vmc)) {  in vmspace_clients_invalidate()
751 		vmc_space_invalidate(vmc, gpa, len, vms->vms_pt_gen);  in vmspace_clients_invalidate()
816 	vm_client_t *vmc;  in vmspace_unmap()  local
830 	for (vmc = list_head(&vms->vms_clients); vmc != NULL;  in vmspace_unmap()
831 	    vmc = list_next(&vms->vms_clients, vmc)) {  in vmspace_unmap()
832 		vmc_space_unmap(vmc, addr, len, vmsm->vmsm_object);  in vmspace_unmap()
984 	vm_client_t *vmc;  in vmspace_client_alloc()  local
986 	vmc = kmem_zalloc(sizeof (vm_client_t), KM_SLEEP);  in vmspace_client_alloc()
987 	vmc->vmc_space = vms;  in vmspace_client_alloc()
988 	mutex_init(&vmc->vmc_lock, NULL, MUTEX_DRIVER, NULL);  in vmspace_client_alloc()
989 	cv_init(&vmc->vmc_cv, NULL, CV_DRIVER, NULL);  in vmspace_client_alloc()
990 	vmc->vmc_state = VCS_IDLE;  in vmspace_client_alloc()
991 	vmc->vmc_cpu_active = -1;  in vmspace_client_alloc()
992 	list_create(&vmc->vmc_held_pages, sizeof (vm_page_t),  in vmspace_client_alloc()
994 	vmc->vmc_track_dirty = vms->vms_track_dirty;  in vmspace_client_alloc()
997 	list_insert_tail(&vms->vms_clients, vmc);  in vmspace_client_alloc()
1000 	return (vmc);  in vmspace_client_alloc()
1027 vmc_activate(vm_client_t *vmc)  in vmc_activate()  argument
1029 	mutex_enter(&vmc->vmc_lock);  in vmc_activate()
1030 	VERIFY0(vmc->vmc_state & VCS_ACTIVE);  in vmc_activate()
1031 	if ((vmc->vmc_state & VCS_ORPHANED) != 0) {  in vmc_activate()
1032 		mutex_exit(&vmc->vmc_lock);  in vmc_activate()
1035 	while ((vmc->vmc_state & VCS_HOLD) != 0) {  in vmc_activate()
1036 		cv_wait(&vmc->vmc_cv, &vmc->vmc_lock);  in vmc_activate()
1038 	vmc->vmc_state |= VCS_ACTIVE;  in vmc_activate()
1047 vmc_deactivate(vm_client_t *vmc)  in vmc_deactivate()  argument
1049 	ASSERT(MUTEX_HELD(&vmc->vmc_lock));  in vmc_deactivate()
1050 	VERIFY(vmc->vmc_state & VCS_ACTIVE);  in vmc_deactivate()
1052 	vmc->vmc_state ^= VCS_ACTIVE;  in vmc_deactivate()
1053 	if ((vmc->vmc_state & VCS_HOLD) != 0) {  in vmc_deactivate()
1054 		cv_broadcast(&vmc->vmc_cv);  in vmc_deactivate()
1056 	mutex_exit(&vmc->vmc_lock);  in vmc_deactivate()
1066 vmc_table_enter(vm_client_t *vmc)  in vmc_table_enter()  argument
1068 	vmspace_t *vms = vmc->vmc_space;  in vmc_table_enter()
1071 	ASSERT0(vmc->vmc_state & (VCS_ACTIVE | VCS_ON_CPU));  in vmc_table_enter()
1072 	ASSERT3S(vmc->vmc_cpu_active, ==, -1);  in vmc_table_enter()
1079 	vmc->vmc_cpu_active = CPU->cpu_id;  in vmc_table_enter()
1080 	vmc->vmc_cpu_gen = gen;  in vmc_table_enter()
1081 	atomic_or_uint(&vmc->vmc_state, VCS_ON_CPU);  in vmc_table_enter()
1092 vmc_table_exit(vm_client_t *vmc)  in vmc_table_exit()  argument
1094 	mutex_enter(&vmc->vmc_lock);  in vmc_table_exit()
1096 	ASSERT(vmc->vmc_state & VCS_ON_CPU);  in vmc_table_exit()
1097 	vmc->vmc_state ^= VCS_ON_CPU;  in vmc_table_exit()
1098 	vmc->vmc_cpu_active = -1;  in vmc_table_exit()
1099 	if ((vmc->vmc_state & VCS_HOLD) != 0) {  in vmc_table_exit()
1100 		cv_broadcast(&vmc->vmc_cv);  in vmc_table_exit()
1103 	mutex_exit(&vmc->vmc_lock);  in vmc_table_exit()
1107 vmc_space_hold(vm_client_t *vmc)  in vmc_space_hold()  argument
1109 	mutex_enter(&vmc->vmc_lock);  in vmc_space_hold()
1110 	VERIFY0(vmc->vmc_state & VCS_HOLD);  in vmc_space_hold()
1117 	atomic_or_uint(&vmc->vmc_state, VCS_HOLD);  in vmc_space_hold()
1120 	while ((vmc->vmc_state & VCS_ACTIVE) != 0) {  in vmc_space_hold()
1121 		cv_wait(&vmc->vmc_cv, &vmc->vmc_lock);  in vmc_space_hold()
1123 	mutex_exit(&vmc->vmc_lock);  in vmc_space_hold()
1127 vmc_space_release(vm_client_t *vmc, bool kick_on_cpu)  in vmc_space_release()  argument
1129 	mutex_enter(&vmc->vmc_lock);  in vmc_space_release()
1130 	VERIFY(vmc->vmc_state & VCS_HOLD);  in vmc_space_release()
1132 	if (kick_on_cpu && (vmc->vmc_state & VCS_ON_CPU) != 0) {  in vmc_space_release()
1133 		poke_cpu(vmc->vmc_cpu_active);  in vmc_space_release()
1135 		while ((vmc->vmc_state & VCS_ON_CPU) != 0) {  in vmc_space_release()
1136 			cv_wait(&vmc->vmc_cv, &vmc->vmc_lock);  in vmc_space_release()
1145 	atomic_and_uint(&vmc->vmc_state, ~VCS_HOLD);  in vmc_space_release()
1146 	cv_broadcast(&vmc->vmc_cv);  in vmc_space_release()
1147 	mutex_exit(&vmc->vmc_lock);  in vmc_space_release()
1151 vmc_space_invalidate(vm_client_t *vmc, uintptr_t addr, size_t size,  in vmc_space_invalidate()  argument
1154 	mutex_enter(&vmc->vmc_lock);  in vmc_space_invalidate()
1155 	VERIFY(vmc->vmc_state & VCS_HOLD);  in vmc_space_invalidate()
1156 	if ((vmc->vmc_state & VCS_ON_CPU) != 0) {  in vmc_space_invalidate()
1162 		if (vmc->vmc_cpu_gen < gen) {  in vmc_space_invalidate()
1163 			poke_cpu(vmc->vmc_cpu_active);  in vmc_space_invalidate()
1165 			while ((vmc->vmc_state & VCS_ON_CPU) != 0) {  in vmc_space_invalidate()
1166 				cv_wait(&vmc->vmc_cv, &vmc->vmc_lock);  in vmc_space_invalidate()
1170 	if (vmc->vmc_inval_func != NULL) {  in vmc_space_invalidate()
1171 		vmc_inval_cb_t func = vmc->vmc_inval_func;  in vmc_space_invalidate()
1172 		void *data = vmc->vmc_inval_data;  in vmc_space_invalidate()
1179 		mutex_exit(&vmc->vmc_lock);  in vmc_space_invalidate()
1181 		mutex_enter(&vmc->vmc_lock);  in vmc_space_invalidate()
1183 	mutex_exit(&vmc->vmc_lock);  in vmc_space_invalidate()
1187 vmc_space_unmap(vm_client_t *vmc, uintptr_t addr, size_t size,  in vmc_space_unmap()  argument
1190 	mutex_enter(&vmc->vmc_lock);  in vmc_space_unmap()
1191 	VERIFY(vmc->vmc_state & VCS_HOLD);  in vmc_space_unmap()
1197 	VERIFY0(vmc->vmc_state & VCS_ON_CPU);  in vmc_space_unmap()
1204 	for (vm_page_t *vmp = list_head(&vmc->vmc_held_pages);  in vmc_space_unmap()
1206 	    vmp = list_next(&vmc->vmc_held_pages, vmc)) {  in vmc_space_unmap()
1225 	mutex_exit(&vmc->vmc_lock);  in vmc_space_unmap()
1229 vmc_space_orphan(vm_client_t *vmc, vmspace_t *vms)  in vmc_space_orphan()  argument
1235 	mutex_enter(&vmc->vmc_lock);  in vmc_space_orphan()
1236 	VERIFY3P(vmc->vmc_space, ==, vms);  in vmc_space_orphan()
1237 	VERIFY0(vmc->vmc_state & VCS_ORPHANED);  in vmc_space_orphan()
1238 	if (vmc->vmc_state & VCS_DESTROY) {  in vmc_space_orphan()
1244 		next = list_next(&vms->vms_clients, vmc);  in vmc_space_orphan()
1252 		for (vm_page_t *vmp = list_head(&vmc->vmc_held_pages);  in vmc_space_orphan()
1254 		    vmp = list_next(&vmc->vmc_held_pages, vmp)) {  in vmc_space_orphan()
1264 		vmc->vmc_state |= VCS_ORPHANED;  in vmc_space_orphan()
1265 		next = list_next(&vms->vms_clients, vmc);  in vmc_space_orphan()
1266 		list_remove(&vms->vms_clients, vmc);  in vmc_space_orphan()
1267 		vmc->vmc_space = NULL;  in vmc_space_orphan()
1269 	mutex_exit(&vmc->vmc_lock);  in vmc_space_orphan()
1277 vmc_hold_ext(vm_client_t *vmc, uintptr_t gpa, int prot, int flags)  in vmc_hold_ext()  argument
1279 	vmspace_t *vms = vmc->vmc_space;  in vmc_hold_ext()
1290 	if (vmc_activate(vmc) != 0) {  in vmc_hold_ext()
1296 		vmc_deactivate(vmc);  in vmc_hold_ext()
1302 	vmp->vmp_client = vmc;  in vmc_hold_ext()
1310 	list_insert_tail(&vmc->vmc_held_pages, vmp);  in vmc_hold_ext()
1311 	vmc_deactivate(vmc);  in vmc_hold_ext()
1320 vmc_hold(vm_client_t *vmc, uintptr_t gpa, int prot)  in vmc_hold()  argument
1322 	return (vmc_hold_ext(vmc, gpa, prot, VPF_DEFAULT));  in vmc_hold()
1326 vmc_fault(vm_client_t *vmc, uintptr_t gpa, int prot)  in vmc_fault()  argument
1328 	vmspace_t *vms = vmc->vmc_space;  in vmc_fault()
1331 	err = vmc_activate(vmc);  in vmc_fault()
1334 		vmc_deactivate(vmc);  in vmc_fault()
1346 vmc_clone(vm_client_t *vmc)  in vmc_clone()  argument
1348 	vmspace_t *vms = vmc->vmc_space;  in vmc_clone()
1358 vmc_set_inval_cb(vm_client_t *vmc, vmc_inval_cb_t func, void *data)  in vmc_set_inval_cb()  argument
1362 	err = vmc_activate(vmc);  in vmc_set_inval_cb()
1364 		vmc->vmc_inval_func = func;  in vmc_set_inval_cb()
1365 		vmc->vmc_inval_data = data;  in vmc_set_inval_cb()
1366 		vmc_deactivate(vmc);  in vmc_set_inval_cb()
1380 vmc_destroy(vm_client_t *vmc)  in vmc_destroy()  argument
1382 	mutex_enter(&vmc->vmc_lock);  in vmc_destroy()
1384 	VERIFY(list_is_empty(&vmc->vmc_held_pages));  in vmc_destroy()
1385 	VERIFY0(vmc->vmc_state & (VCS_ACTIVE | VCS_ON_CPU));  in vmc_destroy()
1387 	if ((vmc->vmc_state & VCS_ORPHANED) == 0) {  in vmc_destroy()
1399 		vmc->vmc_state |= VCS_DESTROY;  in vmc_destroy()
1400 		vms = vmc->vmc_space;  in vmc_destroy()
1401 		mutex_exit(&vmc->vmc_lock);  in vmc_destroy()
1404 		mutex_enter(&vmc->vmc_lock);  in vmc_destroy()
1405 		list_remove(&vms->vms_clients, vmc);  in vmc_destroy()
1412 		mutex_exit(&vmc->vmc_lock);  in vmc_destroy()
1415 		VERIFY3P(vmc->vmc_space, ==, NULL);  in vmc_destroy()
1416 		mutex_exit(&vmc->vmc_lock);  in vmc_destroy()
1419 	mutex_destroy(&vmc->vmc_lock);  in vmc_destroy()
1420 	cv_destroy(&vmc->vmc_cv);  in vmc_destroy()
1421 	list_destroy(&vmc->vmc_held_pages);  in vmc_destroy()
1423 	kmem_free(vmc, sizeof (*vmc));  in vmc_destroy()
1505 vmp_release_inner(vm_page_t *vmp, vm_client_t *vmc)  in vmp_release_inner()  argument
1507 	ASSERT(MUTEX_HELD(&vmc->vmc_lock));  in vmp_release_inner()
1511 	list_remove(&vmc->vmc_held_pages, vmp);  in vmp_release_inner()
1530 		    vmc->vmc_track_dirty) {  in vmp_release_inner()
1531 			vmm_gpt_t *gpt = vmc->vmc_space->vms_gpt;  in vmp_release_inner()
1546 	vm_client_t *vmc = vmp->vmp_client;  in vmp_release()  local
1548 	VERIFY(vmc != NULL);  in vmp_release()
1550 	mutex_enter(&vmc->vmc_lock);  in vmp_release()
1551 	const bool was_unmapped = vmp_release_inner(vmp, vmc);  in vmp_release()
1552 	mutex_exit(&vmc->vmc_lock);  in vmp_release()
1566 	vm_client_t *vmc = vmp->vmp_client;  in vmp_release_chain()  local
1571 	mutex_enter(&vmc->vmc_lock);  in vmp_release_chain()
1576 		ASSERT3P(vmp->vmp_client, ==, vmc);  in vmp_release_chain()
1578 		if (vmp_release_inner(vmp, vmc)) {  in vmp_release_chain()
1583 	mutex_exit(&vmc->vmc_lock);  in vmp_release_chain()
1625 		svma.vmc = NULL;  in vm_segmap_obj()
1660 		svma.vmc = vmspace_client_alloc(vm_get_vmspace(vm));  in vm_segmap_space()