115 static void	vm_object_backing_remove(vm_object_t object);
148 static SYSCTL_NODE(_vm_stats, OID_AUTO, object, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
176 	vm_object_t object;  in vm_object_zdtor()  local
178 	object = (vm_object_t)mem;  in vm_object_zdtor()
179 	KASSERT(object->ref_count == 0,  in vm_object_zdtor()
180 	    ("object %p ref_count = %d", object, object->ref_count));  in vm_object_zdtor()
181 	KASSERT(vm_radix_is_empty(&object->rtree),  in vm_object_zdtor()
182 	    ("object %p has resident pages in its trie", object));  in vm_object_zdtor()
184 	KASSERT(LIST_EMPTY(&object->rvq),  in vm_object_zdtor()
186 	    object));  in vm_object_zdtor()
188 	KASSERT(!vm_object_busied(object),  in vm_object_zdtor()
189 	    ("object %p busy = %d", object, blockcount_read(&object->busy)));  in vm_object_zdtor()
190 	KASSERT(object->resident_page_count == 0,  in vm_object_zdtor()
192 	    object, object->resident_page_count));  in vm_object_zdtor()
193 	KASSERT(atomic_load_int(&object->shadow_count) == 0,  in vm_object_zdtor()
195 	    object, atomic_load_int(&object->shadow_count)));  in vm_object_zdtor()
196 	KASSERT(object->type == OBJT_DEAD,  in vm_object_zdtor()
198 	    object, object->type));  in vm_object_zdtor()
199 	KASSERT(object->charge == 0 && object->cred == NULL,  in vm_object_zdtor()
201 	    object, (uintmax_t)object->charge, object->cred));  in vm_object_zdtor()
208 	vm_object_t object;  in vm_object_zinit()  local
210 	object = (vm_object_t)mem;  in vm_object_zinit()
211 	rw_init_flags(&object->lock, "vmobject", RW_DUPOK | RW_NEW);  in vm_object_zinit()
214 	object->type = OBJT_DEAD;  in vm_object_zinit()
215 	vm_radix_init(&object->rtree);  in vm_object_zinit()
216 	refcount_init(&object->ref_count, 0);  in vm_object_zinit()
217 	blockcount_init(&object->paging_in_progress);  in vm_object_zinit()
218 	blockcount_init(&object->busy);  in vm_object_zinit()
219 	object->resident_page_count = 0;  in vm_object_zinit()
220 	atomic_store_int(&object->shadow_count, 0);  in vm_object_zinit()
221 	object->flags = OBJ_DEAD;  in vm_object_zinit()
224 	TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);  in vm_object_zinit()
231     vm_object_t object, void *handle)  in _vm_object_allocate()  argument
233 	LIST_INIT(&object->shadow_head);  in _vm_object_allocate()
235 	object->type = type;  in _vm_object_allocate()
236 	object->flags = flags;  in _vm_object_allocate()
238 		pctrie_init(&object->un_pager.swp.swp_blks);  in _vm_object_allocate()
239 		object->un_pager.swp.writemappings = 0;  in _vm_object_allocate()
249 	object->pg_color = 0;  in _vm_object_allocate()
250 	object->size = size;  in _vm_object_allocate()
251 	object->domain.dr_policy = NULL;  in _vm_object_allocate()
252 	object->generation = 1;  in _vm_object_allocate()
253 	object->cleangeneration = 1;  in _vm_object_allocate()
254 	refcount_init(&object->ref_count, 1);  in _vm_object_allocate()
255 	object->memattr = VM_MEMATTR_DEFAULT;  in _vm_object_allocate()
256 	object->cred = NULL;  in _vm_object_allocate()
257 	object->charge = 0;  in _vm_object_allocate()
258 	object->handle = handle;  in _vm_object_allocate()
259 	object->backing_object = NULL;  in _vm_object_allocate()
260 	object->backing_object_offset = (vm_ooffset_t) 0;  in _vm_object_allocate()
262 	LIST_INIT(&object->rvq);  in _vm_object_allocate()
264 	umtx_shm_object_init(object);  in _vm_object_allocate()
308 vm_object_clear_flag(vm_object_t object, u_short bits)  in vm_object_clear_flag()  argument
311 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_clear_flag()
312 	object->flags &= ~bits;  in vm_object_clear_flag()
325 vm_object_set_memattr(vm_object_t object, vm_memattr_t memattr)  in vm_object_set_memattr()  argument
328 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_set_memattr()
330 	if (object->type == OBJT_DEAD)  in vm_object_set_memattr()
332 	if (!vm_radix_is_empty(&object->rtree))  in vm_object_set_memattr()
335 	object->memattr = memattr;  in vm_object_set_memattr()
340 vm_object_pip_add(vm_object_t object, short i)  in vm_object_pip_add()  argument
344 		blockcount_acquire(&object->paging_in_progress, i);  in vm_object_pip_add()
348 vm_object_pip_wakeup(vm_object_t object)  in vm_object_pip_wakeup()  argument
351 	vm_object_pip_wakeupn(object, 1);  in vm_object_pip_wakeup()
355 vm_object_pip_wakeupn(vm_object_t object, short i)  in vm_object_pip_wakeupn()  argument
359 		blockcount_release(&object->paging_in_progress, i);  in vm_object_pip_wakeupn()
368 vm_object_pip_sleep(vm_object_t object, const char *waitid)  in vm_object_pip_sleep()  argument
371 	(void)blockcount_sleep(&object->paging_in_progress, &object->lock,  in vm_object_pip_sleep()
376 vm_object_pip_wait(vm_object_t object, const char *waitid)  in vm_object_pip_wait()  argument
379 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_pip_wait()
381 	blockcount_wait(&object->paging_in_progress, &object->lock, waitid,  in vm_object_pip_wait()
386 vm_object_pip_wait_unlocked(vm_object_t object, const char *waitid)  in vm_object_pip_wait_unlocked()  argument
389 	VM_OBJECT_ASSERT_UNLOCKED(object);  in vm_object_pip_wait_unlocked()
391 	blockcount_wait(&object->paging_in_progress, NULL, waitid, PVM);  in vm_object_pip_wait_unlocked()
402 	vm_object_t object;  in vm_object_allocate()  local
428 	object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);  in vm_object_allocate()
429 	_vm_object_allocate(type, size, flags, object, NULL);  in vm_object_allocate()
431 	return (object);  in vm_object_allocate()
437 	vm_object_t object;  in vm_object_allocate_dyn()  local
440 	object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);  in vm_object_allocate_dyn()
441 	_vm_object_allocate(dyntype, size, flags, object, NULL);  in vm_object_allocate_dyn()
443 	return (object);  in vm_object_allocate_dyn()
457 	vm_object_t handle, object;  in vm_object_allocate_anon()  local
465 	object = uma_zalloc(obj_zone, M_WAITOK);  in vm_object_allocate_anon()
467 	    OBJ_ANON | OBJ_ONEMAPPING | OBJ_SWAP, object, handle);  in vm_object_allocate_anon()
468 	object->cred = cred;  in vm_object_allocate_anon()
469 	object->charge = cred != NULL ? charge : 0;  in vm_object_allocate_anon()
470 	return (object);  in vm_object_allocate_anon()
474 vm_object_reference_vnode(vm_object_t object)  in vm_object_reference_vnode()  argument
482 	if (!refcount_acquire_if_gt(&object->ref_count, 0)) {  in vm_object_reference_vnode()
483 		VM_OBJECT_RLOCK(object);  in vm_object_reference_vnode()
484 		old = refcount_acquire(&object->ref_count);  in vm_object_reference_vnode()
485 		if (object->type == OBJT_VNODE && old == 0)  in vm_object_reference_vnode()
486 			vref(object->handle);  in vm_object_reference_vnode()
487 		VM_OBJECT_RUNLOCK(object);  in vm_object_reference_vnode()
497 vm_object_reference(vm_object_t object)  in vm_object_reference()  argument
500 	if (object == NULL)  in vm_object_reference()
503 	if (object->type == OBJT_VNODE)  in vm_object_reference()
504 		vm_object_reference_vnode(object);  in vm_object_reference()
506 		refcount_acquire(&object->ref_count);  in vm_object_reference()
507 	KASSERT((object->flags & OBJ_DEAD) == 0,  in vm_object_reference()
519 vm_object_reference_locked(vm_object_t object)  in vm_object_reference_locked()  argument
523 	VM_OBJECT_ASSERT_LOCKED(object);  in vm_object_reference_locked()
524 	old = refcount_acquire(&object->ref_count);  in vm_object_reference_locked()
525 	if (object->type == OBJT_VNODE && old == 0)  in vm_object_reference_locked()
526 		vref(object->handle);  in vm_object_reference_locked()
527 	KASSERT((object->flags & OBJ_DEAD) == 0,  in vm_object_reference_locked()
535 vm_object_deallocate_vnode(vm_object_t object)  in vm_object_deallocate_vnode()  argument
537 	struct vnode *vp = (struct vnode *) object->handle;  in vm_object_deallocate_vnode()
540 	KASSERT(object->type == OBJT_VNODE,  in vm_object_deallocate_vnode()
545 	last = refcount_release(&object->ref_count);  in vm_object_deallocate_vnode()
546 	VM_OBJECT_RUNLOCK(object);  in vm_object_deallocate_vnode()
552 		umtx_shm_object_terminated(object);  in vm_object_deallocate_vnode()
566 	vm_object_t object;  in vm_object_deallocate_anon()  local
569 	object = LIST_FIRST(&backing_object->shadow_head);  in vm_object_deallocate_anon()
570 	KASSERT(object != NULL &&  in vm_object_deallocate_anon()
575 	KASSERT((object->flags & OBJ_ANON) != 0,  in vm_object_deallocate_anon()
576 	    ("invalid shadow object %p", object));  in vm_object_deallocate_anon()
578 	if (!VM_OBJECT_TRYWLOCK(object)) {  in vm_object_deallocate_anon()
583 		vm_object_pip_add(object, 1);  in vm_object_deallocate_anon()
585 		VM_OBJECT_WLOCK(object);  in vm_object_deallocate_anon()
586 		vm_object_pip_wakeup(object);  in vm_object_deallocate_anon()
593 	if ((object->flags & (OBJ_DEAD | OBJ_COLLAPSING)) != 0 ||  in vm_object_deallocate_anon()
594 	    !refcount_acquire_if_not_zero(&object->ref_count)) {  in vm_object_deallocate_anon()
595 		VM_OBJECT_WUNLOCK(object);  in vm_object_deallocate_anon()
598 	backing_object = object->backing_object;  in vm_object_deallocate_anon()
600 		vm_object_collapse(object);  in vm_object_deallocate_anon()
601 	VM_OBJECT_WUNLOCK(object);  in vm_object_deallocate_anon()
603 	return (object);  in vm_object_deallocate_anon()
618 vm_object_deallocate(vm_object_t object)  in vm_object_deallocate()  argument
623 	while (object != NULL) {  in vm_object_deallocate()
631 		if ((object->flags & OBJ_ANON) == 0)  in vm_object_deallocate()
632 			released = refcount_release_if_gt(&object->ref_count, 1);  in vm_object_deallocate()
634 			released = refcount_release_if_gt(&object->ref_count, 2);  in vm_object_deallocate()
638 		if (object->type == OBJT_VNODE) {  in vm_object_deallocate()
639 			VM_OBJECT_RLOCK(object);  in vm_object_deallocate()
640 			if (object->type == OBJT_VNODE) {  in vm_object_deallocate()
641 				vm_object_deallocate_vnode(object);  in vm_object_deallocate()
644 			VM_OBJECT_RUNLOCK(object);  in vm_object_deallocate()
647 		VM_OBJECT_WLOCK(object);  in vm_object_deallocate()
648 		KASSERT(object->ref_count > 0,  in vm_object_deallocate()
650 		    object->type));  in vm_object_deallocate()
656 		if (!refcount_release(&object->ref_count)) {  in vm_object_deallocate()
657 			if (object->ref_count > 1 ||  in vm_object_deallocate()
658 			    atomic_load_int(&object->shadow_count) == 0) {  in vm_object_deallocate()
659 				if ((object->flags & OBJ_ANON) != 0 &&  in vm_object_deallocate()
660 				    object->ref_count == 1)  in vm_object_deallocate()
661 					vm_object_set_flag(object,  in vm_object_deallocate()
663 				VM_OBJECT_WUNLOCK(object);  in vm_object_deallocate()
668 			object = vm_object_deallocate_anon(object);  in vm_object_deallocate()
676 		umtx_shm_object_terminated(object);  in vm_object_deallocate()
677 		temp = object->backing_object;  in vm_object_deallocate()
679 			KASSERT(object->type == OBJT_SWAP,  in vm_object_deallocate()
680 			    ("shadowed tmpfs v_object 2 %p", object));  in vm_object_deallocate()
681 			vm_object_backing_remove(object);  in vm_object_deallocate()
684 		KASSERT((object->flags & OBJ_DEAD) == 0,  in vm_object_deallocate()
686 		vm_object_set_flag(object, OBJ_DEAD);  in vm_object_deallocate()
687 		vm_object_terminate(object);  in vm_object_deallocate()
688 		object = temp;  in vm_object_deallocate()
693 vm_object_destroy(vm_object_t object)  in vm_object_destroy()  argument
695 	uma_zfree(obj_zone, object);  in vm_object_destroy()
699 vm_object_sub_shadow(vm_object_t object)  in vm_object_sub_shadow()  argument
701 	KASSERT(object->shadow_count >= 1,  in vm_object_sub_shadow()
702 	    ("object %p sub_shadow count zero", object));  in vm_object_sub_shadow()
703 	atomic_subtract_int(&object->shadow_count, 1);  in vm_object_sub_shadow()
707 vm_object_backing_remove_locked(vm_object_t object)  in vm_object_backing_remove_locked()  argument
711 	backing_object = object->backing_object;  in vm_object_backing_remove_locked()
712 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_backing_remove_locked()
715 	KASSERT((object->flags & OBJ_COLLAPSING) == 0,  in vm_object_backing_remove_locked()
719 	if ((object->flags & OBJ_SHADOWLIST) != 0) {  in vm_object_backing_remove_locked()
720 		LIST_REMOVE(object, shadow_list);  in vm_object_backing_remove_locked()
721 		vm_object_clear_flag(object, OBJ_SHADOWLIST);  in vm_object_backing_remove_locked()
723 	object->backing_object = NULL;  in vm_object_backing_remove_locked()
727 vm_object_backing_remove(vm_object_t object)  in vm_object_backing_remove()  argument
731 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_backing_remove()
733 	backing_object = object->backing_object;  in vm_object_backing_remove()
734 	if ((object->flags & OBJ_SHADOWLIST) != 0) {  in vm_object_backing_remove()
736 		vm_object_backing_remove_locked(object);  in vm_object_backing_remove()
739 		object->backing_object = NULL;  in vm_object_backing_remove()
745 vm_object_backing_insert_locked(vm_object_t object, vm_object_t backing_object)  in vm_object_backing_insert_locked()  argument
748 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_backing_insert_locked()
753 		LIST_INSERT_HEAD(&backing_object->shadow_head, object,  in vm_object_backing_insert_locked()
755 		vm_object_set_flag(object, OBJ_SHADOWLIST);  in vm_object_backing_insert_locked()
757 	object->backing_object = backing_object;  in vm_object_backing_insert_locked()
761 vm_object_backing_insert(vm_object_t object, vm_object_t backing_object)  in vm_object_backing_insert()  argument
764 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_backing_insert()
768 		vm_object_backing_insert_locked(object, backing_object);  in vm_object_backing_insert()
771 		object->backing_object = backing_object;  in vm_object_backing_insert()
781 vm_object_backing_insert_ref(vm_object_t object, vm_object_t backing_object)  in vm_object_backing_insert_ref()  argument
784 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_backing_insert_ref()
791 		vm_object_backing_insert_locked(object, backing_object);  in vm_object_backing_insert_ref()
797 		object->backing_object = backing_object;  in vm_object_backing_insert_ref()
805 vm_object_backing_transfer(vm_object_t object, vm_object_t backing_object)  in vm_object_backing_transfer()  argument
813 	vm_object_backing_remove_locked(object);  in vm_object_backing_transfer()
820 		vm_object_backing_insert_locked(object, new_backing_object);  in vm_object_backing_transfer()
828 		object->backing_object = new_backing_object;  in vm_object_backing_transfer()
837 vm_object_collapse_wait(vm_object_t object)  in vm_object_collapse_wait()  argument
840 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_collapse_wait()
842 	while ((object->flags & OBJ_COLLAPSING) != 0) {  in vm_object_collapse_wait()
843 		vm_object_pip_wait(object, "vmcolwait");  in vm_object_collapse_wait()
853 vm_object_backing_collapse_wait(vm_object_t object)  in vm_object_backing_collapse_wait()  argument
857 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_backing_collapse_wait()
860 		backing_object = object->backing_object;  in vm_object_backing_collapse_wait()
867 		VM_OBJECT_WUNLOCK(object);  in vm_object_backing_collapse_wait()
870 		VM_OBJECT_WLOCK(object);  in vm_object_backing_collapse_wait()
883 	vm_object_t object __diagused = objectv;  in vm_object_terminate_single_page()
886 	KASSERT(p->object == object &&  in vm_object_terminate_single_page()
889 	p->object = NULL;  in vm_object_terminate_single_page()
891 		KASSERT((object->flags & OBJ_UNMANAGED) != 0 ||  in vm_object_terminate_single_page()
904 vm_object_terminate_pages(vm_object_t object)  in vm_object_terminate_pages()  argument
906 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_terminate_pages()
913 	if (object->resident_page_count == 0)  in vm_object_terminate_pages()
916 	vm_radix_reclaim_callback(&object->rtree,  in vm_object_terminate_pages()
917 	    vm_object_terminate_single_page, object);  in vm_object_terminate_pages()
918 	object->resident_page_count = 0;  in vm_object_terminate_pages()
919 	if (object->type == OBJT_VNODE)  in vm_object_terminate_pages()
920 		vdrop(object->handle);  in vm_object_terminate_pages()
931 vm_object_terminate(vm_object_t object)  in vm_object_terminate()  argument
934 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_terminate()
935 	KASSERT((object->flags & OBJ_DEAD) != 0,  in vm_object_terminate()
936 	    ("terminating non-dead obj %p", object));  in vm_object_terminate()
937 	KASSERT((object->flags & OBJ_COLLAPSING) == 0,  in vm_object_terminate()
938 	    ("terminating collapsing obj %p", object));  in vm_object_terminate()
939 	KASSERT(object->backing_object == NULL,  in vm_object_terminate()
940 	    ("terminating shadow obj %p", object));  in vm_object_terminate()
949 	vm_object_pip_wait(object, "objtrm");  in vm_object_terminate()
951 	KASSERT(object->ref_count == 0,  in vm_object_terminate()
953 	    object->ref_count));  in vm_object_terminate()
955 	if ((object->flags & OBJ_PG_DTOR) == 0)  in vm_object_terminate()
956 		vm_object_terminate_pages(object);  in vm_object_terminate()
959 	if (__predict_false(!LIST_EMPTY(&object->rvq)))  in vm_object_terminate()
960 		vm_reserv_break_all(object);  in vm_object_terminate()
963 	KASSERT(object->cred == NULL || (object->flags & OBJ_SWAP) != 0,  in vm_object_terminate()
964 	    ("%s: non-swap obj %p has cred", __func__, object));  in vm_object_terminate()
969 	vm_pager_deallocate(object);  in vm_object_terminate()
970 	VM_OBJECT_WUNLOCK(object);  in vm_object_terminate()
972 	vm_object_destroy(object);  in vm_object_terminate()
1048 vm_object_page_clean(vm_object_t object, vm_ooffset_t start, vm_ooffset_t end,  in vm_object_page_clean()  argument
1058 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_page_clean()
1060 	if (!vm_object_mightbedirty(object) || object->resident_page_count == 0)  in vm_object_page_clean()
1068 	tend = (end == 0) ? object->size : OFF_TO_IDX(end + PAGE_MASK);  in vm_object_page_clean()
1069 	allclean = tstart == 0 && tend >= object->size;  in vm_object_page_clean()
1071 	vm_page_iter_init(&pages, object);  in vm_object_page_clean()
1074 	curgeneration = object->generation;  in vm_object_page_clean()
1086 			if (object->generation != curgeneration &&  in vm_object_page_clean()
1097 		if (object->type == OBJT_VNODE) {  in vm_object_page_clean()
1105 			if (object->generation != curgeneration &&  in vm_object_page_clean()
1140 	if (allclean && object->type == OBJT_VNODE)  in vm_object_page_clean()
1141 		object->cleangeneration = curgeneration;  in vm_object_page_clean()
1160 vm_object_sync(vm_object_t object, vm_ooffset_t offset, vm_size_t size,  in vm_object_sync()  argument
1169 	if (object == NULL)  in vm_object_sync()
1173 	VM_OBJECT_WLOCK(object);  in vm_object_sync()
1174 	while ((backing_object = object->backing_object) != NULL) {  in vm_object_sync()
1176 		offset += object->backing_object_offset;  in vm_object_sync()
1177 		VM_OBJECT_WUNLOCK(object);  in vm_object_sync()
1178 		object = backing_object;  in vm_object_sync()
1179 		if (object->size < OFF_TO_IDX(offset + size))  in vm_object_sync()
1180 			size = IDX_TO_OFF(object->size) - offset;  in vm_object_sync()
1194 	if (object->type == OBJT_VNODE &&  in vm_object_sync()
1195 	    vm_object_mightbedirty(object) != 0 &&  in vm_object_sync()
1196 	    ((vp = object->handle)->v_vflag & VV_NOSYNC) == 0) {  in vm_object_sync()
1197 		VM_OBJECT_WUNLOCK(object);  in vm_object_sync()
1201 		    atop(size) == object->size) {  in vm_object_sync()
1215 		VM_OBJECT_WLOCK(object);  in vm_object_sync()
1216 		res = vm_object_page_clean(object, offset, offset + size,  in vm_object_sync()
1218 		VM_OBJECT_WUNLOCK(object);  in vm_object_sync()
1240 		VM_OBJECT_WLOCK(object);  in vm_object_sync()
1242 	if ((object->type == OBJT_VNODE ||  in vm_object_sync()
1243 	     object->type == OBJT_DEVICE) && invalidate) {  in vm_object_sync()
1244 		if (object->type == OBJT_DEVICE)  in vm_object_sync()
1255 		vm_object_page_remove(object, OFF_TO_IDX(offset),  in vm_object_sync()
1258 	VM_OBJECT_WUNLOCK(object);  in vm_object_sync()
1269 vm_object_advice_applies(vm_object_t object, int advice)  in vm_object_advice_applies()  argument
1272 	if ((object->flags & OBJ_UNMANAGED) != 0)  in vm_object_advice_applies()
1276 	return ((object->flags & (OBJ_ONEMAPPING | OBJ_ANON)) ==  in vm_object_advice_applies()
1281 vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex,  in vm_object_madvise_freespace()  argument
1286 		vm_pager_freespace(object, pindex, size);  in vm_object_madvise_freespace()
1310 vm_object_madvise(vm_object_t object, vm_pindex_t pindex, vm_pindex_t end,  in vm_object_madvise()  argument
1318 	if (object == NULL)  in vm_object_madvise()
1321 	vm_page_iter_init(&pages, object);  in vm_object_madvise()
1323 	VM_OBJECT_WLOCK(object);  in vm_object_madvise()
1324 	if (!vm_object_advice_applies(object, advice)) {  in vm_object_madvise()
1325 		VM_OBJECT_WUNLOCK(object);  in vm_object_madvise()
1330 		tobject = object;  in vm_object_madvise()
1344 			if (object->backing_object == NULL) {  in vm_object_madvise()
1347 				vm_object_madvise_freespace(object, advice,  in vm_object_madvise()
1368 				if (tobject != object)  in vm_object_madvise()
1392 			if (object != tobject)  in vm_object_madvise()
1393 				VM_OBJECT_WUNLOCK(object);  in vm_object_madvise()
1411 		if (tobject != object)  in vm_object_madvise()
1414 	VM_OBJECT_WUNLOCK(object);  in vm_object_madvise()
1428 vm_object_shadow(vm_object_t *object, vm_ooffset_t *offset, vm_size_t length,  in vm_object_shadow()  argument
1434 	source = *object;  in vm_object_shadow()
1498 	*object = result;  in vm_object_shadow()
1517 	orig_object = entry->object.vm_object;  in vm_object_split()
1644 	entry->object.vm_object = new_object;  in vm_object_split()
1651 vm_object_collapse_scan_wait(struct pctrie_iter *pages, vm_object_t object,  in vm_object_collapse_scan_wait()  argument
1656 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_collapse_scan_wait()
1657 	backing_object = object->backing_object;  in vm_object_collapse_scan_wait()
1660 	KASSERT(p == NULL || p->object == object || p->object == backing_object,  in vm_object_collapse_scan_wait()
1661 	    ("invalid ownership %p %p %p", p, object, backing_object));  in vm_object_collapse_scan_wait()
1664 		VM_OBJECT_WUNLOCK(object);  in vm_object_collapse_scan_wait()
1667 		VM_OBJECT_WLOCK(object);  in vm_object_collapse_scan_wait()
1668 	} else if (p->object == object) {  in vm_object_collapse_scan_wait()
1671 			VM_OBJECT_WLOCK(object);  in vm_object_collapse_scan_wait()
1673 		VM_OBJECT_WUNLOCK(object);  in vm_object_collapse_scan_wait()
1676 		VM_OBJECT_WLOCK(object);  in vm_object_collapse_scan_wait()
1684 vm_object_collapse_scan(vm_object_t object)  in vm_object_collapse_scan()  argument
1691 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_collapse_scan()
1692 	VM_OBJECT_ASSERT_WLOCKED(object->backing_object);  in vm_object_collapse_scan()
1694 	backing_object = object->backing_object;  in vm_object_collapse_scan()
1695 	backing_offset_index = OFF_TO_IDX(object->backing_object_offset);  in vm_object_collapse_scan()
1706 			next = vm_object_collapse_scan_wait(&pages, object, p);  in vm_object_collapse_scan()
1710 		KASSERT(object->backing_object == backing_object,  in vm_object_collapse_scan()
1712 		    object->backing_object, backing_object));  in vm_object_collapse_scan()
1713 		KASSERT(p->object == backing_object,  in vm_object_collapse_scan()
1715 		    p->object, backing_object));  in vm_object_collapse_scan()
1717 		if (p->pindex < backing_offset_index || object->size <=  in vm_object_collapse_scan()
1738 		pp = vm_page_lookup(object, new_pindex);  in vm_object_collapse_scan()
1747 			next = vm_object_collapse_scan_wait(&pages, object, pp);  in vm_object_collapse_scan()
1762 		if (pp != NULL || vm_pager_has_page(object, new_pindex, NULL,  in vm_object_collapse_scan()
1789 		if (!vm_page_iter_rename(&pages, p, object, new_pindex)) {  in vm_object_collapse_scan()
1791 			next = vm_object_collapse_scan_wait(&pages, object,  in vm_object_collapse_scan()
1804 		vm_reserv_rename(p, object, backing_object,  in vm_object_collapse_scan()
1821 vm_object_collapse(vm_object_t object)  in vm_object_collapse()  argument
1825 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_collapse()
1828 		KASSERT((object->flags & (OBJ_DEAD | OBJ_ANON)) == OBJ_ANON,  in vm_object_collapse()
1836 		backing_object = vm_object_backing_collapse_wait(object);  in vm_object_collapse()
1840 		KASSERT(object->ref_count > 0 &&  in vm_object_collapse()
1841 		    object->ref_count > atomic_load_int(&object->shadow_count),  in vm_object_collapse()
1843 		    object->ref_count, atomic_load_int(&object->shadow_count)));  in vm_object_collapse()
1847 		KASSERT((object->flags & (OBJ_COLLAPSING | OBJ_DEAD)) == 0,  in vm_object_collapse()
1861 			vm_object_pip_add(object, 1);  in vm_object_collapse()
1862 			vm_object_set_flag(object, OBJ_COLLAPSING);  in vm_object_collapse()
1870 			vm_object_collapse_scan(object);  in vm_object_collapse()
1879 			swap_pager_copy(backing_object, object,  in vm_object_collapse()
1880 			    OFF_TO_IDX(object->backing_object_offset), TRUE);  in vm_object_collapse()
1885 			vm_object_clear_flag(object, OBJ_COLLAPSING);  in vm_object_collapse()
1886 			vm_object_backing_transfer(object, backing_object);  in vm_object_collapse()
1887 			object->backing_object_offset +=  in vm_object_collapse()
1889 			VM_OBJECT_WUNLOCK(object);  in vm_object_collapse()
1890 			vm_object_pip_wakeup(object);  in vm_object_collapse()
1907 			VM_OBJECT_WLOCK(object);  in vm_object_collapse()
1916 			if (!swap_pager_scan_all_shadowed(object)) {  in vm_object_collapse()
1926 			vm_object_backing_remove_locked(object);  in vm_object_collapse()
1929 				vm_object_backing_insert_ref(object,  in vm_object_collapse()
1931 				object->backing_object_offset +=  in vm_object_collapse()
1980 vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,  in vm_object_page_remove()  argument
1986 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_page_remove()
1987 	KASSERT((object->flags & OBJ_UNMANAGED) == 0 ||  in vm_object_page_remove()
1989 	    ("vm_object_page_remove: illegal options for object %p", object));  in vm_object_page_remove()
1990 	if (object->resident_page_count == 0)  in vm_object_page_remove()
1992 	vm_object_pip_add(object, 1);  in vm_object_page_remove()
1993 	vm_page_iter_limit_init(&pages, object, end);  in vm_object_page_remove()
2024 				VM_OBJECT_WLOCK(object);  in vm_object_page_remove()
2035 			    object->ref_count != 0)  in vm_object_page_remove()
2049 			    object->ref_count != 0 &&  in vm_object_page_remove()
2058 		    object->ref_count != 0 && !vm_page_try_remove_all(p))  in vm_object_page_remove()
2062 	vm_object_pip_wakeup(object);  in vm_object_page_remove()
2064 	vm_pager_freespace(object, start, (end == 0 ? object->size : end) -  in vm_object_page_remove()
2085 vm_object_page_noreuse(vm_object_t object, vm_pindex_t start, vm_pindex_t end)  in vm_object_page_noreuse()  argument
2090 	VM_OBJECT_ASSERT_LOCKED(object);  in vm_object_page_noreuse()
2091 	KASSERT((object->flags & (OBJ_FICTITIOUS | OBJ_UNMANAGED)) == 0,  in vm_object_page_noreuse()
2092 	    ("vm_object_page_noreuse: illegal object %p", object));  in vm_object_page_noreuse()
2093 	if (object->resident_page_count == 0)  in vm_object_page_noreuse()
2096 	vm_page_iter_limit_init(&pages, object, end);  in vm_object_page_noreuse()
2112 vm_object_populate(vm_object_t object, vm_pindex_t start, vm_pindex_t end)  in vm_object_populate()  argument
2119 	vm_page_iter_init(&pages, object);  in vm_object_populate()
2120 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_populate()
2122 		rv = vm_page_grab_valid_iter(&m, object, pindex,  in vm_object_populate()
2259 vm_object_prepare_buf_pages(vm_object_t object, vm_page_t *ma_dst, int count,  in vm_object_prepare_buf_pages()  argument
2266 	vm_page_iter_init(&pages, object);  in vm_object_prepare_buf_pages()
2267 	VM_OBJECT_ASSERT_LOCKED(object);  in vm_object_prepare_buf_pages()
2275 			m = vm_page_alloc_iter(object, pindex - i - 1,  in vm_object_prepare_buf_pages()
2295 		    (msucc != NULL ? msucc->pindex : object->size) - pindex);  in vm_object_prepare_buf_pages()
2297 			m = vm_page_alloc_iter(object, pindex + i,  in vm_object_prepare_buf_pages()
2309 vm_object_set_writeable_dirty_(vm_object_t object)  in vm_object_set_writeable_dirty_()  argument
2311 	atomic_add_int(&object->generation, 1);  in vm_object_set_writeable_dirty_()
2315 vm_object_mightbedirty_(vm_object_t object)  in vm_object_mightbedirty_()  argument
2317 	return (object->generation != object->cleangeneration);  in vm_object_mightbedirty_()
2329 vm_object_unwire(vm_object_t object, vm_ooffset_t offset, vm_size_t length,  in vm_object_unwire()  argument
2343 	if ((object->flags & OBJ_FICTITIOUS) != 0)  in vm_object_unwire()
2347 	vm_page_iter_init(&pages, object);  in vm_object_unwire()
2350 	VM_OBJECT_RLOCK(object);  in vm_object_unwire()
2359 			tobject = object;  in vm_object_unwire()
2382 			for (tobject = object; locked_depth >= 1;  in vm_object_unwire()
2385 				if (tm->object != tobject)  in vm_object_unwire()
2389 			tobject = tm->object;  in vm_object_unwire()
2402 	for (tobject = object; locked_depth >= 1; locked_depth--) {  in vm_object_unwire()
2415 vm_object_vnode(vm_object_t object)  in vm_object_vnode()  argument
2419 	VM_OBJECT_ASSERT_LOCKED(object);  in vm_object_vnode()
2420 	vm_pager_getvp(object, &vp, NULL);  in vm_object_vnode()
2654 _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry)  in _vm_object_in_map()  argument
2665 			if (_vm_object_in_map(map, object, tmpe)) {  in _vm_object_in_map()
2670 		tmpm = entry->object.sub_map;  in _vm_object_in_map()
2672 			if (_vm_object_in_map(tmpm, object, tmpe)) {  in _vm_object_in_map()
2676 	} else if ((obj = entry->object.vm_object) != NULL) {  in _vm_object_in_map()
2678 			if (obj == object) {  in _vm_object_in_map()
2686 vm_object_in_map(vm_object_t object)  in vm_object_in_map()  argument
2694 		if (_vm_object_in_map(&p->p_vmspace->vm_map, object, 0)) {  in vm_object_in_map()
2700 	if (_vm_object_in_map(kernel_map, object, 0))  in vm_object_in_map()
2707 	vm_object_t object;  in DB_SHOW_COMMAND_FLAGS()  local
2713 	TAILQ_FOREACH(object, &vm_object_list, object_list) {  in DB_SHOW_COMMAND_FLAGS()
2714 		if ((object->flags & OBJ_ANON) != 0) {  in DB_SHOW_COMMAND_FLAGS()
2715 			if (object->ref_count == 0) {  in DB_SHOW_COMMAND_FLAGS()
2718 				    (u_long)object->size);  in DB_SHOW_COMMAND_FLAGS()
2720 			if (!vm_object_in_map(object)) {  in DB_SHOW_COMMAND_FLAGS()
2724 				    object->ref_count, (u_long)object->size,   in DB_SHOW_COMMAND_FLAGS()
2725 				    (u_long)object->size,  in DB_SHOW_COMMAND_FLAGS()
2726 				    (void *)object->backing_object);  in DB_SHOW_COMMAND_FLAGS()
2737 DB_SHOW_COMMAND(object, vm_object_print_static)  in DB_SHOW_COMMAND()  argument
2741 	vm_object_t object = (vm_object_t)addr;  in DB_SHOW_COMMAND()  local
2751 	if (object == NULL)  in DB_SHOW_COMMAND()
2755 	    object, (int)object->type, (uintmax_t)object->size,  in DB_SHOW_COMMAND()
2756 	    object->resident_page_count, object->ref_count, object->flags);  in DB_SHOW_COMMAND()
2758 	    object->cred ? object->cred->cr_ruid : -1,  in DB_SHOW_COMMAND()
2759 	    (uintmax_t)object->charge);  in DB_SHOW_COMMAND()
2761 	    atomic_load_int(&object->shadow_count),  in DB_SHOW_COMMAND()
2762 	    object->backing_object ? object->backing_object->ref_count : 0,  in DB_SHOW_COMMAND()
2763 	    object->backing_object, (uintmax_t)object->backing_object_offset);  in DB_SHOW_COMMAND()
2770 	vm_page_iter_init(&pages, object);  in DB_SHOW_COMMAND()
2810 	vm_object_t object;  in DB_SHOW_COMMAND_FLAGS()  local
2814 	TAILQ_FOREACH(object, &vm_object_list, object_list) {  in DB_SHOW_COMMAND_FLAGS()
2815 		db_printf("new object: %p\n", (void *)object);  in DB_SHOW_COMMAND_FLAGS()
2819 		vm_page_iter_init(&pages, object);  in DB_SHOW_COMMAND_FLAGS()