430 static void	swap_pager_dealloc(vm_object_t object);
437 		swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
441 static void	swap_pager_update_writecount(vm_object_t object,
443 static void	swap_pager_release_writecount(vm_object_t object,
445 static void	swap_pager_freespace_pgo(vm_object_t object, vm_pindex_t start,
491 static daddr_t swp_pager_meta_build(struct pctrie_iter *, vm_object_t object,
544 swblk_lookup(vm_object_t object, vm_pindex_t pindex)  in swblk_lookup()  argument
546 	return (SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks,  in swblk_lookup()
551 swblk_lookup_remove(vm_object_t object, struct swblk *sb)  in swblk_lookup_remove()  argument
553 	SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p);  in swblk_lookup_remove()
557 swblk_is_empty(vm_object_t object)  in swblk_is_empty()  argument
559 	return (pctrie_is_empty(&object->un_pager.swp.swp_blks));  in swblk_is_empty()
570 swblk_iter_init_only(struct pctrie_iter *blks, vm_object_t object)  in swblk_iter_init_only()  argument
572 	VM_OBJECT_ASSERT_LOCKED(object);  in swblk_iter_init_only()
573 	MPASS((object->flags & OBJ_SWAP) != 0);  in swblk_iter_init_only()
574 	pctrie_iter_init(blks, &object->un_pager.swp.swp_blks);  in swblk_iter_init_only()
579 swblk_iter_init(struct pctrie_iter *blks, vm_object_t object,  in swblk_iter_init()  argument
582 	swblk_iter_init_only(blks, object);  in swblk_iter_init()
587 swblk_iter_reinit(struct pctrie_iter *blks, vm_object_t object,  in swblk_iter_reinit()  argument
590 	swblk_iter_init_only(blks, object);  in swblk_iter_reinit()
596 swblk_iter_limit_init(struct pctrie_iter *blks, vm_object_t object,  in swblk_iter_limit_init()  argument
599 	VM_OBJECT_ASSERT_LOCKED(object);  in swblk_iter_limit_init()
600 	MPASS((object->flags & OBJ_SWAP) != 0);  in swblk_iter_limit_init()
601 	pctrie_iter_limit_init(blks, &object->un_pager.swp.swp_blks, limit);  in swblk_iter_limit_init()
768 swap_pager_init_object(vm_object_t object, void *handle, struct ucred *cred,  in swap_pager_init_object()  argument
777 	object->un_pager.swp.writemappings = 0;  in swap_pager_init_object()
778 	object->handle = handle;  in swap_pager_init_object()
780 		object->cred = cred;  in swap_pager_init_object()
781 		object->charge = size;  in swap_pager_init_object()
790 	vm_object_t object;  in swap_pager_alloc_init()  local
797 	object = vm_object_allocate(otype, OFF_TO_IDX(offset +  in swap_pager_alloc_init()
800 	if (!swap_pager_init_object(object, handle, cred, size, offset)) {  in swap_pager_alloc_init()
801 		vm_object_deallocate(object);  in swap_pager_alloc_init()
804 	return (object);  in swap_pager_alloc_init()
822 	vm_object_t object;  in swap_pager_alloc()  local
832 		object = vm_pager_object_lookup(NOBJLIST(handle), handle);  in swap_pager_alloc()
833 		if (object == NULL) {  in swap_pager_alloc()
834 			object = swap_pager_alloc_init(OBJT_SWAP, handle, cred,  in swap_pager_alloc()
836 			if (object != NULL) {  in swap_pager_alloc()
837 				TAILQ_INSERT_TAIL(NOBJLIST(object->handle),  in swap_pager_alloc()
838 				    object, pager_object_list);  in swap_pager_alloc()
843 		object = swap_pager_alloc_init(OBJT_SWAP, handle, cred,  in swap_pager_alloc()
846 	return (object);  in swap_pager_alloc()
860 swap_pager_dealloc(vm_object_t object)  in swap_pager_dealloc()  argument
863 	VM_OBJECT_ASSERT_WLOCKED(object);  in swap_pager_dealloc()
864 	KASSERT((object->flags & OBJ_DEAD) != 0, ("dealloc of reachable obj"));  in swap_pager_dealloc()
870 	if ((object->flags & OBJ_ANON) == 0 && object->handle != NULL) {  in swap_pager_dealloc()
871 		VM_OBJECT_WUNLOCK(object);  in swap_pager_dealloc()
873 		TAILQ_REMOVE(NOBJLIST(object->handle), object,  in swap_pager_dealloc()
876 		VM_OBJECT_WLOCK(object);  in swap_pager_dealloc()
879 	vm_object_pip_wait(object, "swpdea");  in swap_pager_dealloc()
887 	swp_pager_meta_free_all(object);  in swap_pager_dealloc()
888 	object->handle = NULL;  in swap_pager_dealloc()
889 	object->type = OBJT_DEAD;  in swap_pager_dealloc()
894 	if (object->cred != NULL) {  in swap_pager_dealloc()
895 		swap_release_by_cred(object->charge, object->cred);  in swap_pager_dealloc()
896 		object->charge = 0;  in swap_pager_dealloc()
897 		crfree(object->cred);  in swap_pager_dealloc()
898 		object->cred = NULL;  in swap_pager_dealloc()
904 	vm_object_clear_flag(object, OBJ_SWAP);  in swap_pager_dealloc()
1086 swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size,  in swap_pager_freespace()  argument
1089 	MPASS((object->flags & OBJ_SWAP) != 0);  in swap_pager_freespace()
1091 	swp_pager_meta_free(object, start, size, freed);  in swap_pager_freespace()
1095 swap_pager_freespace_pgo(vm_object_t object, vm_pindex_t start, vm_size_t size)  in swap_pager_freespace_pgo()  argument
1097 	MPASS((object->flags & OBJ_SWAP) != 0);  in swap_pager_freespace_pgo()
1099 	swp_pager_meta_free(object, start, size, NULL);  in swap_pager_freespace_pgo()
1111 swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_pindex_t size)  in swap_pager_reserve()  argument
1120 	VM_OBJECT_WLOCK(object);  in swap_pager_reserve()
1121 	swblk_iter_init_only(&blks, object);  in swap_pager_reserve()
1126 			swp_pager_meta_free(object, start, i, NULL);  in swap_pager_reserve()
1127 			VM_OBJECT_WUNLOCK(object);  in swap_pager_reserve()
1131 			addr = swp_pager_meta_build(&blks, object,  in swap_pager_reserve()
1138 	VM_OBJECT_WUNLOCK(object);  in swap_pager_reserve()
1262 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,  in swap_pager_haspage()  argument
1267 	swblk_iter_init_only(&blks, object);  in swap_pager_haspage()
1274 	KASSERT((m->object->flags & OBJ_SWAP) != 0,  in swap_pager_unswapped_acct()
1316 	obj = m->object;  in swap_pager_unswapped()
1334 	sb = swblk_lookup(m->object, m->pindex);  in swap_pager_unswapped()
1343 	swp_pager_free_empty_swblk(m->object, sb);  in swap_pager_unswapped()
1358 swap_pager_getpages_locked(struct pctrie_iter *blks, vm_object_t object,  in swap_pager_getpages_locked()  argument
1364 	VM_OBJECT_ASSERT_WLOCKED(object);  in swap_pager_getpages_locked()
1366 	KASSERT((object->flags & OBJ_SWAP) != 0,  in swap_pager_getpages_locked()
1370 		VM_OBJECT_WUNLOCK(object);  in swap_pager_getpages_locked()
1384 	if ((object->flags & (OBJ_SPLIT | OBJ_DEAD)) != 0) {  in swap_pager_getpages_locked()
1392 	vm_object_prepare_buf_pages(object, bp->b_pages, count, &rbehind,  in swap_pager_getpages_locked()
1399 	    ("no swap blocking containing %p(%jx)", object, (uintmax_t)pindex));  in swap_pager_getpages_locked()
1401 	vm_object_pip_add(object, bp->b_npages);  in swap_pager_getpages_locked()
1402 	VM_OBJECT_WUNLOCK(object);  in swap_pager_getpages_locked()
1442 	VM_OBJECT_WLOCK(object);  in swap_pager_getpages_locked()
1447 		if (VM_OBJECT_SLEEP(object, &object->handle, PSWP,  in swap_pager_getpages_locked()
1454 	VM_OBJECT_WUNLOCK(object);  in swap_pager_getpages_locked()
1474 swap_pager_getpages(vm_object_t object, vm_page_t *ma, int count,  in swap_pager_getpages()  argument
1481 	VM_OBJECT_WLOCK(object);  in swap_pager_getpages()
1482 	swblk_iter_init_only(&blks, object);  in swap_pager_getpages()
1483 	return (swap_pager_getpages_locked(&blks, object, ma, count, rbehind,  in swap_pager_getpages()
1494 swap_pager_getpages_async(vm_object_t object, vm_page_t *ma, int count,  in swap_pager_getpages_async()  argument
1499 	r = swap_pager_getpages(object, ma, count, rbehind, rahead);  in swap_pager_getpages_async()
1538 swap_pager_putpages(vm_object_t object, vm_page_t *ma, int count,  in swap_pager_putpages()  argument
1549 	KASSERT(count == 0 || ma[0]->object == object,  in swap_pager_putpages()
1551 	    __func__, object, ma[0]->object));  in swap_pager_putpages()
1553 	VM_OBJECT_WUNLOCK(object);  in swap_pager_putpages()
1586 		VM_OBJECT_WLOCK(object);  in swap_pager_putpages()
1587 		swblk_iter_init_only(&blks, object);  in swap_pager_putpages()
1591 			KASSERT(mreq->object == object,  in swap_pager_putpages()
1593 			    __func__, mreq->object, object));  in swap_pager_putpages()
1594 			addr = swp_pager_meta_build(&blks, object,  in swap_pager_putpages()
1601 		VM_OBJECT_WUNLOCK(object);  in swap_pager_putpages()
1670 	VM_OBJECT_WLOCK(object);  in swap_pager_putpages()
1685 	vm_object_t object = NULL;  in swp_pager_async_iodone()  local
1711 		object = bp->b_pages[0]->object;  in swp_pager_async_iodone()
1712 		VM_OBJECT_WLOCK(object);  in swp_pager_async_iodone()
1729 			wakeup(&object->handle);  in swp_pager_async_iodone()
1805 	if (object != NULL) {  in swp_pager_async_iodone()
1806 		vm_object_pip_wakeupn(object, bp->b_npages);  in swp_pager_async_iodone()
1807 		VM_OBJECT_WUNLOCK(object);  in swp_pager_async_iodone()
1849 swap_pager_swapped_pages(vm_object_t object)  in swap_pager_swapped_pages()  argument
1856 	VM_OBJECT_ASSERT_LOCKED(object);  in swap_pager_swapped_pages()
1858 	if (swblk_is_empty(object))  in swap_pager_swapped_pages()
1862 	for (sb = swblk_iter_init(&blks, object, 0); sb != NULL;  in swap_pager_swapped_pages()
1879 swap_pager_swapoff_object(struct swdevt *sp, vm_object_t object,  in swap_pager_swapoff_object()  argument
1889 	VM_OBJECT_ASSERT_WLOCKED(object);  in swap_pager_swapoff_object()
1890 	KASSERT((object->flags & OBJ_SWAP) != 0,  in swap_pager_swapoff_object()
1892 	KASSERT((object->flags & OBJ_DEAD) == 0,  in swap_pager_swapoff_object()
1897 	vm_page_iter_init(&pages, object);  in swap_pager_swapoff_object()
1899 	sb = swblk_iter_init(&blks, object, 0);  in swap_pager_swapoff_object()
1920 				VM_OBJECT_SLEEP(object, &object->handle, PSWP,  in swap_pager_swapoff_object()
1938 				m = vm_page_alloc_iter(object, blks.index + i,  in swap_pager_swapoff_object()
1946 			vm_object_pip_add(object, 1);  in swap_pager_swapoff_object()
1948 			rv = swap_pager_getpages_locked(&blks, object, &m, 1,  in swap_pager_swapoff_object()
1954 			VM_OBJECT_WLOCK(object);  in swap_pager_swapoff_object()
1955 			vm_object_pip_wakeupn(object, 1);  in swap_pager_swapoff_object()
1967 			if ((object->flags & OBJ_DEAD) != 0) {  in swap_pager_swapoff_object()
1972 				vm_object_pip_wait(object, "swpoff");  in swap_pager_swapoff_object()
1973 				swp_pager_meta_free_all(object);  in swap_pager_swapoff_object()
1983 			sb = swblk_iter_init(&blks, object, blks.index);  in swap_pager_swapoff_object()
2014 	vm_object_t object;  in swap_pager_swapoff()  local
2024 	TAILQ_FOREACH(object, &vm_object_list, object_list) {  in swap_pager_swapoff()
2025 		if ((object->flags & OBJ_SWAP) == 0)  in swap_pager_swapoff()
2029 		VM_OBJECT_WLOCK(object);  in swap_pager_swapoff()
2034 		if ((object->flags & OBJ_DEAD) != 0)  in swap_pager_swapoff()
2044 		if ((object->flags & OBJ_SWAP) == 0)  in swap_pager_swapoff()
2047 		swap_pager_swapoff_object(sp, object, &bp);  in swap_pager_swapoff()
2049 		VM_OBJECT_WUNLOCK(object);  in swap_pager_swapoff()
2107 swp_pager_free_empty_swblk(vm_object_t object, struct swblk *sb)  in swp_pager_free_empty_swblk()  argument
2111 		swblk_lookup_remove(object, sb);  in swp_pager_free_empty_swblk()
2129 swp_pager_meta_build(struct pctrie_iter *blks, vm_object_t object,  in swp_pager_meta_build()  argument
2138 	VM_OBJECT_ASSERT_WLOCKED(object);  in swp_pager_meta_build()
2158 			VM_OBJECT_WUNLOCK(object);  in swp_pager_meta_build()
2168 			VM_OBJECT_WLOCK(object);  in swp_pager_meta_build()
2169 			sb = swblk_iter_reinit(blks, object, pindex);  in swp_pager_meta_build()
2190 			VM_OBJECT_WUNLOCK(object);  in swp_pager_meta_build()
2200 			VM_OBJECT_WLOCK(object);  in swp_pager_meta_build()
2201 			sb1 = swblk_iter_reinit(blks, object, pindex);  in swp_pager_meta_build()
2318 swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count,  in swp_pager_meta_free()  argument
2329 	VM_OBJECT_ASSERT_WLOCKED(object);  in swp_pager_meta_free()
2332 	if (count == 0 || swblk_is_empty(object))  in swp_pager_meta_free()
2336 	vm_page_iter_init(&pages, object);  in swp_pager_meta_free()
2338 	for (sb = swblk_iter_limit_init(&blks, object, pindex, last),  in swp_pager_meta_free()
2384 swp_pager_meta_free_all(vm_object_t object)  in swp_pager_meta_free_all()  argument
2388 	VM_OBJECT_ASSERT_WLOCKED(object);  in swp_pager_meta_free_all()
2391 	SWAP_PCTRIE_RECLAIM_CALLBACK(&object->un_pager.swp.swp_blks,  in swp_pager_meta_free_all()
2457 swap_pager_seek_data(vm_object_t object, vm_pindex_t pindex)  in swap_pager_seek_data()  argument
2463 	VM_OBJECT_ASSERT_RLOCKED(object);  in swap_pager_seek_data()
2464 	vm_page_iter_init(&pages, object);  in swap_pager_seek_data()
2468 	swblk_iter_init_only(&blks, object);  in swap_pager_seek_data()
2482 		swap_index = object->size;  in swap_pager_seek_data()
2491 swap_pager_seek_hole(vm_object_t object, vm_pindex_t pindex)  in swap_pager_seek_hole()  argument
2497 	VM_OBJECT_ASSERT_RLOCKED(object);  in swap_pager_seek_hole()
2498 	vm_page_iter_init(&pages, object);  in swap_pager_seek_hole()
2499 	swblk_iter_init_only(&blks, object);  in swap_pager_seek_hole()
2513 swap_pager_scan_all_shadowed(vm_object_t object)  in swap_pager_scan_all_shadowed()  argument
2520 	VM_OBJECT_ASSERT_WLOCKED(object);  in swap_pager_scan_all_shadowed()
2521 	VM_OBJECT_ASSERT_WLOCKED(object->backing_object);  in swap_pager_scan_all_shadowed()
2523 	backing_object = object->backing_object;  in swap_pager_scan_all_shadowed()
2528 	KASSERT((object->flags & OBJ_ANON) != 0,  in swap_pager_scan_all_shadowed()
2530 	backing_offset_index = OFF_TO_IDX(object->backing_object_offset);  in swap_pager_scan_all_shadowed()
2532 	    backing_offset_index + object->size);  in swap_pager_scan_all_shadowed()
2533 	vm_page_iter_init(&pages, object);  in swap_pager_scan_all_shadowed()
2535 	swblk_iter_init_only(&blks, object);  in swap_pager_scan_all_shadowed()
3039 	vm_object_t object;  in vmspace_swap_count()  local
3051 		object = cur->object.vm_object;  in vmspace_swap_count()
3052 		if (object == NULL || (object->flags & OBJ_SWAP) == 0)  in vmspace_swap_count()
3054 		VM_OBJECT_RLOCK(object);  in vmspace_swap_count()
3055 		if ((object->flags & OBJ_SWAP) == 0)  in vmspace_swap_count()
3059 		for (sb = swblk_iter_limit_init(&blks, object, pi, e),  in vmspace_swap_count()
3069 		VM_OBJECT_RUNLOCK(object);  in vmspace_swap_count()
3436 swap_pager_update_writecount(vm_object_t object, vm_offset_t start,  in swap_pager_update_writecount()  argument
3440 	VM_OBJECT_WLOCK(object);  in swap_pager_update_writecount()
3441 	KASSERT((object->flags & OBJ_ANON) == 0,  in swap_pager_update_writecount()
3443 	object->un_pager.swp.writemappings += (vm_ooffset_t)end - start;  in swap_pager_update_writecount()
3444 	VM_OBJECT_WUNLOCK(object);  in swap_pager_update_writecount()
3448 swap_pager_release_writecount(vm_object_t object, vm_offset_t start,  in swap_pager_release_writecount()  argument
3452 	VM_OBJECT_WLOCK(object);  in swap_pager_release_writecount()
3453 	KASSERT((object->flags & OBJ_ANON) == 0,  in swap_pager_release_writecount()
3455 	KASSERT(object->un_pager.swp.writemappings >= (vm_ooffset_t)end - start,  in swap_pager_release_writecount()
3456 	    ("swap obj %p writecount %jx dec %jx", object,  in swap_pager_release_writecount()
3457 	    (uintmax_t)object->un_pager.swp.writemappings,  in swap_pager_release_writecount()
3459 	object->un_pager.swp.writemappings -= (vm_ooffset_t)end - start;  in swap_pager_release_writecount()
3460 	VM_OBJECT_WUNLOCK(object);  in swap_pager_release_writecount()