xref: /freebsd/sys/vm/vm_object.c (revision 6189672e600814fec4cc165d25ae050fb2f3a999)
160727d8bSWarner Losh /*-
2796df753SPedro F. Giffuni  * SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU)
351369649SPedro F. Giffuni  *
4df8bae1dSRodney W. Grimes  * Copyright (c) 1991, 1993
5df8bae1dSRodney W. Grimes  *	The Regents of the University of California.  All rights reserved.
6df8bae1dSRodney W. Grimes  *
7df8bae1dSRodney W. Grimes  * This code is derived from software contributed to Berkeley by
8df8bae1dSRodney W. Grimes  * The Mach Operating System project at Carnegie-Mellon University.
9df8bae1dSRodney W. Grimes  *
10df8bae1dSRodney W. Grimes  * Redistribution and use in source and binary forms, with or without
11df8bae1dSRodney W. Grimes  * modification, are permitted provided that the following conditions
12df8bae1dSRodney W. Grimes  * are met:
13df8bae1dSRodney W. Grimes  * 1. Redistributions of source code must retain the above copyright
14df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer.
15df8bae1dSRodney W. Grimes  * 2. Redistributions in binary form must reproduce the above copyright
16df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer in the
17df8bae1dSRodney W. Grimes  *    documentation and/or other materials provided with the distribution.
18fbbd9655SWarner Losh  * 3. Neither the name of the University nor the names of its contributors
19df8bae1dSRodney W. Grimes  *    may be used to endorse or promote products derived from this software
20df8bae1dSRodney W. Grimes  *    without specific prior written permission.
21df8bae1dSRodney W. Grimes  *
22df8bae1dSRodney W. Grimes  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23df8bae1dSRodney W. Grimes  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24df8bae1dSRodney W. Grimes  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25df8bae1dSRodney W. Grimes  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26df8bae1dSRodney W. Grimes  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27df8bae1dSRodney W. Grimes  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28df8bae1dSRodney W. Grimes  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29df8bae1dSRodney W. Grimes  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30df8bae1dSRodney W. Grimes  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31df8bae1dSRodney W. Grimes  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32df8bae1dSRodney W. Grimes  * SUCH DAMAGE.
33df8bae1dSRodney W. Grimes  *
343c4dd356SDavid Greenman  *	from: @(#)vm_object.c	8.5 (Berkeley) 3/22/94
35df8bae1dSRodney W. Grimes  *
36df8bae1dSRodney W. Grimes  *
37df8bae1dSRodney W. Grimes  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
38df8bae1dSRodney W. Grimes  * All rights reserved.
39df8bae1dSRodney W. Grimes  *
40df8bae1dSRodney W. Grimes  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
41df8bae1dSRodney W. Grimes  *
42df8bae1dSRodney W. Grimes  * Permission to use, copy, modify and distribute this software and
43df8bae1dSRodney W. Grimes  * its documentation is hereby granted, provided that both the copyright
44df8bae1dSRodney W. Grimes  * notice and this permission notice appear in all copies of the
45df8bae1dSRodney W. Grimes  * software, derivative works or modified versions, and any portions
46df8bae1dSRodney W. Grimes  * thereof, and that both notices appear in supporting documentation.
47df8bae1dSRodney W. Grimes  *
48df8bae1dSRodney W. Grimes  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
49df8bae1dSRodney W. Grimes  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
50df8bae1dSRodney W. Grimes  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
51df8bae1dSRodney W. Grimes  *
52df8bae1dSRodney W. Grimes  * Carnegie Mellon requests users of this software to return to
53df8bae1dSRodney W. Grimes  *
54df8bae1dSRodney W. Grimes  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
55df8bae1dSRodney W. Grimes  *  School of Computer Science
56df8bae1dSRodney W. Grimes  *  Carnegie Mellon University
57df8bae1dSRodney W. Grimes  *  Pittsburgh PA 15213-3890
58df8bae1dSRodney W. Grimes  *
59df8bae1dSRodney W. Grimes  * any improvements or extensions that they make and grant Carnegie the
60df8bae1dSRodney W. Grimes  * rights to redistribute these changes.
61df8bae1dSRodney W. Grimes  */
62df8bae1dSRodney W. Grimes 
63df8bae1dSRodney W. Grimes /*
64df8bae1dSRodney W. Grimes  *	Virtual memory object module.
65df8bae1dSRodney W. Grimes  */
66df8bae1dSRodney W. Grimes 
67874651b1SDavid E. O'Brien #include <sys/cdefs.h>
68874651b1SDavid E. O'Brien __FBSDID("$FreeBSD$");
69874651b1SDavid E. O'Brien 
70f8a47341SAlan Cox #include "opt_vm.h"
71f8a47341SAlan Cox 
72df8bae1dSRodney W. Grimes #include <sys/param.h>
73df8bae1dSRodney W. Grimes #include <sys/systm.h>
74c99d0c58SMark Johnston #include <sys/blockcount.h>
753f289c3fSJeff Roberson #include <sys/cpuset.h>
76ecfbddf0SKonstantin Belousov #include <sys/limits.h>
77fb919e4dSMark Murray #include <sys/lock.h>
78867a482dSJohn Dyson #include <sys/mman.h>
79cf2819ccSJohn Dyson #include <sys/mount.h>
80b9b7a4beSMatthew Dillon #include <sys/kernel.h>
81f425ab8eSKonstantin Belousov #include <sys/pctrie.h>
82b9b7a4beSMatthew Dillon #include <sys/sysctl.h>
831b367556SJason Evans #include <sys/mutex.h>
84fb919e4dSMark Murray #include <sys/proc.h>		/* for curproc, pageproc */
85cf27e0d1SJeff Roberson #include <sys/refcount.h>
86fb919e4dSMark Murray #include <sys/socket.h>
873364c323SKonstantin Belousov #include <sys/resourcevar.h>
88205be21dSJeff Roberson #include <sys/refcount.h>
8989f6b863SAttilio Rao #include <sys/rwlock.h>
90ff87ae35SJohn Baldwin #include <sys/user.h>
91fb919e4dSMark Murray #include <sys/vnode.h>
92fb919e4dSMark Murray #include <sys/vmmeter.h>
931005a129SJohn Baldwin #include <sys/sx.h>
94df8bae1dSRodney W. Grimes 
95df8bae1dSRodney W. Grimes #include <vm/vm.h>
96efeaf95aSDavid Greenman #include <vm/vm_param.h>
97efeaf95aSDavid Greenman #include <vm/pmap.h>
98efeaf95aSDavid Greenman #include <vm/vm_map.h>
99efeaf95aSDavid Greenman #include <vm/vm_object.h>
100df8bae1dSRodney W. Grimes #include <vm/vm_page.h>
10126f9a767SRodney W. Grimes #include <vm/vm_pageout.h>
1020d94caffSDavid Greenman #include <vm/vm_pager.h>
103e2068d0bSJeff Roberson #include <vm/vm_phys.h>
104e2068d0bSJeff Roberson #include <vm/vm_pagequeue.h>
10505f0fdd2SPoul-Henning Kamp #include <vm/swap_pager.h>
106a1f6d91cSDavid Greenman #include <vm/vm_kern.h>
107efeaf95aSDavid Greenman #include <vm/vm_extern.h>
108774d251dSAttilio Rao #include <vm/vm_radix.h>
109f8a47341SAlan Cox #include <vm/vm_reserv.h>
110670d17b5SJeff Roberson #include <vm/uma.h>
11126f9a767SRodney W. Grimes 
112c53f7aceSDag-Erling Smørgrav static int old_msync;
113c53f7aceSDag-Erling Smørgrav SYSCTL_INT(_vm, OID_AUTO, old_msync, CTLFLAG_RW, &old_msync, 0,
114c53f7aceSDag-Erling Smørgrav     "Use old (insecure) msync behavior");
115c53f7aceSDag-Erling Smørgrav 
116757216f3SKonstantin Belousov static int	vm_object_page_collect_flush(vm_object_t object, vm_page_t p,
11767d0e293SJeff Roberson 		    int pagerflags, int flags, boolean_t *allclean,
118126d6082SKonstantin Belousov 		    boolean_t *eio);
1193280870dSKonstantin Belousov static boolean_t vm_object_page_remove_write(vm_page_t p, int flags,
12067d0e293SJeff Roberson 		    boolean_t *allclean);
12151b867e5SJeff Roberson static void	vm_object_backing_remove(vm_object_t object);
122f6b04d2bSDavid Greenman 
123df8bae1dSRodney W. Grimes /*
124df8bae1dSRodney W. Grimes  *	Virtual memory objects maintain the actual data
125df8bae1dSRodney W. Grimes  *	associated with allocated virtual memory.  A given
126df8bae1dSRodney W. Grimes  *	page of memory exists within exactly one object.
127df8bae1dSRodney W. Grimes  *
128df8bae1dSRodney W. Grimes  *	An object is only deallocated when all "references"
129df8bae1dSRodney W. Grimes  *	are given up.  Only one "reference" to a given
130df8bae1dSRodney W. Grimes  *	region of an object should be writeable.
131df8bae1dSRodney W. Grimes  *
132df8bae1dSRodney W. Grimes  *	Associated with each object is a list of all resident
133df8bae1dSRodney W. Grimes  *	memory pages belonging to that object; this list is
134df8bae1dSRodney W. Grimes  *	maintained by the "vm_page" module, and locked by the object's
135df8bae1dSRodney W. Grimes  *	lock.
136df8bae1dSRodney W. Grimes  *
137df8bae1dSRodney W. Grimes  *	Each object also records a "pager" routine which is
138df8bae1dSRodney W. Grimes  *	used to retrieve (and store) pages to the proper backing
139df8bae1dSRodney W. Grimes  *	storage.  In addition, objects may be backed by other
140df8bae1dSRodney W. Grimes  *	objects from which they were virtual-copied.
141df8bae1dSRodney W. Grimes  *
142df8bae1dSRodney W. Grimes  *	The only items within the object structure which are
143df8bae1dSRodney W. Grimes  *	modified after time of creation are:
144df8bae1dSRodney W. Grimes  *		reference count		locked by object's lock
145df8bae1dSRodney W. Grimes  *		pager routine		locked by object's lock
146df8bae1dSRodney W. Grimes  *
147df8bae1dSRodney W. Grimes  */
148df8bae1dSRodney W. Grimes 
14928f8db14SBruce Evans struct object_q vm_object_list;
150a5698387SAlan Cox struct mtx vm_object_list_mtx;	/* lock for object list and count */
151cccf11b8SAlan Cox 
152cccf11b8SAlan Cox struct vm_object kernel_object_store;
153df8bae1dSRodney W. Grimes 
1547029da5cSPawel Biernacki static SYSCTL_NODE(_vm_stats, OID_AUTO, object, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
1556472ac3dSEd Schouten     "VM object stats");
156604c2bbcSAlan Cox 
157d869a17eSMark Johnston static COUNTER_U64_DEFINE_EARLY(object_collapses);
15811542376SAlan Cox SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, collapses, CTLFLAG_RD,
15911542376SAlan Cox     &object_collapses,
16011542376SAlan Cox     "VM object collapses");
161604c2bbcSAlan Cox 
162d869a17eSMark Johnston static COUNTER_U64_DEFINE_EARLY(object_bypasses);
16311542376SAlan Cox SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, bypasses, CTLFLAG_RD,
16411542376SAlan Cox     &object_bypasses,
16511542376SAlan Cox     "VM object bypasses");
16611542376SAlan Cox 
167d869a17eSMark Johnston static COUNTER_U64_DEFINE_EARLY(object_collapse_waits);
16898087a06SJeff Roberson SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, collapse_waits, CTLFLAG_RD,
16998087a06SJeff Roberson     &object_collapse_waits,
17098087a06SJeff Roberson     "Number of sleeps for collapse");
17198087a06SJeff Roberson 
172670d17b5SJeff Roberson static uma_zone_t obj_zone;
1738355f576SJeff Roberson 
174b23f72e9SBrian Feldman static int vm_object_zinit(void *mem, int size, int flags);
1758355f576SJeff Roberson 
1768355f576SJeff Roberson #ifdef INVARIANTS
1778355f576SJeff Roberson static void vm_object_zdtor(void *mem, int size, void *arg);
1788355f576SJeff Roberson 
1798355f576SJeff Roberson static void
1808355f576SJeff Roberson vm_object_zdtor(void *mem, int size, void *arg)
1818355f576SJeff Roberson {
1828355f576SJeff Roberson 	vm_object_t object;
1838355f576SJeff Roberson 
1848355f576SJeff Roberson 	object = (vm_object_t)mem;
185e735691bSJohn Baldwin 	KASSERT(object->ref_count == 0,
186e735691bSJohn Baldwin 	    ("object %p ref_count = %d", object, object->ref_count));
18743186e53SAlan Cox 	KASSERT(TAILQ_EMPTY(&object->memq),
188198da1b2SAttilio Rao 	    ("object %p has resident pages in its memq", object));
189774d251dSAttilio Rao 	KASSERT(vm_radix_is_empty(&object->rtree),
190774d251dSAttilio Rao 	    ("object %p has resident pages in its trie", object));
191f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0
192f8a47341SAlan Cox 	KASSERT(LIST_EMPTY(&object->rvq),
193f8a47341SAlan Cox 	    ("object %p has reservations",
194f8a47341SAlan Cox 	    object));
195f8a47341SAlan Cox #endif
196c99d0c58SMark Johnston 	KASSERT(!vm_object_busied(object),
197c99d0c58SMark Johnston 	    ("object %p busy = %d", object, blockcount_read(&object->busy)));
1988355f576SJeff Roberson 	KASSERT(object->resident_page_count == 0,
1998355f576SJeff Roberson 	    ("object %p resident_page_count = %d",
2008355f576SJeff Roberson 	    object, object->resident_page_count));
2014cf9f5d8SKonstantin Belousov 	KASSERT(atomic_load_int(&object->shadow_count) == 0,
2028355f576SJeff Roberson 	    ("object %p shadow_count = %d",
2034cf9f5d8SKonstantin Belousov 	    object, atomic_load_int(&object->shadow_count)));
204e735691bSJohn Baldwin 	KASSERT(object->type == OBJT_DEAD,
205e735691bSJohn Baldwin 	    ("object %p has non-dead type %d",
206e735691bSJohn Baldwin 	    object, object->type));
207e1979b45SMark Johnston 	KASSERT(object->charge == 0 && object->cred == NULL,
208e1979b45SMark Johnston 	    ("object %p has non-zero charge %ju (%p)",
209e1979b45SMark Johnston 	    object, (uintmax_t)object->charge, object->cred));
2108355f576SJeff Roberson }
2118355f576SJeff Roberson #endif
2128355f576SJeff Roberson 
213b23f72e9SBrian Feldman static int
214b23f72e9SBrian Feldman vm_object_zinit(void *mem, int size, int flags)
2158355f576SJeff Roberson {
2168355f576SJeff Roberson 	vm_object_t object;
2178355f576SJeff Roberson 
2188355f576SJeff Roberson 	object = (vm_object_t)mem;
219777a36c5SAlan Cox 	rw_init_flags(&object->lock, "vm object", RW_DUPOK | RW_NEW);
2208355f576SJeff Roberson 
2218355f576SJeff Roberson 	/* These are true for any object that has been freed */
222e735691bSJohn Baldwin 	object->type = OBJT_DEAD;
223cd1241fbSKonstantin Belousov 	vm_radix_init(&object->rtree);
22451df5321SJeff Roberson 	refcount_init(&object->ref_count, 0);
225c99d0c58SMark Johnston 	blockcount_init(&object->paging_in_progress);
226c99d0c58SMark Johnston 	blockcount_init(&object->busy);
2278355f576SJeff Roberson 	object->resident_page_count = 0;
2284cf9f5d8SKonstantin Belousov 	atomic_store_int(&object->shadow_count, 0);
229f425ab8eSKonstantin Belousov 	object->flags = OBJ_DEAD;
230e735691bSJohn Baldwin 
231e735691bSJohn Baldwin 	mtx_lock(&vm_object_list_mtx);
232e735691bSJohn Baldwin 	TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);
233e735691bSJohn Baldwin 	mtx_unlock(&vm_object_list_mtx);
234b23f72e9SBrian Feldman 	return (0);
2358355f576SJeff Roberson }
236df8bae1dSRodney W. Grimes 
237a4915c21SAttilio Rao static void
23863967687SJeff Roberson _vm_object_allocate(objtype_t type, vm_pindex_t size, u_short flags,
23967388836SKonstantin Belousov     vm_object_t object, void *handle)
240df8bae1dSRodney W. Grimes {
2410cddd8f0SMatthew Dillon 
242df8bae1dSRodney W. Grimes 	TAILQ_INIT(&object->memq);
2431c500307SAlan Cox 	LIST_INIT(&object->shadow_head);
244a1f6d91cSDavid Greenman 
24524a1cce3SDavid Greenman 	object->type = type;
2464b8365d7SKonstantin Belousov 	object->flags = flags;
2475d32157dSMark Johnston 	if ((flags & OBJ_SWAP) != 0) {
248f425ab8eSKonstantin Belousov 		pctrie_init(&object->un_pager.swp.swp_blks);
2495d32157dSMark Johnston 		object->un_pager.swp.writemappings = 0;
2505d32157dSMark Johnston 	}
251f425ab8eSKonstantin Belousov 
252f425ab8eSKonstantin Belousov 	/*
253f425ab8eSKonstantin Belousov 	 * Ensure that swap_pager_swapoff() iteration over object_list
254f425ab8eSKonstantin Belousov 	 * sees up to date type and pctrie head if it observed
255f425ab8eSKonstantin Belousov 	 * non-dead object.
256f425ab8eSKonstantin Belousov 	 */
257f425ab8eSKonstantin Belousov 	atomic_thread_fence_rel();
258f425ab8eSKonstantin Belousov 
25963967687SJeff Roberson 	object->pg_color = 0;
260df8bae1dSRodney W. Grimes 	object->size = size;
2614c29d2deSMark Johnston 	object->domain.dr_policy = NULL;
262b881da26SAlan Cox 	object->generation = 1;
26367d0e293SJeff Roberson 	object->cleangeneration = 1;
26451df5321SJeff Roberson 	refcount_init(&object->ref_count, 1);
2653153e878SAlan Cox 	object->memattr = VM_MEMATTR_DEFAULT;
266ef694c1aSEdward Tomasz Napierala 	object->cred = NULL;
2673364c323SKonstantin Belousov 	object->charge = 0;
26867388836SKonstantin Belousov 	object->handle = handle;
26924a1cce3SDavid Greenman 	object->backing_object = NULL;
270a316d390SJohn Dyson 	object->backing_object_offset = (vm_ooffset_t) 0;
271f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0
272f8a47341SAlan Cox 	LIST_INIT(&object->rvq);
273f8a47341SAlan Cox #endif
2741bdbd705SKonstantin Belousov 	umtx_shm_object_init(object);
275df8bae1dSRodney W. Grimes }
276df8bae1dSRodney W. Grimes 
277df8bae1dSRodney W. Grimes /*
27826f9a767SRodney W. Grimes  *	vm_object_init:
27926f9a767SRodney W. Grimes  *
28026f9a767SRodney W. Grimes  *	Initialize the VM objects module.
28126f9a767SRodney W. Grimes  */
28226f9a767SRodney W. Grimes void
2831b40f8c0SMatthew Dillon vm_object_init(void)
28426f9a767SRodney W. Grimes {
28526f9a767SRodney W. Grimes 	TAILQ_INIT(&vm_object_list);
2866008862bSJohn Baldwin 	mtx_init(&vm_object_list_mtx, "vm object_list", NULL, MTX_DEF);
2870217125fSDavid Greenman 
28889f6b863SAttilio Rao 	rw_init(&kernel_object->lock, "kernel vm object");
289d1780e8dSKonstantin Belousov 	_vm_object_allocate(OBJT_PHYS, atop(VM_MAX_KERNEL_ADDRESS -
29067388836SKonstantin Belousov 	    VM_MIN_KERNEL_ADDRESS), OBJ_UNMANAGED, kernel_object, NULL);
291f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0
292f8a47341SAlan Cox 	kernel_object->flags |= OBJ_COLORED;
293f8a47341SAlan Cox 	kernel_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS);
294f8a47341SAlan Cox #endif
295a720b31cSKonstantin Belousov 	kernel_object->un_pager.phys.ops = &default_phys_pg_ops;
29626f9a767SRodney W. Grimes 
2978dbca793STor Egge 	/*
2988dbca793STor Egge 	 * The lock portion of struct vm_object must be type stable due
2998dbca793STor Egge 	 * to vm_pageout_fallback_object_lock locking a vm object
3008dbca793STor Egge 	 * without holding any references to it.
301ffae7ea9SKonstantin Belousov 	 *
302ffae7ea9SKonstantin Belousov 	 * paging_in_progress is valid always.  Lockless references to
303ffae7ea9SKonstantin Belousov 	 * the objects may acquire pip and then check OBJ_DEAD.
3048dbca793STor Egge 	 */
3058355f576SJeff Roberson 	obj_zone = uma_zcreate("VM OBJECT", sizeof (struct vm_object), NULL,
3068355f576SJeff Roberson #ifdef INVARIANTS
3078355f576SJeff Roberson 	    vm_object_zdtor,
3088355f576SJeff Roberson #else
3098355f576SJeff Roberson 	    NULL,
3108355f576SJeff Roberson #endif
3115df87b21SJeff Roberson 	    vm_object_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
312774d251dSAttilio Rao 
313cd1241fbSKonstantin Belousov 	vm_radix_zinit();
31499448ed1SJohn Dyson }
31599448ed1SJohn Dyson 
31699448ed1SJohn Dyson void
3171b40f8c0SMatthew Dillon vm_object_clear_flag(vm_object_t object, u_short bits)
3181b40f8c0SMatthew Dillon {
3195440b5a9SAlan Cox 
32089f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
321b06805adSJake Burkholder 	object->flags &= ~bits;
3221b40f8c0SMatthew Dillon }
3231b40f8c0SMatthew Dillon 
3243153e878SAlan Cox /*
3253153e878SAlan Cox  *	Sets the default memory attribute for the specified object.  Pages
3263153e878SAlan Cox  *	that are allocated to this object are by default assigned this memory
3273153e878SAlan Cox  *	attribute.
3283153e878SAlan Cox  *
3293153e878SAlan Cox  *	Presently, this function must be called before any pages are allocated
3303153e878SAlan Cox  *	to the object.  In the future, this requirement may be relaxed for
3313153e878SAlan Cox  *	"default" and "swap" objects.
3323153e878SAlan Cox  */
3333153e878SAlan Cox int
3343153e878SAlan Cox vm_object_set_memattr(vm_object_t object, vm_memattr_t memattr)
3353153e878SAlan Cox {
3363153e878SAlan Cox 
33789f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
3383e7a11caSKonstantin Belousov 
3393e7a11caSKonstantin Belousov 	if (object->type == OBJT_DEAD)
3403e7a11caSKonstantin Belousov 		return (KERN_INVALID_ARGUMENT);
3413153e878SAlan Cox 	if (!TAILQ_EMPTY(&object->memq))
3423153e878SAlan Cox 		return (KERN_FAILURE);
3433e7a11caSKonstantin Belousov 
3443153e878SAlan Cox 	object->memattr = memattr;
3453153e878SAlan Cox 	return (KERN_SUCCESS);
3463153e878SAlan Cox }
3473153e878SAlan Cox 
3481b40f8c0SMatthew Dillon void
3491b40f8c0SMatthew Dillon vm_object_pip_add(vm_object_t object, short i)
3501b40f8c0SMatthew Dillon {
351f279b88dSAlan Cox 
352c99d0c58SMark Johnston 	if (i > 0)
353c99d0c58SMark Johnston 		blockcount_acquire(&object->paging_in_progress, i);
3541b40f8c0SMatthew Dillon }
3551b40f8c0SMatthew Dillon 
3561b40f8c0SMatthew Dillon void
3571b40f8c0SMatthew Dillon vm_object_pip_wakeup(vm_object_t object)
3581b40f8c0SMatthew Dillon {
359f279b88dSAlan Cox 
360c99d0c58SMark Johnston 	vm_object_pip_wakeupn(object, 1);
3611b40f8c0SMatthew Dillon }
3621b40f8c0SMatthew Dillon 
3631b40f8c0SMatthew Dillon void
3641b40f8c0SMatthew Dillon vm_object_pip_wakeupn(vm_object_t object, short i)
3651b40f8c0SMatthew Dillon {
366d647a0edSAlan Cox 
367c99d0c58SMark Johnston 	if (i > 0)
368c99d0c58SMark Johnston 		blockcount_release(&object->paging_in_progress, i);
3691b40f8c0SMatthew Dillon }
3701b40f8c0SMatthew Dillon 
37198087a06SJeff Roberson /*
372c99d0c58SMark Johnston  * Atomically drop the object lock and wait for pip to drain.  This protects
373c99d0c58SMark Johnston  * from sleep/wakeup races due to identity changes.  The lock is not re-acquired
374c99d0c58SMark Johnston  * on return.
37598087a06SJeff Roberson  */
37698087a06SJeff Roberson static void
377eaa17d42SRyan Libby vm_object_pip_sleep(vm_object_t object, const char *waitid)
37898087a06SJeff Roberson {
37998087a06SJeff Roberson 
380c99d0c58SMark Johnston 	(void)blockcount_sleep(&object->paging_in_progress, &object->lock,
381c99d0c58SMark Johnston 	    waitid, PVM | PDROP);
38298087a06SJeff Roberson }
38398087a06SJeff Roberson 
3841b40f8c0SMatthew Dillon void
385eaa17d42SRyan Libby vm_object_pip_wait(vm_object_t object, const char *waitid)
3861b40f8c0SMatthew Dillon {
3871ca58953SAlan Cox 
38889f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
389cf27e0d1SJeff Roberson 
390c99d0c58SMark Johnston 	blockcount_wait(&object->paging_in_progress, &object->lock, waitid,
391c99d0c58SMark Johnston 	    PVM);
3921b40f8c0SMatthew Dillon }
3931b40f8c0SMatthew Dillon 
394cf27e0d1SJeff Roberson void
395eaa17d42SRyan Libby vm_object_pip_wait_unlocked(vm_object_t object, const char *waitid)
396cf27e0d1SJeff Roberson {
397cf27e0d1SJeff Roberson 
398cf27e0d1SJeff Roberson 	VM_OBJECT_ASSERT_UNLOCKED(object);
399cf27e0d1SJeff Roberson 
400c99d0c58SMark Johnston 	blockcount_wait(&object->paging_in_progress, NULL, waitid, PVM);
401cf27e0d1SJeff Roberson }
402cf27e0d1SJeff Roberson 
40326f9a767SRodney W. Grimes /*
40426f9a767SRodney W. Grimes  *	vm_object_allocate:
40526f9a767SRodney W. Grimes  *
40626f9a767SRodney W. Grimes  *	Returns a new object with the given size.
40726f9a767SRodney W. Grimes  */
40826f9a767SRodney W. Grimes vm_object_t
4096395da54SIan Dowse vm_object_allocate(objtype_t type, vm_pindex_t size)
41026f9a767SRodney W. Grimes {
41190688d13SAlan Cox 	vm_object_t object;
41263967687SJeff Roberson 	u_short flags;
41363967687SJeff Roberson 
41463967687SJeff Roberson 	switch (type) {
41563967687SJeff Roberson 	case OBJT_DEAD:
41663967687SJeff Roberson 		panic("vm_object_allocate: can't create OBJT_DEAD");
4174b8365d7SKonstantin Belousov 	case OBJT_SWAP:
4184b8365d7SKonstantin Belousov 		flags = OBJ_COLORED | OBJ_SWAP;
4194b8365d7SKonstantin Belousov 		break;
42063967687SJeff Roberson 	case OBJT_DEVICE:
42163967687SJeff Roberson 	case OBJT_SG:
42263967687SJeff Roberson 		flags = OBJ_FICTITIOUS | OBJ_UNMANAGED;
42363967687SJeff Roberson 		break;
42463967687SJeff Roberson 	case OBJT_MGTDEVICE:
42563967687SJeff Roberson 		flags = OBJ_FICTITIOUS;
42663967687SJeff Roberson 		break;
42763967687SJeff Roberson 	case OBJT_PHYS:
42863967687SJeff Roberson 		flags = OBJ_UNMANAGED;
42963967687SJeff Roberson 		break;
43063967687SJeff Roberson 	case OBJT_VNODE:
43163967687SJeff Roberson 		flags = 0;
43263967687SJeff Roberson 		break;
43363967687SJeff Roberson 	default:
434b730fd30SKonstantin Belousov 		panic("vm_object_allocate: type %d is undefined or dynamic",
435b730fd30SKonstantin Belousov 		    type);
43663967687SJeff Roberson 	}
43763967687SJeff Roberson 	object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);
43867388836SKonstantin Belousov 	_vm_object_allocate(type, size, flags, object, NULL);
43963967687SJeff Roberson 
44063967687SJeff Roberson 	return (object);
44163967687SJeff Roberson }
44263967687SJeff Roberson 
443b730fd30SKonstantin Belousov vm_object_t
444b730fd30SKonstantin Belousov vm_object_allocate_dyn(objtype_t dyntype, vm_pindex_t size, u_short flags)
445b730fd30SKonstantin Belousov {
446b730fd30SKonstantin Belousov 	vm_object_t object;
447b730fd30SKonstantin Belousov 
448b730fd30SKonstantin Belousov 	MPASS(dyntype >= OBJT_FIRST_DYN /* && dyntype < nitems(pagertab) */);
449b730fd30SKonstantin Belousov 	object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);
450b730fd30SKonstantin Belousov 	_vm_object_allocate(dyntype, size, flags, object, NULL);
451b730fd30SKonstantin Belousov 
452b730fd30SKonstantin Belousov 	return (object);
453b730fd30SKonstantin Belousov }
454b730fd30SKonstantin Belousov 
45563967687SJeff Roberson /*
45663967687SJeff Roberson  *	vm_object_allocate_anon:
45763967687SJeff Roberson  *
45863967687SJeff Roberson  *	Returns a new default object of the given size and marked as
45963967687SJeff Roberson  *	anonymous memory for special split/collapse handling.  Color
46063967687SJeff Roberson  *	to be initialized by the caller.
46163967687SJeff Roberson  */
46263967687SJeff Roberson vm_object_t
46367388836SKonstantin Belousov vm_object_allocate_anon(vm_pindex_t size, vm_object_t backing_object,
46467388836SKonstantin Belousov     struct ucred *cred, vm_size_t charge)
46563967687SJeff Roberson {
46667388836SKonstantin Belousov 	vm_object_t handle, object;
46790688d13SAlan Cox 
46867388836SKonstantin Belousov 	if (backing_object == NULL)
46967388836SKonstantin Belousov 		handle = NULL;
47067388836SKonstantin Belousov 	else if ((backing_object->flags & OBJ_ANON) != 0)
47167388836SKonstantin Belousov 		handle = backing_object->handle;
47267388836SKonstantin Belousov 	else
47367388836SKonstantin Belousov 		handle = backing_object;
47467388836SKonstantin Belousov 	object = uma_zalloc(obj_zone, M_WAITOK);
4755d32157dSMark Johnston 	_vm_object_allocate(OBJT_SWAP, size,
4765d32157dSMark Johnston 	    OBJ_ANON | OBJ_ONEMAPPING | OBJ_SWAP, object, handle);
47767388836SKonstantin Belousov 	object->cred = cred;
47867388836SKonstantin Belousov 	object->charge = cred != NULL ? charge : 0;
47990688d13SAlan Cox 	return (object);
48026f9a767SRodney W. Grimes }
48126f9a767SRodney W. Grimes 
48298087a06SJeff Roberson static void
48398087a06SJeff Roberson vm_object_reference_vnode(vm_object_t object)
484df8bae1dSRodney W. Grimes {
485a67d5408SJeff Roberson 	u_int old;
486a67d5408SJeff Roberson 
487a67d5408SJeff Roberson 	/*
48898087a06SJeff Roberson 	 * vnode objects need the lock for the first reference
48998087a06SJeff Roberson 	 * to serialize with vnode_object_deallocate().
490a67d5408SJeff Roberson 	 */
49198087a06SJeff Roberson 	if (!refcount_acquire_if_gt(&object->ref_count, 0)) {
49251df5321SJeff Roberson 		VM_OBJECT_RLOCK(object);
493a67d5408SJeff Roberson 		old = refcount_acquire(&object->ref_count);
4941a0c234eSJeff Roberson 		if (object->type == OBJT_VNODE && old == 0)
4951a0c234eSJeff Roberson 			vref(object->handle);
49651df5321SJeff Roberson 		VM_OBJECT_RUNLOCK(object);
49795e5e988SJohn Dyson 	}
498a67d5408SJeff Roberson }
49995e5e988SJohn Dyson 
50023955314SAlfred Perlstein /*
50198087a06SJeff Roberson  *	vm_object_reference:
50298087a06SJeff Roberson  *
50398087a06SJeff Roberson  *	Acquires a reference to the given object.
50498087a06SJeff Roberson  */
50598087a06SJeff Roberson void
50698087a06SJeff Roberson vm_object_reference(vm_object_t object)
50798087a06SJeff Roberson {
50898087a06SJeff Roberson 
50998087a06SJeff Roberson 	if (object == NULL)
51098087a06SJeff Roberson 		return;
51198087a06SJeff Roberson 
51298087a06SJeff Roberson 	if (object->type == OBJT_VNODE)
51398087a06SJeff Roberson 		vm_object_reference_vnode(object);
51498087a06SJeff Roberson 	else
51598087a06SJeff Roberson 		refcount_acquire(&object->ref_count);
51698087a06SJeff Roberson 	KASSERT((object->flags & OBJ_DEAD) == 0,
51798087a06SJeff Roberson 	    ("vm_object_reference: Referenced dead object."));
51898087a06SJeff Roberson }
51998087a06SJeff Roberson 
52098087a06SJeff Roberson /*
521b921a12bSAlan Cox  *	vm_object_reference_locked:
522b921a12bSAlan Cox  *
523b921a12bSAlan Cox  *	Gets another reference to the given object.
524b921a12bSAlan Cox  *
525b921a12bSAlan Cox  *	The object must be locked.
526b921a12bSAlan Cox  */
527b921a12bSAlan Cox void
528b921a12bSAlan Cox vm_object_reference_locked(vm_object_t object)
529b921a12bSAlan Cox {
530a67d5408SJeff Roberson 	u_int old;
531b921a12bSAlan Cox 
53251df5321SJeff Roberson 	VM_OBJECT_ASSERT_LOCKED(object);
533a67d5408SJeff Roberson 	old = refcount_acquire(&object->ref_count);
5341a0c234eSJeff Roberson 	if (object->type == OBJT_VNODE && old == 0)
5351a0c234eSJeff Roberson 		vref(object->handle);
53698087a06SJeff Roberson 	KASSERT((object->flags & OBJ_DEAD) == 0,
53798087a06SJeff Roberson 	    ("vm_object_reference: Referenced dead object."));
538b921a12bSAlan Cox }
539b921a12bSAlan Cox 
540b921a12bSAlan Cox /*
5419d5abbddSJens Schweikhardt  * Handle deallocating an object of type OBJT_VNODE.
54223955314SAlfred Perlstein  */
54302dd8331SAlan Cox static void
54498087a06SJeff Roberson vm_object_deallocate_vnode(vm_object_t object)
54595e5e988SJohn Dyson {
54695e5e988SJohn Dyson 	struct vnode *vp = (struct vnode *) object->handle;
54726c4e983SJeff Roberson 	bool last;
548219cbf59SEivind Eklund 
5495526d2d9SEivind Eklund 	KASSERT(object->type == OBJT_VNODE,
55098087a06SJeff Roberson 	    ("vm_object_deallocate_vnode: not a vnode object"));
55198087a06SJeff Roberson 	KASSERT(vp != NULL, ("vm_object_deallocate_vnode: missing vp"));
55295e5e988SJohn Dyson 
55326c4e983SJeff Roberson 	/* Object lock to protect handle lookup. */
55426c4e983SJeff Roberson 	last = refcount_release(&object->ref_count);
55526c4e983SJeff Roberson 	VM_OBJECT_RUNLOCK(object);
55626c4e983SJeff Roberson 
55726c4e983SJeff Roberson 	if (!last)
55826c4e983SJeff Roberson 		return;
55926c4e983SJeff Roberson 
560a67d5408SJeff Roberson 	if (!umtx_shm_vnobj_persistent)
5611bdbd705SKonstantin Belousov 		umtx_shm_object_terminated(object);
5621bdbd705SKonstantin Belousov 
56303fa5b34SKonstantin Belousov 	/* vrele may need the vnode lock. */
56447221757SJohn Dyson 	vrele(vp);
56586769ac0SKonstantin Belousov }
566df8bae1dSRodney W. Grimes 
56798087a06SJeff Roberson /*
56898087a06SJeff Roberson  * We dropped a reference on an object and discovered that it had a
56998087a06SJeff Roberson  * single remaining shadow.  This is a sibling of the reference we
57098087a06SJeff Roberson  * dropped.  Attempt to collapse the sibling and backing object.
57198087a06SJeff Roberson  */
57298087a06SJeff Roberson static vm_object_t
57398087a06SJeff Roberson vm_object_deallocate_anon(vm_object_t backing_object)
57498087a06SJeff Roberson {
57598087a06SJeff Roberson 	vm_object_t object;
57698087a06SJeff Roberson 
57798087a06SJeff Roberson 	/* Fetch the final shadow.  */
57898087a06SJeff Roberson 	object = LIST_FIRST(&backing_object->shadow_head);
5794cf9f5d8SKonstantin Belousov 	KASSERT(object != NULL &&
5804cf9f5d8SKonstantin Belousov 	    atomic_load_int(&backing_object->shadow_count) == 1,
58198087a06SJeff Roberson 	    ("vm_object_anon_deallocate: ref_count: %d, shadow_count: %d",
5824cf9f5d8SKonstantin Belousov 	    backing_object->ref_count,
5834cf9f5d8SKonstantin Belousov 	    atomic_load_int(&backing_object->shadow_count)));
5844b8365d7SKonstantin Belousov 	KASSERT((object->flags & OBJ_ANON) != 0,
58598087a06SJeff Roberson 	    ("invalid shadow object %p", object));
58698087a06SJeff Roberson 
58798087a06SJeff Roberson 	if (!VM_OBJECT_TRYWLOCK(object)) {
58898087a06SJeff Roberson 		/*
58998087a06SJeff Roberson 		 * Prevent object from disappearing since we do not have a
59098087a06SJeff Roberson 		 * reference.
59198087a06SJeff Roberson 		 */
59298087a06SJeff Roberson 		vm_object_pip_add(object, 1);
59398087a06SJeff Roberson 		VM_OBJECT_WUNLOCK(backing_object);
59498087a06SJeff Roberson 		VM_OBJECT_WLOCK(object);
59598087a06SJeff Roberson 		vm_object_pip_wakeup(object);
59698087a06SJeff Roberson 	} else
59798087a06SJeff Roberson 		VM_OBJECT_WUNLOCK(backing_object);
59898087a06SJeff Roberson 
59998087a06SJeff Roberson 	/*
60098087a06SJeff Roberson 	 * Check for a collapse/terminate race with the last reference holder.
60198087a06SJeff Roberson 	 */
60298087a06SJeff Roberson 	if ((object->flags & (OBJ_DEAD | OBJ_COLLAPSING)) != 0 ||
60398087a06SJeff Roberson 	    !refcount_acquire_if_not_zero(&object->ref_count)) {
60498087a06SJeff Roberson 		VM_OBJECT_WUNLOCK(object);
60598087a06SJeff Roberson 		return (NULL);
60698087a06SJeff Roberson 	}
60798087a06SJeff Roberson 	backing_object = object->backing_object;
60898087a06SJeff Roberson 	if (backing_object != NULL && (backing_object->flags & OBJ_ANON) != 0)
60998087a06SJeff Roberson 		vm_object_collapse(object);
61098087a06SJeff Roberson 	VM_OBJECT_WUNLOCK(object);
61198087a06SJeff Roberson 
61298087a06SJeff Roberson 	return (object);
61398087a06SJeff Roberson }
61498087a06SJeff Roberson 
615df8bae1dSRodney W. Grimes /*
616df8bae1dSRodney W. Grimes  *	vm_object_deallocate:
617df8bae1dSRodney W. Grimes  *
618df8bae1dSRodney W. Grimes  *	Release a reference to the specified object,
619df8bae1dSRodney W. Grimes  *	gained either through a vm_object_allocate
620df8bae1dSRodney W. Grimes  *	or a vm_object_reference call.  When all references
621df8bae1dSRodney W. Grimes  *	are gone, storage associated with this object
622df8bae1dSRodney W. Grimes  *	may be relinquished.
623df8bae1dSRodney W. Grimes  *
624df8bae1dSRodney W. Grimes  *	No object may be locked.
625df8bae1dSRodney W. Grimes  */
62626f9a767SRodney W. Grimes void
6271b40f8c0SMatthew Dillon vm_object_deallocate(vm_object_t object)
628df8bae1dSRodney W. Grimes {
62998087a06SJeff Roberson 	vm_object_t temp;
63026c4e983SJeff Roberson 	bool released;
631df8bae1dSRodney W. Grimes 
632df8bae1dSRodney W. Grimes 	while (object != NULL) {
63351df5321SJeff Roberson 		/*
63451df5321SJeff Roberson 		 * If the reference count goes to 0 we start calling
63551df5321SJeff Roberson 		 * vm_object_terminate() on the object chain.  A ref count
63651df5321SJeff Roberson 		 * of 1 may be a special case depending on the shadow count
63751df5321SJeff Roberson 		 * being 0 or 1.  These cases require a write lock on the
63851df5321SJeff Roberson 		 * object.
63951df5321SJeff Roberson 		 */
64063967687SJeff Roberson 		if ((object->flags & OBJ_ANON) == 0)
64163967687SJeff Roberson 			released = refcount_release_if_gt(&object->ref_count, 1);
64263967687SJeff Roberson 		else
64351df5321SJeff Roberson 			released = refcount_release_if_gt(&object->ref_count, 2);
64451df5321SJeff Roberson 		if (released)
64551df5321SJeff Roberson 			return;
64651df5321SJeff Roberson 
647a67d5408SJeff Roberson 		if (object->type == OBJT_VNODE) {
64826c4e983SJeff Roberson 			VM_OBJECT_RLOCK(object);
64926c4e983SJeff Roberson 			if (object->type == OBJT_VNODE) {
65098087a06SJeff Roberson 				vm_object_deallocate_vnode(object);
651a67d5408SJeff Roberson 				return;
652a67d5408SJeff Roberson 			}
65326c4e983SJeff Roberson 			VM_OBJECT_RUNLOCK(object);
65426c4e983SJeff Roberson 		}
65526c4e983SJeff Roberson 
65626c4e983SJeff Roberson 		VM_OBJECT_WLOCK(object);
65726c4e983SJeff Roberson 		KASSERT(object->ref_count > 0,
65826c4e983SJeff Roberson 		    ("vm_object_deallocate: object deallocated too many times: %d",
65926c4e983SJeff Roberson 		    object->type));
66026c4e983SJeff Roberson 
661b72b0115SAlan Cox 		/*
66298087a06SJeff Roberson 		 * If this is not the final reference to an anonymous
66398087a06SJeff Roberson 		 * object we may need to collapse the shadow chain.
664b72b0115SAlan Cox 		 */
66598087a06SJeff Roberson 		if (!refcount_release(&object->ref_count)) {
66698087a06SJeff Roberson 			if (object->ref_count > 1 ||
6674cf9f5d8SKonstantin Belousov 			    atomic_load_int(&object->shadow_count) == 0) {
66898087a06SJeff Roberson 				if ((object->flags & OBJ_ANON) != 0 &&
66998087a06SJeff Roberson 				    object->ref_count == 1)
67098087a06SJeff Roberson 					vm_object_set_flag(object,
67198087a06SJeff Roberson 					    OBJ_ONEMAPPING);
67289f6b863SAttilio Rao 				VM_OBJECT_WUNLOCK(object);
67323b186d3SAlan Cox 				return;
67495e5e988SJohn Dyson 			}
67598087a06SJeff Roberson 
67698087a06SJeff Roberson 			/* Handle collapsing last ref on anonymous objects. */
67798087a06SJeff Roberson 			object = vm_object_deallocate_anon(object);
67898087a06SJeff Roberson 			continue;
67998087a06SJeff Roberson 		}
68098087a06SJeff Roberson 
68198087a06SJeff Roberson 		/*
68298087a06SJeff Roberson 		 * Handle the final reference to an object.  We restart
68398087a06SJeff Roberson 		 * the loop with the backing object to avoid recursion.
68498087a06SJeff Roberson 		 */
6851bdbd705SKonstantin Belousov 		umtx_shm_object_terminated(object);
68624a1cce3SDavid Greenman 		temp = object->backing_object;
687c9917419SAlan Cox 		if (temp != NULL) {
6880cb2610eSMark Johnston 			KASSERT(object->type == OBJT_SWAP,
6894bace8e7SKonstantin Belousov 			    ("shadowed tmpfs v_object 2 %p", object));
69051b867e5SJeff Roberson 			vm_object_backing_remove(object);
691de5f6a77SJohn Dyson 		}
69298087a06SJeff Roberson 
69398087a06SJeff Roberson 		KASSERT((object->flags & OBJ_DEAD) == 0,
69498087a06SJeff Roberson 		    ("vm_object_deallocate: Terminating dead object."));
695783a68aaSKonstantin Belousov 		vm_object_set_flag(object, OBJ_DEAD);
696df8bae1dSRodney W. Grimes 		vm_object_terminate(object);
697df8bae1dSRodney W. Grimes 		object = temp;
698df8bae1dSRodney W. Grimes 	}
699df8bae1dSRodney W. Grimes }
700df8bae1dSRodney W. Grimes 
7012ac78f0eSStephan Uphoff void
7022ac78f0eSStephan Uphoff vm_object_destroy(vm_object_t object)
7032ac78f0eSStephan Uphoff {
7042ac78f0eSStephan Uphoff 	uma_zfree(obj_zone, object);
7052ac78f0eSStephan Uphoff }
7062ac78f0eSStephan Uphoff 
70751b867e5SJeff Roberson static void
7084cf9f5d8SKonstantin Belousov vm_object_sub_shadow(vm_object_t object)
7094cf9f5d8SKonstantin Belousov {
7104cf9f5d8SKonstantin Belousov 	KASSERT(object->shadow_count >= 1,
7114cf9f5d8SKonstantin Belousov 	    ("object %p sub_shadow count zero", object));
7124cf9f5d8SKonstantin Belousov 	atomic_subtract_int(&object->shadow_count, 1);
7134cf9f5d8SKonstantin Belousov }
7144cf9f5d8SKonstantin Belousov 
7154cf9f5d8SKonstantin Belousov static void
71651b867e5SJeff Roberson vm_object_backing_remove_locked(vm_object_t object)
71751b867e5SJeff Roberson {
71851b867e5SJeff Roberson 	vm_object_t backing_object;
71951b867e5SJeff Roberson 
72051b867e5SJeff Roberson 	backing_object = object->backing_object;
72151b867e5SJeff Roberson 	VM_OBJECT_ASSERT_WLOCKED(object);
72251b867e5SJeff Roberson 	VM_OBJECT_ASSERT_WLOCKED(backing_object);
72351b867e5SJeff Roberson 
72498087a06SJeff Roberson 	KASSERT((object->flags & OBJ_COLLAPSING) == 0,
72598087a06SJeff Roberson 	    ("vm_object_backing_remove: Removing collapsing object."));
72698087a06SJeff Roberson 
7274cf9f5d8SKonstantin Belousov 	vm_object_sub_shadow(backing_object);
72851b867e5SJeff Roberson 	if ((object->flags & OBJ_SHADOWLIST) != 0) {
72951b867e5SJeff Roberson 		LIST_REMOVE(object, shadow_list);
730630f633fSMark Johnston 		vm_object_clear_flag(object, OBJ_SHADOWLIST);
73151b867e5SJeff Roberson 	}
73251b867e5SJeff Roberson 	object->backing_object = NULL;
73351b867e5SJeff Roberson }
73451b867e5SJeff Roberson 
73551b867e5SJeff Roberson static void
73651b867e5SJeff Roberson vm_object_backing_remove(vm_object_t object)
73751b867e5SJeff Roberson {
73851b867e5SJeff Roberson 	vm_object_t backing_object;
73951b867e5SJeff Roberson 
74051b867e5SJeff Roberson 	VM_OBJECT_ASSERT_WLOCKED(object);
74151b867e5SJeff Roberson 
74251b867e5SJeff Roberson 	backing_object = object->backing_object;
7434cf9f5d8SKonstantin Belousov 	if ((object->flags & OBJ_SHADOWLIST) != 0) {
74451b867e5SJeff Roberson 		VM_OBJECT_WLOCK(backing_object);
74551b867e5SJeff Roberson 		vm_object_backing_remove_locked(object);
74651b867e5SJeff Roberson 		VM_OBJECT_WUNLOCK(backing_object);
7474cf9f5d8SKonstantin Belousov 	} else {
74851b867e5SJeff Roberson 		object->backing_object = NULL;
7494cf9f5d8SKonstantin Belousov 		vm_object_sub_shadow(backing_object);
7504cf9f5d8SKonstantin Belousov 	}
75151b867e5SJeff Roberson }
75251b867e5SJeff Roberson 
75351b867e5SJeff Roberson static void
75451b867e5SJeff Roberson vm_object_backing_insert_locked(vm_object_t object, vm_object_t backing_object)
75551b867e5SJeff Roberson {
75651b867e5SJeff Roberson 
75751b867e5SJeff Roberson 	VM_OBJECT_ASSERT_WLOCKED(object);
75851b867e5SJeff Roberson 
7594cf9f5d8SKonstantin Belousov 	atomic_add_int(&backing_object->shadow_count, 1);
76051b867e5SJeff Roberson 	if ((backing_object->flags & OBJ_ANON) != 0) {
76151b867e5SJeff Roberson 		VM_OBJECT_ASSERT_WLOCKED(backing_object);
76251b867e5SJeff Roberson 		LIST_INSERT_HEAD(&backing_object->shadow_head, object,
76351b867e5SJeff Roberson 		    shadow_list);
764630f633fSMark Johnston 		vm_object_set_flag(object, OBJ_SHADOWLIST);
76551b867e5SJeff Roberson 	}
76651b867e5SJeff Roberson 	object->backing_object = backing_object;
76751b867e5SJeff Roberson }
76851b867e5SJeff Roberson 
76951b867e5SJeff Roberson static void
77051b867e5SJeff Roberson vm_object_backing_insert(vm_object_t object, vm_object_t backing_object)
77151b867e5SJeff Roberson {
77251b867e5SJeff Roberson 
77351b867e5SJeff Roberson 	VM_OBJECT_ASSERT_WLOCKED(object);
77451b867e5SJeff Roberson 
77551b867e5SJeff Roberson 	if ((backing_object->flags & OBJ_ANON) != 0) {
77651b867e5SJeff Roberson 		VM_OBJECT_WLOCK(backing_object);
77751b867e5SJeff Roberson 		vm_object_backing_insert_locked(object, backing_object);
77851b867e5SJeff Roberson 		VM_OBJECT_WUNLOCK(backing_object);
7794cf9f5d8SKonstantin Belousov 	} else {
78051b867e5SJeff Roberson 		object->backing_object = backing_object;
7814cf9f5d8SKonstantin Belousov 		atomic_add_int(&backing_object->shadow_count, 1);
7824cf9f5d8SKonstantin Belousov 	}
78351b867e5SJeff Roberson }
78451b867e5SJeff Roberson 
78598087a06SJeff Roberson /*
78698087a06SJeff Roberson  * Insert an object into a backing_object's shadow list with an additional
78798087a06SJeff Roberson  * reference to the backing_object added.
78898087a06SJeff Roberson  */
78998087a06SJeff Roberson static void
79098087a06SJeff Roberson vm_object_backing_insert_ref(vm_object_t object, vm_object_t backing_object)
79198087a06SJeff Roberson {
79298087a06SJeff Roberson 
79398087a06SJeff Roberson 	VM_OBJECT_ASSERT_WLOCKED(object);
79498087a06SJeff Roberson 
79598087a06SJeff Roberson 	if ((backing_object->flags & OBJ_ANON) != 0) {
79698087a06SJeff Roberson 		VM_OBJECT_WLOCK(backing_object);
79798087a06SJeff Roberson 		KASSERT((backing_object->flags & OBJ_DEAD) == 0,
79898087a06SJeff Roberson 		    ("shadowing dead anonymous object"));
79998087a06SJeff Roberson 		vm_object_reference_locked(backing_object);
80098087a06SJeff Roberson 		vm_object_backing_insert_locked(object, backing_object);
80198087a06SJeff Roberson 		vm_object_clear_flag(backing_object, OBJ_ONEMAPPING);
80298087a06SJeff Roberson 		VM_OBJECT_WUNLOCK(backing_object);
80398087a06SJeff Roberson 	} else {
80498087a06SJeff Roberson 		vm_object_reference(backing_object);
8054cf9f5d8SKonstantin Belousov 		atomic_add_int(&backing_object->shadow_count, 1);
80698087a06SJeff Roberson 		object->backing_object = backing_object;
80798087a06SJeff Roberson 	}
80898087a06SJeff Roberson }
80998087a06SJeff Roberson 
81098087a06SJeff Roberson /*
81198087a06SJeff Roberson  * Transfer a backing reference from backing_object to object.
81298087a06SJeff Roberson  */
81398087a06SJeff Roberson static void
81498087a06SJeff Roberson vm_object_backing_transfer(vm_object_t object, vm_object_t backing_object)
81598087a06SJeff Roberson {
81698087a06SJeff Roberson 	vm_object_t new_backing_object;
81798087a06SJeff Roberson 
81898087a06SJeff Roberson 	/*
81998087a06SJeff Roberson 	 * Note that the reference to backing_object->backing_object
82098087a06SJeff Roberson 	 * moves from within backing_object to within object.
82198087a06SJeff Roberson 	 */
82298087a06SJeff Roberson 	vm_object_backing_remove_locked(object);
82398087a06SJeff Roberson 	new_backing_object = backing_object->backing_object;
82498087a06SJeff Roberson 	if (new_backing_object == NULL)
82598087a06SJeff Roberson 		return;
82698087a06SJeff Roberson 	if ((new_backing_object->flags & OBJ_ANON) != 0) {
82798087a06SJeff Roberson 		VM_OBJECT_WLOCK(new_backing_object);
82898087a06SJeff Roberson 		vm_object_backing_remove_locked(backing_object);
82998087a06SJeff Roberson 		vm_object_backing_insert_locked(object, new_backing_object);
83098087a06SJeff Roberson 		VM_OBJECT_WUNLOCK(new_backing_object);
83198087a06SJeff Roberson 	} else {
8324cf9f5d8SKonstantin Belousov 		/*
8334cf9f5d8SKonstantin Belousov 		 * shadow_count for new_backing_object is left
8344cf9f5d8SKonstantin Belousov 		 * unchanged, its reference provided by backing_object
8354cf9f5d8SKonstantin Belousov 		 * is replaced by object.
8364cf9f5d8SKonstantin Belousov 		 */
83798087a06SJeff Roberson 		object->backing_object = new_backing_object;
83898087a06SJeff Roberson 		backing_object->backing_object = NULL;
83998087a06SJeff Roberson 	}
84098087a06SJeff Roberson }
84198087a06SJeff Roberson 
84298087a06SJeff Roberson /*
84398087a06SJeff Roberson  * Wait for a concurrent collapse to settle.
84498087a06SJeff Roberson  */
84598087a06SJeff Roberson static void
84698087a06SJeff Roberson vm_object_collapse_wait(vm_object_t object)
84798087a06SJeff Roberson {
84898087a06SJeff Roberson 
84998087a06SJeff Roberson 	VM_OBJECT_ASSERT_WLOCKED(object);
85098087a06SJeff Roberson 
85198087a06SJeff Roberson 	while ((object->flags & OBJ_COLLAPSING) != 0) {
85298087a06SJeff Roberson 		vm_object_pip_wait(object, "vmcolwait");
85398087a06SJeff Roberson 		counter_u64_add(object_collapse_waits, 1);
85498087a06SJeff Roberson 	}
85598087a06SJeff Roberson }
85698087a06SJeff Roberson 
85798087a06SJeff Roberson /*
85898087a06SJeff Roberson  * Waits for a backing object to clear a pending collapse and returns
85998087a06SJeff Roberson  * it locked if it is an ANON object.
86098087a06SJeff Roberson  */
86198087a06SJeff Roberson static vm_object_t
86298087a06SJeff Roberson vm_object_backing_collapse_wait(vm_object_t object)
86398087a06SJeff Roberson {
86498087a06SJeff Roberson 	vm_object_t backing_object;
86598087a06SJeff Roberson 
86698087a06SJeff Roberson 	VM_OBJECT_ASSERT_WLOCKED(object);
86798087a06SJeff Roberson 
86898087a06SJeff Roberson 	for (;;) {
86998087a06SJeff Roberson 		backing_object = object->backing_object;
87098087a06SJeff Roberson 		if (backing_object == NULL ||
87198087a06SJeff Roberson 		    (backing_object->flags & OBJ_ANON) == 0)
87298087a06SJeff Roberson 			return (NULL);
87398087a06SJeff Roberson 		VM_OBJECT_WLOCK(backing_object);
87498087a06SJeff Roberson 		if ((backing_object->flags & (OBJ_DEAD | OBJ_COLLAPSING)) == 0)
87598087a06SJeff Roberson 			break;
876cdd02f43SMark Johnston 		VM_OBJECT_WUNLOCK(object);
87798087a06SJeff Roberson 		vm_object_pip_sleep(backing_object, "vmbckwait");
87898087a06SJeff Roberson 		counter_u64_add(object_collapse_waits, 1);
87998087a06SJeff Roberson 		VM_OBJECT_WLOCK(object);
88098087a06SJeff Roberson 	}
88198087a06SJeff Roberson 	return (backing_object);
88298087a06SJeff Roberson }
88351b867e5SJeff Roberson 
8842ac78f0eSStephan Uphoff /*
8857bbdb843SRuslan Bukin  *	vm_object_terminate_pages removes any remaining pageable pages
8867bbdb843SRuslan Bukin  *	from the object and resets the object to an empty state.
8877bbdb843SRuslan Bukin  */
8887bbdb843SRuslan Bukin static void
8897bbdb843SRuslan Bukin vm_object_terminate_pages(vm_object_t object)
8907bbdb843SRuslan Bukin {
8917bbdb843SRuslan Bukin 	vm_page_t p, p_next;
8927bbdb843SRuslan Bukin 
8937bbdb843SRuslan Bukin 	VM_OBJECT_ASSERT_WLOCKED(object);
8947bbdb843SRuslan Bukin 
8957bbdb843SRuslan Bukin 	/*
8967bbdb843SRuslan Bukin 	 * Free any remaining pageable pages.  This also removes them from the
8977bbdb843SRuslan Bukin 	 * paging queues.  However, don't free wired pages, just remove them
8987bbdb843SRuslan Bukin 	 * from the object.  Rather than incrementally removing each page from
8997bbdb843SRuslan Bukin 	 * the object, the page and object are reset to any empty state.
9007bbdb843SRuslan Bukin 	 */
9017bbdb843SRuslan Bukin 	TAILQ_FOREACH_SAFE(p, &object->memq, listq, p_next) {
9027bbdb843SRuslan Bukin 		vm_page_assert_unbusied(p);
903fee2a2faSMark Johnston 		KASSERT(p->object == object &&
904fee2a2faSMark Johnston 		    (p->ref_count & VPRC_OBJREF) != 0,
905fee2a2faSMark Johnston 		    ("vm_object_terminate_pages: page %p is inconsistent", p));
906fee2a2faSMark Johnston 
9077bbdb843SRuslan Bukin 		p->object = NULL;
908fee2a2faSMark Johnston 		if (vm_page_drop(p, VPRC_OBJREF) == VPRC_OBJREF) {
9095cd29d0fSMark Johnston 			VM_CNT_INC(v_pfree);
9105cd29d0fSMark Johnston 			vm_page_free(p);
9114074d642SAlan Cox 		}
912fee2a2faSMark Johnston 	}
9132fcd1ff6SKonstantin Belousov 
9147bbdb843SRuslan Bukin 	/*
9157bbdb843SRuslan Bukin 	 * If the object contained any pages, then reset it to an empty state.
9167bbdb843SRuslan Bukin 	 * None of the object's fields, including "resident_page_count", were
9177bbdb843SRuslan Bukin 	 * modified by the preceding loop.
9187bbdb843SRuslan Bukin 	 */
9197bbdb843SRuslan Bukin 	if (object->resident_page_count != 0) {
9207bbdb843SRuslan Bukin 		vm_radix_reclaim_allnodes(&object->rtree);
9217bbdb843SRuslan Bukin 		TAILQ_INIT(&object->memq);
9227bbdb843SRuslan Bukin 		object->resident_page_count = 0;
9237bbdb843SRuslan Bukin 		if (object->type == OBJT_VNODE)
9247bbdb843SRuslan Bukin 			vdrop(object->handle);
9257bbdb843SRuslan Bukin 	}
9267bbdb843SRuslan Bukin }
9277bbdb843SRuslan Bukin 
9287bbdb843SRuslan Bukin /*
929df8bae1dSRodney W. Grimes  *	vm_object_terminate actually destroys the specified object, freeing
930df8bae1dSRodney W. Grimes  *	up all previously used resources.
931df8bae1dSRodney W. Grimes  *
932df8bae1dSRodney W. Grimes  *	The object must be locked.
9331c7c3c6aSMatthew Dillon  *	This routine may block.
934df8bae1dSRodney W. Grimes  */
93595e5e988SJohn Dyson void
9361b40f8c0SMatthew Dillon vm_object_terminate(vm_object_t object)
937df8bae1dSRodney W. Grimes {
93898087a06SJeff Roberson 
93989f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
940783a68aaSKonstantin Belousov 	KASSERT((object->flags & OBJ_DEAD) != 0,
941783a68aaSKonstantin Belousov 	    ("terminating non-dead obj %p", object));
94298087a06SJeff Roberson 	KASSERT((object->flags & OBJ_COLLAPSING) == 0,
94398087a06SJeff Roberson 	    ("terminating collapsing obj %p", object));
94498087a06SJeff Roberson 	KASSERT(object->backing_object == NULL,
94598087a06SJeff Roberson 	    ("terminating shadow obj %p", object));
946bef608bdSJohn Dyson 
947cf27e0d1SJeff Roberson 	/*
948ffae7ea9SKonstantin Belousov 	 * Wait for the pageout daemon and other current users to be
949ffae7ea9SKonstantin Belousov 	 * done with the object.  Note that new paging_in_progress
950ffae7ea9SKonstantin Belousov 	 * users can come after this wait, but they must check
951ffae7ea9SKonstantin Belousov 	 * OBJ_DEAD flag set (without unlocking the object), and avoid
952ffae7ea9SKonstantin Belousov 	 * the object being terminated.
953cf27e0d1SJeff Roberson 	 */
954cf27e0d1SJeff Roberson 	vm_object_pip_wait(object, "objtrm");
955cf27e0d1SJeff Roberson 
956971dd342SAlfred Perlstein 	KASSERT(object->ref_count == 0,
957971dd342SAlfred Perlstein 	    ("vm_object_terminate: object with references, ref_count=%d",
958971dd342SAlfred Perlstein 	    object->ref_count));
959996c772fSJohn Dyson 
9607bbdb843SRuslan Bukin 	if ((object->flags & OBJ_PG_DTOR) == 0)
9617bbdb843SRuslan Bukin 		vm_object_terminate_pages(object);
962bef608bdSJohn Dyson 
963f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0
964f8a47341SAlan Cox 	if (__predict_false(!LIST_EMPTY(&object->rvq)))
965f8a47341SAlan Cox 		vm_reserv_break_all(object);
966f8a47341SAlan Cox #endif
9677bfda801SAlan Cox 
9680cb2610eSMark Johnston 	KASSERT(object->cred == NULL || (object->flags & OBJ_SWAP) != 0,
969e735691bSJohn Baldwin 	    ("%s: non-swap obj %p has cred", __func__, object));
970e735691bSJohn Baldwin 
9712d8acc0fSJohn Dyson 	/*
9729fcfb650SDavid Greenman 	 * Let the pager know object is dead.
9739fcfb650SDavid Greenman 	 */
9749fcfb650SDavid Greenman 	vm_pager_deallocate(object);
97589f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(object);
9769fcfb650SDavid Greenman 
9772ac78f0eSStephan Uphoff 	vm_object_destroy(object);
97847221757SJohn Dyson }
979df8bae1dSRodney W. Grimes 
980edf93b25SAlan Cox /*
981edf93b25SAlan Cox  * Make the page read-only so that we can clear the object flags.  However, if
982edf93b25SAlan Cox  * this is a nosync mmap then the object is likely to stay dirty so do not
983edf93b25SAlan Cox  * mess with the page and do not clear the object flags.  Returns TRUE if the
984edf93b25SAlan Cox  * page should be flushed, and FALSE otherwise.
985edf93b25SAlan Cox  */
9863280870dSKonstantin Belousov static boolean_t
98767d0e293SJeff Roberson vm_object_page_remove_write(vm_page_t p, int flags, boolean_t *allclean)
9883280870dSKonstantin Belousov {
9893280870dSKonstantin Belousov 
990fff5403fSJeff Roberson 	vm_page_assert_busied(p);
991fff5403fSJeff Roberson 
9923280870dSKonstantin Belousov 	/*
9933280870dSKonstantin Belousov 	 * If we have been asked to skip nosync pages and this is a
9943280870dSKonstantin Belousov 	 * nosync page, skip it.  Note that the object flags were not
9953280870dSKonstantin Belousov 	 * cleared in this case so we do not have to set them.
9963280870dSKonstantin Belousov 	 */
9975cff1f4dSMark Johnston 	if ((flags & OBJPC_NOSYNC) != 0 && (p->a.flags & PGA_NOSYNC) != 0) {
99867d0e293SJeff Roberson 		*allclean = FALSE;
9993280870dSKonstantin Belousov 		return (FALSE);
10003280870dSKonstantin Belousov 	} else {
10013280870dSKonstantin Belousov 		pmap_remove_write(p);
10023280870dSKonstantin Belousov 		return (p->dirty != 0);
10033280870dSKonstantin Belousov 	}
10043280870dSKonstantin Belousov }
10053280870dSKonstantin Belousov 
1006df8bae1dSRodney W. Grimes /*
1007df8bae1dSRodney W. Grimes  *	vm_object_page_clean
1008df8bae1dSRodney W. Grimes  *
10094f79d873SMatthew Dillon  *	Clean all dirty pages in the specified range of object.  Leaves page
10104f79d873SMatthew Dillon  * 	on whatever queue it is currently on.   If NOSYNC is set then do not
1011fff5403fSJeff Roberson  *	write out pages with PGA_NOSYNC set (originally comes from MAP_NOSYNC),
10124f79d873SMatthew Dillon  *	leaving the object dirty.
101326f9a767SRodney W. Grimes  *
10148d34a3bfSKonstantin Belousov  *	For swap objects backing tmpfs regular files, do not flush anything,
10158d34a3bfSKonstantin Belousov  *	but remove write protection on the mapped pages to update mtime through
10168d34a3bfSKonstantin Belousov  *	mmaped writes.
10178d34a3bfSKonstantin Belousov  *
101843b7990eSMatthew Dillon  *	When stuffing pages asynchronously, allow clustering.  XXX we need a
101943b7990eSMatthew Dillon  *	synchronous clustering mode implementation.
102043b7990eSMatthew Dillon  *
102126f9a767SRodney W. Grimes  *	Odd semantics: if start == end, we clean everything.
102226f9a767SRodney W. Grimes  *
102326f9a767SRodney W. Grimes  *	The object must be locked.
1024126d6082SKonstantin Belousov  *
1025126d6082SKonstantin Belousov  *	Returns FALSE if some page from the range was not written, as
1026126d6082SKonstantin Belousov  *	reported by the pager, and TRUE otherwise.
102726f9a767SRodney W. Grimes  */
1028126d6082SKonstantin Belousov boolean_t
102917f3095dSAlan Cox vm_object_page_clean(vm_object_t object, vm_ooffset_t start, vm_ooffset_t end,
1030e239bb97SKonstantin Belousov     int flags)
1031f6b04d2bSDavid Greenman {
1032e239bb97SKonstantin Belousov 	vm_page_t np, p;
103317f3095dSAlan Cox 	vm_pindex_t pi, tend, tstart;
1034126d6082SKonstantin Belousov 	int curgeneration, n, pagerflags;
103567d0e293SJeff Roberson 	boolean_t eio, res, allclean;
1036f6b04d2bSDavid Greenman 
103789f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
1038e5f299ffSKonstantin Belousov 
10398d34a3bfSKonstantin Belousov 	if (!vm_object_mightbedirty(object) || object->resident_page_count == 0)
1040126d6082SKonstantin Belousov 		return (TRUE);
1041f6b04d2bSDavid Greenman 
1042e239bb97SKonstantin Belousov 	pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) != 0 ?
1043e239bb97SKonstantin Belousov 	    VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK;
1044e239bb97SKonstantin Belousov 	pagerflags |= (flags & OBJPC_INVAL) != 0 ? VM_PAGER_PUT_INVAL : 0;
1045e239bb97SKonstantin Belousov 
104617f3095dSAlan Cox 	tstart = OFF_TO_IDX(start);
104717f3095dSAlan Cox 	tend = (end == 0) ? object->size : OFF_TO_IDX(end + PAGE_MASK);
104867d0e293SJeff Roberson 	allclean = tstart == 0 && tend >= object->size;
1049126d6082SKonstantin Belousov 	res = TRUE;
1050f6b04d2bSDavid Greenman 
1051bd7e5f99SJohn Dyson rescan:
10522d8acc0fSJohn Dyson 	curgeneration = object->generation;
10532d8acc0fSJohn Dyson 
105417f3095dSAlan Cox 	for (p = vm_page_find_least(object, tstart); p != NULL; p = np) {
1055bd7e5f99SJohn Dyson 		pi = p->pindex;
1056e239bb97SKonstantin Belousov 		if (pi >= tend)
1057e239bb97SKonstantin Belousov 			break;
1058e239bb97SKonstantin Belousov 		np = TAILQ_NEXT(p, listq);
10590012f373SJeff Roberson 		if (vm_page_none_valid(p))
1060aef922f5SJohn Dyson 			continue;
106163e97555SJeff Roberson 		if (vm_page_busy_acquire(p, VM_ALLOC_WAITFAIL) == 0) {
106267d0e293SJeff Roberson 			if (object->generation != curgeneration &&
106367d0e293SJeff Roberson 			    (flags & OBJPC_SYNC) != 0)
1064e239bb97SKonstantin Belousov 				goto rescan;
1065780636b7SKonstantin Belousov 			np = vm_page_find_least(object, pi);
1066780636b7SKonstantin Belousov 			continue;
1067f6b04d2bSDavid Greenman 		}
106867d0e293SJeff Roberson 		if (!vm_object_page_remove_write(p, flags, &allclean)) {
106963e97555SJeff Roberson 			vm_page_xunbusy(p);
1070bd7e5f99SJohn Dyson 			continue;
107163e97555SJeff Roberson 		}
10728d34a3bfSKonstantin Belousov 		if (object->type == OBJT_VNODE) {
10733280870dSKonstantin Belousov 			n = vm_object_page_collect_flush(object, p, pagerflags,
107467d0e293SJeff Roberson 			    flags, &allclean, &eio);
1075126d6082SKonstantin Belousov 			if (eio) {
1076126d6082SKonstantin Belousov 				res = FALSE;
107767d0e293SJeff Roberson 				allclean = FALSE;
1078126d6082SKonstantin Belousov 			}
107967d0e293SJeff Roberson 			if (object->generation != curgeneration &&
108067d0e293SJeff Roberson 			    (flags & OBJPC_SYNC) != 0)
1081b9b7a4beSMatthew Dillon 				goto rescan;
1082031ec8c1SKonstantin Belousov 
1083031ec8c1SKonstantin Belousov 			/*
1084031ec8c1SKonstantin Belousov 			 * If the VOP_PUTPAGES() did a truncated write, so
1085031ec8c1SKonstantin Belousov 			 * that even the first page of the run is not fully
1086031ec8c1SKonstantin Belousov 			 * written, vm_pageout_flush() returns 0 as the run
1087031ec8c1SKonstantin Belousov 			 * length.  Since the condition that caused truncated
1088031ec8c1SKonstantin Belousov 			 * write may be permanent, e.g. exhausted free space,
1089031ec8c1SKonstantin Belousov 			 * accepting n == 0 would cause an infinite loop.
1090031ec8c1SKonstantin Belousov 			 *
1091031ec8c1SKonstantin Belousov 			 * Forwarding the iterator leaves the unwritten page
1092031ec8c1SKonstantin Belousov 			 * behind, but there is not much we can do there if
1093031ec8c1SKonstantin Belousov 			 * filesystem refuses to write it.
1094031ec8c1SKonstantin Belousov 			 */
1095126d6082SKonstantin Belousov 			if (n == 0) {
1096031ec8c1SKonstantin Belousov 				n = 1;
109767d0e293SJeff Roberson 				allclean = FALSE;
1098126d6082SKonstantin Belousov 			}
10998d34a3bfSKonstantin Belousov 		} else {
11008d34a3bfSKonstantin Belousov 			n = 1;
11018d34a3bfSKonstantin Belousov 			vm_page_xunbusy(p);
11028d34a3bfSKonstantin Belousov 		}
1103e239bb97SKonstantin Belousov 		np = vm_page_find_least(object, pi + n);
1104b9b7a4beSMatthew Dillon 	}
1105b9b7a4beSMatthew Dillon #if 0
1106e239bb97SKonstantin Belousov 	VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC) ? MNT_WAIT : 0);
1107b9b7a4beSMatthew Dillon #endif
1108b9b7a4beSMatthew Dillon 
11098d34a3bfSKonstantin Belousov 	/*
11108d34a3bfSKonstantin Belousov 	 * Leave updating cleangeneration for tmpfs objects to tmpfs
11118d34a3bfSKonstantin Belousov 	 * scan.  It needs to update mtime, which happens for other
11128d34a3bfSKonstantin Belousov 	 * filesystems during page writeouts.
11138d34a3bfSKonstantin Belousov 	 */
11148d34a3bfSKonstantin Belousov 	if (allclean && object->type == OBJT_VNODE)
111567d0e293SJeff Roberson 		object->cleangeneration = curgeneration;
1116126d6082SKonstantin Belousov 	return (res);
1117b9b7a4beSMatthew Dillon }
1118b9b7a4beSMatthew Dillon 
1119b9b7a4beSMatthew Dillon static int
11203280870dSKonstantin Belousov vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags,
112167d0e293SJeff Roberson     int flags, boolean_t *allclean, boolean_t *eio)
1122b9b7a4beSMatthew Dillon {
11233157c503SKonstantin Belousov 	vm_page_t ma[vm_pageout_page_count], p_first, tp;
11243157c503SKonstantin Belousov 	int count, i, mreq, runlen;
1125b9b7a4beSMatthew Dillon 
11267bec141bSKip Macy 	vm_page_lock_assert(p, MA_NOTOWNED);
112763e97555SJeff Roberson 	vm_page_assert_xbusied(p);
112889f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
11293157c503SKonstantin Belousov 
11303157c503SKonstantin Belousov 	count = 1;
11313157c503SKonstantin Belousov 	mreq = 0;
11323157c503SKonstantin Belousov 
11333157c503SKonstantin Belousov 	for (tp = p; count < vm_pageout_page_count; count++) {
11343157c503SKonstantin Belousov 		tp = vm_page_next(tp);
113563e97555SJeff Roberson 		if (tp == NULL || vm_page_tryxbusy(tp) == 0)
1136bd7e5f99SJohn Dyson 			break;
113767d0e293SJeff Roberson 		if (!vm_object_page_remove_write(tp, flags, allclean)) {
113863e97555SJeff Roberson 			vm_page_xunbusy(tp);
1139bd7e5f99SJohn Dyson 			break;
1140bd7e5f99SJohn Dyson 		}
114163e97555SJeff Roberson 	}
1142aef922f5SJohn Dyson 
11433157c503SKonstantin Belousov 	for (p_first = p; count < vm_pageout_page_count; count++) {
11443157c503SKonstantin Belousov 		tp = vm_page_prev(p_first);
114563e97555SJeff Roberson 		if (tp == NULL || vm_page_tryxbusy(tp) == 0)
1146bd7e5f99SJohn Dyson 			break;
114767d0e293SJeff Roberson 		if (!vm_object_page_remove_write(tp, flags, allclean)) {
114863e97555SJeff Roberson 			vm_page_xunbusy(tp);
1149bd7e5f99SJohn Dyson 			break;
115063e97555SJeff Roberson 		}
11513157c503SKonstantin Belousov 		p_first = tp;
11523157c503SKonstantin Belousov 		mreq++;
1153bd7e5f99SJohn Dyson 	}
1154bd7e5f99SJohn Dyson 
11553157c503SKonstantin Belousov 	for (tp = p_first, i = 0; i < count; tp = TAILQ_NEXT(tp, listq), i++)
11563157c503SKonstantin Belousov 		ma[i] = tp;
1157cf2819ccSJohn Dyson 
1158126d6082SKonstantin Belousov 	vm_pageout_flush(ma, count, pagerflags, mreq, &runlen, eio);
11591e8a675cSKonstantin Belousov 	return (runlen);
116026f9a767SRodney W. Grimes }
1161df8bae1dSRodney W. Grimes 
11621efb74fbSJohn Dyson /*
1163950f8459SAlan Cox  * Note that there is absolutely no sense in writing out
1164950f8459SAlan Cox  * anonymous objects, so we track down the vnode object
1165950f8459SAlan Cox  * to write out.
1166950f8459SAlan Cox  * We invalidate (remove) all pages from the address space
1167950f8459SAlan Cox  * for semantic correctness.
1168950f8459SAlan Cox  *
11696bbee8e2SAlan Cox  * If the backing object is a device object with unmanaged pages, then any
11706bbee8e2SAlan Cox  * mappings to the specified range of pages must be removed before this
11716bbee8e2SAlan Cox  * function is called.
11726bbee8e2SAlan Cox  *
1173950f8459SAlan Cox  * Note: certain anonymous maps, such as MAP_NOSYNC maps,
1174950f8459SAlan Cox  * may start out with a NULL object.
1175950f8459SAlan Cox  */
1176126d6082SKonstantin Belousov boolean_t
1177950f8459SAlan Cox vm_object_sync(vm_object_t object, vm_ooffset_t offset, vm_size_t size,
1178950f8459SAlan Cox     boolean_t syncio, boolean_t invalidate)
1179950f8459SAlan Cox {
1180950f8459SAlan Cox 	vm_object_t backing_object;
1181950f8459SAlan Cox 	struct vnode *vp;
11823b582b4eSTor Egge 	struct mount *mp;
1183126d6082SKonstantin Belousov 	int error, flags, fsync_after;
1184126d6082SKonstantin Belousov 	boolean_t res;
1185950f8459SAlan Cox 
1186950f8459SAlan Cox 	if (object == NULL)
1187126d6082SKonstantin Belousov 		return (TRUE);
1188126d6082SKonstantin Belousov 	res = TRUE;
1189126d6082SKonstantin Belousov 	error = 0;
119089f6b863SAttilio Rao 	VM_OBJECT_WLOCK(object);
1191950f8459SAlan Cox 	while ((backing_object = object->backing_object) != NULL) {
119289f6b863SAttilio Rao 		VM_OBJECT_WLOCK(backing_object);
119356e0670fSAlan Cox 		offset += object->backing_object_offset;
119489f6b863SAttilio Rao 		VM_OBJECT_WUNLOCK(object);
1195950f8459SAlan Cox 		object = backing_object;
1196950f8459SAlan Cox 		if (object->size < OFF_TO_IDX(offset + size))
1197950f8459SAlan Cox 			size = IDX_TO_OFF(object->size) - offset;
1198950f8459SAlan Cox 	}
1199950f8459SAlan Cox 	/*
1200950f8459SAlan Cox 	 * Flush pages if writing is allowed, invalidate them
1201950f8459SAlan Cox 	 * if invalidation requested.  Pages undergoing I/O
1202950f8459SAlan Cox 	 * will be ignored by vm_object_page_remove().
1203950f8459SAlan Cox 	 *
1204950f8459SAlan Cox 	 * We cannot lock the vnode and then wait for paging
1205950f8459SAlan Cox 	 * to complete without deadlocking against vm_fault.
1206950f8459SAlan Cox 	 * Instead we simply call vm_object_page_remove() and
1207950f8459SAlan Cox 	 * allow it to block internally on a page-by-page
1208950f8459SAlan Cox 	 * basis when it encounters pages undergoing async
1209950f8459SAlan Cox 	 * I/O.
1210950f8459SAlan Cox 	 */
1211950f8459SAlan Cox 	if (object->type == OBJT_VNODE &&
121267d0e293SJeff Roberson 	    vm_object_mightbedirty(object) != 0 &&
12135bf94937SKonstantin Belousov 	    ((vp = object->handle)->v_vflag & VV_NOSYNC) == 0) {
121489f6b863SAttilio Rao 		VM_OBJECT_WUNLOCK(object);
12153b582b4eSTor Egge 		(void)vn_start_write(vp, &mp, V_WAIT);
1216cb05b60aSAttilio Rao 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
121775ff604aSKonstantin Belousov 		if (syncio && !invalidate && offset == 0 &&
1218d1780e8dSKonstantin Belousov 		    atop(size) == object->size) {
121975ff604aSKonstantin Belousov 			/*
122075ff604aSKonstantin Belousov 			 * If syncing the whole mapping of the file,
122175ff604aSKonstantin Belousov 			 * it is faster to schedule all the writes in
122275ff604aSKonstantin Belousov 			 * async mode, also allowing the clustering,
122375ff604aSKonstantin Belousov 			 * and then wait for i/o to complete.
122475ff604aSKonstantin Belousov 			 */
122575ff604aSKonstantin Belousov 			flags = 0;
122675ff604aSKonstantin Belousov 			fsync_after = TRUE;
122775ff604aSKonstantin Belousov 		} else {
1228950f8459SAlan Cox 			flags = (syncio || invalidate) ? OBJPC_SYNC : 0;
122975ff604aSKonstantin Belousov 			flags |= invalidate ? (OBJPC_SYNC | OBJPC_INVAL) : 0;
123075ff604aSKonstantin Belousov 			fsync_after = FALSE;
123175ff604aSKonstantin Belousov 		}
123289f6b863SAttilio Rao 		VM_OBJECT_WLOCK(object);
1233126d6082SKonstantin Belousov 		res = vm_object_page_clean(object, offset, offset + size,
1234126d6082SKonstantin Belousov 		    flags);
123589f6b863SAttilio Rao 		VM_OBJECT_WUNLOCK(object);
1236*6189672eSKonstantin Belousov 		if (fsync_after) {
1237*6189672eSKonstantin Belousov 			for (;;) {
1238126d6082SKonstantin Belousov 				error = VOP_FSYNC(vp, MNT_WAIT, curthread);
1239*6189672eSKonstantin Belousov 				if (error != ERELOOKUP)
1240*6189672eSKonstantin Belousov 					break;
1241*6189672eSKonstantin Belousov 
1242*6189672eSKonstantin Belousov 				/*
1243*6189672eSKonstantin Belousov 				 * Allow SU/bufdaemon to handle more
1244*6189672eSKonstantin Belousov 				 * dependencies in the meantime.
1245*6189672eSKonstantin Belousov 				 */
1246*6189672eSKonstantin Belousov 				VOP_UNLOCK(vp);
1247*6189672eSKonstantin Belousov 				vn_finished_write(mp);
1248*6189672eSKonstantin Belousov 
1249*6189672eSKonstantin Belousov 				(void)vn_start_write(vp, &mp, V_WAIT);
1250*6189672eSKonstantin Belousov 				vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1251*6189672eSKonstantin Belousov 			}
1252*6189672eSKonstantin Belousov 		}
1253b249ce48SMateusz Guzik 		VOP_UNLOCK(vp);
12543b582b4eSTor Egge 		vn_finished_write(mp);
1255126d6082SKonstantin Belousov 		if (error != 0)
1256126d6082SKonstantin Belousov 			res = FALSE;
125789f6b863SAttilio Rao 		VM_OBJECT_WLOCK(object);
1258950f8459SAlan Cox 	}
1259950f8459SAlan Cox 	if ((object->type == OBJT_VNODE ||
1260950f8459SAlan Cox 	     object->type == OBJT_DEVICE) && invalidate) {
12616bbee8e2SAlan Cox 		if (object->type == OBJT_DEVICE)
12626bbee8e2SAlan Cox 			/*
12636bbee8e2SAlan Cox 			 * The option OBJPR_NOTMAPPED must be passed here
12646bbee8e2SAlan Cox 			 * because vm_object_page_remove() cannot remove
12656bbee8e2SAlan Cox 			 * unmanaged mappings.
12666bbee8e2SAlan Cox 			 */
12676bbee8e2SAlan Cox 			flags = OBJPR_NOTMAPPED;
12686bbee8e2SAlan Cox 		else if (old_msync)
12696195b24aSKonstantin Belousov 			flags = 0;
12706bbee8e2SAlan Cox 		else
12716195b24aSKonstantin Belousov 			flags = OBJPR_CLEANONLY;
12726bbee8e2SAlan Cox 		vm_object_page_remove(object, OFF_TO_IDX(offset),
12736bbee8e2SAlan Cox 		    OFF_TO_IDX(offset + size + PAGE_MASK), flags);
1274950f8459SAlan Cox 	}
127589f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(object);
1276126d6082SKonstantin Belousov 	return (res);
1277950f8459SAlan Cox }
1278950f8459SAlan Cox 
1279950f8459SAlan Cox /*
1280aa3650eaSMark Johnston  * Determine whether the given advice can be applied to the object.  Advice is
1281aa3650eaSMark Johnston  * not applied to unmanaged pages since they never belong to page queues, and
1282aa3650eaSMark Johnston  * since MADV_FREE is destructive, it can apply only to anonymous pages that
1283aa3650eaSMark Johnston  * have been mapped at most once.
1284aa3650eaSMark Johnston  */
1285aa3650eaSMark Johnston static bool
1286aa3650eaSMark Johnston vm_object_advice_applies(vm_object_t object, int advice)
1287aa3650eaSMark Johnston {
1288aa3650eaSMark Johnston 
1289aa3650eaSMark Johnston 	if ((object->flags & OBJ_UNMANAGED) != 0)
1290aa3650eaSMark Johnston 		return (false);
1291aa3650eaSMark Johnston 	if (advice != MADV_FREE)
1292aa3650eaSMark Johnston 		return (true);
129363967687SJeff Roberson 	return ((object->flags & (OBJ_ONEMAPPING | OBJ_ANON)) ==
129463967687SJeff Roberson 	    (OBJ_ONEMAPPING | OBJ_ANON));
1295aa3650eaSMark Johnston }
1296aa3650eaSMark Johnston 
1297aa3650eaSMark Johnston static void
1298aa3650eaSMark Johnston vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex,
1299aa3650eaSMark Johnston     vm_size_t size)
1300aa3650eaSMark Johnston {
1301aa3650eaSMark Johnston 
13021390a5cbSKonstantin Belousov 	if (advice == MADV_FREE)
13031390a5cbSKonstantin Belousov 		vm_pager_freespace(object, pindex, size);
1304aa3650eaSMark Johnston }
1305aa3650eaSMark Johnston 
1306aa3650eaSMark Johnston /*
1307867a482dSJohn Dyson  *	vm_object_madvise:
1308867a482dSJohn Dyson  *
1309867a482dSJohn Dyson  *	Implements the madvise function at the object/page level.
13101c7c3c6aSMatthew Dillon  *
1311193b9358SAlan Cox  *	MADV_WILLNEED	(any object)
1312193b9358SAlan Cox  *
1313193b9358SAlan Cox  *	    Activate the specified pages if they are resident.
1314193b9358SAlan Cox  *
1315193b9358SAlan Cox  *	MADV_DONTNEED	(any object)
1316193b9358SAlan Cox  *
1317193b9358SAlan Cox  *	    Deactivate the specified pages if they are resident.
1318193b9358SAlan Cox  *
13190cb2610eSMark Johnston  *	MADV_FREE	(OBJT_SWAP objects, OBJ_ONEMAPPING only)
1320193b9358SAlan Cox  *
1321193b9358SAlan Cox  *	    Deactivate and clean the specified pages if they are
1322193b9358SAlan Cox  *	    resident.  This permits the process to reuse the pages
1323193b9358SAlan Cox  *	    without faulting or the kernel to reclaim the pages
1324193b9358SAlan Cox  *	    without I/O.
1325867a482dSJohn Dyson  */
1326867a482dSJohn Dyson void
132792a59946SJohn Baldwin vm_object_madvise(vm_object_t object, vm_pindex_t pindex, vm_pindex_t end,
1328c2655a40SMark Johnston     int advice)
1329867a482dSJohn Dyson {
133092a59946SJohn Baldwin 	vm_pindex_t tpindex;
133134567de7SAlan Cox 	vm_object_t backing_object, tobject;
1332aa3650eaSMark Johnston 	vm_page_t m, tm;
1333867a482dSJohn Dyson 
1334867a482dSJohn Dyson 	if (object == NULL)
1335867a482dSJohn Dyson 		return;
1336c2655a40SMark Johnston 
13376e20a165SJohn Dyson relookup:
1338aa3650eaSMark Johnston 	VM_OBJECT_WLOCK(object);
1339aa3650eaSMark Johnston 	if (!vm_object_advice_applies(object, advice)) {
1340aa3650eaSMark Johnston 		VM_OBJECT_WUNLOCK(object);
1341aa3650eaSMark Johnston 		return;
13426e20a165SJohn Dyson 	}
1343aa3650eaSMark Johnston 	for (m = vm_page_find_least(object, pindex); pindex < end; pindex++) {
1344aa3650eaSMark Johnston 		tobject = object;
1345c2655a40SMark Johnston 
13461ce137beSMatthew Dillon 		/*
1347aa3650eaSMark Johnston 		 * If the next page isn't resident in the top-level object, we
1348aa3650eaSMark Johnston 		 * need to search the shadow chain.  When applying MADV_FREE, we
1349aa3650eaSMark Johnston 		 * take care to release any swap space used to store
1350aa3650eaSMark Johnston 		 * non-resident pages.
1351aa3650eaSMark Johnston 		 */
1352aa3650eaSMark Johnston 		if (m == NULL || pindex < m->pindex) {
1353aa3650eaSMark Johnston 			/*
1354aa3650eaSMark Johnston 			 * Optimize a common case: if the top-level object has
1355aa3650eaSMark Johnston 			 * no backing object, we can skip over the non-resident
1356aa3650eaSMark Johnston 			 * range in constant time.
13571ce137beSMatthew Dillon 			 */
1358c2655a40SMark Johnston 			if (object->backing_object == NULL) {
1359c2655a40SMark Johnston 				tpindex = (m != NULL && m->pindex < end) ?
1360c2655a40SMark Johnston 				    m->pindex : end;
1361aa3650eaSMark Johnston 				vm_object_madvise_freespace(object, advice,
1362aa3650eaSMark Johnston 				    pindex, tpindex - pindex);
1363c2655a40SMark Johnston 				if ((pindex = tpindex) == end)
1364c2655a40SMark Johnston 					break;
1365aa3650eaSMark Johnston 				goto next_page;
1366aa3650eaSMark Johnston 			}
1367aa3650eaSMark Johnston 
1368aa3650eaSMark Johnston 			tpindex = pindex;
1369aa3650eaSMark Johnston 			do {
1370aa3650eaSMark Johnston 				vm_object_madvise_freespace(tobject, advice,
1371aa3650eaSMark Johnston 				    tpindex, 1);
13721ce137beSMatthew Dillon 				/*
1373aa3650eaSMark Johnston 				 * Prepare to search the next object in the
1374aa3650eaSMark Johnston 				 * chain.
13751ce137beSMatthew Dillon 				 */
137634567de7SAlan Cox 				backing_object = tobject->backing_object;
137734567de7SAlan Cox 				if (backing_object == NULL)
1378aa3650eaSMark Johnston 					goto next_pindex;
137989f6b863SAttilio Rao 				VM_OBJECT_WLOCK(backing_object);
1380aa3650eaSMark Johnston 				tpindex +=
1381aa3650eaSMark Johnston 				    OFF_TO_IDX(tobject->backing_object_offset);
13829b98b796SAlan Cox 				if (tobject != object)
138389f6b863SAttilio Rao 					VM_OBJECT_WUNLOCK(tobject);
138434567de7SAlan Cox 				tobject = backing_object;
1385aa3650eaSMark Johnston 				if (!vm_object_advice_applies(tobject, advice))
1386aa3650eaSMark Johnston 					goto next_pindex;
1387aa3650eaSMark Johnston 			} while ((tm = vm_page_lookup(tobject, tpindex)) ==
1388aa3650eaSMark Johnston 			    NULL);
1389aa3650eaSMark Johnston 		} else {
1390aa3650eaSMark Johnston next_page:
1391aa3650eaSMark Johnston 			tm = m;
1392aa3650eaSMark Johnston 			m = TAILQ_NEXT(m, listq);
1393c2655a40SMark Johnston 		}
1394c2655a40SMark Johnston 
1395867a482dSJohn Dyson 		/*
13960012f373SJeff Roberson 		 * If the page is not in a normal state, skip it.  The page
13970012f373SJeff Roberson 		 * can not be invalidated while the object lock is held.
1398867a482dSJohn Dyson 		 */
13990012f373SJeff Roberson 		if (!vm_page_all_valid(tm) || vm_page_wired(tm))
1400aa3650eaSMark Johnston 			goto next_pindex;
1401aa3650eaSMark Johnston 		KASSERT((tm->flags & PG_FICTITIOUS) == 0,
1402aa3650eaSMark Johnston 		    ("vm_object_madvise: page %p is fictitious", tm));
1403aa3650eaSMark Johnston 		KASSERT((tm->oflags & VPO_UNMANAGED) == 0,
1404aa3650eaSMark Johnston 		    ("vm_object_madvise: page %p is not managed", tm));
140563e97555SJeff Roberson 		if (vm_page_tryxbusy(tm) == 0) {
1406aa3650eaSMark Johnston 			if (object != tobject)
1407aa3650eaSMark Johnston 				VM_OBJECT_WUNLOCK(object);
1408c2655a40SMark Johnston 			if (advice == MADV_WILLNEED) {
1409b11b56b5SAlan Cox 				/*
1410b11b56b5SAlan Cox 				 * Reference the page before unlocking and
1411b11b56b5SAlan Cox 				 * sleeping so that the page daemon is less
1412b11b56b5SAlan Cox 				 * likely to reclaim it.
1413b11b56b5SAlan Cox 				 */
1414aa3650eaSMark Johnston 				vm_page_aflag_set(tm, PGA_REFERENCED);
1415567e51e1SAlan Cox 			}
141687b64663SMark Johnston 			if (!vm_page_busy_sleep(tm, "madvpo", 0))
141787b64663SMark Johnston 				VM_OBJECT_WUNLOCK(tobject);
14186e20a165SJohn Dyson   			goto relookup;
141934567de7SAlan Cox 		}
1420aa3650eaSMark Johnston 		vm_page_advise(tm, advice);
142163e97555SJeff Roberson 		vm_page_xunbusy(tm);
1422aa3650eaSMark Johnston 		vm_object_madvise_freespace(tobject, advice, tm->pindex, 1);
1423aa3650eaSMark Johnston next_pindex:
14249b98b796SAlan Cox 		if (tobject != object)
142589f6b863SAttilio Rao 			VM_OBJECT_WUNLOCK(tobject);
1426867a482dSJohn Dyson 	}
142789f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(object);
1428867a482dSJohn Dyson }
1429867a482dSJohn Dyson 
1430867a482dSJohn Dyson /*
1431df8bae1dSRodney W. Grimes  *	vm_object_shadow:
1432df8bae1dSRodney W. Grimes  *
1433df8bae1dSRodney W. Grimes  *	Create a new object which is backed by the
1434df8bae1dSRodney W. Grimes  *	specified existing object range.  The source
1435df8bae1dSRodney W. Grimes  *	object reference is deallocated.
1436df8bae1dSRodney W. Grimes  *
1437df8bae1dSRodney W. Grimes  *	The new object and offset into that object
1438df8bae1dSRodney W. Grimes  *	are returned in the source parameters.
1439df8bae1dSRodney W. Grimes  */
144026f9a767SRodney W. Grimes void
144167388836SKonstantin Belousov vm_object_shadow(vm_object_t *object, vm_ooffset_t *offset, vm_size_t length,
144267388836SKonstantin Belousov     struct ucred *cred, bool shared)
1443df8bae1dSRodney W. Grimes {
1444d031cff1SMatthew Dillon 	vm_object_t source;
1445d031cff1SMatthew Dillon 	vm_object_t result;
1446df8bae1dSRodney W. Grimes 
1447df8bae1dSRodney W. Grimes 	source = *object;
1448df8bae1dSRodney W. Grimes 
1449df8bae1dSRodney W. Grimes 	/*
14509a2f6362SAlan Cox 	 * Don't create the new object if the old object isn't shared.
145163967687SJeff Roberson 	 *
145263967687SJeff Roberson 	 * If we hold the only reference we can guarantee that it won't
145363967687SJeff Roberson 	 * increase while we have the map locked.  Otherwise the race is
145463967687SJeff Roberson 	 * harmless and we will end up with an extra shadow object that
145563967687SJeff Roberson 	 * will be collapsed later.
14569a2f6362SAlan Cox 	 */
145763967687SJeff Roberson 	if (source != NULL && source->ref_count == 1 &&
145832362449SKonstantin Belousov 	    (source->flags & OBJ_ANON) != 0)
14599a2f6362SAlan Cox 		return;
14609a2f6362SAlan Cox 
14619a2f6362SAlan Cox 	/*
1462570a2f4aSAlan Cox 	 * Allocate a new object with the given length.
1463df8bae1dSRodney W. Grimes 	 */
146467388836SKonstantin Belousov 	result = vm_object_allocate_anon(atop(length), source, cred, length);
1465df8bae1dSRodney W. Grimes 
1466df8bae1dSRodney W. Grimes 	/*
146751b867e5SJeff Roberson 	 * Store the offset into the source object, and fix up the offset into
146851b867e5SJeff Roberson 	 * the new object.
146951b867e5SJeff Roberson 	 */
147051b867e5SJeff Roberson 	result->backing_object_offset = *offset;
147151b867e5SJeff Roberson 
147267388836SKonstantin Belousov 	if (shared || source != NULL) {
147367388836SKonstantin Belousov 		VM_OBJECT_WLOCK(result);
147467388836SKonstantin Belousov 
147551b867e5SJeff Roberson 		/*
147667388836SKonstantin Belousov 		 * The new object shadows the source object, adding a
147767388836SKonstantin Belousov 		 * reference to it.  Our caller changes his reference
147867388836SKonstantin Belousov 		 * to point to the new object, removing a reference to
147967388836SKonstantin Belousov 		 * the source object.  Net result: no change of
148067388836SKonstantin Belousov 		 * reference count, unless the caller needs to add one
148167388836SKonstantin Belousov 		 * more reference due to forking a shared map entry.
148267388836SKonstantin Belousov 		 */
148367388836SKonstantin Belousov 		if (shared) {
148467388836SKonstantin Belousov 			vm_object_reference_locked(result);
148567388836SKonstantin Belousov 			vm_object_clear_flag(result, OBJ_ONEMAPPING);
148667388836SKonstantin Belousov 		}
148767388836SKonstantin Belousov 
148867388836SKonstantin Belousov 		/*
148967388836SKonstantin Belousov 		 * Try to optimize the result object's page color when
149067388836SKonstantin Belousov 		 * shadowing in order to maintain page coloring
149167388836SKonstantin Belousov 		 * consistency in the combined shadowed object.
1492df8bae1dSRodney W. Grimes 		 */
1493570a2f4aSAlan Cox 		if (source != NULL) {
149451b867e5SJeff Roberson 			vm_object_backing_insert(result, source);
14953f289c3fSJeff Roberson 			result->domain = source->domain;
1496f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0
1497630f633fSMark Johnston 			vm_object_set_flag(result,
1498630f633fSMark Johnston 			    (source->flags & OBJ_COLORED));
149967388836SKonstantin Belousov 			result->pg_color = (source->pg_color +
150067388836SKonstantin Belousov 			    OFF_TO_IDX(*offset)) & ((1 << (VM_NFREEORDER -
150167388836SKonstantin Belousov 			    1)) - 1);
1502f8a47341SAlan Cox #endif
150367388836SKonstantin Belousov 		}
150451b867e5SJeff Roberson 		VM_OBJECT_WUNLOCK(result);
1505de5f6a77SJohn Dyson 	}
1506df8bae1dSRodney W. Grimes 
1507df8bae1dSRodney W. Grimes 	/*
1508df8bae1dSRodney W. Grimes 	 * Return the new things
1509df8bae1dSRodney W. Grimes 	 */
1510df8bae1dSRodney W. Grimes 	*offset = 0;
1511df8bae1dSRodney W. Grimes 	*object = result;
1512df8bae1dSRodney W. Grimes }
1513df8bae1dSRodney W. Grimes 
1514c5aaa06dSAlan Cox /*
1515c5aaa06dSAlan Cox  *	vm_object_split:
1516c5aaa06dSAlan Cox  *
1517c5aaa06dSAlan Cox  * Split the pages in a map entry into a new object.  This affords
1518c5aaa06dSAlan Cox  * easier removal of unused pages, and keeps object inheritance from
1519c5aaa06dSAlan Cox  * being a negative impact on memory usage.
1520c5aaa06dSAlan Cox  */
1521c5aaa06dSAlan Cox void
1522c5aaa06dSAlan Cox vm_object_split(vm_map_entry_t entry)
1523c5aaa06dSAlan Cox {
1524fff19e0eSMark Johnston 	vm_page_t m, m_next;
152598087a06SJeff Roberson 	vm_object_t orig_object, new_object, backing_object;
152673000556SAlan Cox 	vm_pindex_t idx, offidxstart;
152773000556SAlan Cox 	vm_size_t size;
1528c5aaa06dSAlan Cox 
1529c5aaa06dSAlan Cox 	orig_object = entry->object.vm_object;
153098087a06SJeff Roberson 	KASSERT((orig_object->flags & OBJ_ONEMAPPING) != 0,
153198087a06SJeff Roberson 	    ("vm_object_split:  Splitting object with multiple mappings."));
153263967687SJeff Roberson 	if ((orig_object->flags & OBJ_ANON) == 0)
1533c5aaa06dSAlan Cox 		return;
1534c5aaa06dSAlan Cox 	if (orig_object->ref_count <= 1)
1535c5aaa06dSAlan Cox 		return;
153689f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(orig_object);
1537c5aaa06dSAlan Cox 
15384da9f125SAlan Cox 	offidxstart = OFF_TO_IDX(entry->offset);
153995442adfSAlan Cox 	size = atop(entry->end - entry->start);
1540c5aaa06dSAlan Cox 
154167388836SKonstantin Belousov 	new_object = vm_object_allocate_anon(size, orig_object,
154267388836SKonstantin Belousov 	    orig_object->cred, ptoa(size));
1543c5aaa06dSAlan Cox 
1544c5474b8fSAlan Cox 	/*
154598087a06SJeff Roberson 	 * We must wait for the orig_object to complete any in-progress
154698087a06SJeff Roberson 	 * collapse so that the swap blocks are stable below.  The
154798087a06SJeff Roberson 	 * additional reference on backing_object by new object will
154898087a06SJeff Roberson 	 * prevent further collapse operations until split completes.
154998087a06SJeff Roberson 	 */
155098087a06SJeff Roberson 	VM_OBJECT_WLOCK(orig_object);
155198087a06SJeff Roberson 	vm_object_collapse_wait(orig_object);
155298087a06SJeff Roberson 
155398087a06SJeff Roberson 	/*
1554c5474b8fSAlan Cox 	 * At this point, the new object is still private, so the order in
1555c5474b8fSAlan Cox 	 * which the original and new objects are locked does not matter.
1556c5474b8fSAlan Cox 	 */
155789f6b863SAttilio Rao 	VM_OBJECT_WLOCK(new_object);
15583f289c3fSJeff Roberson 	new_object->domain = orig_object->domain;
155998087a06SJeff Roberson 	backing_object = orig_object->backing_object;
156098087a06SJeff Roberson 	if (backing_object != NULL) {
156198087a06SJeff Roberson 		vm_object_backing_insert_ref(new_object, backing_object);
1562c5aaa06dSAlan Cox 		new_object->backing_object_offset =
15634da9f125SAlan Cox 		    orig_object->backing_object_offset + entry->offset;
1564c5aaa06dSAlan Cox 	}
1565ef694c1aSEdward Tomasz Napierala 	if (orig_object->cred != NULL) {
1566ef694c1aSEdward Tomasz Napierala 		crhold(orig_object->cred);
15673364c323SKonstantin Belousov 		KASSERT(orig_object->charge >= ptoa(size),
15683364c323SKonstantin Belousov 		    ("orig_object->charge < 0"));
15693364c323SKonstantin Belousov 		orig_object->charge -= ptoa(size);
15703364c323SKonstantin Belousov 	}
157198087a06SJeff Roberson 
157298087a06SJeff Roberson 	/*
157398087a06SJeff Roberson 	 * Mark the split operation so that swap_pager_getpages() knows
157498087a06SJeff Roberson 	 * that the object is in transition.
157598087a06SJeff Roberson 	 */
157698087a06SJeff Roberson 	vm_object_set_flag(orig_object, OBJ_SPLIT);
1577aec9e7d8SMark Johnston #ifdef INVARIANTS
1578aec9e7d8SMark Johnston 	idx = 0;
1579aec9e7d8SMark Johnston #endif
1580c5aaa06dSAlan Cox retry:
1581b382c10aSKonstantin Belousov 	m = vm_page_find_least(orig_object, offidxstart);
1582aec9e7d8SMark Johnston 	KASSERT(m == NULL || idx <= m->pindex - offidxstart,
1583aec9e7d8SMark Johnston 	    ("%s: object %p was repopulated", __func__, orig_object));
158473000556SAlan Cox 	for (; m != NULL && (idx = m->pindex - offidxstart) < size;
158573000556SAlan Cox 	    m = m_next) {
158673000556SAlan Cox 		m_next = TAILQ_NEXT(m, listq);
1587c5aaa06dSAlan Cox 
1588c5aaa06dSAlan Cox 		/*
1589c5aaa06dSAlan Cox 		 * We must wait for pending I/O to complete before we can
1590c5aaa06dSAlan Cox 		 * rename the page.
1591c5aaa06dSAlan Cox 		 *
1592c5aaa06dSAlan Cox 		 * We do not have to VM_PROT_NONE the page as mappings should
1593c5aaa06dSAlan Cox 		 * not be changed by this operation.
1594c5aaa06dSAlan Cox 		 */
159563e97555SJeff Roberson 		if (vm_page_tryxbusy(m) == 0) {
159689f6b863SAttilio Rao 			VM_OBJECT_WUNLOCK(new_object);
159787b64663SMark Johnston 			if (vm_page_busy_sleep(m, "spltwt", 0))
159887b64663SMark Johnston 				VM_OBJECT_WLOCK(orig_object);
159989f6b863SAttilio Rao 			VM_OBJECT_WLOCK(new_object);
1600c5aaa06dSAlan Cox 			goto retry;
1601de33beddSAlan Cox 		}
1602e946b949SAttilio Rao 
16034bf95d00SJeff Roberson 		/*
16044bf95d00SJeff Roberson 		 * The page was left invalid.  Likely placed there by
16054bf95d00SJeff Roberson 		 * an incomplete fault.  Just remove and ignore.
16064bf95d00SJeff Roberson 		 */
16074bf95d00SJeff Roberson 		if (vm_page_none_valid(m)) {
16084bf95d00SJeff Roberson 			if (vm_page_remove(m))
16094bf95d00SJeff Roberson 				vm_page_free(m);
16104bf95d00SJeff Roberson 			continue;
16114bf95d00SJeff Roberson 		}
16124bf95d00SJeff Roberson 
16133453bca8SAlan Cox 		/* vm_page_rename() will dirty the page. */
1614e946b949SAttilio Rao 		if (vm_page_rename(m, new_object, idx)) {
161563e97555SJeff Roberson 			vm_page_xunbusy(m);
1616e946b949SAttilio Rao 			VM_OBJECT_WUNLOCK(new_object);
1617e946b949SAttilio Rao 			VM_OBJECT_WUNLOCK(orig_object);
16188d6fbbb8SJeff Roberson 			vm_radix_wait();
1619e946b949SAttilio Rao 			VM_OBJECT_WLOCK(orig_object);
1620e946b949SAttilio Rao 			VM_OBJECT_WLOCK(new_object);
1621e946b949SAttilio Rao 			goto retry;
1622e946b949SAttilio Rao 		}
162363e97555SJeff Roberson 
1624b5f359b7SAlan Cox #if VM_NRESERVLEVEL > 0
1625b5f359b7SAlan Cox 		/*
1626b5f359b7SAlan Cox 		 * If some of the reservation's allocated pages remain with
1627b5f359b7SAlan Cox 		 * the original object, then transferring the reservation to
1628b5f359b7SAlan Cox 		 * the new object is neither particularly beneficial nor
1629b5f359b7SAlan Cox 		 * particularly harmful as compared to leaving the reservation
1630b5f359b7SAlan Cox 		 * with the original object.  If, however, all of the
1631b5f359b7SAlan Cox 		 * reservation's allocated pages are transferred to the new
1632b5f359b7SAlan Cox 		 * object, then transferring the reservation is typically
1633b5f359b7SAlan Cox 		 * beneficial.  Determining which of these two cases applies
1634b5f359b7SAlan Cox 		 * would be more costly than unconditionally renaming the
1635b5f359b7SAlan Cox 		 * reservation.
1636b5f359b7SAlan Cox 		 */
1637b5f359b7SAlan Cox 		vm_reserv_rename(m, new_object, orig_object, offidxstart);
1638b5f359b7SAlan Cox #endif
1639c5aaa06dSAlan Cox 	}
1640fff19e0eSMark Johnston 
1641c5aaa06dSAlan Cox 	/*
1642c7c8dd7eSAlan Cox 	 * swap_pager_copy() can sleep, in which case the orig_object's
1643c7c8dd7eSAlan Cox 	 * and new_object's locks are released and reacquired.
1644c5aaa06dSAlan Cox 	 */
1645c5aaa06dSAlan Cox 	swap_pager_copy(orig_object, new_object, offidxstart, 0);
1646fff19e0eSMark Johnston 
1647fff19e0eSMark Johnston 	TAILQ_FOREACH(m, &new_object->memq, listq)
1648fff19e0eSMark Johnston 		vm_page_xunbusy(m);
1649fff19e0eSMark Johnston 
165098087a06SJeff Roberson 	vm_object_clear_flag(orig_object, OBJ_SPLIT);
165189f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(orig_object);
165289f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(new_object);
1653c5aaa06dSAlan Cox 	entry->object.vm_object = new_object;
1654c5aaa06dSAlan Cox 	entry->offset = 0LL;
1655c5aaa06dSAlan Cox 	vm_object_deallocate(orig_object);
165689f6b863SAttilio Rao 	VM_OBJECT_WLOCK(new_object);
1657c5aaa06dSAlan Cox }
1658c5aaa06dSAlan Cox 
165999a1570aSKonstantin Belousov static vm_page_t
166098087a06SJeff Roberson vm_object_collapse_scan_wait(vm_object_t object, vm_page_t p)
166199a1570aSKonstantin Belousov {
166299a1570aSKonstantin Belousov 	vm_object_t backing_object;
166399a1570aSKonstantin Belousov 
166499a1570aSKonstantin Belousov 	VM_OBJECT_ASSERT_WLOCKED(object);
166599a1570aSKonstantin Belousov 	backing_object = object->backing_object;
166699a1570aSKonstantin Belousov 	VM_OBJECT_ASSERT_WLOCKED(backing_object);
166799a1570aSKonstantin Belousov 
166899a1570aSKonstantin Belousov 	KASSERT(p == NULL || p->object == object || p->object == backing_object,
166999a1570aSKonstantin Belousov 	    ("invalid ownership %p %p %p", p, object, backing_object));
16708d6fbbb8SJeff Roberson 	/* The page is only NULL when rename fails. */
16714cdea4a8SJeff Roberson 	if (p == NULL) {
16726a14746cSRyan Libby 		VM_OBJECT_WUNLOCK(object);
16736a14746cSRyan Libby 		VM_OBJECT_WUNLOCK(backing_object);
16748d6fbbb8SJeff Roberson 		vm_radix_wait();
167599a1570aSKonstantin Belousov 		VM_OBJECT_WLOCK(object);
167687b64663SMark Johnston 	} else if (p->object == object) {
167787b64663SMark Johnston 		VM_OBJECT_WUNLOCK(backing_object);
167887b64663SMark Johnston 		if (vm_page_busy_sleep(p, "vmocol", 0))
167987b64663SMark Johnston 			VM_OBJECT_WLOCK(object);
168087b64663SMark Johnston 	} else {
168187b64663SMark Johnston 		VM_OBJECT_WUNLOCK(object);
168287b64663SMark Johnston 		if (!vm_page_busy_sleep(p, "vmocol", 0))
168387b64663SMark Johnston 			VM_OBJECT_WUNLOCK(backing_object);
168487b64663SMark Johnston 		VM_OBJECT_WLOCK(object);
168587b64663SMark Johnston 	}
168699a1570aSKonstantin Belousov 	VM_OBJECT_WLOCK(backing_object);
168799a1570aSKonstantin Belousov 	return (TAILQ_FIRST(&backing_object->memq));
168899a1570aSKonstantin Belousov }
168999a1570aSKonstantin Belousov 
169099a1570aSKonstantin Belousov static bool
16914cc8daf7SConrad Meyer vm_object_scan_all_shadowed(vm_object_t object)
16924cc8daf7SConrad Meyer {
16934cc8daf7SConrad Meyer 	vm_object_t backing_object;
16944cc8daf7SConrad Meyer 	vm_page_t p, pp;
169577d6fd97SKonstantin Belousov 	vm_pindex_t backing_offset_index, new_pindex, pi, ps;
16964cc8daf7SConrad Meyer 
16974cc8daf7SConrad Meyer 	VM_OBJECT_ASSERT_WLOCKED(object);
16984cc8daf7SConrad Meyer 	VM_OBJECT_ASSERT_WLOCKED(object->backing_object);
16994cc8daf7SConrad Meyer 
17004cc8daf7SConrad Meyer 	backing_object = object->backing_object;
17014cc8daf7SConrad Meyer 
170263967687SJeff Roberson 	if ((backing_object->flags & OBJ_ANON) == 0)
17034cc8daf7SConrad Meyer 		return (false);
17044cc8daf7SConrad Meyer 
170577d6fd97SKonstantin Belousov 	pi = backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
170677d6fd97SKonstantin Belousov 	p = vm_page_find_least(backing_object, pi);
170777d6fd97SKonstantin Belousov 	ps = swap_pager_find_least(backing_object, pi);
17084cc8daf7SConrad Meyer 
17094cc8daf7SConrad Meyer 	/*
171077d6fd97SKonstantin Belousov 	 * Only check pages inside the parent object's range and
171177d6fd97SKonstantin Belousov 	 * inside the parent object's mapping of the backing object.
17124cc8daf7SConrad Meyer 	 */
171377d6fd97SKonstantin Belousov 	for (;; pi++) {
171477d6fd97SKonstantin Belousov 		if (p != NULL && p->pindex < pi)
171577d6fd97SKonstantin Belousov 			p = TAILQ_NEXT(p, listq);
171677d6fd97SKonstantin Belousov 		if (ps < pi)
171777d6fd97SKonstantin Belousov 			ps = swap_pager_find_least(backing_object, pi);
171877d6fd97SKonstantin Belousov 		if (p == NULL && ps >= backing_object->size)
171977d6fd97SKonstantin Belousov 			break;
172077d6fd97SKonstantin Belousov 		else if (p == NULL)
172177d6fd97SKonstantin Belousov 			pi = ps;
172277d6fd97SKonstantin Belousov 		else
172377d6fd97SKonstantin Belousov 			pi = MIN(p->pindex, ps);
172477d6fd97SKonstantin Belousov 
172577d6fd97SKonstantin Belousov 		new_pindex = pi - backing_offset_index;
172677d6fd97SKonstantin Belousov 		if (new_pindex >= object->size)
172777d6fd97SKonstantin Belousov 			break;
17284cc8daf7SConrad Meyer 
1729cd0047f3SKonstantin Belousov 		if (p != NULL) {
17304cc8daf7SConrad Meyer 			/*
1731cd0047f3SKonstantin Belousov 			 * If the backing object page is busy a
1732cd0047f3SKonstantin Belousov 			 * grandparent or older page may still be
1733cd0047f3SKonstantin Belousov 			 * undergoing CoW.  It is not safe to collapse
1734cd0047f3SKonstantin Belousov 			 * the backing object until it is quiesced.
173558447749SJeff Roberson 			 */
1736cd0047f3SKonstantin Belousov 			if (vm_page_tryxbusy(p) == 0)
173758447749SJeff Roberson 				return (false);
173858447749SJeff Roberson 
173958447749SJeff Roberson 			/*
1740cd0047f3SKonstantin Belousov 			 * We raced with the fault handler that left
1741cd0047f3SKonstantin Belousov 			 * newly allocated invalid page on the object
1742cd0047f3SKonstantin Belousov 			 * queue and retried.
1743cd0047f3SKonstantin Belousov 			 */
1744cd0047f3SKonstantin Belousov 			if (!vm_page_all_valid(p))
1745cd0047f3SKonstantin Belousov 				goto unbusy_ret;
1746cd0047f3SKonstantin Belousov 		}
1747cd0047f3SKonstantin Belousov 
1748cd0047f3SKonstantin Belousov 		/*
17494cc8daf7SConrad Meyer 		 * See if the parent has the page or if the parent's object
17504cc8daf7SConrad Meyer 		 * pager has the page.  If the parent has the page but the page
17514cc8daf7SConrad Meyer 		 * is not valid, the parent's object pager must have the page.
17524cc8daf7SConrad Meyer 		 *
17534cc8daf7SConrad Meyer 		 * If this fails, the parent does not completely shadow the
17544cc8daf7SConrad Meyer 		 * object and we might as well give up now.
17554cc8daf7SConrad Meyer 		 */
17564cc8daf7SConrad Meyer 		pp = vm_page_lookup(object, new_pindex);
1757cd0047f3SKonstantin Belousov 
17580012f373SJeff Roberson 		/*
1759cd0047f3SKonstantin Belousov 		 * The valid check here is stable due to object lock
1760cd0047f3SKonstantin Belousov 		 * being required to clear valid and initiate paging.
1761cd0047f3SKonstantin Belousov 		 * Busy of p disallows fault handler to validate pp.
17620012f373SJeff Roberson 		 */
17630012f373SJeff Roberson 		if ((pp == NULL || vm_page_none_valid(pp)) &&
17644cc8daf7SConrad Meyer 		    !vm_pager_has_page(object, new_pindex, NULL, NULL))
1765cd0047f3SKonstantin Belousov 			goto unbusy_ret;
1766cd0047f3SKonstantin Belousov 		if (p != NULL)
1767cd0047f3SKonstantin Belousov 			vm_page_xunbusy(p);
17684cc8daf7SConrad Meyer 	}
17694cc8daf7SConrad Meyer 	return (true);
1770cd0047f3SKonstantin Belousov 
1771cd0047f3SKonstantin Belousov unbusy_ret:
1772cd0047f3SKonstantin Belousov 	if (p != NULL)
1773cd0047f3SKonstantin Belousov 		vm_page_xunbusy(p);
1774cd0047f3SKonstantin Belousov 	return (false);
17754cc8daf7SConrad Meyer }
17764cc8daf7SConrad Meyer 
177798087a06SJeff Roberson static void
177898087a06SJeff Roberson vm_object_collapse_scan(vm_object_t object)
17792ad1a3f7SMatthew Dillon {
17802ad1a3f7SMatthew Dillon 	vm_object_t backing_object;
178199a1570aSKonstantin Belousov 	vm_page_t next, p, pp;
178299a1570aSKonstantin Belousov 	vm_pindex_t backing_offset_index, new_pindex;
17832ad1a3f7SMatthew Dillon 
178489f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
178589f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object->backing_object);
17862ad1a3f7SMatthew Dillon 
17872ad1a3f7SMatthew Dillon 	backing_object = object->backing_object;
17882ad1a3f7SMatthew Dillon 	backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
17892ad1a3f7SMatthew Dillon 
17902ad1a3f7SMatthew Dillon 	/*
17912ad1a3f7SMatthew Dillon 	 * Our scan
17922ad1a3f7SMatthew Dillon 	 */
17934cc8daf7SConrad Meyer 	for (p = TAILQ_FIRST(&backing_object->memq); p != NULL; p = next) {
179499a1570aSKonstantin Belousov 		next = TAILQ_NEXT(p, listq);
179599a1570aSKonstantin Belousov 		new_pindex = p->pindex - backing_offset_index;
17962ad1a3f7SMatthew Dillon 
17972ad1a3f7SMatthew Dillon 		/*
17982ad1a3f7SMatthew Dillon 		 * Check for busy page
17992ad1a3f7SMatthew Dillon 		 */
180063e97555SJeff Roberson 		if (vm_page_tryxbusy(p) == 0) {
180198087a06SJeff Roberson 			next = vm_object_collapse_scan_wait(object, p);
18022ad1a3f7SMatthew Dillon 			continue;
18032ad1a3f7SMatthew Dillon 		}
18042ad1a3f7SMatthew Dillon 
180598087a06SJeff Roberson 		KASSERT(object->backing_object == backing_object,
180698087a06SJeff Roberson 		    ("vm_object_collapse_scan: backing object mismatch %p != %p",
180798087a06SJeff Roberson 		    object->backing_object, backing_object));
180899a1570aSKonstantin Belousov 		KASSERT(p->object == backing_object,
180998087a06SJeff Roberson 		    ("vm_object_collapse_scan: object mismatch %p != %p",
181098087a06SJeff Roberson 		    p->object, backing_object));
18112ad1a3f7SMatthew Dillon 
181299a1570aSKonstantin Belousov 		if (p->pindex < backing_offset_index ||
181399a1570aSKonstantin Belousov 		    new_pindex >= object->size) {
18141390a5cbSKonstantin Belousov 			vm_pager_freespace(backing_object, p->pindex, 1);
1815e946b949SAttilio Rao 
1816f6d89838SAlan Cox 			KASSERT(!pmap_page_is_mapped(p),
1817f6d89838SAlan Cox 			    ("freeing mapped page %p", p));
18180fd977b3SMark Johnston 			if (vm_page_remove(p))
18192ad1a3f7SMatthew Dillon 				vm_page_free(p);
18202ad1a3f7SMatthew Dillon 			continue;
18212ad1a3f7SMatthew Dillon 		}
18222ad1a3f7SMatthew Dillon 
1823cd0047f3SKonstantin Belousov 		if (!vm_page_all_valid(p)) {
1824cd0047f3SKonstantin Belousov 			KASSERT(!pmap_page_is_mapped(p),
1825cd0047f3SKonstantin Belousov 			    ("freeing mapped page %p", p));
1826cd0047f3SKonstantin Belousov 			if (vm_page_remove(p))
1827cd0047f3SKonstantin Belousov 				vm_page_free(p);
1828cd0047f3SKonstantin Belousov 			continue;
1829cd0047f3SKonstantin Belousov 		}
1830cd0047f3SKonstantin Belousov 
18312ad1a3f7SMatthew Dillon 		pp = vm_page_lookup(object, new_pindex);
183263e97555SJeff Roberson 		if (pp != NULL && vm_page_tryxbusy(pp) == 0) {
183363e97555SJeff Roberson 			vm_page_xunbusy(p);
1834e18cc7bfSMax Laier 			/*
18354cc8daf7SConrad Meyer 			 * The page in the parent is busy and possibly not
18364cc8daf7SConrad Meyer 			 * (yet) valid.  Until its state is finalized by the
18374cc8daf7SConrad Meyer 			 * busy bit owner, we can't tell whether it shadows the
183898087a06SJeff Roberson 			 * original page.
1839e18cc7bfSMax Laier 			 */
184098087a06SJeff Roberson 			next = vm_object_collapse_scan_wait(object, pp);
1841e18cc7bfSMax Laier 			continue;
1842e18cc7bfSMax Laier 		}
184399a1570aSKonstantin Belousov 
18444bf95d00SJeff Roberson 		if (pp != NULL && vm_page_none_valid(pp)) {
18454bf95d00SJeff Roberson 			/*
18464bf95d00SJeff Roberson 			 * The page was invalid in the parent.  Likely placed
18474bf95d00SJeff Roberson 			 * there by an incomplete fault.  Just remove and
18484bf95d00SJeff Roberson 			 * ignore.  p can replace it.
18494bf95d00SJeff Roberson 			 */
18504bf95d00SJeff Roberson 			if (vm_page_remove(pp))
18514bf95d00SJeff Roberson 				vm_page_free(pp);
18524bf95d00SJeff Roberson 			pp = NULL;
18534bf95d00SJeff Roberson 		}
185499a1570aSKonstantin Belousov 
18554cc8daf7SConrad Meyer 		if (pp != NULL || vm_pager_has_page(object, new_pindex, NULL,
18564cc8daf7SConrad Meyer 			NULL)) {
185799a1570aSKonstantin Belousov 			/*
18584cc8daf7SConrad Meyer 			 * The page already exists in the parent OR swap exists
18594cc8daf7SConrad Meyer 			 * for this location in the parent.  Leave the parent's
18604cc8daf7SConrad Meyer 			 * page alone.  Destroy the original page from the
18614cc8daf7SConrad Meyer 			 * backing object.
186299a1570aSKonstantin Belousov 			 */
18631390a5cbSKonstantin Belousov 			vm_pager_freespace(backing_object, p->pindex, 1);
1864f6d89838SAlan Cox 			KASSERT(!pmap_page_is_mapped(p),
1865f6d89838SAlan Cox 			    ("freeing mapped page %p", p));
18660fd977b3SMark Johnston 			if (vm_page_remove(p))
18672ad1a3f7SMatthew Dillon 				vm_page_free(p);
186863e97555SJeff Roberson 			if (pp != NULL)
186963e97555SJeff Roberson 				vm_page_xunbusy(pp);
18702ad1a3f7SMatthew Dillon 			continue;
18712ad1a3f7SMatthew Dillon 		}
18722ad1a3f7SMatthew Dillon 
1873e946b949SAttilio Rao 		/*
18744cc8daf7SConrad Meyer 		 * Page does not exist in parent, rename the page from the
18754cc8daf7SConrad Meyer 		 * backing object to the main object.
1876e946b949SAttilio Rao 		 *
18774cc8daf7SConrad Meyer 		 * If the page was mapped to a process, it can remain mapped
18783453bca8SAlan Cox 		 * through the rename.  vm_page_rename() will dirty the page.
1879e946b949SAttilio Rao 		 */
1880e946b949SAttilio Rao 		if (vm_page_rename(p, object, new_pindex)) {
188163e97555SJeff Roberson 			vm_page_xunbusy(p);
188298087a06SJeff Roberson 			next = vm_object_collapse_scan_wait(object, NULL);
1883e946b949SAttilio Rao 			continue;
1884e946b949SAttilio Rao 		}
188514a5dc17SAttilio Rao 
188614a5dc17SAttilio Rao 		/* Use the old pindex to free the right page. */
18871390a5cbSKonstantin Belousov 		vm_pager_freespace(backing_object, new_pindex +
18881390a5cbSKonstantin Belousov 		    backing_offset_index, 1);
1889e946b949SAttilio Rao 
1890f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0
1891f8a47341SAlan Cox 		/*
1892f8a47341SAlan Cox 		 * Rename the reservation.
1893f8a47341SAlan Cox 		 */
1894f8a47341SAlan Cox 		vm_reserv_rename(p, object, backing_object,
1895f8a47341SAlan Cox 		    backing_offset_index);
1896f8a47341SAlan Cox #endif
18978da1c098SJeff Roberson 		vm_page_xunbusy(p);
18982ad1a3f7SMatthew Dillon 	}
18992fe6e4d7SDavid Greenman 	return;
19002fe6e4d7SDavid Greenman }
19012fe6e4d7SDavid Greenman 
1902df8bae1dSRodney W. Grimes /*
1903df8bae1dSRodney W. Grimes  *	vm_object_collapse:
1904df8bae1dSRodney W. Grimes  *
1905df8bae1dSRodney W. Grimes  *	Collapse an object with the object backing it.
1906df8bae1dSRodney W. Grimes  *	Pages in the backing object are moved into the
1907df8bae1dSRodney W. Grimes  *	parent, and the backing object is deallocated.
1908df8bae1dSRodney W. Grimes  */
190926f9a767SRodney W. Grimes void
19101b40f8c0SMatthew Dillon vm_object_collapse(vm_object_t object)
1911df8bae1dSRodney W. Grimes {
191298f139daSKonstantin Belousov 	vm_object_t backing_object, new_backing_object;
191398f139daSKonstantin Belousov 
191489f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
191523955314SAlfred Perlstein 
1916df8bae1dSRodney W. Grimes 	while (TRUE) {
191798087a06SJeff Roberson 		KASSERT((object->flags & (OBJ_DEAD | OBJ_ANON)) == OBJ_ANON,
191898087a06SJeff Roberson 		    ("collapsing invalid object"));
191998087a06SJeff Roberson 
1920df8bae1dSRodney W. Grimes 		/*
192198087a06SJeff Roberson 		 * Wait for the backing_object to finish any pending
192298087a06SJeff Roberson 		 * collapse so that the caller sees the shortest possible
192398087a06SJeff Roberson 		 * shadow chain.
1924df8bae1dSRodney W. Grimes 		 */
192598087a06SJeff Roberson 		backing_object = vm_object_backing_collapse_wait(object);
192698087a06SJeff Roberson 		if (backing_object == NULL)
192798087a06SJeff Roberson 			return;
192898087a06SJeff Roberson 
192998087a06SJeff Roberson 		KASSERT(object->ref_count > 0 &&
19304cf9f5d8SKonstantin Belousov 		    object->ref_count > atomic_load_int(&object->shadow_count),
193198087a06SJeff Roberson 		    ("collapse with invalid ref %d or shadow %d count.",
19324cf9f5d8SKonstantin Belousov 		    object->ref_count, atomic_load_int(&object->shadow_count)));
193398087a06SJeff Roberson 		KASSERT((backing_object->flags &
193498087a06SJeff Roberson 		    (OBJ_COLLAPSING | OBJ_DEAD)) == 0,
193598087a06SJeff Roberson 		    ("vm_object_collapse: Backing object already collapsing."));
193698087a06SJeff Roberson 		KASSERT((object->flags & (OBJ_COLLAPSING | OBJ_DEAD)) == 0,
193798087a06SJeff Roberson 		    ("vm_object_collapse: object is already collapsing."));
1938df8bae1dSRodney W. Grimes 
1939f919ebdeSDavid Greenman 		/*
194098087a06SJeff Roberson 		 * We know that we can either collapse the backing object if
194198087a06SJeff Roberson 		 * the parent is the only reference to it, or (perhaps) have
19422ad1a3f7SMatthew Dillon 		 * the parent bypass the object if the parent happens to shadow
19432ad1a3f7SMatthew Dillon 		 * all the resident pages in the entire backing object.
1944df8bae1dSRodney W. Grimes 		 */
1945df8bae1dSRodney W. Grimes 		if (backing_object->ref_count == 1) {
19464cf9f5d8SKonstantin Belousov 			KASSERT(atomic_load_int(&backing_object->shadow_count)
19474cf9f5d8SKonstantin Belousov 			    == 1,
194898087a06SJeff Roberson 			    ("vm_object_collapse: shadow_count: %d",
19494cf9f5d8SKonstantin Belousov 			    atomic_load_int(&backing_object->shadow_count)));
1950aa9bc3b1SKonstantin Belousov 			vm_object_pip_add(object, 1);
195198087a06SJeff Roberson 			vm_object_set_flag(object, OBJ_COLLAPSING);
1952aa9bc3b1SKonstantin Belousov 			vm_object_pip_add(backing_object, 1);
195398087a06SJeff Roberson 			vm_object_set_flag(backing_object, OBJ_DEAD);
1954aa9bc3b1SKonstantin Belousov 
1955df8bae1dSRodney W. Grimes 			/*
19562ad1a3f7SMatthew Dillon 			 * If there is exactly one reference to the backing
19572ad1a3f7SMatthew Dillon 			 * object, we can collapse it into the parent.
1958df8bae1dSRodney W. Grimes 			 */
195998087a06SJeff Roberson 			vm_object_collapse_scan(object);
1960df8bae1dSRodney W. Grimes 
1961f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0
1962f8a47341SAlan Cox 			/*
1963f8a47341SAlan Cox 			 * Break any reservations from backing_object.
1964f8a47341SAlan Cox 			 */
1965f8a47341SAlan Cox 			if (__predict_false(!LIST_EMPTY(&backing_object->rvq)))
1966f8a47341SAlan Cox 				vm_reserv_break_all(backing_object);
1967f8a47341SAlan Cox #endif
1968f8a47341SAlan Cox 
1969df8bae1dSRodney W. Grimes 			/*
1970df8bae1dSRodney W. Grimes 			 * Move the pager from backing_object to object.
1971fff19e0eSMark Johnston 			 *
1972fff19e0eSMark Johnston 			 * swap_pager_copy() can sleep, in which case the
1973fff19e0eSMark Johnston 			 * backing_object's and object's locks are released and
1974fff19e0eSMark Johnston 			 * reacquired.
1975df8bae1dSRodney W. Grimes 			 */
1976fff19e0eSMark Johnston 			swap_pager_copy(backing_object, object,
19771c7c3c6aSMatthew Dillon 			    OFF_TO_IDX(object->backing_object_offset), TRUE);
197898087a06SJeff Roberson 
1979df8bae1dSRodney W. Grimes 			/*
1980df8bae1dSRodney W. Grimes 			 * Object now shadows whatever backing_object did.
1981df8bae1dSRodney W. Grimes 			 */
198298087a06SJeff Roberson 			vm_object_clear_flag(object, OBJ_COLLAPSING);
198398087a06SJeff Roberson 			vm_object_backing_transfer(object, backing_object);
19842ad1a3f7SMatthew Dillon 			object->backing_object_offset +=
19852ad1a3f7SMatthew Dillon 			    backing_object->backing_object_offset;
198698087a06SJeff Roberson 			VM_OBJECT_WUNLOCK(object);
198798087a06SJeff Roberson 			vm_object_pip_wakeup(object);
19882ad1a3f7SMatthew Dillon 
1989df8bae1dSRodney W. Grimes 			/*
1990df8bae1dSRodney W. Grimes 			 * Discard backing_object.
1991df8bae1dSRodney W. Grimes 			 *
19920d94caffSDavid Greenman 			 * Since the backing object has no pages, no pager left,
19930d94caffSDavid Greenman 			 * and no object references within it, all that is
19940d94caffSDavid Greenman 			 * necessary is to dispose of it.
1995df8bae1dSRodney W. Grimes 			 */
19969b4d473aSKonstantin Belousov 			KASSERT(backing_object->ref_count == 1, (
19979b4d473aSKonstantin Belousov "backing_object %p was somehow re-referenced during collapse!",
19989b4d473aSKonstantin Belousov 			    backing_object));
1999aa9bc3b1SKonstantin Belousov 			vm_object_pip_wakeup(backing_object);
200098087a06SJeff Roberson 			(void)refcount_release(&backing_object->ref_count);
200198087a06SJeff Roberson 			vm_object_terminate(backing_object);
200211542376SAlan Cox 			counter_u64_add(object_collapses, 1);
200398087a06SJeff Roberson 			VM_OBJECT_WLOCK(object);
20040d94caffSDavid Greenman 		} else {
2005df8bae1dSRodney W. Grimes 			/*
20062ad1a3f7SMatthew Dillon 			 * If we do not entirely shadow the backing object,
20072ad1a3f7SMatthew Dillon 			 * there is nothing we can do so we give up.
200898087a06SJeff Roberson 			 *
200998087a06SJeff Roberson 			 * The object lock and backing_object lock must not
201098087a06SJeff Roberson 			 * be dropped during this sequence.
2011df8bae1dSRodney W. Grimes 			 */
201258447749SJeff Roberson 			if (!vm_object_scan_all_shadowed(object)) {
201389f6b863SAttilio Rao 				VM_OBJECT_WUNLOCK(backing_object);
20142ad1a3f7SMatthew Dillon 				break;
201524a1cce3SDavid Greenman 			}
2016df8bae1dSRodney W. Grimes 
2017df8bae1dSRodney W. Grimes 			/*
20180d94caffSDavid Greenman 			 * Make the parent shadow the next object in the
20190d94caffSDavid Greenman 			 * chain.  Deallocating backing_object will not remove
20200d94caffSDavid Greenman 			 * it, since its reference count is at least 2.
2021df8bae1dSRodney W. Grimes 			 */
202251b867e5SJeff Roberson 			vm_object_backing_remove_locked(object);
202395e5e988SJohn Dyson 			new_backing_object = backing_object->backing_object;
202451b867e5SJeff Roberson 			if (new_backing_object != NULL) {
202598087a06SJeff Roberson 				vm_object_backing_insert_ref(object,
202651b867e5SJeff Roberson 				    new_backing_object);
202795e5e988SJohn Dyson 				object->backing_object_offset +=
202895e5e988SJohn Dyson 				    backing_object->backing_object_offset;
2029de5f6a77SJohn Dyson 			}
2030df8bae1dSRodney W. Grimes 
2031df8bae1dSRodney W. Grimes 			/*
20320d94caffSDavid Greenman 			 * Drop the reference count on backing_object. Since
203322ec553fSAlan Cox 			 * its ref_count was at least 2, it will not vanish.
2034df8bae1dSRodney W. Grimes 			 */
203598087a06SJeff Roberson 			(void)refcount_release(&backing_object->ref_count);
203698087a06SJeff Roberson 			KASSERT(backing_object->ref_count >= 1, (
203798087a06SJeff Roberson "backing_object %p was somehow dereferenced during collapse!",
203898087a06SJeff Roberson 			    backing_object));
203989f6b863SAttilio Rao 			VM_OBJECT_WUNLOCK(backing_object);
204011542376SAlan Cox 			counter_u64_add(object_bypasses, 1);
2041df8bae1dSRodney W. Grimes 		}
2042df8bae1dSRodney W. Grimes 
2043df8bae1dSRodney W. Grimes 		/*
2044df8bae1dSRodney W. Grimes 		 * Try again with this object's new backing object.
2045df8bae1dSRodney W. Grimes 		 */
2046df8bae1dSRodney W. Grimes 	}
2047df8bae1dSRodney W. Grimes }
2048df8bae1dSRodney W. Grimes 
2049df8bae1dSRodney W. Grimes /*
2050bff99f0dSAlan Cox  *	vm_object_page_remove:
2051df8bae1dSRodney W. Grimes  *
205268855966SAlan Cox  *	For the given object, either frees or invalidates each of the
20536bbee8e2SAlan Cox  *	specified pages.  In general, a page is freed.  However, if a page is
20546bbee8e2SAlan Cox  *	wired for any reason other than the existence of a managed, wired
20556bbee8e2SAlan Cox  *	mapping, then it may be invalidated but not removed from the object.
20566bbee8e2SAlan Cox  *	Pages are specified by the given range ["start", "end") and the option
20576bbee8e2SAlan Cox  *	OBJPR_CLEANONLY.  As a special case, if "end" is zero, then the range
20586bbee8e2SAlan Cox  *	extends from "start" to the end of the object.  If the option
20596bbee8e2SAlan Cox  *	OBJPR_CLEANONLY is specified, then only the non-dirty pages within the
20606bbee8e2SAlan Cox  *	specified range are affected.  If the option OBJPR_NOTMAPPED is
20616bbee8e2SAlan Cox  *	specified, then the pages within the specified range must have no
20626bbee8e2SAlan Cox  *	mappings.  Otherwise, if this option is not specified, any mappings to
20636bbee8e2SAlan Cox  *	the specified pages are removed before the pages are freed or
20646bbee8e2SAlan Cox  *	invalidated.
206568855966SAlan Cox  *
20666bbee8e2SAlan Cox  *	In general, this operation should only be performed on objects that
20676bbee8e2SAlan Cox  *	contain managed pages.  There are, however, two exceptions.  First, it
20686bbee8e2SAlan Cox  *	is performed on the kernel and kmem objects by vm_map_entry_delete().
20696bbee8e2SAlan Cox  *	Second, it is used by msync(..., MS_INVALIDATE) to invalidate device-
20706bbee8e2SAlan Cox  *	backed pages.  In both of these cases, the option OBJPR_CLEANONLY must
20716bbee8e2SAlan Cox  *	not be specified and the option OBJPR_NOTMAPPED must be specified.
2072df8bae1dSRodney W. Grimes  *
2073df8bae1dSRodney W. Grimes  *	The object must be locked.
2074df8bae1dSRodney W. Grimes  */
207526f9a767SRodney W. Grimes void
2076ecde4b32SAlan Cox vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
20776bbee8e2SAlan Cox     int options)
2078df8bae1dSRodney W. Grimes {
2079d031cff1SMatthew Dillon 	vm_page_t p, next;
2080df8bae1dSRodney W. Grimes 
208189f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
208228634820SAlan Cox 	KASSERT((object->flags & OBJ_UNMANAGED) == 0 ||
20836bbee8e2SAlan Cox 	    (options & (OBJPR_CLEANONLY | OBJPR_NOTMAPPED)) == OBJPR_NOTMAPPED,
20846bbee8e2SAlan Cox 	    ("vm_object_page_remove: illegal options for object %p", object));
2085ecde4b32SAlan Cox 	if (object->resident_page_count == 0)
20867667839aSAlan Cox 		return;
2087d474eaaaSDoug Rabson 	vm_object_pip_add(object, 1);
208826f9a767SRodney W. Grimes again:
2089b382c10aSKonstantin Belousov 	p = vm_page_find_least(object, start);
20902965a453SKip Macy 
209175741c04SAlan Cox 	/*
20926bbee8e2SAlan Cox 	 * Here, the variable "p" is either (1) the page with the least pindex
20936bbee8e2SAlan Cox 	 * greater than or equal to the parameter "start" or (2) NULL.
209475741c04SAlan Cox 	 */
20956bbee8e2SAlan Cox 	for (; p != NULL && (p->pindex < end || end == 0); p = next) {
2096b18bfc3dSJohn Dyson 		next = TAILQ_NEXT(p, listq);
209775741c04SAlan Cox 
209859677d3cSAlan Cox 		/*
2099d28af1abSMark Johnston 		 * Skip invalid pages if asked to do so.  Try to avoid acquiring
2100d28af1abSMark Johnston 		 * the busy lock, as some consumers rely on this to avoid
2101d28af1abSMark Johnston 		 * deadlocks.
2102d28af1abSMark Johnston 		 *
2103d28af1abSMark Johnston 		 * A thread may concurrently transition the page from invalid to
2104d28af1abSMark Johnston 		 * valid using only the busy lock, so the result of this check
2105d28af1abSMark Johnston 		 * is immediately stale.  It is up to consumers to handle this,
2106d28af1abSMark Johnston 		 * for instance by ensuring that all invalid->valid transitions
2107d28af1abSMark Johnston 		 * happen with a mutex held, as may be possible for a
2108d28af1abSMark Johnston 		 * filesystem.
2109d28af1abSMark Johnston 		 */
2110d28af1abSMark Johnston 		if ((options & OBJPR_VALIDONLY) != 0 && vm_page_none_valid(p))
2111d28af1abSMark Johnston 			continue;
2112d28af1abSMark Johnston 
2113d28af1abSMark Johnston 		/*
21146bbee8e2SAlan Cox 		 * If the page is wired for any reason besides the existence
21156bbee8e2SAlan Cox 		 * of managed, wired mappings, then it cannot be freed.  For
21166bbee8e2SAlan Cox 		 * example, fictitious pages, which represent device memory,
21176bbee8e2SAlan Cox 		 * are inherently wired and cannot be freed.  They can,
21186bbee8e2SAlan Cox 		 * however, be invalidated if the option OBJPR_CLEANONLY is
21196bbee8e2SAlan Cox 		 * not specified.
212059677d3cSAlan Cox 		 */
212163e97555SJeff Roberson 		if (vm_page_tryxbusy(p) == 0) {
212287b64663SMark Johnston 			if (vm_page_busy_sleep(p, "vmopar", 0))
212387b64663SMark Johnston 				VM_OBJECT_WLOCK(object);
2124fee2a2faSMark Johnston 			goto again;
2125fee2a2faSMark Johnston 		}
2126d28af1abSMark Johnston 		if ((options & OBJPR_VALIDONLY) != 0 && vm_page_none_valid(p)) {
2127d28af1abSMark Johnston 			vm_page_xunbusy(p);
2128d28af1abSMark Johnston 			continue;
2129d28af1abSMark Johnston 		}
2130d842aa51SMark Johnston 		if (vm_page_wired(p)) {
2131fee2a2faSMark Johnston wired:
2132cf060942SAlan Cox 			if ((options & OBJPR_NOTMAPPED) == 0 &&
2133cf060942SAlan Cox 			    object->ref_count != 0)
21344fec79beSAlan Cox 				pmap_remove_all(p);
21356bbee8e2SAlan Cox 			if ((options & OBJPR_CLEANONLY) == 0) {
21360012f373SJeff Roberson 				vm_page_invalid(p);
2137a28042d1SAlan Cox 				vm_page_undirty(p);
2138a28042d1SAlan Cox 			}
213963e97555SJeff Roberson 			vm_page_xunbusy(p);
214093c5d3a4SKonstantin Belousov 			continue;
21410d94caffSDavid Greenman 		}
214268855966SAlan Cox 		KASSERT((p->flags & PG_FICTITIOUS) == 0,
214368855966SAlan Cox 		    ("vm_object_page_remove: page %p is fictitious", p));
21440012f373SJeff Roberson 		if ((options & OBJPR_CLEANONLY) != 0 &&
21450012f373SJeff Roberson 		    !vm_page_none_valid(p)) {
2146cf060942SAlan Cox 			if ((options & OBJPR_NOTMAPPED) == 0 &&
2147fee2a2faSMark Johnston 			    object->ref_count != 0 &&
2148fee2a2faSMark Johnston 			    !vm_page_try_remove_write(p))
2149fee2a2faSMark Johnston 				goto wired;
215063e97555SJeff Roberson 			if (p->dirty != 0) {
215163e97555SJeff Roberson 				vm_page_xunbusy(p);
215293c5d3a4SKonstantin Belousov 				continue;
21532965a453SKip Macy 			}
215463e97555SJeff Roberson 		}
2155fee2a2faSMark Johnston 		if ((options & OBJPR_NOTMAPPED) == 0 &&
2156fee2a2faSMark Johnston 		    object->ref_count != 0 && !vm_page_try_remove_all(p))
2157fee2a2faSMark Johnston 			goto wired;
21585cd29d0fSMark Johnston 		vm_page_free(p);
21592965a453SKip Macy 	}
2160f919ebdeSDavid Greenman 	vm_object_pip_wakeup(object);
216184242cf6SMark Johnston 
21621390a5cbSKonstantin Belousov 	vm_pager_freespace(object, start, (end == 0 ? object->size : end) -
21631390a5cbSKonstantin Belousov 	    start);
2164c0503609SDavid Greenman }
2165df8bae1dSRodney W. Grimes 
2166df8bae1dSRodney W. Grimes /*
21673138cd36SMark Johnston  *	vm_object_page_noreuse:
2168936c09acSJohn Baldwin  *
21693138cd36SMark Johnston  *	For the given object, attempt to move the specified pages to
21703138cd36SMark Johnston  *	the head of the inactive queue.  This bypasses regular LRU
21713138cd36SMark Johnston  *	operation and allows the pages to be reused quickly under memory
21723138cd36SMark Johnston  *	pressure.  If a page is wired for any reason, then it will not
21733138cd36SMark Johnston  *	be queued.  Pages are specified by the range ["start", "end").
21743138cd36SMark Johnston  *	As a special case, if "end" is zero, then the range extends from
21753138cd36SMark Johnston  *	"start" to the end of the object.
2176936c09acSJohn Baldwin  *
2177936c09acSJohn Baldwin  *	This operation should only be performed on objects that
217828634820SAlan Cox  *	contain non-fictitious, managed pages.
2179936c09acSJohn Baldwin  *
2180936c09acSJohn Baldwin  *	The object must be locked.
2181936c09acSJohn Baldwin  */
2182936c09acSJohn Baldwin void
21833138cd36SMark Johnston vm_object_page_noreuse(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
2184936c09acSJohn Baldwin {
2185936c09acSJohn Baldwin 	vm_page_t p, next;
2186936c09acSJohn Baldwin 
218752d1addaSAlan Cox 	VM_OBJECT_ASSERT_LOCKED(object);
218828634820SAlan Cox 	KASSERT((object->flags & (OBJ_FICTITIOUS | OBJ_UNMANAGED)) == 0,
21893138cd36SMark Johnston 	    ("vm_object_page_noreuse: illegal object %p", object));
2190936c09acSJohn Baldwin 	if (object->resident_page_count == 0)
2191936c09acSJohn Baldwin 		return;
2192936c09acSJohn Baldwin 	p = vm_page_find_least(object, start);
2193936c09acSJohn Baldwin 
2194936c09acSJohn Baldwin 	/*
2195936c09acSJohn Baldwin 	 * Here, the variable "p" is either (1) the page with the least pindex
2196936c09acSJohn Baldwin 	 * greater than or equal to the parameter "start" or (2) NULL.
2197936c09acSJohn Baldwin 	 */
2198936c09acSJohn Baldwin 	for (; p != NULL && (p->pindex < end || end == 0); p = next) {
2199936c09acSJohn Baldwin 		next = TAILQ_NEXT(p, listq);
22003138cd36SMark Johnston 		vm_page_deactivate_noreuse(p);
2201936c09acSJohn Baldwin 	}
2202936c09acSJohn Baldwin }
2203936c09acSJohn Baldwin 
2204936c09acSJohn Baldwin /*
2205387aabc5SAlan Cox  *	Populate the specified range of the object with valid pages.  Returns
2206387aabc5SAlan Cox  *	TRUE if the range is successfully populated and FALSE otherwise.
2207387aabc5SAlan Cox  *
2208387aabc5SAlan Cox  *	Note: This function should be optimized to pass a larger array of
2209387aabc5SAlan Cox  *	pages to vm_pager_get_pages() before it is applied to a non-
2210387aabc5SAlan Cox  *	OBJT_DEVICE object.
2211387aabc5SAlan Cox  *
2212387aabc5SAlan Cox  *	The object must be locked.
2213387aabc5SAlan Cox  */
2214387aabc5SAlan Cox boolean_t
2215387aabc5SAlan Cox vm_object_populate(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
2216387aabc5SAlan Cox {
2217093c7f39SGleb Smirnoff 	vm_page_t m;
2218387aabc5SAlan Cox 	vm_pindex_t pindex;
2219387aabc5SAlan Cox 	int rv;
2220387aabc5SAlan Cox 
222189f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
2222387aabc5SAlan Cox 	for (pindex = start; pindex < end; pindex++) {
2223c7575748SJeff Roberson 		rv = vm_page_grab_valid(&m, object, pindex, VM_ALLOC_NORMAL);
2224c7575748SJeff Roberson 		if (rv != VM_PAGER_OK)
2225387aabc5SAlan Cox 			break;
2226c7575748SJeff Roberson 
2227387aabc5SAlan Cox 		/*
2228387aabc5SAlan Cox 		 * Keep "m" busy because a subsequent iteration may unlock
2229387aabc5SAlan Cox 		 * the object.
2230387aabc5SAlan Cox 		 */
2231387aabc5SAlan Cox 	}
2232387aabc5SAlan Cox 	if (pindex > start) {
2233387aabc5SAlan Cox 		m = vm_page_lookup(object, start);
2234387aabc5SAlan Cox 		while (m != NULL && m->pindex < pindex) {
2235c7aebda8SAttilio Rao 			vm_page_xunbusy(m);
2236387aabc5SAlan Cox 			m = TAILQ_NEXT(m, listq);
2237387aabc5SAlan Cox 		}
2238387aabc5SAlan Cox 	}
2239387aabc5SAlan Cox 	return (pindex == end);
2240387aabc5SAlan Cox }
2241387aabc5SAlan Cox 
2242387aabc5SAlan Cox /*
2243df8bae1dSRodney W. Grimes  *	Routine:	vm_object_coalesce
2244df8bae1dSRodney W. Grimes  *	Function:	Coalesces two objects backing up adjoining
2245df8bae1dSRodney W. Grimes  *			regions of memory into a single object.
2246df8bae1dSRodney W. Grimes  *
2247df8bae1dSRodney W. Grimes  *	returns TRUE if objects were combined.
2248df8bae1dSRodney W. Grimes  *
2249df8bae1dSRodney W. Grimes  *	NOTE:	Only works at the moment if the second object is NULL -
2250df8bae1dSRodney W. Grimes  *		if it's not, which object do we lock first?
2251df8bae1dSRodney W. Grimes  *
2252df8bae1dSRodney W. Grimes  *	Parameters:
2253df8bae1dSRodney W. Grimes  *		prev_object	First object to coalesce
2254df8bae1dSRodney W. Grimes  *		prev_offset	Offset into prev_object
2255df8bae1dSRodney W. Grimes  *		prev_size	Size of reference to prev_object
225657a21abaSAlan Cox  *		next_size	Size of reference to the second object
22573364c323SKonstantin Belousov  *		reserved	Indicator that extension region has
22583364c323SKonstantin Belousov  *				swap accounted for
2259df8bae1dSRodney W. Grimes  *
2260df8bae1dSRodney W. Grimes  *	Conditions:
2261df8bae1dSRodney W. Grimes  *	The object must *not* be locked.
2262df8bae1dSRodney W. Grimes  */
22630d94caffSDavid Greenman boolean_t
226457a21abaSAlan Cox vm_object_coalesce(vm_object_t prev_object, vm_ooffset_t prev_offset,
22653364c323SKonstantin Belousov     vm_size_t prev_size, vm_size_t next_size, boolean_t reserved)
2266df8bae1dSRodney W. Grimes {
2267ea41812fSAlan Cox 	vm_pindex_t next_pindex;
2268df8bae1dSRodney W. Grimes 
226900e1854aSAlan Cox 	if (prev_object == NULL)
2270df8bae1dSRodney W. Grimes 		return (TRUE);
227163967687SJeff Roberson 	if ((prev_object->flags & OBJ_ANON) == 0)
227230dcfc09SJohn Dyson 		return (FALSE);
227330dcfc09SJohn Dyson 
227463967687SJeff Roberson 	VM_OBJECT_WLOCK(prev_object);
2275df8bae1dSRodney W. Grimes 	/*
227698087a06SJeff Roberson 	 * Try to collapse the object first.
2277df8bae1dSRodney W. Grimes 	 */
2278df8bae1dSRodney W. Grimes 	vm_object_collapse(prev_object);
2279df8bae1dSRodney W. Grimes 
2280df8bae1dSRodney W. Grimes 	/*
22810d94caffSDavid Greenman 	 * Can't coalesce if: . more than one reference . paged out . shadows
22820d94caffSDavid Greenman 	 * another object . has a copy elsewhere (any of which mean that the
22830d94caffSDavid Greenman 	 * pages not mapped to prev_entry may be in use anyway)
2284df8bae1dSRodney W. Grimes 	 */
22858cc7e047SJohn Dyson 	if (prev_object->backing_object != NULL) {
228689f6b863SAttilio Rao 		VM_OBJECT_WUNLOCK(prev_object);
2287df8bae1dSRodney W. Grimes 		return (FALSE);
2288df8bae1dSRodney W. Grimes 	}
2289a316d390SJohn Dyson 
2290a316d390SJohn Dyson 	prev_size >>= PAGE_SHIFT;
2291a316d390SJohn Dyson 	next_size >>= PAGE_SHIFT;
229257a21abaSAlan Cox 	next_pindex = OFF_TO_IDX(prev_offset) + prev_size;
22938cc7e047SJohn Dyson 
22940e48e068SMark Johnston 	if (prev_object->ref_count > 1 &&
22950e48e068SMark Johnston 	    prev_object->size != next_pindex &&
22960e48e068SMark Johnston 	    (prev_object->flags & OBJ_ONEMAPPING) == 0) {
229789f6b863SAttilio Rao 		VM_OBJECT_WUNLOCK(prev_object);
22988cc7e047SJohn Dyson 		return (FALSE);
22998cc7e047SJohn Dyson 	}
23008cc7e047SJohn Dyson 
2301df8bae1dSRodney W. Grimes 	/*
23023364c323SKonstantin Belousov 	 * Account for the charge.
23033364c323SKonstantin Belousov 	 */
2304ef694c1aSEdward Tomasz Napierala 	if (prev_object->cred != NULL) {
23053364c323SKonstantin Belousov 		/*
23063364c323SKonstantin Belousov 		 * If prev_object was charged, then this mapping,
2307763df3ecSPedro F. Giffuni 		 * although not charged now, may become writable
2308ef694c1aSEdward Tomasz Napierala 		 * later. Non-NULL cred in the object would prevent
23093364c323SKonstantin Belousov 		 * swap reservation during enabling of the write
23103364c323SKonstantin Belousov 		 * access, so reserve swap now. Failed reservation
23113364c323SKonstantin Belousov 		 * cause allocation of the separate object for the map
23123364c323SKonstantin Belousov 		 * entry, and swap reservation for this entry is
23133364c323SKonstantin Belousov 		 * managed in appropriate time.
23143364c323SKonstantin Belousov 		 */
2315ef694c1aSEdward Tomasz Napierala 		if (!reserved && !swap_reserve_by_cred(ptoa(next_size),
2316ef694c1aSEdward Tomasz Napierala 		    prev_object->cred)) {
23179f790a17SKonstantin Belousov 			VM_OBJECT_WUNLOCK(prev_object);
23183364c323SKonstantin Belousov 			return (FALSE);
23193364c323SKonstantin Belousov 		}
23203364c323SKonstantin Belousov 		prev_object->charge += ptoa(next_size);
23213364c323SKonstantin Belousov 	}
23223364c323SKonstantin Belousov 
23233364c323SKonstantin Belousov 	/*
23240d94caffSDavid Greenman 	 * Remove any pages that may still be in the object from a previous
23250d94caffSDavid Greenman 	 * deallocation.
2326df8bae1dSRodney W. Grimes 	 */
2327ea41812fSAlan Cox 	if (next_pindex < prev_object->size) {
23286bbee8e2SAlan Cox 		vm_object_page_remove(prev_object, next_pindex, next_pindex +
23296bbee8e2SAlan Cox 		    next_size, 0);
23303364c323SKonstantin Belousov #if 0
2331ef694c1aSEdward Tomasz Napierala 		if (prev_object->cred != NULL) {
23323364c323SKonstantin Belousov 			KASSERT(prev_object->charge >=
23333364c323SKonstantin Belousov 			    ptoa(prev_object->size - next_pindex),
23343364c323SKonstantin Belousov 			    ("object %p overcharged 1 %jx %jx", prev_object,
23353364c323SKonstantin Belousov 				(uintmax_t)next_pindex, (uintmax_t)next_size));
23363364c323SKonstantin Belousov 			prev_object->charge -= ptoa(prev_object->size -
23373364c323SKonstantin Belousov 			    next_pindex);
23383364c323SKonstantin Belousov 		}
23393364c323SKonstantin Belousov #endif
2340ea41812fSAlan Cox 	}
2341df8bae1dSRodney W. Grimes 
2342df8bae1dSRodney W. Grimes 	/*
2343df8bae1dSRodney W. Grimes 	 * Extend the object if necessary.
2344df8bae1dSRodney W. Grimes 	 */
2345ea41812fSAlan Cox 	if (next_pindex + next_size > prev_object->size)
2346ea41812fSAlan Cox 		prev_object->size = next_pindex + next_size;
2347df8bae1dSRodney W. Grimes 
234889f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(prev_object);
2349df8bae1dSRodney W. Grimes 	return (TRUE);
2350df8bae1dSRodney W. Grimes }
2351df8bae1dSRodney W. Grimes 
23527a5a6352SMatthew Dillon void
2353180bcaa4SKonstantin Belousov vm_object_set_writeable_dirty_(vm_object_t object)
23547a5a6352SMatthew Dillon {
235567d0e293SJeff Roberson 	atomic_add_int(&object->generation, 1);
23567a5a6352SMatthew Dillon }
23577a5a6352SMatthew Dillon 
2358c23c555bSKonstantin Belousov bool
2359c23c555bSKonstantin Belousov vm_object_mightbedirty_(vm_object_t object)
2360c23c555bSKonstantin Belousov {
2361c23c555bSKonstantin Belousov 	return (object->generation != object->cleangeneration);
2362c23c555bSKonstantin Belousov }
2363c23c555bSKonstantin Belousov 
236403462509SAlan Cox /*
236503462509SAlan Cox  *	vm_object_unwire:
236603462509SAlan Cox  *
236703462509SAlan Cox  *	For each page offset within the specified range of the given object,
236803462509SAlan Cox  *	find the highest-level page in the shadow chain and unwire it.  A page
236903462509SAlan Cox  *	must exist at every page offset, and the highest-level page must be
237003462509SAlan Cox  *	wired.
237103462509SAlan Cox  */
237203462509SAlan Cox void
237303462509SAlan Cox vm_object_unwire(vm_object_t object, vm_ooffset_t offset, vm_size_t length,
237403462509SAlan Cox     uint8_t queue)
237503462509SAlan Cox {
237620e4afbfSKonstantin Belousov 	vm_object_t tobject, t1object;
237703462509SAlan Cox 	vm_page_t m, tm;
237803462509SAlan Cox 	vm_pindex_t end_pindex, pindex, tpindex;
237903462509SAlan Cox 	int depth, locked_depth;
238003462509SAlan Cox 
238103462509SAlan Cox 	KASSERT((offset & PAGE_MASK) == 0,
238203462509SAlan Cox 	    ("vm_object_unwire: offset is not page aligned"));
238303462509SAlan Cox 	KASSERT((length & PAGE_MASK) == 0,
238403462509SAlan Cox 	    ("vm_object_unwire: length is not a multiple of PAGE_SIZE"));
238503462509SAlan Cox 	/* The wired count of a fictitious page never changes. */
238603462509SAlan Cox 	if ((object->flags & OBJ_FICTITIOUS) != 0)
238703462509SAlan Cox 		return;
238803462509SAlan Cox 	pindex = OFF_TO_IDX(offset);
238903462509SAlan Cox 	end_pindex = pindex + atop(length);
239020e4afbfSKonstantin Belousov again:
239103462509SAlan Cox 	locked_depth = 1;
239203462509SAlan Cox 	VM_OBJECT_RLOCK(object);
239303462509SAlan Cox 	m = vm_page_find_least(object, pindex);
239403462509SAlan Cox 	while (pindex < end_pindex) {
239503462509SAlan Cox 		if (m == NULL || pindex < m->pindex) {
239603462509SAlan Cox 			/*
239703462509SAlan Cox 			 * The first object in the shadow chain doesn't
239803462509SAlan Cox 			 * contain a page at the current index.  Therefore,
239903462509SAlan Cox 			 * the page must exist in a backing object.
240003462509SAlan Cox 			 */
240103462509SAlan Cox 			tobject = object;
240203462509SAlan Cox 			tpindex = pindex;
240303462509SAlan Cox 			depth = 0;
240403462509SAlan Cox 			do {
240503462509SAlan Cox 				tpindex +=
240603462509SAlan Cox 				    OFF_TO_IDX(tobject->backing_object_offset);
240703462509SAlan Cox 				tobject = tobject->backing_object;
240803462509SAlan Cox 				KASSERT(tobject != NULL,
240903462509SAlan Cox 				    ("vm_object_unwire: missing page"));
241003462509SAlan Cox 				if ((tobject->flags & OBJ_FICTITIOUS) != 0)
241103462509SAlan Cox 					goto next_page;
241203462509SAlan Cox 				depth++;
241303462509SAlan Cox 				if (depth == locked_depth) {
241403462509SAlan Cox 					locked_depth++;
241503462509SAlan Cox 					VM_OBJECT_RLOCK(tobject);
241603462509SAlan Cox 				}
241703462509SAlan Cox 			} while ((tm = vm_page_lookup(tobject, tpindex)) ==
241803462509SAlan Cox 			    NULL);
241903462509SAlan Cox 		} else {
242003462509SAlan Cox 			tm = m;
242103462509SAlan Cox 			m = TAILQ_NEXT(m, listq);
242203462509SAlan Cox 		}
242363e97555SJeff Roberson 		if (vm_page_trysbusy(tm) == 0) {
242487e93ea6SMark Johnston 			for (tobject = object; locked_depth >= 1;
242520e4afbfSKonstantin Belousov 			    locked_depth--) {
242620e4afbfSKonstantin Belousov 				t1object = tobject->backing_object;
242787e93ea6SMark Johnston 				if (tm->object != tobject)
242820e4afbfSKonstantin Belousov 					VM_OBJECT_RUNLOCK(tobject);
242920e4afbfSKonstantin Belousov 				tobject = t1object;
243020e4afbfSKonstantin Belousov 			}
243187b64663SMark Johnston 			tobject = tm->object;
243287b64663SMark Johnston 			if (!vm_page_busy_sleep(tm, "unwbo",
243387b64663SMark Johnston 			    VM_ALLOC_IGN_SBUSY))
243487b64663SMark Johnston 				VM_OBJECT_RUNLOCK(tobject);
243520e4afbfSKonstantin Belousov 			goto again;
243620e4afbfSKonstantin Belousov 		}
243703462509SAlan Cox 		vm_page_unwire(tm, queue);
243863e97555SJeff Roberson 		vm_page_sunbusy(tm);
243903462509SAlan Cox next_page:
244003462509SAlan Cox 		pindex++;
244103462509SAlan Cox 	}
244203462509SAlan Cox 	/* Release the accumulated object locks. */
244320e4afbfSKonstantin Belousov 	for (tobject = object; locked_depth >= 1; locked_depth--) {
244420e4afbfSKonstantin Belousov 		t1object = tobject->backing_object;
244520e4afbfSKonstantin Belousov 		VM_OBJECT_RUNLOCK(tobject);
244620e4afbfSKonstantin Belousov 		tobject = t1object;
244703462509SAlan Cox 	}
244803462509SAlan Cox }
244903462509SAlan Cox 
24500951bd36SEric van Gyzen /*
24510951bd36SEric van Gyzen  * Return the vnode for the given object, or NULL if none exists.
24520951bd36SEric van Gyzen  * For tmpfs objects, the function may return NULL if there is
24530951bd36SEric van Gyzen  * no vnode allocated at the time of the call.
24540951bd36SEric van Gyzen  */
245563e4c6cdSEric van Gyzen struct vnode *
245663e4c6cdSEric van Gyzen vm_object_vnode(vm_object_t object)
245763e4c6cdSEric van Gyzen {
24580951bd36SEric van Gyzen 	struct vnode *vp;
245963e4c6cdSEric van Gyzen 
246063e4c6cdSEric van Gyzen 	VM_OBJECT_ASSERT_LOCKED(object);
2461a7c198a2SKonstantin Belousov 	vm_pager_getvp(object, &vp, NULL);
24620951bd36SEric van Gyzen 	return (vp);
246363e4c6cdSEric van Gyzen }
246463e4c6cdSEric van Gyzen 
2465205be21dSJeff Roberson /*
2466205be21dSJeff Roberson  * Busy the vm object.  This prevents new pages belonging to the object from
2467205be21dSJeff Roberson  * becoming busy.  Existing pages persist as busy.  Callers are responsible
2468205be21dSJeff Roberson  * for checking page state before proceeding.
2469205be21dSJeff Roberson  */
2470205be21dSJeff Roberson void
2471205be21dSJeff Roberson vm_object_busy(vm_object_t obj)
2472205be21dSJeff Roberson {
2473205be21dSJeff Roberson 
2474205be21dSJeff Roberson 	VM_OBJECT_ASSERT_LOCKED(obj);
2475205be21dSJeff Roberson 
2476c99d0c58SMark Johnston 	blockcount_acquire(&obj->busy, 1);
2477205be21dSJeff Roberson 	/* The fence is required to order loads of page busy. */
2478205be21dSJeff Roberson 	atomic_thread_fence_acq_rel();
2479205be21dSJeff Roberson }
2480205be21dSJeff Roberson 
2481205be21dSJeff Roberson void
2482205be21dSJeff Roberson vm_object_unbusy(vm_object_t obj)
2483205be21dSJeff Roberson {
2484205be21dSJeff Roberson 
2485c99d0c58SMark Johnston 	blockcount_release(&obj->busy, 1);
2486205be21dSJeff Roberson }
2487205be21dSJeff Roberson 
2488205be21dSJeff Roberson void
2489205be21dSJeff Roberson vm_object_busy_wait(vm_object_t obj, const char *wmesg)
2490205be21dSJeff Roberson {
2491205be21dSJeff Roberson 
2492205be21dSJeff Roberson 	VM_OBJECT_ASSERT_UNLOCKED(obj);
2493205be21dSJeff Roberson 
2494c99d0c58SMark Johnston 	(void)blockcount_sleep(&obj->busy, NULL, wmesg, PVM);
2495205be21dSJeff Roberson }
2496205be21dSJeff Roberson 
2497cd37afd8SRick Macklem /*
2498cd37afd8SRick Macklem  * This function aims to determine if the object is mapped,
2499cd37afd8SRick Macklem  * specifically, if it is referenced by a vm_map_entry.  Because
2500cd37afd8SRick Macklem  * objects occasionally acquire transient references that do not
2501cd37afd8SRick Macklem  * represent a mapping, the method used here is inexact.  However, it
2502cd37afd8SRick Macklem  * has very low overhead and is good enough for the advisory
2503cd37afd8SRick Macklem  * vm.vmtotal sysctl.
2504cd37afd8SRick Macklem  */
2505cd37afd8SRick Macklem bool
2506cd37afd8SRick Macklem vm_object_is_active(vm_object_t obj)
2507cd37afd8SRick Macklem {
2508cd37afd8SRick Macklem 
25094cf9f5d8SKonstantin Belousov 	return (obj->ref_count > atomic_load_int(&obj->shadow_count));
2510cd37afd8SRick Macklem }
2511cd37afd8SRick Macklem 
2512ff87ae35SJohn Baldwin static int
251342812cccSKonstantin Belousov vm_object_list_handler(struct sysctl_req *req, bool swap_only)
2514ff87ae35SJohn Baldwin {
25150ecee546SKonstantin Belousov 	struct kinfo_vmobject *kvo;
2516ff87ae35SJohn Baldwin 	char *fullpath, *freepath;
2517ff87ae35SJohn Baldwin 	struct vnode *vp;
2518ff87ae35SJohn Baldwin 	struct vattr va;
2519ff87ae35SJohn Baldwin 	vm_object_t obj;
2520ff87ae35SJohn Baldwin 	vm_page_t m;
2521ecfbddf0SKonstantin Belousov 	u_long sp;
2522ff87ae35SJohn Baldwin 	int count, error;
2523ff87ae35SJohn Baldwin 
2524ff87ae35SJohn Baldwin 	if (req->oldptr == NULL) {
2525ff87ae35SJohn Baldwin 		/*
2526ff87ae35SJohn Baldwin 		 * If an old buffer has not been provided, generate an
2527ff87ae35SJohn Baldwin 		 * estimate of the space needed for a subsequent call.
2528ff87ae35SJohn Baldwin 		 */
2529ff87ae35SJohn Baldwin 		mtx_lock(&vm_object_list_mtx);
2530ff87ae35SJohn Baldwin 		count = 0;
2531ff87ae35SJohn Baldwin 		TAILQ_FOREACH(obj, &vm_object_list, object_list) {
2532ff87ae35SJohn Baldwin 			if (obj->type == OBJT_DEAD)
2533ff87ae35SJohn Baldwin 				continue;
2534ff87ae35SJohn Baldwin 			count++;
2535ff87ae35SJohn Baldwin 		}
2536ff87ae35SJohn Baldwin 		mtx_unlock(&vm_object_list_mtx);
2537ff87ae35SJohn Baldwin 		return (SYSCTL_OUT(req, NULL, sizeof(struct kinfo_vmobject) *
2538ff87ae35SJohn Baldwin 		    count * 11 / 10));
2539ff87ae35SJohn Baldwin 	}
2540ff87ae35SJohn Baldwin 
2541b050ee6cSMark Johnston 	kvo = malloc(sizeof(*kvo), M_TEMP, M_WAITOK | M_ZERO);
2542ff87ae35SJohn Baldwin 	error = 0;
2543ff87ae35SJohn Baldwin 
2544ff87ae35SJohn Baldwin 	/*
2545ff87ae35SJohn Baldwin 	 * VM objects are type stable and are never removed from the
2546ff87ae35SJohn Baldwin 	 * list once added.  This allows us to safely read obj->object_list
2547ff87ae35SJohn Baldwin 	 * after reacquiring the VM object lock.
2548ff87ae35SJohn Baldwin 	 */
2549ff87ae35SJohn Baldwin 	mtx_lock(&vm_object_list_mtx);
2550ff87ae35SJohn Baldwin 	TAILQ_FOREACH(obj, &vm_object_list, object_list) {
255142812cccSKonstantin Belousov 		if (obj->type == OBJT_DEAD ||
255242812cccSKonstantin Belousov 		    (swap_only && (obj->flags & (OBJ_ANON | OBJ_SWAP)) == 0))
2553ff87ae35SJohn Baldwin 			continue;
2554ff87ae35SJohn Baldwin 		VM_OBJECT_RLOCK(obj);
255542812cccSKonstantin Belousov 		if (obj->type == OBJT_DEAD ||
255642812cccSKonstantin Belousov 		    (swap_only && (obj->flags & (OBJ_ANON | OBJ_SWAP)) == 0)) {
2557ff87ae35SJohn Baldwin 			VM_OBJECT_RUNLOCK(obj);
2558ff87ae35SJohn Baldwin 			continue;
2559ff87ae35SJohn Baldwin 		}
2560ff87ae35SJohn Baldwin 		mtx_unlock(&vm_object_list_mtx);
25610ecee546SKonstantin Belousov 		kvo->kvo_size = ptoa(obj->size);
25620ecee546SKonstantin Belousov 		kvo->kvo_resident = obj->resident_page_count;
25630ecee546SKonstantin Belousov 		kvo->kvo_ref_count = obj->ref_count;
25644cf9f5d8SKonstantin Belousov 		kvo->kvo_shadow_count = atomic_load_int(&obj->shadow_count);
25650ecee546SKonstantin Belousov 		kvo->kvo_memattr = obj->memattr;
25660ecee546SKonstantin Belousov 		kvo->kvo_active = 0;
25670ecee546SKonstantin Belousov 		kvo->kvo_inactive = 0;
25687738118eSKonstantin Belousov 		if (!swap_only) {
2569ff87ae35SJohn Baldwin 			TAILQ_FOREACH(m, &obj->memq, listq) {
2570ff87ae35SJohn Baldwin 				/*
2571ff87ae35SJohn Baldwin 				 * A page may belong to the object but be
2572ff87ae35SJohn Baldwin 				 * dequeued and set to PQ_NONE while the
2573ff87ae35SJohn Baldwin 				 * object lock is not held.  This makes the
2574ff87ae35SJohn Baldwin 				 * reads of m->queue below racy, and we do not
2575ff87ae35SJohn Baldwin 				 * count pages set to PQ_NONE.  However, this
2576ff87ae35SJohn Baldwin 				 * sysctl is only meant to give an
2577ff87ae35SJohn Baldwin 				 * approximation of the system anyway.
2578ff87ae35SJohn Baldwin 				 */
25795cff1f4dSMark Johnston 				if (m->a.queue == PQ_ACTIVE)
25800ecee546SKonstantin Belousov 					kvo->kvo_active++;
25815cff1f4dSMark Johnston 				else if (m->a.queue == PQ_INACTIVE)
25820ecee546SKonstantin Belousov 					kvo->kvo_inactive++;
2583ff87ae35SJohn Baldwin 			}
25847738118eSKonstantin Belousov 		}
2585ff87ae35SJohn Baldwin 
25860ecee546SKonstantin Belousov 		kvo->kvo_vn_fileid = 0;
25870ecee546SKonstantin Belousov 		kvo->kvo_vn_fsid = 0;
25880ecee546SKonstantin Belousov 		kvo->kvo_vn_fsid_freebsd11 = 0;
2589ff87ae35SJohn Baldwin 		freepath = NULL;
2590ff87ae35SJohn Baldwin 		fullpath = "";
25917738118eSKonstantin Belousov 		vp = NULL;
25927738118eSKonstantin Belousov 		kvo->kvo_type = vm_object_kvme_type(obj, swap_only ? NULL : &vp);
2593ecfbddf0SKonstantin Belousov 		if (vp != NULL) {
2594ff87ae35SJohn Baldwin 			vref(vp);
2595ecfbddf0SKonstantin Belousov 		} else if ((obj->flags & OBJ_ANON) != 0) {
25967f3c78fbSMark Johnston 			MPASS(kvo->kvo_type == KVME_TYPE_SWAP);
2597ecfbddf0SKonstantin Belousov 			kvo->kvo_me = (uintptr_t)obj;
2598ecfbddf0SKonstantin Belousov 			/* tmpfs objs are reported as vnodes */
2599ecfbddf0SKonstantin Belousov 			kvo->kvo_backing_obj = (uintptr_t)obj->backing_object;
2600ecfbddf0SKonstantin Belousov 			sp = swap_pager_swapped_pages(obj);
2601ecfbddf0SKonstantin Belousov 			kvo->kvo_swapped = sp > UINT32_MAX ? UINT32_MAX : sp;
2602ecfbddf0SKonstantin Belousov 		}
2603ff87ae35SJohn Baldwin 		VM_OBJECT_RUNLOCK(obj);
2604ff87ae35SJohn Baldwin 		if (vp != NULL) {
2605feabaaf9SMateusz Guzik 			vn_fullpath(vp, &fullpath, &freepath);
2606ff87ae35SJohn Baldwin 			vn_lock(vp, LK_SHARED | LK_RETRY);
2607ff87ae35SJohn Baldwin 			if (VOP_GETATTR(vp, &va, curthread->td_ucred) == 0) {
26080ecee546SKonstantin Belousov 				kvo->kvo_vn_fileid = va.va_fileid;
26090ecee546SKonstantin Belousov 				kvo->kvo_vn_fsid = va.va_fsid;
26100ecee546SKonstantin Belousov 				kvo->kvo_vn_fsid_freebsd11 = va.va_fsid;
261169921123SKonstantin Belousov 								/* truncate */
2612ff87ae35SJohn Baldwin 			}
2613ff87ae35SJohn Baldwin 			vput(vp);
2614ff87ae35SJohn Baldwin 		}
2615ff87ae35SJohn Baldwin 
26160ecee546SKonstantin Belousov 		strlcpy(kvo->kvo_path, fullpath, sizeof(kvo->kvo_path));
2617ff87ae35SJohn Baldwin 		if (freepath != NULL)
2618ff87ae35SJohn Baldwin 			free(freepath, M_TEMP);
2619ff87ae35SJohn Baldwin 
2620ff87ae35SJohn Baldwin 		/* Pack record size down */
26210ecee546SKonstantin Belousov 		kvo->kvo_structsize = offsetof(struct kinfo_vmobject, kvo_path)
26220ecee546SKonstantin Belousov 		    + strlen(kvo->kvo_path) + 1;
26230ecee546SKonstantin Belousov 		kvo->kvo_structsize = roundup(kvo->kvo_structsize,
2624ff87ae35SJohn Baldwin 		    sizeof(uint64_t));
26250ecee546SKonstantin Belousov 		error = SYSCTL_OUT(req, kvo, kvo->kvo_structsize);
2626350fc36bSKonstantin Belousov 		maybe_yield();
2627ff87ae35SJohn Baldwin 		mtx_lock(&vm_object_list_mtx);
2628ff87ae35SJohn Baldwin 		if (error)
2629ff87ae35SJohn Baldwin 			break;
2630ff87ae35SJohn Baldwin 	}
2631ff87ae35SJohn Baldwin 	mtx_unlock(&vm_object_list_mtx);
26320ecee546SKonstantin Belousov 	free(kvo, M_TEMP);
2633ff87ae35SJohn Baldwin 	return (error);
2634ff87ae35SJohn Baldwin }
26351b610624SKonstantin Belousov 
26361b610624SKonstantin Belousov static int
26371b610624SKonstantin Belousov sysctl_vm_object_list(SYSCTL_HANDLER_ARGS)
26381b610624SKonstantin Belousov {
263942812cccSKonstantin Belousov 	return (vm_object_list_handler(req, false));
26401b610624SKonstantin Belousov }
26411b610624SKonstantin Belousov 
2642ff87ae35SJohn Baldwin SYSCTL_PROC(_vm, OID_AUTO, objects, CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_SKIP |
2643ff87ae35SJohn Baldwin     CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_object_list, "S,kinfo_vmobject",
2644ff87ae35SJohn Baldwin     "List of VM objects");
2645ff87ae35SJohn Baldwin 
264642812cccSKonstantin Belousov static int
264742812cccSKonstantin Belousov sysctl_vm_object_list_swap(SYSCTL_HANDLER_ARGS)
264842812cccSKonstantin Belousov {
264942812cccSKonstantin Belousov 	return (vm_object_list_handler(req, true));
265042812cccSKonstantin Belousov }
265142812cccSKonstantin Belousov 
265242812cccSKonstantin Belousov /*
265342812cccSKonstantin Belousov  * This sysctl returns list of the anonymous or swap objects. Intent
265442812cccSKonstantin Belousov  * is to provide stripped optimized list useful to analyze swap use.
265542812cccSKonstantin Belousov  * Since technically non-swap (default) objects participate in the
265642812cccSKonstantin Belousov  * shadow chains, and are converted to swap type as needed by swap
265742812cccSKonstantin Belousov  * pager, we must report them.
265842812cccSKonstantin Belousov  */
265942812cccSKonstantin Belousov SYSCTL_PROC(_vm, OID_AUTO, swap_objects,
266042812cccSKonstantin Belousov     CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_SKIP | CTLFLAG_MPSAFE, NULL, 0,
266142812cccSKonstantin Belousov     sysctl_vm_object_list_swap, "S,kinfo_vmobject",
266242812cccSKonstantin Belousov     "List of swap VM objects");
266342812cccSKonstantin Belousov 
2664c7c34a24SBruce Evans #include "opt_ddb.h"
2665c3cb3e12SDavid Greenman #ifdef DDB
2666c7c34a24SBruce Evans #include <sys/kernel.h>
2667c7c34a24SBruce Evans 
2668ce9edcf5SPoul-Henning Kamp #include <sys/cons.h>
2669c7c34a24SBruce Evans 
2670c7c34a24SBruce Evans #include <ddb/ddb.h>
2671c7c34a24SBruce Evans 
2672cac597e4SBruce Evans static int
26731b40f8c0SMatthew Dillon _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry)
2674a1f6d91cSDavid Greenman {
2675a1f6d91cSDavid Greenman 	vm_map_t tmpm;
2676a1f6d91cSDavid Greenman 	vm_map_entry_t tmpe;
2677a1f6d91cSDavid Greenman 	vm_object_t obj;
2678a1f6d91cSDavid Greenman 
2679a1f6d91cSDavid Greenman 	if (map == 0)
2680a1f6d91cSDavid Greenman 		return 0;
2681a1f6d91cSDavid Greenman 
2682a1f6d91cSDavid Greenman 	if (entry == 0) {
26832288078cSDoug Moore 		VM_MAP_ENTRY_FOREACH(tmpe, map) {
2684a1f6d91cSDavid Greenman 			if (_vm_object_in_map(map, object, tmpe)) {
2685a1f6d91cSDavid Greenman 				return 1;
2686a1f6d91cSDavid Greenman 			}
2687a1f6d91cSDavid Greenman 		}
26889fdfe602SMatthew Dillon 	} else if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
26899fdfe602SMatthew Dillon 		tmpm = entry->object.sub_map;
26902288078cSDoug Moore 		VM_MAP_ENTRY_FOREACH(tmpe, tmpm) {
2691a1f6d91cSDavid Greenman 			if (_vm_object_in_map(tmpm, object, tmpe)) {
2692a1f6d91cSDavid Greenman 				return 1;
2693a1f6d91cSDavid Greenman 			}
2694a1f6d91cSDavid Greenman 		}
26958aef1712SMatthew Dillon 	} else if ((obj = entry->object.vm_object) != NULL) {
269624a1cce3SDavid Greenman 		for (; obj; obj = obj->backing_object)
2697a1f6d91cSDavid Greenman 			if (obj == object) {
2698a1f6d91cSDavid Greenman 				return 1;
2699a1f6d91cSDavid Greenman 			}
2700a1f6d91cSDavid Greenman 	}
2701a1f6d91cSDavid Greenman 	return 0;
2702a1f6d91cSDavid Greenman }
2703a1f6d91cSDavid Greenman 
2704cac597e4SBruce Evans static int
27051b40f8c0SMatthew Dillon vm_object_in_map(vm_object_t object)
2706a1f6d91cSDavid Greenman {
2707a1f6d91cSDavid Greenman 	struct proc *p;
27081005a129SJohn Baldwin 
270960517fd1SJohn Baldwin 	/* sx_slock(&allproc_lock); */
2710f67af5c9SXin LI 	FOREACH_PROC_IN_SYSTEM(p) {
2711a1f6d91cSDavid Greenman 		if (!p->p_vmspace /* || (p->p_flag & (P_SYSTEM|P_WEXIT)) */)
2712a1f6d91cSDavid Greenman 			continue;
2713553629ebSJake Burkholder 		if (_vm_object_in_map(&p->p_vmspace->vm_map, object, 0)) {
271460517fd1SJohn Baldwin 			/* sx_sunlock(&allproc_lock); */
2715a1f6d91cSDavid Greenman 			return 1;
2716a1f6d91cSDavid Greenman 		}
2717553629ebSJake Burkholder 	}
271860517fd1SJohn Baldwin 	/* sx_sunlock(&allproc_lock); */
2719a1f6d91cSDavid Greenman 	if (_vm_object_in_map(kernel_map, object, 0))
2720a1f6d91cSDavid Greenman 		return 1;
2721a1f6d91cSDavid Greenman 	return 0;
2722a1f6d91cSDavid Greenman }
2723a1f6d91cSDavid Greenman 
2724c84c5e00SMitchell Horne DB_SHOW_COMMAND_FLAGS(vmochk, vm_object_check, DB_CMD_MEMSAFE)
2725f708ef1bSPoul-Henning Kamp {
2726a1f6d91cSDavid Greenman 	vm_object_t object;
2727a1f6d91cSDavid Greenman 
2728a1f6d91cSDavid Greenman 	/*
2729a1f6d91cSDavid Greenman 	 * make sure that internal objs are in a map somewhere
2730a1f6d91cSDavid Greenman 	 * and none have zero ref counts.
2731a1f6d91cSDavid Greenman 	 */
2732cc64b484SAlfred Perlstein 	TAILQ_FOREACH(object, &vm_object_list, object_list) {
273332362449SKonstantin Belousov 		if ((object->flags & OBJ_ANON) != 0) {
2734a1f6d91cSDavid Greenman 			if (object->ref_count == 0) {
27353efc015bSPeter Wemm 				db_printf("vmochk: internal obj has zero ref count: %ld\n",
27363efc015bSPeter Wemm 					(long)object->size);
2737a1f6d91cSDavid Greenman 			}
2738a1f6d91cSDavid Greenman 			if (!vm_object_in_map(object)) {
2739fc62ef1fSBruce Evans 				db_printf(
2740fc62ef1fSBruce Evans 			"vmochk: internal obj is not in a map: "
2741fc62ef1fSBruce Evans 			"ref: %d, size: %lu: 0x%lx, backing_object: %p\n",
2742fc62ef1fSBruce Evans 				    object->ref_count, (u_long)object->size,
2743fc62ef1fSBruce Evans 				    (u_long)object->size,
2744fc62ef1fSBruce Evans 				    (void *)object->backing_object);
2745a1f6d91cSDavid Greenman 			}
2746a1f6d91cSDavid Greenman 		}
27478cc8c586SEric van Gyzen 		if (db_pager_quit)
27488cc8c586SEric van Gyzen 			return;
2749a1f6d91cSDavid Greenman 	}
2750a1f6d91cSDavid Greenman }
2751a1f6d91cSDavid Greenman 
275226f9a767SRodney W. Grimes /*
2753df8bae1dSRodney W. Grimes  *	vm_object_print:	[ debug ]
2754df8bae1dSRodney W. Grimes  */
2755c7c34a24SBruce Evans DB_SHOW_COMMAND(object, vm_object_print_static)
2756df8bae1dSRodney W. Grimes {
2757c7c34a24SBruce Evans 	/* XXX convert args. */
2758c7c34a24SBruce Evans 	vm_object_t object = (vm_object_t)addr;
2759c7c34a24SBruce Evans 	boolean_t full = have_addr;
2760c7c34a24SBruce Evans 
2761d031cff1SMatthew Dillon 	vm_page_t p;
2762df8bae1dSRodney W. Grimes 
2763c7c34a24SBruce Evans 	/* XXX count is an (unused) arg.  Avoid shadowing it. */
2764c7c34a24SBruce Evans #define	count	was_count
2765c7c34a24SBruce Evans 
2766d031cff1SMatthew Dillon 	int count;
2767df8bae1dSRodney W. Grimes 
2768df8bae1dSRodney W. Grimes 	if (object == NULL)
2769df8bae1dSRodney W. Grimes 		return;
2770df8bae1dSRodney W. Grimes 
2771eb95adefSBruce Evans 	db_iprintf(
2772ef694c1aSEdward Tomasz Napierala 	    "Object %p: type=%d, size=0x%jx, res=%d, ref=%d, flags=0x%x ruid %d charge %jx\n",
2773e47cd172SMaxime Henrion 	    object, (int)object->type, (uintmax_t)object->size,
27743364c323SKonstantin Belousov 	    object->resident_page_count, object->ref_count, object->flags,
2775ef694c1aSEdward Tomasz Napierala 	    object->cred ? object->cred->cr_ruid : -1, (uintmax_t)object->charge);
2776e47cd172SMaxime Henrion 	db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%jx\n",
27774cf9f5d8SKonstantin Belousov 	    atomic_load_int(&object->shadow_count),
2778eb95adefSBruce Evans 	    object->backing_object ? object->backing_object->ref_count : 0,
2779e47cd172SMaxime Henrion 	    object->backing_object, (uintmax_t)object->backing_object_offset);
2780df8bae1dSRodney W. Grimes 
2781df8bae1dSRodney W. Grimes 	if (!full)
2782df8bae1dSRodney W. Grimes 		return;
2783df8bae1dSRodney W. Grimes 
2784c7c34a24SBruce Evans 	db_indent += 2;
2785df8bae1dSRodney W. Grimes 	count = 0;
2786fc2ffbe6SPoul-Henning Kamp 	TAILQ_FOREACH(p, &object->memq, listq) {
2787df8bae1dSRodney W. Grimes 		if (count == 0)
2788c7c34a24SBruce Evans 			db_iprintf("memory:=");
2789df8bae1dSRodney W. Grimes 		else if (count == 6) {
2790c7c34a24SBruce Evans 			db_printf("\n");
2791c7c34a24SBruce Evans 			db_iprintf(" ...");
2792df8bae1dSRodney W. Grimes 			count = 0;
2793df8bae1dSRodney W. Grimes 		} else
2794c7c34a24SBruce Evans 			db_printf(",");
2795df8bae1dSRodney W. Grimes 		count++;
2796df8bae1dSRodney W. Grimes 
2797e47cd172SMaxime Henrion 		db_printf("(off=0x%jx,page=0x%jx)",
2798e47cd172SMaxime Henrion 		    (uintmax_t)p->pindex, (uintmax_t)VM_PAGE_TO_PHYS(p));
27998cc8c586SEric van Gyzen 
28008cc8c586SEric van Gyzen 		if (db_pager_quit)
28018cc8c586SEric van Gyzen 			break;
2802df8bae1dSRodney W. Grimes 	}
2803df8bae1dSRodney W. Grimes 	if (count != 0)
2804c7c34a24SBruce Evans 		db_printf("\n");
2805c7c34a24SBruce Evans 	db_indent -= 2;
2806df8bae1dSRodney W. Grimes }
28075070c7f8SJohn Dyson 
2808c7c34a24SBruce Evans /* XXX. */
2809c7c34a24SBruce Evans #undef count
2810c7c34a24SBruce Evans 
2811c7c34a24SBruce Evans /* XXX need this non-static entry for calling from vm_map_print. */
28125070c7f8SJohn Dyson void
28131b40f8c0SMatthew Dillon vm_object_print(
28141b40f8c0SMatthew Dillon         /* db_expr_t */ long addr,
28151b40f8c0SMatthew Dillon 	boolean_t have_addr,
28161b40f8c0SMatthew Dillon 	/* db_expr_t */ long count,
28171b40f8c0SMatthew Dillon 	char *modif)
2818c7c34a24SBruce Evans {
2819c7c34a24SBruce Evans 	vm_object_print_static(addr, have_addr, count, modif);
2820c7c34a24SBruce Evans }
2821c7c34a24SBruce Evans 
2822c84c5e00SMitchell Horne DB_SHOW_COMMAND_FLAGS(vmopag, vm_object_print_pages, DB_CMD_MEMSAFE)
28235070c7f8SJohn Dyson {
28245070c7f8SJohn Dyson 	vm_object_t object;
2825bb2ac86fSKonstantin Belousov 	vm_pindex_t fidx;
2826bb2ac86fSKonstantin Belousov 	vm_paddr_t pa;
2827bb2ac86fSKonstantin Belousov 	vm_page_t m, prev_m;
282818048b6eSStephen J. Kiernan 	int rcount;
2829cc64b484SAlfred Perlstein 
2830cc64b484SAlfred Perlstein 	TAILQ_FOREACH(object, &vm_object_list, object_list) {
2831fc62ef1fSBruce Evans 		db_printf("new object: %p\n", (void *)object);
283218048b6eSStephen J. Kiernan 		if (db_pager_quit)
28335070c7f8SJohn Dyson 			return;
283418048b6eSStephen J. Kiernan 
28355070c7f8SJohn Dyson 		rcount = 0;
28365070c7f8SJohn Dyson 		fidx = 0;
2837bb2ac86fSKonstantin Belousov 		pa = -1;
2838bb2ac86fSKonstantin Belousov 		TAILQ_FOREACH(m, &object->memq, listq) {
2839bb2ac86fSKonstantin Belousov 			if ((prev_m = TAILQ_PREV(m, pglist, listq)) != NULL &&
2840bb2ac86fSKonstantin Belousov 			    prev_m->pindex + 1 != m->pindex) {
28415070c7f8SJohn Dyson 				if (rcount) {
28423efc015bSPeter Wemm 					db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
28433efc015bSPeter Wemm 						(long)fidx, rcount, (long)pa);
284418048b6eSStephen J. Kiernan 					if (db_pager_quit)
28455070c7f8SJohn Dyson 						return;
28465070c7f8SJohn Dyson 					rcount = 0;
28475070c7f8SJohn Dyson 				}
28485070c7f8SJohn Dyson 			}
28495070c7f8SJohn Dyson 			if (rcount &&
28505070c7f8SJohn Dyson 				(VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) {
28515070c7f8SJohn Dyson 				++rcount;
28525070c7f8SJohn Dyson 				continue;
28535070c7f8SJohn Dyson 			}
28545070c7f8SJohn Dyson 			if (rcount) {
28552446e4f0SAlan Cox 				db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
28563efc015bSPeter Wemm 					(long)fidx, rcount, (long)pa);
285718048b6eSStephen J. Kiernan 				if (db_pager_quit)
28585070c7f8SJohn Dyson 					return;
28595070c7f8SJohn Dyson 			}
2860bb2ac86fSKonstantin Belousov 			fidx = m->pindex;
28615070c7f8SJohn Dyson 			pa = VM_PAGE_TO_PHYS(m);
28625070c7f8SJohn Dyson 			rcount = 1;
28635070c7f8SJohn Dyson 		}
28645070c7f8SJohn Dyson 		if (rcount) {
28653efc015bSPeter Wemm 			db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
28663efc015bSPeter Wemm 				(long)fidx, rcount, (long)pa);
286718048b6eSStephen J. Kiernan 			if (db_pager_quit)
28685070c7f8SJohn Dyson 				return;
28695070c7f8SJohn Dyson 		}
28705070c7f8SJohn Dyson 	}
28715070c7f8SJohn Dyson }
2872c3cb3e12SDavid Greenman #endif /* DDB */
2873