xref: /freebsd/sys/vm/vm_object.c (revision 18048b6e3cb647f2a18e1e51df1cf86a90fc6802)
160727d8bSWarner Losh /*-
2796df753SPedro F. Giffuni  * SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU)
351369649SPedro F. Giffuni  *
4df8bae1dSRodney W. Grimes  * Copyright (c) 1991, 1993
5df8bae1dSRodney W. Grimes  *	The Regents of the University of California.  All rights reserved.
6df8bae1dSRodney W. Grimes  *
7df8bae1dSRodney W. Grimes  * This code is derived from software contributed to Berkeley by
8df8bae1dSRodney W. Grimes  * The Mach Operating System project at Carnegie-Mellon University.
9df8bae1dSRodney W. Grimes  *
10df8bae1dSRodney W. Grimes  * Redistribution and use in source and binary forms, with or without
11df8bae1dSRodney W. Grimes  * modification, are permitted provided that the following conditions
12df8bae1dSRodney W. Grimes  * are met:
13df8bae1dSRodney W. Grimes  * 1. Redistributions of source code must retain the above copyright
14df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer.
15df8bae1dSRodney W. Grimes  * 2. Redistributions in binary form must reproduce the above copyright
16df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer in the
17df8bae1dSRodney W. Grimes  *    documentation and/or other materials provided with the distribution.
18fbbd9655SWarner Losh  * 3. Neither the name of the University nor the names of its contributors
19df8bae1dSRodney W. Grimes  *    may be used to endorse or promote products derived from this software
20df8bae1dSRodney W. Grimes  *    without specific prior written permission.
21df8bae1dSRodney W. Grimes  *
22df8bae1dSRodney W. Grimes  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23df8bae1dSRodney W. Grimes  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24df8bae1dSRodney W. Grimes  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25df8bae1dSRodney W. Grimes  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26df8bae1dSRodney W. Grimes  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27df8bae1dSRodney W. Grimes  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28df8bae1dSRodney W. Grimes  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29df8bae1dSRodney W. Grimes  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30df8bae1dSRodney W. Grimes  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31df8bae1dSRodney W. Grimes  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32df8bae1dSRodney W. Grimes  * SUCH DAMAGE.
33df8bae1dSRodney W. Grimes  *
343c4dd356SDavid Greenman  *	from: @(#)vm_object.c	8.5 (Berkeley) 3/22/94
35df8bae1dSRodney W. Grimes  *
36df8bae1dSRodney W. Grimes  *
37df8bae1dSRodney W. Grimes  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
38df8bae1dSRodney W. Grimes  * All rights reserved.
39df8bae1dSRodney W. Grimes  *
40df8bae1dSRodney W. Grimes  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
41df8bae1dSRodney W. Grimes  *
42df8bae1dSRodney W. Grimes  * Permission to use, copy, modify and distribute this software and
43df8bae1dSRodney W. Grimes  * its documentation is hereby granted, provided that both the copyright
44df8bae1dSRodney W. Grimes  * notice and this permission notice appear in all copies of the
45df8bae1dSRodney W. Grimes  * software, derivative works or modified versions, and any portions
46df8bae1dSRodney W. Grimes  * thereof, and that both notices appear in supporting documentation.
47df8bae1dSRodney W. Grimes  *
48df8bae1dSRodney W. Grimes  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
49df8bae1dSRodney W. Grimes  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
50df8bae1dSRodney W. Grimes  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
51df8bae1dSRodney W. Grimes  *
52df8bae1dSRodney W. Grimes  * Carnegie Mellon requests users of this software to return to
53df8bae1dSRodney W. Grimes  *
54df8bae1dSRodney W. Grimes  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
55df8bae1dSRodney W. Grimes  *  School of Computer Science
56df8bae1dSRodney W. Grimes  *  Carnegie Mellon University
57df8bae1dSRodney W. Grimes  *  Pittsburgh PA 15213-3890
58df8bae1dSRodney W. Grimes  *
59df8bae1dSRodney W. Grimes  * any improvements or extensions that they make and grant Carnegie the
60df8bae1dSRodney W. Grimes  * rights to redistribute these changes.
61df8bae1dSRodney W. Grimes  */
62df8bae1dSRodney W. Grimes 
63df8bae1dSRodney W. Grimes /*
64df8bae1dSRodney W. Grimes  *	Virtual memory object module.
65df8bae1dSRodney W. Grimes  */
66df8bae1dSRodney W. Grimes 
67874651b1SDavid E. O'Brien #include <sys/cdefs.h>
68874651b1SDavid E. O'Brien __FBSDID("$FreeBSD$");
69874651b1SDavid E. O'Brien 
70f8a47341SAlan Cox #include "opt_vm.h"
71f8a47341SAlan Cox 
72df8bae1dSRodney W. Grimes #include <sys/param.h>
73df8bae1dSRodney W. Grimes #include <sys/systm.h>
74c99d0c58SMark Johnston #include <sys/blockcount.h>
753f289c3fSJeff Roberson #include <sys/cpuset.h>
76ecfbddf0SKonstantin Belousov #include <sys/limits.h>
77fb919e4dSMark Murray #include <sys/lock.h>
78867a482dSJohn Dyson #include <sys/mman.h>
79cf2819ccSJohn Dyson #include <sys/mount.h>
80b9b7a4beSMatthew Dillon #include <sys/kernel.h>
81f425ab8eSKonstantin Belousov #include <sys/pctrie.h>
82b9b7a4beSMatthew Dillon #include <sys/sysctl.h>
831b367556SJason Evans #include <sys/mutex.h>
84fb919e4dSMark Murray #include <sys/proc.h>		/* for curproc, pageproc */
85cf27e0d1SJeff Roberson #include <sys/refcount.h>
86fb919e4dSMark Murray #include <sys/socket.h>
873364c323SKonstantin Belousov #include <sys/resourcevar.h>
88205be21dSJeff Roberson #include <sys/refcount.h>
8989f6b863SAttilio Rao #include <sys/rwlock.h>
90ff87ae35SJohn Baldwin #include <sys/user.h>
91fb919e4dSMark Murray #include <sys/vnode.h>
92fb919e4dSMark Murray #include <sys/vmmeter.h>
931005a129SJohn Baldwin #include <sys/sx.h>
94df8bae1dSRodney W. Grimes 
95df8bae1dSRodney W. Grimes #include <vm/vm.h>
96efeaf95aSDavid Greenman #include <vm/vm_param.h>
97efeaf95aSDavid Greenman #include <vm/pmap.h>
98efeaf95aSDavid Greenman #include <vm/vm_map.h>
99efeaf95aSDavid Greenman #include <vm/vm_object.h>
100df8bae1dSRodney W. Grimes #include <vm/vm_page.h>
10126f9a767SRodney W. Grimes #include <vm/vm_pageout.h>
1020d94caffSDavid Greenman #include <vm/vm_pager.h>
103e2068d0bSJeff Roberson #include <vm/vm_phys.h>
104e2068d0bSJeff Roberson #include <vm/vm_pagequeue.h>
10505f0fdd2SPoul-Henning Kamp #include <vm/swap_pager.h>
106a1f6d91cSDavid Greenman #include <vm/vm_kern.h>
107efeaf95aSDavid Greenman #include <vm/vm_extern.h>
108774d251dSAttilio Rao #include <vm/vm_radix.h>
109f8a47341SAlan Cox #include <vm/vm_reserv.h>
110670d17b5SJeff Roberson #include <vm/uma.h>
11126f9a767SRodney W. Grimes 
112c53f7aceSDag-Erling Smørgrav static int old_msync;
113c53f7aceSDag-Erling Smørgrav SYSCTL_INT(_vm, OID_AUTO, old_msync, CTLFLAG_RW, &old_msync, 0,
114c53f7aceSDag-Erling Smørgrav     "Use old (insecure) msync behavior");
115c53f7aceSDag-Erling Smørgrav 
116757216f3SKonstantin Belousov static int	vm_object_page_collect_flush(vm_object_t object, vm_page_t p,
11767d0e293SJeff Roberson 		    int pagerflags, int flags, boolean_t *allclean,
118126d6082SKonstantin Belousov 		    boolean_t *eio);
1193280870dSKonstantin Belousov static boolean_t vm_object_page_remove_write(vm_page_t p, int flags,
12067d0e293SJeff Roberson 		    boolean_t *allclean);
12151b867e5SJeff Roberson static void	vm_object_backing_remove(vm_object_t object);
122f6b04d2bSDavid Greenman 
123df8bae1dSRodney W. Grimes /*
124df8bae1dSRodney W. Grimes  *	Virtual memory objects maintain the actual data
125df8bae1dSRodney W. Grimes  *	associated with allocated virtual memory.  A given
126df8bae1dSRodney W. Grimes  *	page of memory exists within exactly one object.
127df8bae1dSRodney W. Grimes  *
128df8bae1dSRodney W. Grimes  *	An object is only deallocated when all "references"
129df8bae1dSRodney W. Grimes  *	are given up.  Only one "reference" to a given
130df8bae1dSRodney W. Grimes  *	region of an object should be writeable.
131df8bae1dSRodney W. Grimes  *
132df8bae1dSRodney W. Grimes  *	Associated with each object is a list of all resident
133df8bae1dSRodney W. Grimes  *	memory pages belonging to that object; this list is
134df8bae1dSRodney W. Grimes  *	maintained by the "vm_page" module, and locked by the object's
135df8bae1dSRodney W. Grimes  *	lock.
136df8bae1dSRodney W. Grimes  *
137df8bae1dSRodney W. Grimes  *	Each object also records a "pager" routine which is
138df8bae1dSRodney W. Grimes  *	used to retrieve (and store) pages to the proper backing
139df8bae1dSRodney W. Grimes  *	storage.  In addition, objects may be backed by other
140df8bae1dSRodney W. Grimes  *	objects from which they were virtual-copied.
141df8bae1dSRodney W. Grimes  *
142df8bae1dSRodney W. Grimes  *	The only items within the object structure which are
143df8bae1dSRodney W. Grimes  *	modified after time of creation are:
144df8bae1dSRodney W. Grimes  *		reference count		locked by object's lock
145df8bae1dSRodney W. Grimes  *		pager routine		locked by object's lock
146df8bae1dSRodney W. Grimes  *
147df8bae1dSRodney W. Grimes  */
148df8bae1dSRodney W. Grimes 
14928f8db14SBruce Evans struct object_q vm_object_list;
150a5698387SAlan Cox struct mtx vm_object_list_mtx;	/* lock for object list and count */
151cccf11b8SAlan Cox 
152cccf11b8SAlan Cox struct vm_object kernel_object_store;
153df8bae1dSRodney W. Grimes 
1547029da5cSPawel Biernacki static SYSCTL_NODE(_vm_stats, OID_AUTO, object, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
1556472ac3dSEd Schouten     "VM object stats");
156604c2bbcSAlan Cox 
157d869a17eSMark Johnston static COUNTER_U64_DEFINE_EARLY(object_collapses);
15811542376SAlan Cox SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, collapses, CTLFLAG_RD,
15911542376SAlan Cox     &object_collapses,
16011542376SAlan Cox     "VM object collapses");
161604c2bbcSAlan Cox 
162d869a17eSMark Johnston static COUNTER_U64_DEFINE_EARLY(object_bypasses);
16311542376SAlan Cox SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, bypasses, CTLFLAG_RD,
16411542376SAlan Cox     &object_bypasses,
16511542376SAlan Cox     "VM object bypasses");
16611542376SAlan Cox 
167d869a17eSMark Johnston static COUNTER_U64_DEFINE_EARLY(object_collapse_waits);
16898087a06SJeff Roberson SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, collapse_waits, CTLFLAG_RD,
16998087a06SJeff Roberson     &object_collapse_waits,
17098087a06SJeff Roberson     "Number of sleeps for collapse");
17198087a06SJeff Roberson 
172670d17b5SJeff Roberson static uma_zone_t obj_zone;
1738355f576SJeff Roberson 
174b23f72e9SBrian Feldman static int vm_object_zinit(void *mem, int size, int flags);
1758355f576SJeff Roberson 
1768355f576SJeff Roberson #ifdef INVARIANTS
1778355f576SJeff Roberson static void vm_object_zdtor(void *mem, int size, void *arg);
1788355f576SJeff Roberson 
1798355f576SJeff Roberson static void
1808355f576SJeff Roberson vm_object_zdtor(void *mem, int size, void *arg)
1818355f576SJeff Roberson {
1828355f576SJeff Roberson 	vm_object_t object;
1838355f576SJeff Roberson 
1848355f576SJeff Roberson 	object = (vm_object_t)mem;
185e735691bSJohn Baldwin 	KASSERT(object->ref_count == 0,
186e735691bSJohn Baldwin 	    ("object %p ref_count = %d", object, object->ref_count));
18743186e53SAlan Cox 	KASSERT(TAILQ_EMPTY(&object->memq),
188198da1b2SAttilio Rao 	    ("object %p has resident pages in its memq", object));
189774d251dSAttilio Rao 	KASSERT(vm_radix_is_empty(&object->rtree),
190774d251dSAttilio Rao 	    ("object %p has resident pages in its trie", object));
191f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0
192f8a47341SAlan Cox 	KASSERT(LIST_EMPTY(&object->rvq),
193f8a47341SAlan Cox 	    ("object %p has reservations",
194f8a47341SAlan Cox 	    object));
195f8a47341SAlan Cox #endif
196c99d0c58SMark Johnston 	KASSERT(!vm_object_busied(object),
197c99d0c58SMark Johnston 	    ("object %p busy = %d", object, blockcount_read(&object->busy)));
1988355f576SJeff Roberson 	KASSERT(object->resident_page_count == 0,
1998355f576SJeff Roberson 	    ("object %p resident_page_count = %d",
2008355f576SJeff Roberson 	    object, object->resident_page_count));
2018355f576SJeff Roberson 	KASSERT(object->shadow_count == 0,
2028355f576SJeff Roberson 	    ("object %p shadow_count = %d",
2038355f576SJeff Roberson 	    object, object->shadow_count));
204e735691bSJohn Baldwin 	KASSERT(object->type == OBJT_DEAD,
205e735691bSJohn Baldwin 	    ("object %p has non-dead type %d",
206e735691bSJohn Baldwin 	    object, object->type));
2078355f576SJeff Roberson }
2088355f576SJeff Roberson #endif
2098355f576SJeff Roberson 
210b23f72e9SBrian Feldman static int
211b23f72e9SBrian Feldman vm_object_zinit(void *mem, int size, int flags)
2128355f576SJeff Roberson {
2138355f576SJeff Roberson 	vm_object_t object;
2148355f576SJeff Roberson 
2158355f576SJeff Roberson 	object = (vm_object_t)mem;
216777a36c5SAlan Cox 	rw_init_flags(&object->lock, "vm object", RW_DUPOK | RW_NEW);
2178355f576SJeff Roberson 
2188355f576SJeff Roberson 	/* These are true for any object that has been freed */
219e735691bSJohn Baldwin 	object->type = OBJT_DEAD;
220cd1241fbSKonstantin Belousov 	vm_radix_init(&object->rtree);
22151df5321SJeff Roberson 	refcount_init(&object->ref_count, 0);
222c99d0c58SMark Johnston 	blockcount_init(&object->paging_in_progress);
223c99d0c58SMark Johnston 	blockcount_init(&object->busy);
2248355f576SJeff Roberson 	object->resident_page_count = 0;
2258355f576SJeff Roberson 	object->shadow_count = 0;
226f425ab8eSKonstantin Belousov 	object->flags = OBJ_DEAD;
227e735691bSJohn Baldwin 
228e735691bSJohn Baldwin 	mtx_lock(&vm_object_list_mtx);
229e735691bSJohn Baldwin 	TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);
230e735691bSJohn Baldwin 	mtx_unlock(&vm_object_list_mtx);
231b23f72e9SBrian Feldman 	return (0);
2328355f576SJeff Roberson }
233df8bae1dSRodney W. Grimes 
234a4915c21SAttilio Rao static void
23563967687SJeff Roberson _vm_object_allocate(objtype_t type, vm_pindex_t size, u_short flags,
23667388836SKonstantin Belousov     vm_object_t object, void *handle)
237df8bae1dSRodney W. Grimes {
2380cddd8f0SMatthew Dillon 
239df8bae1dSRodney W. Grimes 	TAILQ_INIT(&object->memq);
2401c500307SAlan Cox 	LIST_INIT(&object->shadow_head);
241a1f6d91cSDavid Greenman 
24224a1cce3SDavid Greenman 	object->type = type;
2434b8365d7SKonstantin Belousov 	object->flags = flags;
2444b8365d7SKonstantin Belousov 	if ((flags & OBJ_SWAP) != 0)
245f425ab8eSKonstantin Belousov 		pctrie_init(&object->un_pager.swp.swp_blks);
246f425ab8eSKonstantin Belousov 
247f425ab8eSKonstantin Belousov 	/*
248f425ab8eSKonstantin Belousov 	 * Ensure that swap_pager_swapoff() iteration over object_list
249f425ab8eSKonstantin Belousov 	 * sees up to date type and pctrie head if it observed
250f425ab8eSKonstantin Belousov 	 * non-dead object.
251f425ab8eSKonstantin Belousov 	 */
252f425ab8eSKonstantin Belousov 	atomic_thread_fence_rel();
253f425ab8eSKonstantin Belousov 
25463967687SJeff Roberson 	object->pg_color = 0;
255df8bae1dSRodney W. Grimes 	object->size = size;
2564c29d2deSMark Johnston 	object->domain.dr_policy = NULL;
257b881da26SAlan Cox 	object->generation = 1;
25867d0e293SJeff Roberson 	object->cleangeneration = 1;
25951df5321SJeff Roberson 	refcount_init(&object->ref_count, 1);
2603153e878SAlan Cox 	object->memattr = VM_MEMATTR_DEFAULT;
261ef694c1aSEdward Tomasz Napierala 	object->cred = NULL;
2623364c323SKonstantin Belousov 	object->charge = 0;
26367388836SKonstantin Belousov 	object->handle = handle;
26424a1cce3SDavid Greenman 	object->backing_object = NULL;
265a316d390SJohn Dyson 	object->backing_object_offset = (vm_ooffset_t) 0;
266f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0
267f8a47341SAlan Cox 	LIST_INIT(&object->rvq);
268f8a47341SAlan Cox #endif
2691bdbd705SKonstantin Belousov 	umtx_shm_object_init(object);
270df8bae1dSRodney W. Grimes }
271df8bae1dSRodney W. Grimes 
272df8bae1dSRodney W. Grimes /*
27326f9a767SRodney W. Grimes  *	vm_object_init:
27426f9a767SRodney W. Grimes  *
27526f9a767SRodney W. Grimes  *	Initialize the VM objects module.
27626f9a767SRodney W. Grimes  */
27726f9a767SRodney W. Grimes void
2781b40f8c0SMatthew Dillon vm_object_init(void)
27926f9a767SRodney W. Grimes {
28026f9a767SRodney W. Grimes 	TAILQ_INIT(&vm_object_list);
2816008862bSJohn Baldwin 	mtx_init(&vm_object_list_mtx, "vm object_list", NULL, MTX_DEF);
2820217125fSDavid Greenman 
28389f6b863SAttilio Rao 	rw_init(&kernel_object->lock, "kernel vm object");
284d1780e8dSKonstantin Belousov 	_vm_object_allocate(OBJT_PHYS, atop(VM_MAX_KERNEL_ADDRESS -
28567388836SKonstantin Belousov 	    VM_MIN_KERNEL_ADDRESS), OBJ_UNMANAGED, kernel_object, NULL);
286f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0
287f8a47341SAlan Cox 	kernel_object->flags |= OBJ_COLORED;
288f8a47341SAlan Cox 	kernel_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS);
289f8a47341SAlan Cox #endif
290a720b31cSKonstantin Belousov 	kernel_object->un_pager.phys.ops = &default_phys_pg_ops;
29126f9a767SRodney W. Grimes 
2928dbca793STor Egge 	/*
2938dbca793STor Egge 	 * The lock portion of struct vm_object must be type stable due
2948dbca793STor Egge 	 * to vm_pageout_fallback_object_lock locking a vm object
2958dbca793STor Egge 	 * without holding any references to it.
296ffae7ea9SKonstantin Belousov 	 *
297ffae7ea9SKonstantin Belousov 	 * paging_in_progress is valid always.  Lockless references to
298ffae7ea9SKonstantin Belousov 	 * the objects may acquire pip and then check OBJ_DEAD.
2998dbca793STor Egge 	 */
3008355f576SJeff Roberson 	obj_zone = uma_zcreate("VM OBJECT", sizeof (struct vm_object), NULL,
3018355f576SJeff Roberson #ifdef INVARIANTS
3028355f576SJeff Roberson 	    vm_object_zdtor,
3038355f576SJeff Roberson #else
3048355f576SJeff Roberson 	    NULL,
3058355f576SJeff Roberson #endif
3065df87b21SJeff Roberson 	    vm_object_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
307774d251dSAttilio Rao 
308cd1241fbSKonstantin Belousov 	vm_radix_zinit();
30999448ed1SJohn Dyson }
31099448ed1SJohn Dyson 
31199448ed1SJohn Dyson void
3121b40f8c0SMatthew Dillon vm_object_clear_flag(vm_object_t object, u_short bits)
3131b40f8c0SMatthew Dillon {
3145440b5a9SAlan Cox 
31589f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
316b06805adSJake Burkholder 	object->flags &= ~bits;
3171b40f8c0SMatthew Dillon }
3181b40f8c0SMatthew Dillon 
3193153e878SAlan Cox /*
3203153e878SAlan Cox  *	Sets the default memory attribute for the specified object.  Pages
3213153e878SAlan Cox  *	that are allocated to this object are by default assigned this memory
3223153e878SAlan Cox  *	attribute.
3233153e878SAlan Cox  *
3243153e878SAlan Cox  *	Presently, this function must be called before any pages are allocated
3253153e878SAlan Cox  *	to the object.  In the future, this requirement may be relaxed for
3263153e878SAlan Cox  *	"default" and "swap" objects.
3273153e878SAlan Cox  */
3283153e878SAlan Cox int
3293153e878SAlan Cox vm_object_set_memattr(vm_object_t object, vm_memattr_t memattr)
3303153e878SAlan Cox {
3313153e878SAlan Cox 
33289f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
3333e7a11caSKonstantin Belousov 
3343e7a11caSKonstantin Belousov 	if (object->type == OBJT_DEAD)
3353e7a11caSKonstantin Belousov 		return (KERN_INVALID_ARGUMENT);
3363153e878SAlan Cox 	if (!TAILQ_EMPTY(&object->memq))
3373153e878SAlan Cox 		return (KERN_FAILURE);
3383e7a11caSKonstantin Belousov 
3393153e878SAlan Cox 	object->memattr = memattr;
3403153e878SAlan Cox 	return (KERN_SUCCESS);
3413153e878SAlan Cox }
3423153e878SAlan Cox 
3431b40f8c0SMatthew Dillon void
3441b40f8c0SMatthew Dillon vm_object_pip_add(vm_object_t object, short i)
3451b40f8c0SMatthew Dillon {
346f279b88dSAlan Cox 
347c99d0c58SMark Johnston 	if (i > 0)
348c99d0c58SMark Johnston 		blockcount_acquire(&object->paging_in_progress, i);
3491b40f8c0SMatthew Dillon }
3501b40f8c0SMatthew Dillon 
3511b40f8c0SMatthew Dillon void
3521b40f8c0SMatthew Dillon vm_object_pip_wakeup(vm_object_t object)
3531b40f8c0SMatthew Dillon {
354f279b88dSAlan Cox 
355c99d0c58SMark Johnston 	vm_object_pip_wakeupn(object, 1);
3561b40f8c0SMatthew Dillon }
3571b40f8c0SMatthew Dillon 
3581b40f8c0SMatthew Dillon void
3591b40f8c0SMatthew Dillon vm_object_pip_wakeupn(vm_object_t object, short i)
3601b40f8c0SMatthew Dillon {
361d647a0edSAlan Cox 
362c99d0c58SMark Johnston 	if (i > 0)
363c99d0c58SMark Johnston 		blockcount_release(&object->paging_in_progress, i);
3641b40f8c0SMatthew Dillon }
3651b40f8c0SMatthew Dillon 
36698087a06SJeff Roberson /*
367c99d0c58SMark Johnston  * Atomically drop the object lock and wait for pip to drain.  This protects
368c99d0c58SMark Johnston  * from sleep/wakeup races due to identity changes.  The lock is not re-acquired
369c99d0c58SMark Johnston  * on return.
37098087a06SJeff Roberson  */
37198087a06SJeff Roberson static void
372eaa17d42SRyan Libby vm_object_pip_sleep(vm_object_t object, const char *waitid)
37398087a06SJeff Roberson {
37498087a06SJeff Roberson 
375c99d0c58SMark Johnston 	(void)blockcount_sleep(&object->paging_in_progress, &object->lock,
376c99d0c58SMark Johnston 	    waitid, PVM | PDROP);
37798087a06SJeff Roberson }
37898087a06SJeff Roberson 
3791b40f8c0SMatthew Dillon void
380eaa17d42SRyan Libby vm_object_pip_wait(vm_object_t object, const char *waitid)
3811b40f8c0SMatthew Dillon {
3821ca58953SAlan Cox 
38389f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
384cf27e0d1SJeff Roberson 
385c99d0c58SMark Johnston 	blockcount_wait(&object->paging_in_progress, &object->lock, waitid,
386c99d0c58SMark Johnston 	    PVM);
3871b40f8c0SMatthew Dillon }
3881b40f8c0SMatthew Dillon 
389cf27e0d1SJeff Roberson void
390eaa17d42SRyan Libby vm_object_pip_wait_unlocked(vm_object_t object, const char *waitid)
391cf27e0d1SJeff Roberson {
392cf27e0d1SJeff Roberson 
393cf27e0d1SJeff Roberson 	VM_OBJECT_ASSERT_UNLOCKED(object);
394cf27e0d1SJeff Roberson 
395c99d0c58SMark Johnston 	blockcount_wait(&object->paging_in_progress, NULL, waitid, PVM);
396cf27e0d1SJeff Roberson }
397cf27e0d1SJeff Roberson 
39826f9a767SRodney W. Grimes /*
39926f9a767SRodney W. Grimes  *	vm_object_allocate:
40026f9a767SRodney W. Grimes  *
40126f9a767SRodney W. Grimes  *	Returns a new object with the given size.
40226f9a767SRodney W. Grimes  */
40326f9a767SRodney W. Grimes vm_object_t
4046395da54SIan Dowse vm_object_allocate(objtype_t type, vm_pindex_t size)
40526f9a767SRodney W. Grimes {
40690688d13SAlan Cox 	vm_object_t object;
40763967687SJeff Roberson 	u_short flags;
40863967687SJeff Roberson 
40963967687SJeff Roberson 	switch (type) {
41063967687SJeff Roberson 	case OBJT_DEAD:
41163967687SJeff Roberson 		panic("vm_object_allocate: can't create OBJT_DEAD");
41263967687SJeff Roberson 	case OBJT_DEFAULT:
41363967687SJeff Roberson 		flags = OBJ_COLORED;
41463967687SJeff Roberson 		break;
4154b8365d7SKonstantin Belousov 	case OBJT_SWAP:
4164b8365d7SKonstantin Belousov 		flags = OBJ_COLORED | OBJ_SWAP;
4174b8365d7SKonstantin Belousov 		break;
41863967687SJeff Roberson 	case OBJT_DEVICE:
41963967687SJeff Roberson 	case OBJT_SG:
42063967687SJeff Roberson 		flags = OBJ_FICTITIOUS | OBJ_UNMANAGED;
42163967687SJeff Roberson 		break;
42263967687SJeff Roberson 	case OBJT_MGTDEVICE:
42363967687SJeff Roberson 		flags = OBJ_FICTITIOUS;
42463967687SJeff Roberson 		break;
42563967687SJeff Roberson 	case OBJT_PHYS:
42663967687SJeff Roberson 		flags = OBJ_UNMANAGED;
42763967687SJeff Roberson 		break;
42863967687SJeff Roberson 	case OBJT_VNODE:
42963967687SJeff Roberson 		flags = 0;
43063967687SJeff Roberson 		break;
43163967687SJeff Roberson 	default:
432b730fd30SKonstantin Belousov 		panic("vm_object_allocate: type %d is undefined or dynamic",
433b730fd30SKonstantin Belousov 		    type);
43463967687SJeff Roberson 	}
43563967687SJeff Roberson 	object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);
43667388836SKonstantin Belousov 	_vm_object_allocate(type, size, flags, object, NULL);
43763967687SJeff Roberson 
43863967687SJeff Roberson 	return (object);
43963967687SJeff Roberson }
44063967687SJeff Roberson 
441b730fd30SKonstantin Belousov vm_object_t
442b730fd30SKonstantin Belousov vm_object_allocate_dyn(objtype_t dyntype, vm_pindex_t size, u_short flags)
443b730fd30SKonstantin Belousov {
444b730fd30SKonstantin Belousov 	vm_object_t object;
445b730fd30SKonstantin Belousov 
446b730fd30SKonstantin Belousov 	MPASS(dyntype >= OBJT_FIRST_DYN /* && dyntype < nitems(pagertab) */);
447b730fd30SKonstantin Belousov 	object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);
448b730fd30SKonstantin Belousov 	_vm_object_allocate(dyntype, size, flags, object, NULL);
449b730fd30SKonstantin Belousov 
450b730fd30SKonstantin Belousov 	return (object);
451b730fd30SKonstantin Belousov }
452b730fd30SKonstantin Belousov 
45363967687SJeff Roberson /*
45463967687SJeff Roberson  *	vm_object_allocate_anon:
45563967687SJeff Roberson  *
45663967687SJeff Roberson  *	Returns a new default object of the given size and marked as
45763967687SJeff Roberson  *	anonymous memory for special split/collapse handling.  Color
45863967687SJeff Roberson  *	to be initialized by the caller.
45963967687SJeff Roberson  */
46063967687SJeff Roberson vm_object_t
46167388836SKonstantin Belousov vm_object_allocate_anon(vm_pindex_t size, vm_object_t backing_object,
46267388836SKonstantin Belousov     struct ucred *cred, vm_size_t charge)
46363967687SJeff Roberson {
46467388836SKonstantin Belousov 	vm_object_t handle, object;
46590688d13SAlan Cox 
46667388836SKonstantin Belousov 	if (backing_object == NULL)
46767388836SKonstantin Belousov 		handle = NULL;
46867388836SKonstantin Belousov 	else if ((backing_object->flags & OBJ_ANON) != 0)
46967388836SKonstantin Belousov 		handle = backing_object->handle;
47067388836SKonstantin Belousov 	else
47167388836SKonstantin Belousov 		handle = backing_object;
47267388836SKonstantin Belousov 	object = uma_zalloc(obj_zone, M_WAITOK);
47363967687SJeff Roberson 	_vm_object_allocate(OBJT_DEFAULT, size, OBJ_ANON | OBJ_ONEMAPPING,
47467388836SKonstantin Belousov 	    object, handle);
47567388836SKonstantin Belousov 	object->cred = cred;
47667388836SKonstantin Belousov 	object->charge = cred != NULL ? charge : 0;
47790688d13SAlan Cox 	return (object);
47826f9a767SRodney W. Grimes }
47926f9a767SRodney W. Grimes 
48098087a06SJeff Roberson static void
48198087a06SJeff Roberson vm_object_reference_vnode(vm_object_t object)
482df8bae1dSRodney W. Grimes {
483a67d5408SJeff Roberson 	u_int old;
484a67d5408SJeff Roberson 
485a67d5408SJeff Roberson 	/*
48698087a06SJeff Roberson 	 * vnode objects need the lock for the first reference
48798087a06SJeff Roberson 	 * to serialize with vnode_object_deallocate().
488a67d5408SJeff Roberson 	 */
48998087a06SJeff Roberson 	if (!refcount_acquire_if_gt(&object->ref_count, 0)) {
49051df5321SJeff Roberson 		VM_OBJECT_RLOCK(object);
491a67d5408SJeff Roberson 		old = refcount_acquire(&object->ref_count);
4921a0c234eSJeff Roberson 		if (object->type == OBJT_VNODE && old == 0)
4931a0c234eSJeff Roberson 			vref(object->handle);
49451df5321SJeff Roberson 		VM_OBJECT_RUNLOCK(object);
49595e5e988SJohn Dyson 	}
496a67d5408SJeff Roberson }
49795e5e988SJohn Dyson 
49823955314SAlfred Perlstein /*
49998087a06SJeff Roberson  *	vm_object_reference:
50098087a06SJeff Roberson  *
50198087a06SJeff Roberson  *	Acquires a reference to the given object.
50298087a06SJeff Roberson  */
50398087a06SJeff Roberson void
50498087a06SJeff Roberson vm_object_reference(vm_object_t object)
50598087a06SJeff Roberson {
50698087a06SJeff Roberson 
50798087a06SJeff Roberson 	if (object == NULL)
50898087a06SJeff Roberson 		return;
50998087a06SJeff Roberson 
51098087a06SJeff Roberson 	if (object->type == OBJT_VNODE)
51198087a06SJeff Roberson 		vm_object_reference_vnode(object);
51298087a06SJeff Roberson 	else
51398087a06SJeff Roberson 		refcount_acquire(&object->ref_count);
51498087a06SJeff Roberson 	KASSERT((object->flags & OBJ_DEAD) == 0,
51598087a06SJeff Roberson 	    ("vm_object_reference: Referenced dead object."));
51698087a06SJeff Roberson }
51798087a06SJeff Roberson 
51898087a06SJeff Roberson /*
519b921a12bSAlan Cox  *	vm_object_reference_locked:
520b921a12bSAlan Cox  *
521b921a12bSAlan Cox  *	Gets another reference to the given object.
522b921a12bSAlan Cox  *
523b921a12bSAlan Cox  *	The object must be locked.
524b921a12bSAlan Cox  */
525b921a12bSAlan Cox void
526b921a12bSAlan Cox vm_object_reference_locked(vm_object_t object)
527b921a12bSAlan Cox {
528a67d5408SJeff Roberson 	u_int old;
529b921a12bSAlan Cox 
53051df5321SJeff Roberson 	VM_OBJECT_ASSERT_LOCKED(object);
531a67d5408SJeff Roberson 	old = refcount_acquire(&object->ref_count);
5321a0c234eSJeff Roberson 	if (object->type == OBJT_VNODE && old == 0)
5331a0c234eSJeff Roberson 		vref(object->handle);
53498087a06SJeff Roberson 	KASSERT((object->flags & OBJ_DEAD) == 0,
53598087a06SJeff Roberson 	    ("vm_object_reference: Referenced dead object."));
536b921a12bSAlan Cox }
537b921a12bSAlan Cox 
538b921a12bSAlan Cox /*
5399d5abbddSJens Schweikhardt  * Handle deallocating an object of type OBJT_VNODE.
54023955314SAlfred Perlstein  */
54102dd8331SAlan Cox static void
54298087a06SJeff Roberson vm_object_deallocate_vnode(vm_object_t object)
54395e5e988SJohn Dyson {
54495e5e988SJohn Dyson 	struct vnode *vp = (struct vnode *) object->handle;
54526c4e983SJeff Roberson 	bool last;
546219cbf59SEivind Eklund 
5475526d2d9SEivind Eklund 	KASSERT(object->type == OBJT_VNODE,
54898087a06SJeff Roberson 	    ("vm_object_deallocate_vnode: not a vnode object"));
54998087a06SJeff Roberson 	KASSERT(vp != NULL, ("vm_object_deallocate_vnode: missing vp"));
55095e5e988SJohn Dyson 
55126c4e983SJeff Roberson 	/* Object lock to protect handle lookup. */
55226c4e983SJeff Roberson 	last = refcount_release(&object->ref_count);
55326c4e983SJeff Roberson 	VM_OBJECT_RUNLOCK(object);
55426c4e983SJeff Roberson 
55526c4e983SJeff Roberson 	if (!last)
55626c4e983SJeff Roberson 		return;
55726c4e983SJeff Roberson 
558a67d5408SJeff Roberson 	if (!umtx_shm_vnobj_persistent)
5591bdbd705SKonstantin Belousov 		umtx_shm_object_terminated(object);
5601bdbd705SKonstantin Belousov 
56103fa5b34SKonstantin Belousov 	/* vrele may need the vnode lock. */
56247221757SJohn Dyson 	vrele(vp);
56386769ac0SKonstantin Belousov }
564df8bae1dSRodney W. Grimes 
56598087a06SJeff Roberson /*
56698087a06SJeff Roberson  * We dropped a reference on an object and discovered that it had a
56798087a06SJeff Roberson  * single remaining shadow.  This is a sibling of the reference we
56898087a06SJeff Roberson  * dropped.  Attempt to collapse the sibling and backing object.
56998087a06SJeff Roberson  */
57098087a06SJeff Roberson static vm_object_t
57198087a06SJeff Roberson vm_object_deallocate_anon(vm_object_t backing_object)
57298087a06SJeff Roberson {
57398087a06SJeff Roberson 	vm_object_t object;
57498087a06SJeff Roberson 
57598087a06SJeff Roberson 	/* Fetch the final shadow.  */
57698087a06SJeff Roberson 	object = LIST_FIRST(&backing_object->shadow_head);
57798087a06SJeff Roberson 	KASSERT(object != NULL && backing_object->shadow_count == 1,
57898087a06SJeff Roberson 	    ("vm_object_anon_deallocate: ref_count: %d, shadow_count: %d",
57998087a06SJeff Roberson 	    backing_object->ref_count, backing_object->shadow_count));
5804b8365d7SKonstantin Belousov 	KASSERT((object->flags & OBJ_ANON) != 0,
58198087a06SJeff Roberson 	    ("invalid shadow object %p", object));
58298087a06SJeff Roberson 
58398087a06SJeff Roberson 	if (!VM_OBJECT_TRYWLOCK(object)) {
58498087a06SJeff Roberson 		/*
58598087a06SJeff Roberson 		 * Prevent object from disappearing since we do not have a
58698087a06SJeff Roberson 		 * reference.
58798087a06SJeff Roberson 		 */
58898087a06SJeff Roberson 		vm_object_pip_add(object, 1);
58998087a06SJeff Roberson 		VM_OBJECT_WUNLOCK(backing_object);
59098087a06SJeff Roberson 		VM_OBJECT_WLOCK(object);
59198087a06SJeff Roberson 		vm_object_pip_wakeup(object);
59298087a06SJeff Roberson 	} else
59398087a06SJeff Roberson 		VM_OBJECT_WUNLOCK(backing_object);
59498087a06SJeff Roberson 
59598087a06SJeff Roberson 	/*
59698087a06SJeff Roberson 	 * Check for a collapse/terminate race with the last reference holder.
59798087a06SJeff Roberson 	 */
59898087a06SJeff Roberson 	if ((object->flags & (OBJ_DEAD | OBJ_COLLAPSING)) != 0 ||
59998087a06SJeff Roberson 	    !refcount_acquire_if_not_zero(&object->ref_count)) {
60098087a06SJeff Roberson 		VM_OBJECT_WUNLOCK(object);
60198087a06SJeff Roberson 		return (NULL);
60298087a06SJeff Roberson 	}
60398087a06SJeff Roberson 	backing_object = object->backing_object;
60498087a06SJeff Roberson 	if (backing_object != NULL && (backing_object->flags & OBJ_ANON) != 0)
60598087a06SJeff Roberson 		vm_object_collapse(object);
60698087a06SJeff Roberson 	VM_OBJECT_WUNLOCK(object);
60798087a06SJeff Roberson 
60898087a06SJeff Roberson 	return (object);
60998087a06SJeff Roberson }
61098087a06SJeff Roberson 
611df8bae1dSRodney W. Grimes /*
612df8bae1dSRodney W. Grimes  *	vm_object_deallocate:
613df8bae1dSRodney W. Grimes  *
614df8bae1dSRodney W. Grimes  *	Release a reference to the specified object,
615df8bae1dSRodney W. Grimes  *	gained either through a vm_object_allocate
616df8bae1dSRodney W. Grimes  *	or a vm_object_reference call.  When all references
617df8bae1dSRodney W. Grimes  *	are gone, storage associated with this object
618df8bae1dSRodney W. Grimes  *	may be relinquished.
619df8bae1dSRodney W. Grimes  *
620df8bae1dSRodney W. Grimes  *	No object may be locked.
621df8bae1dSRodney W. Grimes  */
62226f9a767SRodney W. Grimes void
6231b40f8c0SMatthew Dillon vm_object_deallocate(vm_object_t object)
624df8bae1dSRodney W. Grimes {
62598087a06SJeff Roberson 	vm_object_t temp;
62626c4e983SJeff Roberson 	bool released;
627df8bae1dSRodney W. Grimes 
628df8bae1dSRodney W. Grimes 	while (object != NULL) {
62951df5321SJeff Roberson 		/*
63051df5321SJeff Roberson 		 * If the reference count goes to 0 we start calling
63151df5321SJeff Roberson 		 * vm_object_terminate() on the object chain.  A ref count
63251df5321SJeff Roberson 		 * of 1 may be a special case depending on the shadow count
63351df5321SJeff Roberson 		 * being 0 or 1.  These cases require a write lock on the
63451df5321SJeff Roberson 		 * object.
63551df5321SJeff Roberson 		 */
63663967687SJeff Roberson 		if ((object->flags & OBJ_ANON) == 0)
63763967687SJeff Roberson 			released = refcount_release_if_gt(&object->ref_count, 1);
63863967687SJeff Roberson 		else
63951df5321SJeff Roberson 			released = refcount_release_if_gt(&object->ref_count, 2);
64051df5321SJeff Roberson 		if (released)
64151df5321SJeff Roberson 			return;
64251df5321SJeff Roberson 
643a67d5408SJeff Roberson 		if (object->type == OBJT_VNODE) {
64426c4e983SJeff Roberson 			VM_OBJECT_RLOCK(object);
64526c4e983SJeff Roberson 			if (object->type == OBJT_VNODE) {
64698087a06SJeff Roberson 				vm_object_deallocate_vnode(object);
647a67d5408SJeff Roberson 				return;
648a67d5408SJeff Roberson 			}
64926c4e983SJeff Roberson 			VM_OBJECT_RUNLOCK(object);
65026c4e983SJeff Roberson 		}
65126c4e983SJeff Roberson 
65226c4e983SJeff Roberson 		VM_OBJECT_WLOCK(object);
65326c4e983SJeff Roberson 		KASSERT(object->ref_count > 0,
65426c4e983SJeff Roberson 		    ("vm_object_deallocate: object deallocated too many times: %d",
65526c4e983SJeff Roberson 		    object->type));
65626c4e983SJeff Roberson 
657b72b0115SAlan Cox 		/*
65898087a06SJeff Roberson 		 * If this is not the final reference to an anonymous
65998087a06SJeff Roberson 		 * object we may need to collapse the shadow chain.
660b72b0115SAlan Cox 		 */
66198087a06SJeff Roberson 		if (!refcount_release(&object->ref_count)) {
66298087a06SJeff Roberson 			if (object->ref_count > 1 ||
66398087a06SJeff Roberson 			    object->shadow_count == 0) {
66498087a06SJeff Roberson 				if ((object->flags & OBJ_ANON) != 0 &&
66598087a06SJeff Roberson 				    object->ref_count == 1)
66698087a06SJeff Roberson 					vm_object_set_flag(object,
66798087a06SJeff Roberson 					    OBJ_ONEMAPPING);
66889f6b863SAttilio Rao 				VM_OBJECT_WUNLOCK(object);
66923b186d3SAlan Cox 				return;
67095e5e988SJohn Dyson 			}
67198087a06SJeff Roberson 
67298087a06SJeff Roberson 			/* Handle collapsing last ref on anonymous objects. */
67398087a06SJeff Roberson 			object = vm_object_deallocate_anon(object);
67498087a06SJeff Roberson 			continue;
67598087a06SJeff Roberson 		}
67698087a06SJeff Roberson 
67798087a06SJeff Roberson 		/*
67898087a06SJeff Roberson 		 * Handle the final reference to an object.  We restart
67998087a06SJeff Roberson 		 * the loop with the backing object to avoid recursion.
68098087a06SJeff Roberson 		 */
6811bdbd705SKonstantin Belousov 		umtx_shm_object_terminated(object);
68224a1cce3SDavid Greenman 		temp = object->backing_object;
683c9917419SAlan Cox 		if (temp != NULL) {
6847079449bSKonstantin Belousov 			KASSERT(object->type == OBJT_DEFAULT ||
6857079449bSKonstantin Belousov 			    object->type == OBJT_SWAP,
6864bace8e7SKonstantin Belousov 			    ("shadowed tmpfs v_object 2 %p", object));
68751b867e5SJeff Roberson 			vm_object_backing_remove(object);
688de5f6a77SJohn Dyson 		}
68998087a06SJeff Roberson 
69098087a06SJeff Roberson 		KASSERT((object->flags & OBJ_DEAD) == 0,
69198087a06SJeff Roberson 		    ("vm_object_deallocate: Terminating dead object."));
692783a68aaSKonstantin Belousov 		vm_object_set_flag(object, OBJ_DEAD);
693df8bae1dSRodney W. Grimes 		vm_object_terminate(object);
694df8bae1dSRodney W. Grimes 		object = temp;
695df8bae1dSRodney W. Grimes 	}
696df8bae1dSRodney W. Grimes }
697df8bae1dSRodney W. Grimes 
698df8bae1dSRodney W. Grimes /*
6992ac78f0eSStephan Uphoff  *	vm_object_destroy removes the object from the global object list
7002ac78f0eSStephan Uphoff  *      and frees the space for the object.
7012ac78f0eSStephan Uphoff  */
7022ac78f0eSStephan Uphoff void
7032ac78f0eSStephan Uphoff vm_object_destroy(vm_object_t object)
7042ac78f0eSStephan Uphoff {
7052ac78f0eSStephan Uphoff 
7062ac78f0eSStephan Uphoff 	/*
7073364c323SKonstantin Belousov 	 * Release the allocation charge.
7083364c323SKonstantin Belousov 	 */
709ef694c1aSEdward Tomasz Napierala 	if (object->cred != NULL) {
710ef694c1aSEdward Tomasz Napierala 		swap_release_by_cred(object->charge, object->cred);
7113364c323SKonstantin Belousov 		object->charge = 0;
712ef694c1aSEdward Tomasz Napierala 		crfree(object->cred);
713ef694c1aSEdward Tomasz Napierala 		object->cred = NULL;
7143364c323SKonstantin Belousov 	}
7153364c323SKonstantin Belousov 
7163364c323SKonstantin Belousov 	/*
7172ac78f0eSStephan Uphoff 	 * Free the space for the object.
7182ac78f0eSStephan Uphoff 	 */
7192ac78f0eSStephan Uphoff 	uma_zfree(obj_zone, object);
7202ac78f0eSStephan Uphoff }
7212ac78f0eSStephan Uphoff 
72251b867e5SJeff Roberson static void
72351b867e5SJeff Roberson vm_object_backing_remove_locked(vm_object_t object)
72451b867e5SJeff Roberson {
72551b867e5SJeff Roberson 	vm_object_t backing_object;
72651b867e5SJeff Roberson 
72751b867e5SJeff Roberson 	backing_object = object->backing_object;
72851b867e5SJeff Roberson 	VM_OBJECT_ASSERT_WLOCKED(object);
72951b867e5SJeff Roberson 	VM_OBJECT_ASSERT_WLOCKED(backing_object);
73051b867e5SJeff Roberson 
73198087a06SJeff Roberson 	KASSERT((object->flags & OBJ_COLLAPSING) == 0,
73298087a06SJeff Roberson 	    ("vm_object_backing_remove: Removing collapsing object."));
73398087a06SJeff Roberson 
73451b867e5SJeff Roberson 	if ((object->flags & OBJ_SHADOWLIST) != 0) {
73551b867e5SJeff Roberson 		LIST_REMOVE(object, shadow_list);
73651b867e5SJeff Roberson 		backing_object->shadow_count--;
73751b867e5SJeff Roberson 		object->flags &= ~OBJ_SHADOWLIST;
73851b867e5SJeff Roberson 	}
73951b867e5SJeff Roberson 	object->backing_object = NULL;
74051b867e5SJeff Roberson }
74151b867e5SJeff Roberson 
74251b867e5SJeff Roberson static void
74351b867e5SJeff Roberson vm_object_backing_remove(vm_object_t object)
74451b867e5SJeff Roberson {
74551b867e5SJeff Roberson 	vm_object_t backing_object;
74651b867e5SJeff Roberson 
74751b867e5SJeff Roberson 	VM_OBJECT_ASSERT_WLOCKED(object);
74851b867e5SJeff Roberson 
74951b867e5SJeff Roberson 	if ((object->flags & OBJ_SHADOWLIST) != 0) {
75051b867e5SJeff Roberson 		backing_object = object->backing_object;
75151b867e5SJeff Roberson 		VM_OBJECT_WLOCK(backing_object);
75251b867e5SJeff Roberson 		vm_object_backing_remove_locked(object);
75351b867e5SJeff Roberson 		VM_OBJECT_WUNLOCK(backing_object);
75451b867e5SJeff Roberson 	} else
75551b867e5SJeff Roberson 		object->backing_object = NULL;
75651b867e5SJeff Roberson }
75751b867e5SJeff Roberson 
75851b867e5SJeff Roberson static void
75951b867e5SJeff Roberson vm_object_backing_insert_locked(vm_object_t object, vm_object_t backing_object)
76051b867e5SJeff Roberson {
76151b867e5SJeff Roberson 
76251b867e5SJeff Roberson 	VM_OBJECT_ASSERT_WLOCKED(object);
76351b867e5SJeff Roberson 
76451b867e5SJeff Roberson 	if ((backing_object->flags & OBJ_ANON) != 0) {
76551b867e5SJeff Roberson 		VM_OBJECT_ASSERT_WLOCKED(backing_object);
76651b867e5SJeff Roberson 		LIST_INSERT_HEAD(&backing_object->shadow_head, object,
76751b867e5SJeff Roberson 		    shadow_list);
76851b867e5SJeff Roberson 		backing_object->shadow_count++;
76951b867e5SJeff Roberson 		object->flags |= OBJ_SHADOWLIST;
77051b867e5SJeff Roberson 	}
77151b867e5SJeff Roberson 	object->backing_object = backing_object;
77251b867e5SJeff Roberson }
77351b867e5SJeff Roberson 
77451b867e5SJeff Roberson static void
77551b867e5SJeff Roberson vm_object_backing_insert(vm_object_t object, vm_object_t backing_object)
77651b867e5SJeff Roberson {
77751b867e5SJeff Roberson 
77851b867e5SJeff Roberson 	VM_OBJECT_ASSERT_WLOCKED(object);
77951b867e5SJeff Roberson 
78051b867e5SJeff Roberson 	if ((backing_object->flags & OBJ_ANON) != 0) {
78151b867e5SJeff Roberson 		VM_OBJECT_WLOCK(backing_object);
78251b867e5SJeff Roberson 		vm_object_backing_insert_locked(object, backing_object);
78351b867e5SJeff Roberson 		VM_OBJECT_WUNLOCK(backing_object);
78451b867e5SJeff Roberson 	} else
78551b867e5SJeff Roberson 		object->backing_object = backing_object;
78651b867e5SJeff Roberson }
78751b867e5SJeff Roberson 
78898087a06SJeff Roberson /*
78998087a06SJeff Roberson  * Insert an object into a backing_object's shadow list with an additional
79098087a06SJeff Roberson  * reference to the backing_object added.
79198087a06SJeff Roberson  */
79298087a06SJeff Roberson static void
79398087a06SJeff Roberson vm_object_backing_insert_ref(vm_object_t object, vm_object_t backing_object)
79498087a06SJeff Roberson {
79598087a06SJeff Roberson 
79698087a06SJeff Roberson 	VM_OBJECT_ASSERT_WLOCKED(object);
79798087a06SJeff Roberson 
79898087a06SJeff Roberson 	if ((backing_object->flags & OBJ_ANON) != 0) {
79998087a06SJeff Roberson 		VM_OBJECT_WLOCK(backing_object);
80098087a06SJeff Roberson 		KASSERT((backing_object->flags & OBJ_DEAD) == 0,
80198087a06SJeff Roberson 		    ("shadowing dead anonymous object"));
80298087a06SJeff Roberson 		vm_object_reference_locked(backing_object);
80398087a06SJeff Roberson 		vm_object_backing_insert_locked(object, backing_object);
80498087a06SJeff Roberson 		vm_object_clear_flag(backing_object, OBJ_ONEMAPPING);
80598087a06SJeff Roberson 		VM_OBJECT_WUNLOCK(backing_object);
80698087a06SJeff Roberson 	} else {
80798087a06SJeff Roberson 		vm_object_reference(backing_object);
80898087a06SJeff Roberson 		object->backing_object = backing_object;
80998087a06SJeff Roberson 	}
81098087a06SJeff Roberson }
81198087a06SJeff Roberson 
81298087a06SJeff Roberson /*
81398087a06SJeff Roberson  * Transfer a backing reference from backing_object to object.
81498087a06SJeff Roberson  */
81598087a06SJeff Roberson static void
81698087a06SJeff Roberson vm_object_backing_transfer(vm_object_t object, vm_object_t backing_object)
81798087a06SJeff Roberson {
81898087a06SJeff Roberson 	vm_object_t new_backing_object;
81998087a06SJeff Roberson 
82098087a06SJeff Roberson 	/*
82198087a06SJeff Roberson 	 * Note that the reference to backing_object->backing_object
82298087a06SJeff Roberson 	 * moves from within backing_object to within object.
82398087a06SJeff Roberson 	 */
82498087a06SJeff Roberson 	vm_object_backing_remove_locked(object);
82598087a06SJeff Roberson 	new_backing_object = backing_object->backing_object;
82698087a06SJeff Roberson 	if (new_backing_object == NULL)
82798087a06SJeff Roberson 		return;
82898087a06SJeff Roberson 	if ((new_backing_object->flags & OBJ_ANON) != 0) {
82998087a06SJeff Roberson 		VM_OBJECT_WLOCK(new_backing_object);
83098087a06SJeff Roberson 		vm_object_backing_remove_locked(backing_object);
83198087a06SJeff Roberson 		vm_object_backing_insert_locked(object, new_backing_object);
83298087a06SJeff Roberson 		VM_OBJECT_WUNLOCK(new_backing_object);
83398087a06SJeff Roberson 	} else {
83498087a06SJeff Roberson 		object->backing_object = new_backing_object;
83598087a06SJeff Roberson 		backing_object->backing_object = NULL;
83698087a06SJeff Roberson 	}
83798087a06SJeff Roberson }
83898087a06SJeff Roberson 
83998087a06SJeff Roberson /*
84098087a06SJeff Roberson  * Wait for a concurrent collapse to settle.
84198087a06SJeff Roberson  */
84298087a06SJeff Roberson static void
84398087a06SJeff Roberson vm_object_collapse_wait(vm_object_t object)
84498087a06SJeff Roberson {
84598087a06SJeff Roberson 
84698087a06SJeff Roberson 	VM_OBJECT_ASSERT_WLOCKED(object);
84798087a06SJeff Roberson 
84898087a06SJeff Roberson 	while ((object->flags & OBJ_COLLAPSING) != 0) {
84998087a06SJeff Roberson 		vm_object_pip_wait(object, "vmcolwait");
85098087a06SJeff Roberson 		counter_u64_add(object_collapse_waits, 1);
85198087a06SJeff Roberson 	}
85298087a06SJeff Roberson }
85398087a06SJeff Roberson 
85498087a06SJeff Roberson /*
85598087a06SJeff Roberson  * Waits for a backing object to clear a pending collapse and returns
85698087a06SJeff Roberson  * it locked if it is an ANON object.
85798087a06SJeff Roberson  */
85898087a06SJeff Roberson static vm_object_t
85998087a06SJeff Roberson vm_object_backing_collapse_wait(vm_object_t object)
86098087a06SJeff Roberson {
86198087a06SJeff Roberson 	vm_object_t backing_object;
86298087a06SJeff Roberson 
86398087a06SJeff Roberson 	VM_OBJECT_ASSERT_WLOCKED(object);
86498087a06SJeff Roberson 
86598087a06SJeff Roberson 	for (;;) {
86698087a06SJeff Roberson 		backing_object = object->backing_object;
86798087a06SJeff Roberson 		if (backing_object == NULL ||
86898087a06SJeff Roberson 		    (backing_object->flags & OBJ_ANON) == 0)
86998087a06SJeff Roberson 			return (NULL);
87098087a06SJeff Roberson 		VM_OBJECT_WLOCK(backing_object);
87198087a06SJeff Roberson 		if ((backing_object->flags & (OBJ_DEAD | OBJ_COLLAPSING)) == 0)
87298087a06SJeff Roberson 			break;
873cdd02f43SMark Johnston 		VM_OBJECT_WUNLOCK(object);
87498087a06SJeff Roberson 		vm_object_pip_sleep(backing_object, "vmbckwait");
87598087a06SJeff Roberson 		counter_u64_add(object_collapse_waits, 1);
87698087a06SJeff Roberson 		VM_OBJECT_WLOCK(object);
87798087a06SJeff Roberson 	}
87898087a06SJeff Roberson 	return (backing_object);
87998087a06SJeff Roberson }
88051b867e5SJeff Roberson 
8812ac78f0eSStephan Uphoff /*
8827bbdb843SRuslan Bukin  *	vm_object_terminate_pages removes any remaining pageable pages
8837bbdb843SRuslan Bukin  *	from the object and resets the object to an empty state.
8847bbdb843SRuslan Bukin  */
8857bbdb843SRuslan Bukin static void
8867bbdb843SRuslan Bukin vm_object_terminate_pages(vm_object_t object)
8877bbdb843SRuslan Bukin {
8887bbdb843SRuslan Bukin 	vm_page_t p, p_next;
8897bbdb843SRuslan Bukin 
8907bbdb843SRuslan Bukin 	VM_OBJECT_ASSERT_WLOCKED(object);
8917bbdb843SRuslan Bukin 
8927bbdb843SRuslan Bukin 	/*
8937bbdb843SRuslan Bukin 	 * Free any remaining pageable pages.  This also removes them from the
8947bbdb843SRuslan Bukin 	 * paging queues.  However, don't free wired pages, just remove them
8957bbdb843SRuslan Bukin 	 * from the object.  Rather than incrementally removing each page from
8967bbdb843SRuslan Bukin 	 * the object, the page and object are reset to any empty state.
8977bbdb843SRuslan Bukin 	 */
8987bbdb843SRuslan Bukin 	TAILQ_FOREACH_SAFE(p, &object->memq, listq, p_next) {
8997bbdb843SRuslan Bukin 		vm_page_assert_unbusied(p);
900fee2a2faSMark Johnston 		KASSERT(p->object == object &&
901fee2a2faSMark Johnston 		    (p->ref_count & VPRC_OBJREF) != 0,
902fee2a2faSMark Johnston 		    ("vm_object_terminate_pages: page %p is inconsistent", p));
903fee2a2faSMark Johnston 
9047bbdb843SRuslan Bukin 		p->object = NULL;
905fee2a2faSMark Johnston 		if (vm_page_drop(p, VPRC_OBJREF) == VPRC_OBJREF) {
9065cd29d0fSMark Johnston 			VM_CNT_INC(v_pfree);
9075cd29d0fSMark Johnston 			vm_page_free(p);
9084074d642SAlan Cox 		}
909fee2a2faSMark Johnston 	}
9102fcd1ff6SKonstantin Belousov 
9117bbdb843SRuslan Bukin 	/*
9127bbdb843SRuslan Bukin 	 * If the object contained any pages, then reset it to an empty state.
9137bbdb843SRuslan Bukin 	 * None of the object's fields, including "resident_page_count", were
9147bbdb843SRuslan Bukin 	 * modified by the preceding loop.
9157bbdb843SRuslan Bukin 	 */
9167bbdb843SRuslan Bukin 	if (object->resident_page_count != 0) {
9177bbdb843SRuslan Bukin 		vm_radix_reclaim_allnodes(&object->rtree);
9187bbdb843SRuslan Bukin 		TAILQ_INIT(&object->memq);
9197bbdb843SRuslan Bukin 		object->resident_page_count = 0;
9207bbdb843SRuslan Bukin 		if (object->type == OBJT_VNODE)
9217bbdb843SRuslan Bukin 			vdrop(object->handle);
9227bbdb843SRuslan Bukin 	}
9237bbdb843SRuslan Bukin }
9247bbdb843SRuslan Bukin 
9257bbdb843SRuslan Bukin /*
926df8bae1dSRodney W. Grimes  *	vm_object_terminate actually destroys the specified object, freeing
927df8bae1dSRodney W. Grimes  *	up all previously used resources.
928df8bae1dSRodney W. Grimes  *
929df8bae1dSRodney W. Grimes  *	The object must be locked.
9301c7c3c6aSMatthew Dillon  *	This routine may block.
931df8bae1dSRodney W. Grimes  */
93295e5e988SJohn Dyson void
9331b40f8c0SMatthew Dillon vm_object_terminate(vm_object_t object)
934df8bae1dSRodney W. Grimes {
93598087a06SJeff Roberson 
93689f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
937783a68aaSKonstantin Belousov 	KASSERT((object->flags & OBJ_DEAD) != 0,
938783a68aaSKonstantin Belousov 	    ("terminating non-dead obj %p", object));
93998087a06SJeff Roberson 	KASSERT((object->flags & OBJ_COLLAPSING) == 0,
94098087a06SJeff Roberson 	    ("terminating collapsing obj %p", object));
94198087a06SJeff Roberson 	KASSERT(object->backing_object == NULL,
94298087a06SJeff Roberson 	    ("terminating shadow obj %p", object));
943bef608bdSJohn Dyson 
944cf27e0d1SJeff Roberson 	/*
945ffae7ea9SKonstantin Belousov 	 * Wait for the pageout daemon and other current users to be
946ffae7ea9SKonstantin Belousov 	 * done with the object.  Note that new paging_in_progress
947ffae7ea9SKonstantin Belousov 	 * users can come after this wait, but they must check
948ffae7ea9SKonstantin Belousov 	 * OBJ_DEAD flag set (without unlocking the object), and avoid
949ffae7ea9SKonstantin Belousov 	 * the object being terminated.
950cf27e0d1SJeff Roberson 	 */
951cf27e0d1SJeff Roberson 	vm_object_pip_wait(object, "objtrm");
952cf27e0d1SJeff Roberson 
953971dd342SAlfred Perlstein 	KASSERT(object->ref_count == 0,
954971dd342SAlfred Perlstein 	    ("vm_object_terminate: object with references, ref_count=%d",
955971dd342SAlfred Perlstein 	    object->ref_count));
956996c772fSJohn Dyson 
9577bbdb843SRuslan Bukin 	if ((object->flags & OBJ_PG_DTOR) == 0)
9587bbdb843SRuslan Bukin 		vm_object_terminate_pages(object);
959bef608bdSJohn Dyson 
960f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0
961f8a47341SAlan Cox 	if (__predict_false(!LIST_EMPTY(&object->rvq)))
962f8a47341SAlan Cox 		vm_reserv_break_all(object);
963f8a47341SAlan Cox #endif
9647bfda801SAlan Cox 
965e735691bSJohn Baldwin 	KASSERT(object->cred == NULL || object->type == OBJT_DEFAULT ||
9667079449bSKonstantin Belousov 	    (object->flags & OBJ_SWAP) != 0,
967e735691bSJohn Baldwin 	    ("%s: non-swap obj %p has cred", __func__, object));
968e735691bSJohn Baldwin 
9692d8acc0fSJohn Dyson 	/*
9709fcfb650SDavid Greenman 	 * Let the pager know object is dead.
9719fcfb650SDavid Greenman 	 */
9729fcfb650SDavid Greenman 	vm_pager_deallocate(object);
97389f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(object);
9749fcfb650SDavid Greenman 
9752ac78f0eSStephan Uphoff 	vm_object_destroy(object);
97647221757SJohn Dyson }
977df8bae1dSRodney W. Grimes 
978edf93b25SAlan Cox /*
979edf93b25SAlan Cox  * Make the page read-only so that we can clear the object flags.  However, if
980edf93b25SAlan Cox  * this is a nosync mmap then the object is likely to stay dirty so do not
981edf93b25SAlan Cox  * mess with the page and do not clear the object flags.  Returns TRUE if the
982edf93b25SAlan Cox  * page should be flushed, and FALSE otherwise.
983edf93b25SAlan Cox  */
9843280870dSKonstantin Belousov static boolean_t
98567d0e293SJeff Roberson vm_object_page_remove_write(vm_page_t p, int flags, boolean_t *allclean)
9863280870dSKonstantin Belousov {
9873280870dSKonstantin Belousov 
988fff5403fSJeff Roberson 	vm_page_assert_busied(p);
989fff5403fSJeff Roberson 
9903280870dSKonstantin Belousov 	/*
9913280870dSKonstantin Belousov 	 * If we have been asked to skip nosync pages and this is a
9923280870dSKonstantin Belousov 	 * nosync page, skip it.  Note that the object flags were not
9933280870dSKonstantin Belousov 	 * cleared in this case so we do not have to set them.
9943280870dSKonstantin Belousov 	 */
9955cff1f4dSMark Johnston 	if ((flags & OBJPC_NOSYNC) != 0 && (p->a.flags & PGA_NOSYNC) != 0) {
99667d0e293SJeff Roberson 		*allclean = FALSE;
9973280870dSKonstantin Belousov 		return (FALSE);
9983280870dSKonstantin Belousov 	} else {
9993280870dSKonstantin Belousov 		pmap_remove_write(p);
10003280870dSKonstantin Belousov 		return (p->dirty != 0);
10013280870dSKonstantin Belousov 	}
10023280870dSKonstantin Belousov }
10033280870dSKonstantin Belousov 
1004df8bae1dSRodney W. Grimes /*
1005df8bae1dSRodney W. Grimes  *	vm_object_page_clean
1006df8bae1dSRodney W. Grimes  *
10074f79d873SMatthew Dillon  *	Clean all dirty pages in the specified range of object.  Leaves page
10084f79d873SMatthew Dillon  * 	on whatever queue it is currently on.   If NOSYNC is set then do not
1009fff5403fSJeff Roberson  *	write out pages with PGA_NOSYNC set (originally comes from MAP_NOSYNC),
10104f79d873SMatthew Dillon  *	leaving the object dirty.
101126f9a767SRodney W. Grimes  *
10128d34a3bfSKonstantin Belousov  *	For swap objects backing tmpfs regular files, do not flush anything,
10138d34a3bfSKonstantin Belousov  *	but remove write protection on the mapped pages to update mtime through
10148d34a3bfSKonstantin Belousov  *	mmaped writes.
10158d34a3bfSKonstantin Belousov  *
101643b7990eSMatthew Dillon  *	When stuffing pages asynchronously, allow clustering.  XXX we need a
101743b7990eSMatthew Dillon  *	synchronous clustering mode implementation.
101843b7990eSMatthew Dillon  *
101926f9a767SRodney W. Grimes  *	Odd semantics: if start == end, we clean everything.
102026f9a767SRodney W. Grimes  *
102126f9a767SRodney W. Grimes  *	The object must be locked.
1022126d6082SKonstantin Belousov  *
1023126d6082SKonstantin Belousov  *	Returns FALSE if some page from the range was not written, as
1024126d6082SKonstantin Belousov  *	reported by the pager, and TRUE otherwise.
102526f9a767SRodney W. Grimes  */
1026126d6082SKonstantin Belousov boolean_t
102717f3095dSAlan Cox vm_object_page_clean(vm_object_t object, vm_ooffset_t start, vm_ooffset_t end,
1028e239bb97SKonstantin Belousov     int flags)
1029f6b04d2bSDavid Greenman {
1030e239bb97SKonstantin Belousov 	vm_page_t np, p;
103117f3095dSAlan Cox 	vm_pindex_t pi, tend, tstart;
1032126d6082SKonstantin Belousov 	int curgeneration, n, pagerflags;
103367d0e293SJeff Roberson 	boolean_t eio, res, allclean;
1034f6b04d2bSDavid Greenman 
103589f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
1036e5f299ffSKonstantin Belousov 
10378d34a3bfSKonstantin Belousov 	if (!vm_object_mightbedirty(object) || object->resident_page_count == 0)
1038126d6082SKonstantin Belousov 		return (TRUE);
1039f6b04d2bSDavid Greenman 
1040e239bb97SKonstantin Belousov 	pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) != 0 ?
1041e239bb97SKonstantin Belousov 	    VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK;
1042e239bb97SKonstantin Belousov 	pagerflags |= (flags & OBJPC_INVAL) != 0 ? VM_PAGER_PUT_INVAL : 0;
1043e239bb97SKonstantin Belousov 
104417f3095dSAlan Cox 	tstart = OFF_TO_IDX(start);
104517f3095dSAlan Cox 	tend = (end == 0) ? object->size : OFF_TO_IDX(end + PAGE_MASK);
104667d0e293SJeff Roberson 	allclean = tstart == 0 && tend >= object->size;
1047126d6082SKonstantin Belousov 	res = TRUE;
1048f6b04d2bSDavid Greenman 
1049bd7e5f99SJohn Dyson rescan:
10502d8acc0fSJohn Dyson 	curgeneration = object->generation;
10512d8acc0fSJohn Dyson 
105217f3095dSAlan Cox 	for (p = vm_page_find_least(object, tstart); p != NULL; p = np) {
1053bd7e5f99SJohn Dyson 		pi = p->pindex;
1054e239bb97SKonstantin Belousov 		if (pi >= tend)
1055e239bb97SKonstantin Belousov 			break;
1056e239bb97SKonstantin Belousov 		np = TAILQ_NEXT(p, listq);
10570012f373SJeff Roberson 		if (vm_page_none_valid(p))
1058aef922f5SJohn Dyson 			continue;
105963e97555SJeff Roberson 		if (vm_page_busy_acquire(p, VM_ALLOC_WAITFAIL) == 0) {
106067d0e293SJeff Roberson 			if (object->generation != curgeneration &&
106167d0e293SJeff Roberson 			    (flags & OBJPC_SYNC) != 0)
1062e239bb97SKonstantin Belousov 				goto rescan;
1063780636b7SKonstantin Belousov 			np = vm_page_find_least(object, pi);
1064780636b7SKonstantin Belousov 			continue;
1065f6b04d2bSDavid Greenman 		}
106667d0e293SJeff Roberson 		if (!vm_object_page_remove_write(p, flags, &allclean)) {
106763e97555SJeff Roberson 			vm_page_xunbusy(p);
1068bd7e5f99SJohn Dyson 			continue;
106963e97555SJeff Roberson 		}
10708d34a3bfSKonstantin Belousov 		if (object->type == OBJT_VNODE) {
10713280870dSKonstantin Belousov 			n = vm_object_page_collect_flush(object, p, pagerflags,
107267d0e293SJeff Roberson 			    flags, &allclean, &eio);
1073126d6082SKonstantin Belousov 			if (eio) {
1074126d6082SKonstantin Belousov 				res = FALSE;
107567d0e293SJeff Roberson 				allclean = FALSE;
1076126d6082SKonstantin Belousov 			}
107767d0e293SJeff Roberson 			if (object->generation != curgeneration &&
107867d0e293SJeff Roberson 			    (flags & OBJPC_SYNC) != 0)
1079b9b7a4beSMatthew Dillon 				goto rescan;
1080031ec8c1SKonstantin Belousov 
1081031ec8c1SKonstantin Belousov 			/*
1082031ec8c1SKonstantin Belousov 			 * If the VOP_PUTPAGES() did a truncated write, so
1083031ec8c1SKonstantin Belousov 			 * that even the first page of the run is not fully
1084031ec8c1SKonstantin Belousov 			 * written, vm_pageout_flush() returns 0 as the run
1085031ec8c1SKonstantin Belousov 			 * length.  Since the condition that caused truncated
1086031ec8c1SKonstantin Belousov 			 * write may be permanent, e.g. exhausted free space,
1087031ec8c1SKonstantin Belousov 			 * accepting n == 0 would cause an infinite loop.
1088031ec8c1SKonstantin Belousov 			 *
1089031ec8c1SKonstantin Belousov 			 * Forwarding the iterator leaves the unwritten page
1090031ec8c1SKonstantin Belousov 			 * behind, but there is not much we can do there if
1091031ec8c1SKonstantin Belousov 			 * filesystem refuses to write it.
1092031ec8c1SKonstantin Belousov 			 */
1093126d6082SKonstantin Belousov 			if (n == 0) {
1094031ec8c1SKonstantin Belousov 				n = 1;
109567d0e293SJeff Roberson 				allclean = FALSE;
1096126d6082SKonstantin Belousov 			}
10978d34a3bfSKonstantin Belousov 		} else {
10988d34a3bfSKonstantin Belousov 			n = 1;
10998d34a3bfSKonstantin Belousov 			vm_page_xunbusy(p);
11008d34a3bfSKonstantin Belousov 		}
1101e239bb97SKonstantin Belousov 		np = vm_page_find_least(object, pi + n);
1102b9b7a4beSMatthew Dillon 	}
1103b9b7a4beSMatthew Dillon #if 0
1104e239bb97SKonstantin Belousov 	VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC) ? MNT_WAIT : 0);
1105b9b7a4beSMatthew Dillon #endif
1106b9b7a4beSMatthew Dillon 
11078d34a3bfSKonstantin Belousov 	/*
11088d34a3bfSKonstantin Belousov 	 * Leave updating cleangeneration for tmpfs objects to tmpfs
11098d34a3bfSKonstantin Belousov 	 * scan.  It needs to update mtime, which happens for other
11108d34a3bfSKonstantin Belousov 	 * filesystems during page writeouts.
11118d34a3bfSKonstantin Belousov 	 */
11128d34a3bfSKonstantin Belousov 	if (allclean && object->type == OBJT_VNODE)
111367d0e293SJeff Roberson 		object->cleangeneration = curgeneration;
1114126d6082SKonstantin Belousov 	return (res);
1115b9b7a4beSMatthew Dillon }
1116b9b7a4beSMatthew Dillon 
1117b9b7a4beSMatthew Dillon static int
11183280870dSKonstantin Belousov vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags,
111967d0e293SJeff Roberson     int flags, boolean_t *allclean, boolean_t *eio)
1120b9b7a4beSMatthew Dillon {
11213157c503SKonstantin Belousov 	vm_page_t ma[vm_pageout_page_count], p_first, tp;
11223157c503SKonstantin Belousov 	int count, i, mreq, runlen;
1123b9b7a4beSMatthew Dillon 
11247bec141bSKip Macy 	vm_page_lock_assert(p, MA_NOTOWNED);
112563e97555SJeff Roberson 	vm_page_assert_xbusied(p);
112689f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
11273157c503SKonstantin Belousov 
11283157c503SKonstantin Belousov 	count = 1;
11293157c503SKonstantin Belousov 	mreq = 0;
11303157c503SKonstantin Belousov 
11313157c503SKonstantin Belousov 	for (tp = p; count < vm_pageout_page_count; count++) {
11323157c503SKonstantin Belousov 		tp = vm_page_next(tp);
113363e97555SJeff Roberson 		if (tp == NULL || vm_page_tryxbusy(tp) == 0)
1134bd7e5f99SJohn Dyson 			break;
113567d0e293SJeff Roberson 		if (!vm_object_page_remove_write(tp, flags, allclean)) {
113663e97555SJeff Roberson 			vm_page_xunbusy(tp);
1137bd7e5f99SJohn Dyson 			break;
1138bd7e5f99SJohn Dyson 		}
113963e97555SJeff Roberson 	}
1140aef922f5SJohn Dyson 
11413157c503SKonstantin Belousov 	for (p_first = p; count < vm_pageout_page_count; count++) {
11423157c503SKonstantin Belousov 		tp = vm_page_prev(p_first);
114363e97555SJeff Roberson 		if (tp == NULL || vm_page_tryxbusy(tp) == 0)
1144bd7e5f99SJohn Dyson 			break;
114567d0e293SJeff Roberson 		if (!vm_object_page_remove_write(tp, flags, allclean)) {
114663e97555SJeff Roberson 			vm_page_xunbusy(tp);
1147bd7e5f99SJohn Dyson 			break;
114863e97555SJeff Roberson 		}
11493157c503SKonstantin Belousov 		p_first = tp;
11503157c503SKonstantin Belousov 		mreq++;
1151bd7e5f99SJohn Dyson 	}
1152bd7e5f99SJohn Dyson 
11533157c503SKonstantin Belousov 	for (tp = p_first, i = 0; i < count; tp = TAILQ_NEXT(tp, listq), i++)
11543157c503SKonstantin Belousov 		ma[i] = tp;
1155cf2819ccSJohn Dyson 
1156126d6082SKonstantin Belousov 	vm_pageout_flush(ma, count, pagerflags, mreq, &runlen, eio);
11571e8a675cSKonstantin Belousov 	return (runlen);
115826f9a767SRodney W. Grimes }
1159df8bae1dSRodney W. Grimes 
11601efb74fbSJohn Dyson /*
1161950f8459SAlan Cox  * Note that there is absolutely no sense in writing out
1162950f8459SAlan Cox  * anonymous objects, so we track down the vnode object
1163950f8459SAlan Cox  * to write out.
1164950f8459SAlan Cox  * We invalidate (remove) all pages from the address space
1165950f8459SAlan Cox  * for semantic correctness.
1166950f8459SAlan Cox  *
11676bbee8e2SAlan Cox  * If the backing object is a device object with unmanaged pages, then any
11686bbee8e2SAlan Cox  * mappings to the specified range of pages must be removed before this
11696bbee8e2SAlan Cox  * function is called.
11706bbee8e2SAlan Cox  *
1171950f8459SAlan Cox  * Note: certain anonymous maps, such as MAP_NOSYNC maps,
1172950f8459SAlan Cox  * may start out with a NULL object.
1173950f8459SAlan Cox  */
1174126d6082SKonstantin Belousov boolean_t
1175950f8459SAlan Cox vm_object_sync(vm_object_t object, vm_ooffset_t offset, vm_size_t size,
1176950f8459SAlan Cox     boolean_t syncio, boolean_t invalidate)
1177950f8459SAlan Cox {
1178950f8459SAlan Cox 	vm_object_t backing_object;
1179950f8459SAlan Cox 	struct vnode *vp;
11803b582b4eSTor Egge 	struct mount *mp;
1181126d6082SKonstantin Belousov 	int error, flags, fsync_after;
1182126d6082SKonstantin Belousov 	boolean_t res;
1183950f8459SAlan Cox 
1184950f8459SAlan Cox 	if (object == NULL)
1185126d6082SKonstantin Belousov 		return (TRUE);
1186126d6082SKonstantin Belousov 	res = TRUE;
1187126d6082SKonstantin Belousov 	error = 0;
118889f6b863SAttilio Rao 	VM_OBJECT_WLOCK(object);
1189950f8459SAlan Cox 	while ((backing_object = object->backing_object) != NULL) {
119089f6b863SAttilio Rao 		VM_OBJECT_WLOCK(backing_object);
119156e0670fSAlan Cox 		offset += object->backing_object_offset;
119289f6b863SAttilio Rao 		VM_OBJECT_WUNLOCK(object);
1193950f8459SAlan Cox 		object = backing_object;
1194950f8459SAlan Cox 		if (object->size < OFF_TO_IDX(offset + size))
1195950f8459SAlan Cox 			size = IDX_TO_OFF(object->size) - offset;
1196950f8459SAlan Cox 	}
1197950f8459SAlan Cox 	/*
1198950f8459SAlan Cox 	 * Flush pages if writing is allowed, invalidate them
1199950f8459SAlan Cox 	 * if invalidation requested.  Pages undergoing I/O
1200950f8459SAlan Cox 	 * will be ignored by vm_object_page_remove().
1201950f8459SAlan Cox 	 *
1202950f8459SAlan Cox 	 * We cannot lock the vnode and then wait for paging
1203950f8459SAlan Cox 	 * to complete without deadlocking against vm_fault.
1204950f8459SAlan Cox 	 * Instead we simply call vm_object_page_remove() and
1205950f8459SAlan Cox 	 * allow it to block internally on a page-by-page
1206950f8459SAlan Cox 	 * basis when it encounters pages undergoing async
1207950f8459SAlan Cox 	 * I/O.
1208950f8459SAlan Cox 	 */
1209950f8459SAlan Cox 	if (object->type == OBJT_VNODE &&
121067d0e293SJeff Roberson 	    vm_object_mightbedirty(object) != 0 &&
12115bf94937SKonstantin Belousov 	    ((vp = object->handle)->v_vflag & VV_NOSYNC) == 0) {
121289f6b863SAttilio Rao 		VM_OBJECT_WUNLOCK(object);
12133b582b4eSTor Egge 		(void) vn_start_write(vp, &mp, V_WAIT);
1214cb05b60aSAttilio Rao 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
121575ff604aSKonstantin Belousov 		if (syncio && !invalidate && offset == 0 &&
1216d1780e8dSKonstantin Belousov 		    atop(size) == object->size) {
121775ff604aSKonstantin Belousov 			/*
121875ff604aSKonstantin Belousov 			 * If syncing the whole mapping of the file,
121975ff604aSKonstantin Belousov 			 * it is faster to schedule all the writes in
122075ff604aSKonstantin Belousov 			 * async mode, also allowing the clustering,
122175ff604aSKonstantin Belousov 			 * and then wait for i/o to complete.
122275ff604aSKonstantin Belousov 			 */
122375ff604aSKonstantin Belousov 			flags = 0;
122475ff604aSKonstantin Belousov 			fsync_after = TRUE;
122575ff604aSKonstantin Belousov 		} else {
1226950f8459SAlan Cox 			flags = (syncio || invalidate) ? OBJPC_SYNC : 0;
122775ff604aSKonstantin Belousov 			flags |= invalidate ? (OBJPC_SYNC | OBJPC_INVAL) : 0;
122875ff604aSKonstantin Belousov 			fsync_after = FALSE;
122975ff604aSKonstantin Belousov 		}
123089f6b863SAttilio Rao 		VM_OBJECT_WLOCK(object);
1231126d6082SKonstantin Belousov 		res = vm_object_page_clean(object, offset, offset + size,
1232126d6082SKonstantin Belousov 		    flags);
123389f6b863SAttilio Rao 		VM_OBJECT_WUNLOCK(object);
123475ff604aSKonstantin Belousov 		if (fsync_after)
1235126d6082SKonstantin Belousov 			error = VOP_FSYNC(vp, MNT_WAIT, curthread);
1236b249ce48SMateusz Guzik 		VOP_UNLOCK(vp);
12373b582b4eSTor Egge 		vn_finished_write(mp);
1238126d6082SKonstantin Belousov 		if (error != 0)
1239126d6082SKonstantin Belousov 			res = FALSE;
124089f6b863SAttilio Rao 		VM_OBJECT_WLOCK(object);
1241950f8459SAlan Cox 	}
1242950f8459SAlan Cox 	if ((object->type == OBJT_VNODE ||
1243950f8459SAlan Cox 	     object->type == OBJT_DEVICE) && invalidate) {
12446bbee8e2SAlan Cox 		if (object->type == OBJT_DEVICE)
12456bbee8e2SAlan Cox 			/*
12466bbee8e2SAlan Cox 			 * The option OBJPR_NOTMAPPED must be passed here
12476bbee8e2SAlan Cox 			 * because vm_object_page_remove() cannot remove
12486bbee8e2SAlan Cox 			 * unmanaged mappings.
12496bbee8e2SAlan Cox 			 */
12506bbee8e2SAlan Cox 			flags = OBJPR_NOTMAPPED;
12516bbee8e2SAlan Cox 		else if (old_msync)
12526195b24aSKonstantin Belousov 			flags = 0;
12536bbee8e2SAlan Cox 		else
12546195b24aSKonstantin Belousov 			flags = OBJPR_CLEANONLY;
12556bbee8e2SAlan Cox 		vm_object_page_remove(object, OFF_TO_IDX(offset),
12566bbee8e2SAlan Cox 		    OFF_TO_IDX(offset + size + PAGE_MASK), flags);
1257950f8459SAlan Cox 	}
125889f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(object);
1259126d6082SKonstantin Belousov 	return (res);
1260950f8459SAlan Cox }
1261950f8459SAlan Cox 
1262950f8459SAlan Cox /*
1263aa3650eaSMark Johnston  * Determine whether the given advice can be applied to the object.  Advice is
1264aa3650eaSMark Johnston  * not applied to unmanaged pages since they never belong to page queues, and
1265aa3650eaSMark Johnston  * since MADV_FREE is destructive, it can apply only to anonymous pages that
1266aa3650eaSMark Johnston  * have been mapped at most once.
1267aa3650eaSMark Johnston  */
1268aa3650eaSMark Johnston static bool
1269aa3650eaSMark Johnston vm_object_advice_applies(vm_object_t object, int advice)
1270aa3650eaSMark Johnston {
1271aa3650eaSMark Johnston 
1272aa3650eaSMark Johnston 	if ((object->flags & OBJ_UNMANAGED) != 0)
1273aa3650eaSMark Johnston 		return (false);
1274aa3650eaSMark Johnston 	if (advice != MADV_FREE)
1275aa3650eaSMark Johnston 		return (true);
127663967687SJeff Roberson 	return ((object->flags & (OBJ_ONEMAPPING | OBJ_ANON)) ==
127763967687SJeff Roberson 	    (OBJ_ONEMAPPING | OBJ_ANON));
1278aa3650eaSMark Johnston }
1279aa3650eaSMark Johnston 
1280aa3650eaSMark Johnston static void
1281aa3650eaSMark Johnston vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex,
1282aa3650eaSMark Johnston     vm_size_t size)
1283aa3650eaSMark Johnston {
1284aa3650eaSMark Johnston 
12851390a5cbSKonstantin Belousov 	if (advice == MADV_FREE)
12861390a5cbSKonstantin Belousov 		vm_pager_freespace(object, pindex, size);
1287aa3650eaSMark Johnston }
1288aa3650eaSMark Johnston 
1289aa3650eaSMark Johnston /*
1290867a482dSJohn Dyson  *	vm_object_madvise:
1291867a482dSJohn Dyson  *
1292867a482dSJohn Dyson  *	Implements the madvise function at the object/page level.
12931c7c3c6aSMatthew Dillon  *
1294193b9358SAlan Cox  *	MADV_WILLNEED	(any object)
1295193b9358SAlan Cox  *
1296193b9358SAlan Cox  *	    Activate the specified pages if they are resident.
1297193b9358SAlan Cox  *
1298193b9358SAlan Cox  *	MADV_DONTNEED	(any object)
1299193b9358SAlan Cox  *
1300193b9358SAlan Cox  *	    Deactivate the specified pages if they are resident.
1301193b9358SAlan Cox  *
1302193b9358SAlan Cox  *	MADV_FREE	(OBJT_DEFAULT/OBJT_SWAP objects,
1303193b9358SAlan Cox  *			 OBJ_ONEMAPPING only)
1304193b9358SAlan Cox  *
1305193b9358SAlan Cox  *	    Deactivate and clean the specified pages if they are
1306193b9358SAlan Cox  *	    resident.  This permits the process to reuse the pages
1307193b9358SAlan Cox  *	    without faulting or the kernel to reclaim the pages
1308193b9358SAlan Cox  *	    without I/O.
1309867a482dSJohn Dyson  */
1310867a482dSJohn Dyson void
131192a59946SJohn Baldwin vm_object_madvise(vm_object_t object, vm_pindex_t pindex, vm_pindex_t end,
1312c2655a40SMark Johnston     int advice)
1313867a482dSJohn Dyson {
131492a59946SJohn Baldwin 	vm_pindex_t tpindex;
131534567de7SAlan Cox 	vm_object_t backing_object, tobject;
1316aa3650eaSMark Johnston 	vm_page_t m, tm;
1317867a482dSJohn Dyson 
1318867a482dSJohn Dyson 	if (object == NULL)
1319867a482dSJohn Dyson 		return;
1320c2655a40SMark Johnston 
13216e20a165SJohn Dyson relookup:
1322aa3650eaSMark Johnston 	VM_OBJECT_WLOCK(object);
1323aa3650eaSMark Johnston 	if (!vm_object_advice_applies(object, advice)) {
1324aa3650eaSMark Johnston 		VM_OBJECT_WUNLOCK(object);
1325aa3650eaSMark Johnston 		return;
13266e20a165SJohn Dyson 	}
1327aa3650eaSMark Johnston 	for (m = vm_page_find_least(object, pindex); pindex < end; pindex++) {
1328aa3650eaSMark Johnston 		tobject = object;
1329c2655a40SMark Johnston 
13301ce137beSMatthew Dillon 		/*
1331aa3650eaSMark Johnston 		 * If the next page isn't resident in the top-level object, we
1332aa3650eaSMark Johnston 		 * need to search the shadow chain.  When applying MADV_FREE, we
1333aa3650eaSMark Johnston 		 * take care to release any swap space used to store
1334aa3650eaSMark Johnston 		 * non-resident pages.
1335aa3650eaSMark Johnston 		 */
1336aa3650eaSMark Johnston 		if (m == NULL || pindex < m->pindex) {
1337aa3650eaSMark Johnston 			/*
1338aa3650eaSMark Johnston 			 * Optimize a common case: if the top-level object has
1339aa3650eaSMark Johnston 			 * no backing object, we can skip over the non-resident
1340aa3650eaSMark Johnston 			 * range in constant time.
13411ce137beSMatthew Dillon 			 */
1342c2655a40SMark Johnston 			if (object->backing_object == NULL) {
1343c2655a40SMark Johnston 				tpindex = (m != NULL && m->pindex < end) ?
1344c2655a40SMark Johnston 				    m->pindex : end;
1345aa3650eaSMark Johnston 				vm_object_madvise_freespace(object, advice,
1346aa3650eaSMark Johnston 				    pindex, tpindex - pindex);
1347c2655a40SMark Johnston 				if ((pindex = tpindex) == end)
1348c2655a40SMark Johnston 					break;
1349aa3650eaSMark Johnston 				goto next_page;
1350aa3650eaSMark Johnston 			}
1351aa3650eaSMark Johnston 
1352aa3650eaSMark Johnston 			tpindex = pindex;
1353aa3650eaSMark Johnston 			do {
1354aa3650eaSMark Johnston 				vm_object_madvise_freespace(tobject, advice,
1355aa3650eaSMark Johnston 				    tpindex, 1);
13561ce137beSMatthew Dillon 				/*
1357aa3650eaSMark Johnston 				 * Prepare to search the next object in the
1358aa3650eaSMark Johnston 				 * chain.
13591ce137beSMatthew Dillon 				 */
136034567de7SAlan Cox 				backing_object = tobject->backing_object;
136134567de7SAlan Cox 				if (backing_object == NULL)
1362aa3650eaSMark Johnston 					goto next_pindex;
136389f6b863SAttilio Rao 				VM_OBJECT_WLOCK(backing_object);
1364aa3650eaSMark Johnston 				tpindex +=
1365aa3650eaSMark Johnston 				    OFF_TO_IDX(tobject->backing_object_offset);
13669b98b796SAlan Cox 				if (tobject != object)
136789f6b863SAttilio Rao 					VM_OBJECT_WUNLOCK(tobject);
136834567de7SAlan Cox 				tobject = backing_object;
1369aa3650eaSMark Johnston 				if (!vm_object_advice_applies(tobject, advice))
1370aa3650eaSMark Johnston 					goto next_pindex;
1371aa3650eaSMark Johnston 			} while ((tm = vm_page_lookup(tobject, tpindex)) ==
1372aa3650eaSMark Johnston 			    NULL);
1373aa3650eaSMark Johnston 		} else {
1374aa3650eaSMark Johnston next_page:
1375aa3650eaSMark Johnston 			tm = m;
1376aa3650eaSMark Johnston 			m = TAILQ_NEXT(m, listq);
1377c2655a40SMark Johnston 		}
1378c2655a40SMark Johnston 
1379867a482dSJohn Dyson 		/*
13800012f373SJeff Roberson 		 * If the page is not in a normal state, skip it.  The page
13810012f373SJeff Roberson 		 * can not be invalidated while the object lock is held.
1382867a482dSJohn Dyson 		 */
13830012f373SJeff Roberson 		if (!vm_page_all_valid(tm) || vm_page_wired(tm))
1384aa3650eaSMark Johnston 			goto next_pindex;
1385aa3650eaSMark Johnston 		KASSERT((tm->flags & PG_FICTITIOUS) == 0,
1386aa3650eaSMark Johnston 		    ("vm_object_madvise: page %p is fictitious", tm));
1387aa3650eaSMark Johnston 		KASSERT((tm->oflags & VPO_UNMANAGED) == 0,
1388aa3650eaSMark Johnston 		    ("vm_object_madvise: page %p is not managed", tm));
138963e97555SJeff Roberson 		if (vm_page_tryxbusy(tm) == 0) {
1390aa3650eaSMark Johnston 			if (object != tobject)
1391aa3650eaSMark Johnston 				VM_OBJECT_WUNLOCK(object);
1392c2655a40SMark Johnston 			if (advice == MADV_WILLNEED) {
1393b11b56b5SAlan Cox 				/*
1394b11b56b5SAlan Cox 				 * Reference the page before unlocking and
1395b11b56b5SAlan Cox 				 * sleeping so that the page daemon is less
1396b11b56b5SAlan Cox 				 * likely to reclaim it.
1397b11b56b5SAlan Cox 				 */
1398aa3650eaSMark Johnston 				vm_page_aflag_set(tm, PGA_REFERENCED);
1399567e51e1SAlan Cox 			}
140087b64663SMark Johnston 			if (!vm_page_busy_sleep(tm, "madvpo", 0))
140187b64663SMark Johnston 				VM_OBJECT_WUNLOCK(tobject);
14026e20a165SJohn Dyson   			goto relookup;
140334567de7SAlan Cox 		}
1404aa3650eaSMark Johnston 		vm_page_advise(tm, advice);
140563e97555SJeff Roberson 		vm_page_xunbusy(tm);
1406aa3650eaSMark Johnston 		vm_object_madvise_freespace(tobject, advice, tm->pindex, 1);
1407aa3650eaSMark Johnston next_pindex:
14089b98b796SAlan Cox 		if (tobject != object)
140989f6b863SAttilio Rao 			VM_OBJECT_WUNLOCK(tobject);
1410867a482dSJohn Dyson 	}
141189f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(object);
1412867a482dSJohn Dyson }
1413867a482dSJohn Dyson 
1414867a482dSJohn Dyson /*
1415df8bae1dSRodney W. Grimes  *	vm_object_shadow:
1416df8bae1dSRodney W. Grimes  *
1417df8bae1dSRodney W. Grimes  *	Create a new object which is backed by the
1418df8bae1dSRodney W. Grimes  *	specified existing object range.  The source
1419df8bae1dSRodney W. Grimes  *	object reference is deallocated.
1420df8bae1dSRodney W. Grimes  *
1421df8bae1dSRodney W. Grimes  *	The new object and offset into that object
1422df8bae1dSRodney W. Grimes  *	are returned in the source parameters.
1423df8bae1dSRodney W. Grimes  */
142426f9a767SRodney W. Grimes void
142567388836SKonstantin Belousov vm_object_shadow(vm_object_t *object, vm_ooffset_t *offset, vm_size_t length,
142667388836SKonstantin Belousov     struct ucred *cred, bool shared)
1427df8bae1dSRodney W. Grimes {
1428d031cff1SMatthew Dillon 	vm_object_t source;
1429d031cff1SMatthew Dillon 	vm_object_t result;
1430df8bae1dSRodney W. Grimes 
1431df8bae1dSRodney W. Grimes 	source = *object;
1432df8bae1dSRodney W. Grimes 
1433df8bae1dSRodney W. Grimes 	/*
14349a2f6362SAlan Cox 	 * Don't create the new object if the old object isn't shared.
143563967687SJeff Roberson 	 *
143663967687SJeff Roberson 	 * If we hold the only reference we can guarantee that it won't
143763967687SJeff Roberson 	 * increase while we have the map locked.  Otherwise the race is
143863967687SJeff Roberson 	 * harmless and we will end up with an extra shadow object that
143963967687SJeff Roberson 	 * will be collapsed later.
14409a2f6362SAlan Cox 	 */
144163967687SJeff Roberson 	if (source != NULL && source->ref_count == 1 &&
144232362449SKonstantin Belousov 	    (source->flags & OBJ_ANON) != 0)
14439a2f6362SAlan Cox 		return;
14449a2f6362SAlan Cox 
14459a2f6362SAlan Cox 	/*
1446570a2f4aSAlan Cox 	 * Allocate a new object with the given length.
1447df8bae1dSRodney W. Grimes 	 */
144867388836SKonstantin Belousov 	result = vm_object_allocate_anon(atop(length), source, cred, length);
1449df8bae1dSRodney W. Grimes 
1450df8bae1dSRodney W. Grimes 	/*
145151b867e5SJeff Roberson 	 * Store the offset into the source object, and fix up the offset into
145251b867e5SJeff Roberson 	 * the new object.
145351b867e5SJeff Roberson 	 */
145451b867e5SJeff Roberson 	result->backing_object_offset = *offset;
145551b867e5SJeff Roberson 
145667388836SKonstantin Belousov 	if (shared || source != NULL) {
145767388836SKonstantin Belousov 		VM_OBJECT_WLOCK(result);
145867388836SKonstantin Belousov 
145951b867e5SJeff Roberson 		/*
146067388836SKonstantin Belousov 		 * The new object shadows the source object, adding a
146167388836SKonstantin Belousov 		 * reference to it.  Our caller changes his reference
146267388836SKonstantin Belousov 		 * to point to the new object, removing a reference to
146367388836SKonstantin Belousov 		 * the source object.  Net result: no change of
146467388836SKonstantin Belousov 		 * reference count, unless the caller needs to add one
146567388836SKonstantin Belousov 		 * more reference due to forking a shared map entry.
146667388836SKonstantin Belousov 		 */
146767388836SKonstantin Belousov 		if (shared) {
146867388836SKonstantin Belousov 			vm_object_reference_locked(result);
146967388836SKonstantin Belousov 			vm_object_clear_flag(result, OBJ_ONEMAPPING);
147067388836SKonstantin Belousov 		}
147167388836SKonstantin Belousov 
147267388836SKonstantin Belousov 		/*
147367388836SKonstantin Belousov 		 * Try to optimize the result object's page color when
147467388836SKonstantin Belousov 		 * shadowing in order to maintain page coloring
147567388836SKonstantin Belousov 		 * consistency in the combined shadowed object.
1476df8bae1dSRodney W. Grimes 		 */
1477570a2f4aSAlan Cox 		if (source != NULL) {
147851b867e5SJeff Roberson 			vm_object_backing_insert(result, source);
14793f289c3fSJeff Roberson 			result->domain = source->domain;
1480f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0
14817b54b1a9SAlan Cox 			result->flags |= source->flags & OBJ_COLORED;
148267388836SKonstantin Belousov 			result->pg_color = (source->pg_color +
148367388836SKonstantin Belousov 			    OFF_TO_IDX(*offset)) & ((1 << (VM_NFREEORDER -
148467388836SKonstantin Belousov 			    1)) - 1);
1485f8a47341SAlan Cox #endif
148667388836SKonstantin Belousov 		}
148751b867e5SJeff Roberson 		VM_OBJECT_WUNLOCK(result);
1488de5f6a77SJohn Dyson 	}
1489df8bae1dSRodney W. Grimes 
1490df8bae1dSRodney W. Grimes 	/*
1491df8bae1dSRodney W. Grimes 	 * Return the new things
1492df8bae1dSRodney W. Grimes 	 */
1493df8bae1dSRodney W. Grimes 	*offset = 0;
1494df8bae1dSRodney W. Grimes 	*object = result;
1495df8bae1dSRodney W. Grimes }
1496df8bae1dSRodney W. Grimes 
1497c5aaa06dSAlan Cox /*
1498c5aaa06dSAlan Cox  *	vm_object_split:
1499c5aaa06dSAlan Cox  *
1500c5aaa06dSAlan Cox  * Split the pages in a map entry into a new object.  This affords
1501c5aaa06dSAlan Cox  * easier removal of unused pages, and keeps object inheritance from
1502c5aaa06dSAlan Cox  * being a negative impact on memory usage.
1503c5aaa06dSAlan Cox  */
1504c5aaa06dSAlan Cox void
1505c5aaa06dSAlan Cox vm_object_split(vm_map_entry_t entry)
1506c5aaa06dSAlan Cox {
1507aec9e7d8SMark Johnston 	vm_page_t m, m_busy, m_next;
150898087a06SJeff Roberson 	vm_object_t orig_object, new_object, backing_object;
150973000556SAlan Cox 	vm_pindex_t idx, offidxstart;
151073000556SAlan Cox 	vm_size_t size;
1511c5aaa06dSAlan Cox 
1512c5aaa06dSAlan Cox 	orig_object = entry->object.vm_object;
151398087a06SJeff Roberson 	KASSERT((orig_object->flags & OBJ_ONEMAPPING) != 0,
151498087a06SJeff Roberson 	    ("vm_object_split:  Splitting object with multiple mappings."));
151563967687SJeff Roberson 	if ((orig_object->flags & OBJ_ANON) == 0)
1516c5aaa06dSAlan Cox 		return;
1517c5aaa06dSAlan Cox 	if (orig_object->ref_count <= 1)
1518c5aaa06dSAlan Cox 		return;
151989f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(orig_object);
1520c5aaa06dSAlan Cox 
15214da9f125SAlan Cox 	offidxstart = OFF_TO_IDX(entry->offset);
152295442adfSAlan Cox 	size = atop(entry->end - entry->start);
1523c5aaa06dSAlan Cox 
15244da9f125SAlan Cox 	/*
15254da9f125SAlan Cox 	 * If swap_pager_copy() is later called, it will convert new_object
15264da9f125SAlan Cox 	 * into a swap object.
15274da9f125SAlan Cox 	 */
152867388836SKonstantin Belousov 	new_object = vm_object_allocate_anon(size, orig_object,
152967388836SKonstantin Belousov 	    orig_object->cred, ptoa(size));
1530c5aaa06dSAlan Cox 
1531c5474b8fSAlan Cox 	/*
153298087a06SJeff Roberson 	 * We must wait for the orig_object to complete any in-progress
153398087a06SJeff Roberson 	 * collapse so that the swap blocks are stable below.  The
153498087a06SJeff Roberson 	 * additional reference on backing_object by new object will
153598087a06SJeff Roberson 	 * prevent further collapse operations until split completes.
153698087a06SJeff Roberson 	 */
153798087a06SJeff Roberson 	VM_OBJECT_WLOCK(orig_object);
153898087a06SJeff Roberson 	vm_object_collapse_wait(orig_object);
153998087a06SJeff Roberson 
154098087a06SJeff Roberson 	/*
1541c5474b8fSAlan Cox 	 * At this point, the new object is still private, so the order in
1542c5474b8fSAlan Cox 	 * which the original and new objects are locked does not matter.
1543c5474b8fSAlan Cox 	 */
154489f6b863SAttilio Rao 	VM_OBJECT_WLOCK(new_object);
15453f289c3fSJeff Roberson 	new_object->domain = orig_object->domain;
154698087a06SJeff Roberson 	backing_object = orig_object->backing_object;
154798087a06SJeff Roberson 	if (backing_object != NULL) {
154898087a06SJeff Roberson 		vm_object_backing_insert_ref(new_object, backing_object);
1549c5aaa06dSAlan Cox 		new_object->backing_object_offset =
15504da9f125SAlan Cox 		    orig_object->backing_object_offset + entry->offset;
1551c5aaa06dSAlan Cox 	}
1552ef694c1aSEdward Tomasz Napierala 	if (orig_object->cred != NULL) {
1553ef694c1aSEdward Tomasz Napierala 		crhold(orig_object->cred);
15543364c323SKonstantin Belousov 		KASSERT(orig_object->charge >= ptoa(size),
15553364c323SKonstantin Belousov 		    ("orig_object->charge < 0"));
15563364c323SKonstantin Belousov 		orig_object->charge -= ptoa(size);
15573364c323SKonstantin Belousov 	}
155898087a06SJeff Roberson 
155998087a06SJeff Roberson 	/*
156098087a06SJeff Roberson 	 * Mark the split operation so that swap_pager_getpages() knows
156198087a06SJeff Roberson 	 * that the object is in transition.
156298087a06SJeff Roberson 	 */
156398087a06SJeff Roberson 	vm_object_set_flag(orig_object, OBJ_SPLIT);
1564aec9e7d8SMark Johnston 	m_busy = NULL;
1565aec9e7d8SMark Johnston #ifdef INVARIANTS
1566aec9e7d8SMark Johnston 	idx = 0;
1567aec9e7d8SMark Johnston #endif
1568c5aaa06dSAlan Cox retry:
1569b382c10aSKonstantin Belousov 	m = vm_page_find_least(orig_object, offidxstart);
1570aec9e7d8SMark Johnston 	KASSERT(m == NULL || idx <= m->pindex - offidxstart,
1571aec9e7d8SMark Johnston 	    ("%s: object %p was repopulated", __func__, orig_object));
157273000556SAlan Cox 	for (; m != NULL && (idx = m->pindex - offidxstart) < size;
157373000556SAlan Cox 	    m = m_next) {
157473000556SAlan Cox 		m_next = TAILQ_NEXT(m, listq);
1575c5aaa06dSAlan Cox 
1576c5aaa06dSAlan Cox 		/*
1577c5aaa06dSAlan Cox 		 * We must wait for pending I/O to complete before we can
1578c5aaa06dSAlan Cox 		 * rename the page.
1579c5aaa06dSAlan Cox 		 *
1580c5aaa06dSAlan Cox 		 * We do not have to VM_PROT_NONE the page as mappings should
1581c5aaa06dSAlan Cox 		 * not be changed by this operation.
1582c5aaa06dSAlan Cox 		 */
158363e97555SJeff Roberson 		if (vm_page_tryxbusy(m) == 0) {
158489f6b863SAttilio Rao 			VM_OBJECT_WUNLOCK(new_object);
158587b64663SMark Johnston 			if (vm_page_busy_sleep(m, "spltwt", 0))
158687b64663SMark Johnston 				VM_OBJECT_WLOCK(orig_object);
158789f6b863SAttilio Rao 			VM_OBJECT_WLOCK(new_object);
1588c5aaa06dSAlan Cox 			goto retry;
1589de33beddSAlan Cox 		}
1590e946b949SAttilio Rao 
15914bf95d00SJeff Roberson 		/*
15924bf95d00SJeff Roberson 		 * The page was left invalid.  Likely placed there by
15934bf95d00SJeff Roberson 		 * an incomplete fault.  Just remove and ignore.
15944bf95d00SJeff Roberson 		 */
15954bf95d00SJeff Roberson 		if (vm_page_none_valid(m)) {
15964bf95d00SJeff Roberson 			if (vm_page_remove(m))
15974bf95d00SJeff Roberson 				vm_page_free(m);
15984bf95d00SJeff Roberson 			continue;
15994bf95d00SJeff Roberson 		}
16004bf95d00SJeff Roberson 
16013453bca8SAlan Cox 		/* vm_page_rename() will dirty the page. */
1602e946b949SAttilio Rao 		if (vm_page_rename(m, new_object, idx)) {
160363e97555SJeff Roberson 			vm_page_xunbusy(m);
1604e946b949SAttilio Rao 			VM_OBJECT_WUNLOCK(new_object);
1605e946b949SAttilio Rao 			VM_OBJECT_WUNLOCK(orig_object);
16068d6fbbb8SJeff Roberson 			vm_radix_wait();
1607e946b949SAttilio Rao 			VM_OBJECT_WLOCK(orig_object);
1608e946b949SAttilio Rao 			VM_OBJECT_WLOCK(new_object);
1609e946b949SAttilio Rao 			goto retry;
1610e946b949SAttilio Rao 		}
161163e97555SJeff Roberson 
1612b5f359b7SAlan Cox #if VM_NRESERVLEVEL > 0
1613b5f359b7SAlan Cox 		/*
1614b5f359b7SAlan Cox 		 * If some of the reservation's allocated pages remain with
1615b5f359b7SAlan Cox 		 * the original object, then transferring the reservation to
1616b5f359b7SAlan Cox 		 * the new object is neither particularly beneficial nor
1617b5f359b7SAlan Cox 		 * particularly harmful as compared to leaving the reservation
1618b5f359b7SAlan Cox 		 * with the original object.  If, however, all of the
1619b5f359b7SAlan Cox 		 * reservation's allocated pages are transferred to the new
1620b5f359b7SAlan Cox 		 * object, then transferring the reservation is typically
1621b5f359b7SAlan Cox 		 * beneficial.  Determining which of these two cases applies
1622b5f359b7SAlan Cox 		 * would be more costly than unconditionally renaming the
1623b5f359b7SAlan Cox 		 * reservation.
1624b5f359b7SAlan Cox 		 */
1625b5f359b7SAlan Cox 		vm_reserv_rename(m, new_object, orig_object, offidxstart);
1626b5f359b7SAlan Cox #endif
1627aec9e7d8SMark Johnston 
1628aec9e7d8SMark Johnston 		/*
1629aec9e7d8SMark Johnston 		 * orig_object's type may change while sleeping, so keep track
1630aec9e7d8SMark Johnston 		 * of the beginning of the busied range.
1631aec9e7d8SMark Johnston 		 */
16328da1c098SJeff Roberson 		if (orig_object->type != OBJT_SWAP)
16338da1c098SJeff Roberson 			vm_page_xunbusy(m);
1634aec9e7d8SMark Johnston 		else if (m_busy == NULL)
1635aec9e7d8SMark Johnston 			m_busy = m;
1636c5aaa06dSAlan Cox 	}
16374b8365d7SKonstantin Belousov 	if ((orig_object->flags & OBJ_SWAP) != 0) {
1638c5aaa06dSAlan Cox 		/*
1639c7c8dd7eSAlan Cox 		 * swap_pager_copy() can sleep, in which case the orig_object's
1640c7c8dd7eSAlan Cox 		 * and new_object's locks are released and reacquired.
1641c5aaa06dSAlan Cox 		 */
1642c5aaa06dSAlan Cox 		swap_pager_copy(orig_object, new_object, offidxstart, 0);
1643aec9e7d8SMark Johnston 		if (m_busy != NULL)
1644aec9e7d8SMark Johnston 			TAILQ_FOREACH_FROM(m_busy, &new_object->memq, listq)
1645aec9e7d8SMark Johnston 				vm_page_xunbusy(m_busy);
1646c5aaa06dSAlan Cox 	}
164798087a06SJeff Roberson 	vm_object_clear_flag(orig_object, OBJ_SPLIT);
164889f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(orig_object);
164989f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(new_object);
1650c5aaa06dSAlan Cox 	entry->object.vm_object = new_object;
1651c5aaa06dSAlan Cox 	entry->offset = 0LL;
1652c5aaa06dSAlan Cox 	vm_object_deallocate(orig_object);
165389f6b863SAttilio Rao 	VM_OBJECT_WLOCK(new_object);
1654c5aaa06dSAlan Cox }
1655c5aaa06dSAlan Cox 
165699a1570aSKonstantin Belousov static vm_page_t
165798087a06SJeff Roberson vm_object_collapse_scan_wait(vm_object_t object, vm_page_t p)
165899a1570aSKonstantin Belousov {
165999a1570aSKonstantin Belousov 	vm_object_t backing_object;
166099a1570aSKonstantin Belousov 
166199a1570aSKonstantin Belousov 	VM_OBJECT_ASSERT_WLOCKED(object);
166299a1570aSKonstantin Belousov 	backing_object = object->backing_object;
166399a1570aSKonstantin Belousov 	VM_OBJECT_ASSERT_WLOCKED(backing_object);
166499a1570aSKonstantin Belousov 
166599a1570aSKonstantin Belousov 	KASSERT(p == NULL || p->object == object || p->object == backing_object,
166699a1570aSKonstantin Belousov 	    ("invalid ownership %p %p %p", p, object, backing_object));
16678d6fbbb8SJeff Roberson 	/* The page is only NULL when rename fails. */
16684cdea4a8SJeff Roberson 	if (p == NULL) {
16696a14746cSRyan Libby 		VM_OBJECT_WUNLOCK(object);
16706a14746cSRyan Libby 		VM_OBJECT_WUNLOCK(backing_object);
16718d6fbbb8SJeff Roberson 		vm_radix_wait();
167299a1570aSKonstantin Belousov 		VM_OBJECT_WLOCK(object);
167387b64663SMark Johnston 	} else if (p->object == object) {
167487b64663SMark Johnston 		VM_OBJECT_WUNLOCK(backing_object);
167587b64663SMark Johnston 		if (vm_page_busy_sleep(p, "vmocol", 0))
167687b64663SMark Johnston 			VM_OBJECT_WLOCK(object);
167787b64663SMark Johnston 	} else {
167887b64663SMark Johnston 		VM_OBJECT_WUNLOCK(object);
167987b64663SMark Johnston 		if (!vm_page_busy_sleep(p, "vmocol", 0))
168087b64663SMark Johnston 			VM_OBJECT_WUNLOCK(backing_object);
168187b64663SMark Johnston 		VM_OBJECT_WLOCK(object);
168287b64663SMark Johnston 	}
168399a1570aSKonstantin Belousov 	VM_OBJECT_WLOCK(backing_object);
168499a1570aSKonstantin Belousov 	return (TAILQ_FIRST(&backing_object->memq));
168599a1570aSKonstantin Belousov }
168699a1570aSKonstantin Belousov 
168799a1570aSKonstantin Belousov static bool
16884cc8daf7SConrad Meyer vm_object_scan_all_shadowed(vm_object_t object)
16894cc8daf7SConrad Meyer {
16904cc8daf7SConrad Meyer 	vm_object_t backing_object;
16914cc8daf7SConrad Meyer 	vm_page_t p, pp;
169277d6fd97SKonstantin Belousov 	vm_pindex_t backing_offset_index, new_pindex, pi, ps;
16934cc8daf7SConrad Meyer 
16944cc8daf7SConrad Meyer 	VM_OBJECT_ASSERT_WLOCKED(object);
16954cc8daf7SConrad Meyer 	VM_OBJECT_ASSERT_WLOCKED(object->backing_object);
16964cc8daf7SConrad Meyer 
16974cc8daf7SConrad Meyer 	backing_object = object->backing_object;
16984cc8daf7SConrad Meyer 
169963967687SJeff Roberson 	if ((backing_object->flags & OBJ_ANON) == 0)
17004cc8daf7SConrad Meyer 		return (false);
17014cc8daf7SConrad Meyer 
170277d6fd97SKonstantin Belousov 	pi = backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
170377d6fd97SKonstantin Belousov 	p = vm_page_find_least(backing_object, pi);
170477d6fd97SKonstantin Belousov 	ps = swap_pager_find_least(backing_object, pi);
17054cc8daf7SConrad Meyer 
17064cc8daf7SConrad Meyer 	/*
170777d6fd97SKonstantin Belousov 	 * Only check pages inside the parent object's range and
170877d6fd97SKonstantin Belousov 	 * inside the parent object's mapping of the backing object.
17094cc8daf7SConrad Meyer 	 */
171077d6fd97SKonstantin Belousov 	for (;; pi++) {
171177d6fd97SKonstantin Belousov 		if (p != NULL && p->pindex < pi)
171277d6fd97SKonstantin Belousov 			p = TAILQ_NEXT(p, listq);
171377d6fd97SKonstantin Belousov 		if (ps < pi)
171477d6fd97SKonstantin Belousov 			ps = swap_pager_find_least(backing_object, pi);
171577d6fd97SKonstantin Belousov 		if (p == NULL && ps >= backing_object->size)
171677d6fd97SKonstantin Belousov 			break;
171777d6fd97SKonstantin Belousov 		else if (p == NULL)
171877d6fd97SKonstantin Belousov 			pi = ps;
171977d6fd97SKonstantin Belousov 		else
172077d6fd97SKonstantin Belousov 			pi = MIN(p->pindex, ps);
172177d6fd97SKonstantin Belousov 
172277d6fd97SKonstantin Belousov 		new_pindex = pi - backing_offset_index;
172377d6fd97SKonstantin Belousov 		if (new_pindex >= object->size)
172477d6fd97SKonstantin Belousov 			break;
17254cc8daf7SConrad Meyer 
1726cd0047f3SKonstantin Belousov 		if (p != NULL) {
17274cc8daf7SConrad Meyer 			/*
1728cd0047f3SKonstantin Belousov 			 * If the backing object page is busy a
1729cd0047f3SKonstantin Belousov 			 * grandparent or older page may still be
1730cd0047f3SKonstantin Belousov 			 * undergoing CoW.  It is not safe to collapse
1731cd0047f3SKonstantin Belousov 			 * the backing object until it is quiesced.
173258447749SJeff Roberson 			 */
1733cd0047f3SKonstantin Belousov 			if (vm_page_tryxbusy(p) == 0)
173458447749SJeff Roberson 				return (false);
173558447749SJeff Roberson 
173658447749SJeff Roberson 			/*
1737cd0047f3SKonstantin Belousov 			 * We raced with the fault handler that left
1738cd0047f3SKonstantin Belousov 			 * newly allocated invalid page on the object
1739cd0047f3SKonstantin Belousov 			 * queue and retried.
1740cd0047f3SKonstantin Belousov 			 */
1741cd0047f3SKonstantin Belousov 			if (!vm_page_all_valid(p))
1742cd0047f3SKonstantin Belousov 				goto unbusy_ret;
1743cd0047f3SKonstantin Belousov 		}
1744cd0047f3SKonstantin Belousov 
1745cd0047f3SKonstantin Belousov 		/*
17464cc8daf7SConrad Meyer 		 * See if the parent has the page or if the parent's object
17474cc8daf7SConrad Meyer 		 * pager has the page.  If the parent has the page but the page
17484cc8daf7SConrad Meyer 		 * is not valid, the parent's object pager must have the page.
17494cc8daf7SConrad Meyer 		 *
17504cc8daf7SConrad Meyer 		 * If this fails, the parent does not completely shadow the
17514cc8daf7SConrad Meyer 		 * object and we might as well give up now.
17524cc8daf7SConrad Meyer 		 */
17534cc8daf7SConrad Meyer 		pp = vm_page_lookup(object, new_pindex);
1754cd0047f3SKonstantin Belousov 
17550012f373SJeff Roberson 		/*
1756cd0047f3SKonstantin Belousov 		 * The valid check here is stable due to object lock
1757cd0047f3SKonstantin Belousov 		 * being required to clear valid and initiate paging.
1758cd0047f3SKonstantin Belousov 		 * Busy of p disallows fault handler to validate pp.
17590012f373SJeff Roberson 		 */
17600012f373SJeff Roberson 		if ((pp == NULL || vm_page_none_valid(pp)) &&
17614cc8daf7SConrad Meyer 		    !vm_pager_has_page(object, new_pindex, NULL, NULL))
1762cd0047f3SKonstantin Belousov 			goto unbusy_ret;
1763cd0047f3SKonstantin Belousov 		if (p != NULL)
1764cd0047f3SKonstantin Belousov 			vm_page_xunbusy(p);
17654cc8daf7SConrad Meyer 	}
17664cc8daf7SConrad Meyer 	return (true);
1767cd0047f3SKonstantin Belousov 
1768cd0047f3SKonstantin Belousov unbusy_ret:
1769cd0047f3SKonstantin Belousov 	if (p != NULL)
1770cd0047f3SKonstantin Belousov 		vm_page_xunbusy(p);
1771cd0047f3SKonstantin Belousov 	return (false);
17724cc8daf7SConrad Meyer }
17734cc8daf7SConrad Meyer 
177498087a06SJeff Roberson static void
177598087a06SJeff Roberson vm_object_collapse_scan(vm_object_t object)
17762ad1a3f7SMatthew Dillon {
17772ad1a3f7SMatthew Dillon 	vm_object_t backing_object;
177899a1570aSKonstantin Belousov 	vm_page_t next, p, pp;
177999a1570aSKonstantin Belousov 	vm_pindex_t backing_offset_index, new_pindex;
17802ad1a3f7SMatthew Dillon 
178189f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
178289f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object->backing_object);
17832ad1a3f7SMatthew Dillon 
17842ad1a3f7SMatthew Dillon 	backing_object = object->backing_object;
17852ad1a3f7SMatthew Dillon 	backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
17862ad1a3f7SMatthew Dillon 
17872ad1a3f7SMatthew Dillon 	/*
17882ad1a3f7SMatthew Dillon 	 * Our scan
17892ad1a3f7SMatthew Dillon 	 */
17904cc8daf7SConrad Meyer 	for (p = TAILQ_FIRST(&backing_object->memq); p != NULL; p = next) {
179199a1570aSKonstantin Belousov 		next = TAILQ_NEXT(p, listq);
179299a1570aSKonstantin Belousov 		new_pindex = p->pindex - backing_offset_index;
17932ad1a3f7SMatthew Dillon 
17942ad1a3f7SMatthew Dillon 		/*
17952ad1a3f7SMatthew Dillon 		 * Check for busy page
17962ad1a3f7SMatthew Dillon 		 */
179763e97555SJeff Roberson 		if (vm_page_tryxbusy(p) == 0) {
179898087a06SJeff Roberson 			next = vm_object_collapse_scan_wait(object, p);
17992ad1a3f7SMatthew Dillon 			continue;
18002ad1a3f7SMatthew Dillon 		}
18012ad1a3f7SMatthew Dillon 
180298087a06SJeff Roberson 		KASSERT(object->backing_object == backing_object,
180398087a06SJeff Roberson 		    ("vm_object_collapse_scan: backing object mismatch %p != %p",
180498087a06SJeff Roberson 		    object->backing_object, backing_object));
180599a1570aSKonstantin Belousov 		KASSERT(p->object == backing_object,
180698087a06SJeff Roberson 		    ("vm_object_collapse_scan: object mismatch %p != %p",
180798087a06SJeff Roberson 		    p->object, backing_object));
18082ad1a3f7SMatthew Dillon 
180999a1570aSKonstantin Belousov 		if (p->pindex < backing_offset_index ||
181099a1570aSKonstantin Belousov 		    new_pindex >= object->size) {
18111390a5cbSKonstantin Belousov 			vm_pager_freespace(backing_object, p->pindex, 1);
1812e946b949SAttilio Rao 
1813f6d89838SAlan Cox 			KASSERT(!pmap_page_is_mapped(p),
1814f6d89838SAlan Cox 			    ("freeing mapped page %p", p));
18150fd977b3SMark Johnston 			if (vm_page_remove(p))
18162ad1a3f7SMatthew Dillon 				vm_page_free(p);
18172ad1a3f7SMatthew Dillon 			continue;
18182ad1a3f7SMatthew Dillon 		}
18192ad1a3f7SMatthew Dillon 
1820cd0047f3SKonstantin Belousov 		if (!vm_page_all_valid(p)) {
1821cd0047f3SKonstantin Belousov 			KASSERT(!pmap_page_is_mapped(p),
1822cd0047f3SKonstantin Belousov 			    ("freeing mapped page %p", p));
1823cd0047f3SKonstantin Belousov 			if (vm_page_remove(p))
1824cd0047f3SKonstantin Belousov 				vm_page_free(p);
1825cd0047f3SKonstantin Belousov 			continue;
1826cd0047f3SKonstantin Belousov 		}
1827cd0047f3SKonstantin Belousov 
18282ad1a3f7SMatthew Dillon 		pp = vm_page_lookup(object, new_pindex);
182963e97555SJeff Roberson 		if (pp != NULL && vm_page_tryxbusy(pp) == 0) {
183063e97555SJeff Roberson 			vm_page_xunbusy(p);
1831e18cc7bfSMax Laier 			/*
18324cc8daf7SConrad Meyer 			 * The page in the parent is busy and possibly not
18334cc8daf7SConrad Meyer 			 * (yet) valid.  Until its state is finalized by the
18344cc8daf7SConrad Meyer 			 * busy bit owner, we can't tell whether it shadows the
183598087a06SJeff Roberson 			 * original page.
1836e18cc7bfSMax Laier 			 */
183798087a06SJeff Roberson 			next = vm_object_collapse_scan_wait(object, pp);
1838e18cc7bfSMax Laier 			continue;
1839e18cc7bfSMax Laier 		}
184099a1570aSKonstantin Belousov 
18414bf95d00SJeff Roberson 		if (pp != NULL && vm_page_none_valid(pp)) {
18424bf95d00SJeff Roberson 			/*
18434bf95d00SJeff Roberson 			 * The page was invalid in the parent.  Likely placed
18444bf95d00SJeff Roberson 			 * there by an incomplete fault.  Just remove and
18454bf95d00SJeff Roberson 			 * ignore.  p can replace it.
18464bf95d00SJeff Roberson 			 */
18474bf95d00SJeff Roberson 			if (vm_page_remove(pp))
18484bf95d00SJeff Roberson 				vm_page_free(pp);
18494bf95d00SJeff Roberson 			pp = NULL;
18504bf95d00SJeff Roberson 		}
185199a1570aSKonstantin Belousov 
18524cc8daf7SConrad Meyer 		if (pp != NULL || vm_pager_has_page(object, new_pindex, NULL,
18534cc8daf7SConrad Meyer 			NULL)) {
185499a1570aSKonstantin Belousov 			/*
18554cc8daf7SConrad Meyer 			 * The page already exists in the parent OR swap exists
18564cc8daf7SConrad Meyer 			 * for this location in the parent.  Leave the parent's
18574cc8daf7SConrad Meyer 			 * page alone.  Destroy the original page from the
18584cc8daf7SConrad Meyer 			 * backing object.
185999a1570aSKonstantin Belousov 			 */
18601390a5cbSKonstantin Belousov 			vm_pager_freespace(backing_object, p->pindex, 1);
1861f6d89838SAlan Cox 			KASSERT(!pmap_page_is_mapped(p),
1862f6d89838SAlan Cox 			    ("freeing mapped page %p", p));
18630fd977b3SMark Johnston 			if (vm_page_remove(p))
18642ad1a3f7SMatthew Dillon 				vm_page_free(p);
186563e97555SJeff Roberson 			if (pp != NULL)
186663e97555SJeff Roberson 				vm_page_xunbusy(pp);
18672ad1a3f7SMatthew Dillon 			continue;
18682ad1a3f7SMatthew Dillon 		}
18692ad1a3f7SMatthew Dillon 
1870e946b949SAttilio Rao 		/*
18714cc8daf7SConrad Meyer 		 * Page does not exist in parent, rename the page from the
18724cc8daf7SConrad Meyer 		 * backing object to the main object.
1873e946b949SAttilio Rao 		 *
18744cc8daf7SConrad Meyer 		 * If the page was mapped to a process, it can remain mapped
18753453bca8SAlan Cox 		 * through the rename.  vm_page_rename() will dirty the page.
1876e946b949SAttilio Rao 		 */
1877e946b949SAttilio Rao 		if (vm_page_rename(p, object, new_pindex)) {
187863e97555SJeff Roberson 			vm_page_xunbusy(p);
187998087a06SJeff Roberson 			next = vm_object_collapse_scan_wait(object, NULL);
1880e946b949SAttilio Rao 			continue;
1881e946b949SAttilio Rao 		}
188214a5dc17SAttilio Rao 
188314a5dc17SAttilio Rao 		/* Use the old pindex to free the right page. */
18841390a5cbSKonstantin Belousov 		vm_pager_freespace(backing_object, new_pindex +
18851390a5cbSKonstantin Belousov 		    backing_offset_index, 1);
1886e946b949SAttilio Rao 
1887f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0
1888f8a47341SAlan Cox 		/*
1889f8a47341SAlan Cox 		 * Rename the reservation.
1890f8a47341SAlan Cox 		 */
1891f8a47341SAlan Cox 		vm_reserv_rename(p, object, backing_object,
1892f8a47341SAlan Cox 		    backing_offset_index);
1893f8a47341SAlan Cox #endif
18948da1c098SJeff Roberson 		vm_page_xunbusy(p);
18952ad1a3f7SMatthew Dillon 	}
18962fe6e4d7SDavid Greenman 	return;
18972fe6e4d7SDavid Greenman }
18982fe6e4d7SDavid Greenman 
1899df8bae1dSRodney W. Grimes /*
1900df8bae1dSRodney W. Grimes  *	vm_object_collapse:
1901df8bae1dSRodney W. Grimes  *
1902df8bae1dSRodney W. Grimes  *	Collapse an object with the object backing it.
1903df8bae1dSRodney W. Grimes  *	Pages in the backing object are moved into the
1904df8bae1dSRodney W. Grimes  *	parent, and the backing object is deallocated.
1905df8bae1dSRodney W. Grimes  */
190626f9a767SRodney W. Grimes void
19071b40f8c0SMatthew Dillon vm_object_collapse(vm_object_t object)
1908df8bae1dSRodney W. Grimes {
190998f139daSKonstantin Belousov 	vm_object_t backing_object, new_backing_object;
191098f139daSKonstantin Belousov 
191189f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
191223955314SAlfred Perlstein 
1913df8bae1dSRodney W. Grimes 	while (TRUE) {
191498087a06SJeff Roberson 		KASSERT((object->flags & (OBJ_DEAD | OBJ_ANON)) == OBJ_ANON,
191598087a06SJeff Roberson 		    ("collapsing invalid object"));
191698087a06SJeff Roberson 
1917df8bae1dSRodney W. Grimes 		/*
191898087a06SJeff Roberson 		 * Wait for the backing_object to finish any pending
191998087a06SJeff Roberson 		 * collapse so that the caller sees the shortest possible
192098087a06SJeff Roberson 		 * shadow chain.
1921df8bae1dSRodney W. Grimes 		 */
192298087a06SJeff Roberson 		backing_object = vm_object_backing_collapse_wait(object);
192398087a06SJeff Roberson 		if (backing_object == NULL)
192498087a06SJeff Roberson 			return;
192598087a06SJeff Roberson 
192698087a06SJeff Roberson 		KASSERT(object->ref_count > 0 &&
192798087a06SJeff Roberson 		    object->ref_count > object->shadow_count,
192898087a06SJeff Roberson 		    ("collapse with invalid ref %d or shadow %d count.",
192998087a06SJeff Roberson 		    object->ref_count, object->shadow_count));
193098087a06SJeff Roberson 		KASSERT((backing_object->flags &
193198087a06SJeff Roberson 		    (OBJ_COLLAPSING | OBJ_DEAD)) == 0,
193298087a06SJeff Roberson 		    ("vm_object_collapse: Backing object already collapsing."));
193398087a06SJeff Roberson 		KASSERT((object->flags & (OBJ_COLLAPSING | OBJ_DEAD)) == 0,
193498087a06SJeff Roberson 		    ("vm_object_collapse: object is already collapsing."));
1935df8bae1dSRodney W. Grimes 
1936f919ebdeSDavid Greenman 		/*
193798087a06SJeff Roberson 		 * We know that we can either collapse the backing object if
193898087a06SJeff Roberson 		 * the parent is the only reference to it, or (perhaps) have
19392ad1a3f7SMatthew Dillon 		 * the parent bypass the object if the parent happens to shadow
19402ad1a3f7SMatthew Dillon 		 * all the resident pages in the entire backing object.
1941df8bae1dSRodney W. Grimes 		 */
1942df8bae1dSRodney W. Grimes 		if (backing_object->ref_count == 1) {
194398087a06SJeff Roberson 			KASSERT(backing_object->shadow_count == 1,
194498087a06SJeff Roberson 			    ("vm_object_collapse: shadow_count: %d",
194598087a06SJeff Roberson 			    backing_object->shadow_count));
1946aa9bc3b1SKonstantin Belousov 			vm_object_pip_add(object, 1);
194798087a06SJeff Roberson 			vm_object_set_flag(object, OBJ_COLLAPSING);
1948aa9bc3b1SKonstantin Belousov 			vm_object_pip_add(backing_object, 1);
194998087a06SJeff Roberson 			vm_object_set_flag(backing_object, OBJ_DEAD);
1950aa9bc3b1SKonstantin Belousov 
1951df8bae1dSRodney W. Grimes 			/*
19522ad1a3f7SMatthew Dillon 			 * If there is exactly one reference to the backing
19532ad1a3f7SMatthew Dillon 			 * object, we can collapse it into the parent.
1954df8bae1dSRodney W. Grimes 			 */
195598087a06SJeff Roberson 			vm_object_collapse_scan(object);
1956df8bae1dSRodney W. Grimes 
1957f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0
1958f8a47341SAlan Cox 			/*
1959f8a47341SAlan Cox 			 * Break any reservations from backing_object.
1960f8a47341SAlan Cox 			 */
1961f8a47341SAlan Cox 			if (__predict_false(!LIST_EMPTY(&backing_object->rvq)))
1962f8a47341SAlan Cox 				vm_reserv_break_all(backing_object);
1963f8a47341SAlan Cox #endif
1964f8a47341SAlan Cox 
1965df8bae1dSRodney W. Grimes 			/*
1966df8bae1dSRodney W. Grimes 			 * Move the pager from backing_object to object.
1967df8bae1dSRodney W. Grimes 			 */
19684b8365d7SKonstantin Belousov 			if ((backing_object->flags & OBJ_SWAP) != 0) {
196924a1cce3SDavid Greenman 				/*
1970c7c8dd7eSAlan Cox 				 * swap_pager_copy() can sleep, in which case
1971c7c8dd7eSAlan Cox 				 * the backing_object's and object's locks are
1972c7c8dd7eSAlan Cox 				 * released and reacquired.
1973571a1e92SAttilio Rao 				 * Since swap_pager_copy() is being asked to
197498087a06SJeff Roberson 				 * destroy backing_object, it will change the
197598087a06SJeff Roberson 				 * type to OBJT_DEFAULT.
197624a1cce3SDavid Greenman 				 */
19771c7c3c6aSMatthew Dillon 				swap_pager_copy(
19781c7c3c6aSMatthew Dillon 				    backing_object,
19791c7c3c6aSMatthew Dillon 				    object,
19801c7c3c6aSMatthew Dillon 				    OFF_TO_IDX(object->backing_object_offset), TRUE);
1981c0503609SDavid Greenman 			}
198298087a06SJeff Roberson 
1983df8bae1dSRodney W. Grimes 			/*
1984df8bae1dSRodney W. Grimes 			 * Object now shadows whatever backing_object did.
1985df8bae1dSRodney W. Grimes 			 */
198698087a06SJeff Roberson 			vm_object_clear_flag(object, OBJ_COLLAPSING);
198798087a06SJeff Roberson 			vm_object_backing_transfer(object, backing_object);
19882ad1a3f7SMatthew Dillon 			object->backing_object_offset +=
19892ad1a3f7SMatthew Dillon 			    backing_object->backing_object_offset;
199098087a06SJeff Roberson 			VM_OBJECT_WUNLOCK(object);
199198087a06SJeff Roberson 			vm_object_pip_wakeup(object);
19922ad1a3f7SMatthew Dillon 
1993df8bae1dSRodney W. Grimes 			/*
1994df8bae1dSRodney W. Grimes 			 * Discard backing_object.
1995df8bae1dSRodney W. Grimes 			 *
19960d94caffSDavid Greenman 			 * Since the backing object has no pages, no pager left,
19970d94caffSDavid Greenman 			 * and no object references within it, all that is
19980d94caffSDavid Greenman 			 * necessary is to dispose of it.
1999df8bae1dSRodney W. Grimes 			 */
20009b4d473aSKonstantin Belousov 			KASSERT(backing_object->ref_count == 1, (
20019b4d473aSKonstantin Belousov "backing_object %p was somehow re-referenced during collapse!",
20029b4d473aSKonstantin Belousov 			    backing_object));
2003aa9bc3b1SKonstantin Belousov 			vm_object_pip_wakeup(backing_object);
200498087a06SJeff Roberson 			(void)refcount_release(&backing_object->ref_count);
200598087a06SJeff Roberson 			vm_object_terminate(backing_object);
200611542376SAlan Cox 			counter_u64_add(object_collapses, 1);
200798087a06SJeff Roberson 			VM_OBJECT_WLOCK(object);
20080d94caffSDavid Greenman 		} else {
2009df8bae1dSRodney W. Grimes 			/*
20102ad1a3f7SMatthew Dillon 			 * If we do not entirely shadow the backing object,
20112ad1a3f7SMatthew Dillon 			 * there is nothing we can do so we give up.
201298087a06SJeff Roberson 			 *
201398087a06SJeff Roberson 			 * The object lock and backing_object lock must not
201498087a06SJeff Roberson 			 * be dropped during this sequence.
2015df8bae1dSRodney W. Grimes 			 */
201658447749SJeff Roberson 			if (!vm_object_scan_all_shadowed(object)) {
201789f6b863SAttilio Rao 				VM_OBJECT_WUNLOCK(backing_object);
20182ad1a3f7SMatthew Dillon 				break;
201924a1cce3SDavid Greenman 			}
2020df8bae1dSRodney W. Grimes 
2021df8bae1dSRodney W. Grimes 			/*
20220d94caffSDavid Greenman 			 * Make the parent shadow the next object in the
20230d94caffSDavid Greenman 			 * chain.  Deallocating backing_object will not remove
20240d94caffSDavid Greenman 			 * it, since its reference count is at least 2.
2025df8bae1dSRodney W. Grimes 			 */
202651b867e5SJeff Roberson 			vm_object_backing_remove_locked(object);
202795e5e988SJohn Dyson 			new_backing_object = backing_object->backing_object;
202851b867e5SJeff Roberson 			if (new_backing_object != NULL) {
202998087a06SJeff Roberson 				vm_object_backing_insert_ref(object,
203051b867e5SJeff Roberson 				    new_backing_object);
203195e5e988SJohn Dyson 				object->backing_object_offset +=
203295e5e988SJohn Dyson 				    backing_object->backing_object_offset;
2033de5f6a77SJohn Dyson 			}
2034df8bae1dSRodney W. Grimes 
2035df8bae1dSRodney W. Grimes 			/*
20360d94caffSDavid Greenman 			 * Drop the reference count on backing_object. Since
203722ec553fSAlan Cox 			 * its ref_count was at least 2, it will not vanish.
2038df8bae1dSRodney W. Grimes 			 */
203998087a06SJeff Roberson 			(void)refcount_release(&backing_object->ref_count);
204098087a06SJeff Roberson 			KASSERT(backing_object->ref_count >= 1, (
204198087a06SJeff Roberson "backing_object %p was somehow dereferenced during collapse!",
204298087a06SJeff Roberson 			    backing_object));
204389f6b863SAttilio Rao 			VM_OBJECT_WUNLOCK(backing_object);
204411542376SAlan Cox 			counter_u64_add(object_bypasses, 1);
2045df8bae1dSRodney W. Grimes 		}
2046df8bae1dSRodney W. Grimes 
2047df8bae1dSRodney W. Grimes 		/*
2048df8bae1dSRodney W. Grimes 		 * Try again with this object's new backing object.
2049df8bae1dSRodney W. Grimes 		 */
2050df8bae1dSRodney W. Grimes 	}
2051df8bae1dSRodney W. Grimes }
2052df8bae1dSRodney W. Grimes 
2053df8bae1dSRodney W. Grimes /*
2054bff99f0dSAlan Cox  *	vm_object_page_remove:
2055df8bae1dSRodney W. Grimes  *
205668855966SAlan Cox  *	For the given object, either frees or invalidates each of the
20576bbee8e2SAlan Cox  *	specified pages.  In general, a page is freed.  However, if a page is
20586bbee8e2SAlan Cox  *	wired for any reason other than the existence of a managed, wired
20596bbee8e2SAlan Cox  *	mapping, then it may be invalidated but not removed from the object.
20606bbee8e2SAlan Cox  *	Pages are specified by the given range ["start", "end") and the option
20616bbee8e2SAlan Cox  *	OBJPR_CLEANONLY.  As a special case, if "end" is zero, then the range
20626bbee8e2SAlan Cox  *	extends from "start" to the end of the object.  If the option
20636bbee8e2SAlan Cox  *	OBJPR_CLEANONLY is specified, then only the non-dirty pages within the
20646bbee8e2SAlan Cox  *	specified range are affected.  If the option OBJPR_NOTMAPPED is
20656bbee8e2SAlan Cox  *	specified, then the pages within the specified range must have no
20666bbee8e2SAlan Cox  *	mappings.  Otherwise, if this option is not specified, any mappings to
20676bbee8e2SAlan Cox  *	the specified pages are removed before the pages are freed or
20686bbee8e2SAlan Cox  *	invalidated.
206968855966SAlan Cox  *
20706bbee8e2SAlan Cox  *	In general, this operation should only be performed on objects that
20716bbee8e2SAlan Cox  *	contain managed pages.  There are, however, two exceptions.  First, it
20726bbee8e2SAlan Cox  *	is performed on the kernel and kmem objects by vm_map_entry_delete().
20736bbee8e2SAlan Cox  *	Second, it is used by msync(..., MS_INVALIDATE) to invalidate device-
20746bbee8e2SAlan Cox  *	backed pages.  In both of these cases, the option OBJPR_CLEANONLY must
20756bbee8e2SAlan Cox  *	not be specified and the option OBJPR_NOTMAPPED must be specified.
2076df8bae1dSRodney W. Grimes  *
2077df8bae1dSRodney W. Grimes  *	The object must be locked.
2078df8bae1dSRodney W. Grimes  */
207926f9a767SRodney W. Grimes void
2080ecde4b32SAlan Cox vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
20816bbee8e2SAlan Cox     int options)
2082df8bae1dSRodney W. Grimes {
2083d031cff1SMatthew Dillon 	vm_page_t p, next;
2084df8bae1dSRodney W. Grimes 
208589f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
208628634820SAlan Cox 	KASSERT((object->flags & OBJ_UNMANAGED) == 0 ||
20876bbee8e2SAlan Cox 	    (options & (OBJPR_CLEANONLY | OBJPR_NOTMAPPED)) == OBJPR_NOTMAPPED,
20886bbee8e2SAlan Cox 	    ("vm_object_page_remove: illegal options for object %p", object));
2089ecde4b32SAlan Cox 	if (object->resident_page_count == 0)
20907667839aSAlan Cox 		return;
2091d474eaaaSDoug Rabson 	vm_object_pip_add(object, 1);
209226f9a767SRodney W. Grimes again:
2093b382c10aSKonstantin Belousov 	p = vm_page_find_least(object, start);
20942965a453SKip Macy 
209575741c04SAlan Cox 	/*
20966bbee8e2SAlan Cox 	 * Here, the variable "p" is either (1) the page with the least pindex
20976bbee8e2SAlan Cox 	 * greater than or equal to the parameter "start" or (2) NULL.
209875741c04SAlan Cox 	 */
20996bbee8e2SAlan Cox 	for (; p != NULL && (p->pindex < end || end == 0); p = next) {
2100b18bfc3dSJohn Dyson 		next = TAILQ_NEXT(p, listq);
210175741c04SAlan Cox 
210259677d3cSAlan Cox 		/*
2103d28af1abSMark Johnston 		 * Skip invalid pages if asked to do so.  Try to avoid acquiring
2104d28af1abSMark Johnston 		 * the busy lock, as some consumers rely on this to avoid
2105d28af1abSMark Johnston 		 * deadlocks.
2106d28af1abSMark Johnston 		 *
2107d28af1abSMark Johnston 		 * A thread may concurrently transition the page from invalid to
2108d28af1abSMark Johnston 		 * valid using only the busy lock, so the result of this check
2109d28af1abSMark Johnston 		 * is immediately stale.  It is up to consumers to handle this,
2110d28af1abSMark Johnston 		 * for instance by ensuring that all invalid->valid transitions
2111d28af1abSMark Johnston 		 * happen with a mutex held, as may be possible for a
2112d28af1abSMark Johnston 		 * filesystem.
2113d28af1abSMark Johnston 		 */
2114d28af1abSMark Johnston 		if ((options & OBJPR_VALIDONLY) != 0 && vm_page_none_valid(p))
2115d28af1abSMark Johnston 			continue;
2116d28af1abSMark Johnston 
2117d28af1abSMark Johnston 		/*
21186bbee8e2SAlan Cox 		 * If the page is wired for any reason besides the existence
21196bbee8e2SAlan Cox 		 * of managed, wired mappings, then it cannot be freed.  For
21206bbee8e2SAlan Cox 		 * example, fictitious pages, which represent device memory,
21216bbee8e2SAlan Cox 		 * are inherently wired and cannot be freed.  They can,
21226bbee8e2SAlan Cox 		 * however, be invalidated if the option OBJPR_CLEANONLY is
21236bbee8e2SAlan Cox 		 * not specified.
212459677d3cSAlan Cox 		 */
212563e97555SJeff Roberson 		if (vm_page_tryxbusy(p) == 0) {
212687b64663SMark Johnston 			if (vm_page_busy_sleep(p, "vmopar", 0))
212787b64663SMark Johnston 				VM_OBJECT_WLOCK(object);
2128fee2a2faSMark Johnston 			goto again;
2129fee2a2faSMark Johnston 		}
2130d28af1abSMark Johnston 		if ((options & OBJPR_VALIDONLY) != 0 && vm_page_none_valid(p)) {
2131d28af1abSMark Johnston 			vm_page_xunbusy(p);
2132d28af1abSMark Johnston 			continue;
2133d28af1abSMark Johnston 		}
2134d842aa51SMark Johnston 		if (vm_page_wired(p)) {
2135fee2a2faSMark Johnston wired:
2136cf060942SAlan Cox 			if ((options & OBJPR_NOTMAPPED) == 0 &&
2137cf060942SAlan Cox 			    object->ref_count != 0)
21384fec79beSAlan Cox 				pmap_remove_all(p);
21396bbee8e2SAlan Cox 			if ((options & OBJPR_CLEANONLY) == 0) {
21400012f373SJeff Roberson 				vm_page_invalid(p);
2141a28042d1SAlan Cox 				vm_page_undirty(p);
2142a28042d1SAlan Cox 			}
214363e97555SJeff Roberson 			vm_page_xunbusy(p);
214493c5d3a4SKonstantin Belousov 			continue;
21450d94caffSDavid Greenman 		}
214668855966SAlan Cox 		KASSERT((p->flags & PG_FICTITIOUS) == 0,
214768855966SAlan Cox 		    ("vm_object_page_remove: page %p is fictitious", p));
21480012f373SJeff Roberson 		if ((options & OBJPR_CLEANONLY) != 0 &&
21490012f373SJeff Roberson 		    !vm_page_none_valid(p)) {
2150cf060942SAlan Cox 			if ((options & OBJPR_NOTMAPPED) == 0 &&
2151fee2a2faSMark Johnston 			    object->ref_count != 0 &&
2152fee2a2faSMark Johnston 			    !vm_page_try_remove_write(p))
2153fee2a2faSMark Johnston 				goto wired;
215463e97555SJeff Roberson 			if (p->dirty != 0) {
215563e97555SJeff Roberson 				vm_page_xunbusy(p);
215693c5d3a4SKonstantin Belousov 				continue;
21572965a453SKip Macy 			}
215863e97555SJeff Roberson 		}
2159fee2a2faSMark Johnston 		if ((options & OBJPR_NOTMAPPED) == 0 &&
2160fee2a2faSMark Johnston 		    object->ref_count != 0 && !vm_page_try_remove_all(p))
2161fee2a2faSMark Johnston 			goto wired;
21625cd29d0fSMark Johnston 		vm_page_free(p);
21632965a453SKip Macy 	}
2164f919ebdeSDavid Greenman 	vm_object_pip_wakeup(object);
216584242cf6SMark Johnston 
21661390a5cbSKonstantin Belousov 	vm_pager_freespace(object, start, (end == 0 ? object->size : end) -
21671390a5cbSKonstantin Belousov 	    start);
2168c0503609SDavid Greenman }
2169df8bae1dSRodney W. Grimes 
2170df8bae1dSRodney W. Grimes /*
21713138cd36SMark Johnston  *	vm_object_page_noreuse:
2172936c09acSJohn Baldwin  *
21733138cd36SMark Johnston  *	For the given object, attempt to move the specified pages to
21743138cd36SMark Johnston  *	the head of the inactive queue.  This bypasses regular LRU
21753138cd36SMark Johnston  *	operation and allows the pages to be reused quickly under memory
21763138cd36SMark Johnston  *	pressure.  If a page is wired for any reason, then it will not
21773138cd36SMark Johnston  *	be queued.  Pages are specified by the range ["start", "end").
21783138cd36SMark Johnston  *	As a special case, if "end" is zero, then the range extends from
21793138cd36SMark Johnston  *	"start" to the end of the object.
2180936c09acSJohn Baldwin  *
2181936c09acSJohn Baldwin  *	This operation should only be performed on objects that
218228634820SAlan Cox  *	contain non-fictitious, managed pages.
2183936c09acSJohn Baldwin  *
2184936c09acSJohn Baldwin  *	The object must be locked.
2185936c09acSJohn Baldwin  */
2186936c09acSJohn Baldwin void
21873138cd36SMark Johnston vm_object_page_noreuse(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
2188936c09acSJohn Baldwin {
2189936c09acSJohn Baldwin 	vm_page_t p, next;
2190936c09acSJohn Baldwin 
219152d1addaSAlan Cox 	VM_OBJECT_ASSERT_LOCKED(object);
219228634820SAlan Cox 	KASSERT((object->flags & (OBJ_FICTITIOUS | OBJ_UNMANAGED)) == 0,
21933138cd36SMark Johnston 	    ("vm_object_page_noreuse: illegal object %p", object));
2194936c09acSJohn Baldwin 	if (object->resident_page_count == 0)
2195936c09acSJohn Baldwin 		return;
2196936c09acSJohn Baldwin 	p = vm_page_find_least(object, start);
2197936c09acSJohn Baldwin 
2198936c09acSJohn Baldwin 	/*
2199936c09acSJohn Baldwin 	 * Here, the variable "p" is either (1) the page with the least pindex
2200936c09acSJohn Baldwin 	 * greater than or equal to the parameter "start" or (2) NULL.
2201936c09acSJohn Baldwin 	 */
2202936c09acSJohn Baldwin 	for (; p != NULL && (p->pindex < end || end == 0); p = next) {
2203936c09acSJohn Baldwin 		next = TAILQ_NEXT(p, listq);
22043138cd36SMark Johnston 		vm_page_deactivate_noreuse(p);
2205936c09acSJohn Baldwin 	}
2206936c09acSJohn Baldwin }
2207936c09acSJohn Baldwin 
2208936c09acSJohn Baldwin /*
2209387aabc5SAlan Cox  *	Populate the specified range of the object with valid pages.  Returns
2210387aabc5SAlan Cox  *	TRUE if the range is successfully populated and FALSE otherwise.
2211387aabc5SAlan Cox  *
2212387aabc5SAlan Cox  *	Note: This function should be optimized to pass a larger array of
2213387aabc5SAlan Cox  *	pages to vm_pager_get_pages() before it is applied to a non-
2214387aabc5SAlan Cox  *	OBJT_DEVICE object.
2215387aabc5SAlan Cox  *
2216387aabc5SAlan Cox  *	The object must be locked.
2217387aabc5SAlan Cox  */
2218387aabc5SAlan Cox boolean_t
2219387aabc5SAlan Cox vm_object_populate(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
2220387aabc5SAlan Cox {
2221093c7f39SGleb Smirnoff 	vm_page_t m;
2222387aabc5SAlan Cox 	vm_pindex_t pindex;
2223387aabc5SAlan Cox 	int rv;
2224387aabc5SAlan Cox 
222589f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
2226387aabc5SAlan Cox 	for (pindex = start; pindex < end; pindex++) {
2227c7575748SJeff Roberson 		rv = vm_page_grab_valid(&m, object, pindex, VM_ALLOC_NORMAL);
2228c7575748SJeff Roberson 		if (rv != VM_PAGER_OK)
2229387aabc5SAlan Cox 			break;
2230c7575748SJeff Roberson 
2231387aabc5SAlan Cox 		/*
2232387aabc5SAlan Cox 		 * Keep "m" busy because a subsequent iteration may unlock
2233387aabc5SAlan Cox 		 * the object.
2234387aabc5SAlan Cox 		 */
2235387aabc5SAlan Cox 	}
2236387aabc5SAlan Cox 	if (pindex > start) {
2237387aabc5SAlan Cox 		m = vm_page_lookup(object, start);
2238387aabc5SAlan Cox 		while (m != NULL && m->pindex < pindex) {
2239c7aebda8SAttilio Rao 			vm_page_xunbusy(m);
2240387aabc5SAlan Cox 			m = TAILQ_NEXT(m, listq);
2241387aabc5SAlan Cox 		}
2242387aabc5SAlan Cox 	}
2243387aabc5SAlan Cox 	return (pindex == end);
2244387aabc5SAlan Cox }
2245387aabc5SAlan Cox 
2246387aabc5SAlan Cox /*
2247df8bae1dSRodney W. Grimes  *	Routine:	vm_object_coalesce
2248df8bae1dSRodney W. Grimes  *	Function:	Coalesces two objects backing up adjoining
2249df8bae1dSRodney W. Grimes  *			regions of memory into a single object.
2250df8bae1dSRodney W. Grimes  *
2251df8bae1dSRodney W. Grimes  *	returns TRUE if objects were combined.
2252df8bae1dSRodney W. Grimes  *
2253df8bae1dSRodney W. Grimes  *	NOTE:	Only works at the moment if the second object is NULL -
2254df8bae1dSRodney W. Grimes  *		if it's not, which object do we lock first?
2255df8bae1dSRodney W. Grimes  *
2256df8bae1dSRodney W. Grimes  *	Parameters:
2257df8bae1dSRodney W. Grimes  *		prev_object	First object to coalesce
2258df8bae1dSRodney W. Grimes  *		prev_offset	Offset into prev_object
2259df8bae1dSRodney W. Grimes  *		prev_size	Size of reference to prev_object
226057a21abaSAlan Cox  *		next_size	Size of reference to the second object
22613364c323SKonstantin Belousov  *		reserved	Indicator that extension region has
22623364c323SKonstantin Belousov  *				swap accounted for
2263df8bae1dSRodney W. Grimes  *
2264df8bae1dSRodney W. Grimes  *	Conditions:
2265df8bae1dSRodney W. Grimes  *	The object must *not* be locked.
2266df8bae1dSRodney W. Grimes  */
22670d94caffSDavid Greenman boolean_t
226857a21abaSAlan Cox vm_object_coalesce(vm_object_t prev_object, vm_ooffset_t prev_offset,
22693364c323SKonstantin Belousov     vm_size_t prev_size, vm_size_t next_size, boolean_t reserved)
2270df8bae1dSRodney W. Grimes {
2271ea41812fSAlan Cox 	vm_pindex_t next_pindex;
2272df8bae1dSRodney W. Grimes 
227300e1854aSAlan Cox 	if (prev_object == NULL)
2274df8bae1dSRodney W. Grimes 		return (TRUE);
227563967687SJeff Roberson 	if ((prev_object->flags & OBJ_ANON) == 0)
227630dcfc09SJohn Dyson 		return (FALSE);
227730dcfc09SJohn Dyson 
227863967687SJeff Roberson 	VM_OBJECT_WLOCK(prev_object);
2279df8bae1dSRodney W. Grimes 	/*
228098087a06SJeff Roberson 	 * Try to collapse the object first.
2281df8bae1dSRodney W. Grimes 	 */
2282df8bae1dSRodney W. Grimes 	vm_object_collapse(prev_object);
2283df8bae1dSRodney W. Grimes 
2284df8bae1dSRodney W. Grimes 	/*
22850d94caffSDavid Greenman 	 * Can't coalesce if: . more than one reference . paged out . shadows
22860d94caffSDavid Greenman 	 * another object . has a copy elsewhere (any of which mean that the
22870d94caffSDavid Greenman 	 * pages not mapped to prev_entry may be in use anyway)
2288df8bae1dSRodney W. Grimes 	 */
22898cc7e047SJohn Dyson 	if (prev_object->backing_object != NULL) {
229089f6b863SAttilio Rao 		VM_OBJECT_WUNLOCK(prev_object);
2291df8bae1dSRodney W. Grimes 		return (FALSE);
2292df8bae1dSRodney W. Grimes 	}
2293a316d390SJohn Dyson 
2294a316d390SJohn Dyson 	prev_size >>= PAGE_SHIFT;
2295a316d390SJohn Dyson 	next_size >>= PAGE_SHIFT;
229657a21abaSAlan Cox 	next_pindex = OFF_TO_IDX(prev_offset) + prev_size;
22978cc7e047SJohn Dyson 
22980e48e068SMark Johnston 	if (prev_object->ref_count > 1 &&
22990e48e068SMark Johnston 	    prev_object->size != next_pindex &&
23000e48e068SMark Johnston 	    (prev_object->flags & OBJ_ONEMAPPING) == 0) {
230189f6b863SAttilio Rao 		VM_OBJECT_WUNLOCK(prev_object);
23028cc7e047SJohn Dyson 		return (FALSE);
23038cc7e047SJohn Dyson 	}
23048cc7e047SJohn Dyson 
2305df8bae1dSRodney W. Grimes 	/*
23063364c323SKonstantin Belousov 	 * Account for the charge.
23073364c323SKonstantin Belousov 	 */
2308ef694c1aSEdward Tomasz Napierala 	if (prev_object->cred != NULL) {
23093364c323SKonstantin Belousov 		/*
23103364c323SKonstantin Belousov 		 * If prev_object was charged, then this mapping,
2311763df3ecSPedro F. Giffuni 		 * although not charged now, may become writable
2312ef694c1aSEdward Tomasz Napierala 		 * later. Non-NULL cred in the object would prevent
23133364c323SKonstantin Belousov 		 * swap reservation during enabling of the write
23143364c323SKonstantin Belousov 		 * access, so reserve swap now. Failed reservation
23153364c323SKonstantin Belousov 		 * cause allocation of the separate object for the map
23163364c323SKonstantin Belousov 		 * entry, and swap reservation for this entry is
23173364c323SKonstantin Belousov 		 * managed in appropriate time.
23183364c323SKonstantin Belousov 		 */
2319ef694c1aSEdward Tomasz Napierala 		if (!reserved && !swap_reserve_by_cred(ptoa(next_size),
2320ef694c1aSEdward Tomasz Napierala 		    prev_object->cred)) {
23219f790a17SKonstantin Belousov 			VM_OBJECT_WUNLOCK(prev_object);
23223364c323SKonstantin Belousov 			return (FALSE);
23233364c323SKonstantin Belousov 		}
23243364c323SKonstantin Belousov 		prev_object->charge += ptoa(next_size);
23253364c323SKonstantin Belousov 	}
23263364c323SKonstantin Belousov 
23273364c323SKonstantin Belousov 	/*
23280d94caffSDavid Greenman 	 * Remove any pages that may still be in the object from a previous
23290d94caffSDavid Greenman 	 * deallocation.
2330df8bae1dSRodney W. Grimes 	 */
2331ea41812fSAlan Cox 	if (next_pindex < prev_object->size) {
23326bbee8e2SAlan Cox 		vm_object_page_remove(prev_object, next_pindex, next_pindex +
23336bbee8e2SAlan Cox 		    next_size, 0);
23343364c323SKonstantin Belousov #if 0
2335ef694c1aSEdward Tomasz Napierala 		if (prev_object->cred != NULL) {
23363364c323SKonstantin Belousov 			KASSERT(prev_object->charge >=
23373364c323SKonstantin Belousov 			    ptoa(prev_object->size - next_pindex),
23383364c323SKonstantin Belousov 			    ("object %p overcharged 1 %jx %jx", prev_object,
23393364c323SKonstantin Belousov 				(uintmax_t)next_pindex, (uintmax_t)next_size));
23403364c323SKonstantin Belousov 			prev_object->charge -= ptoa(prev_object->size -
23413364c323SKonstantin Belousov 			    next_pindex);
23423364c323SKonstantin Belousov 		}
23433364c323SKonstantin Belousov #endif
2344ea41812fSAlan Cox 	}
2345df8bae1dSRodney W. Grimes 
2346df8bae1dSRodney W. Grimes 	/*
2347df8bae1dSRodney W. Grimes 	 * Extend the object if necessary.
2348df8bae1dSRodney W. Grimes 	 */
2349ea41812fSAlan Cox 	if (next_pindex + next_size > prev_object->size)
2350ea41812fSAlan Cox 		prev_object->size = next_pindex + next_size;
2351df8bae1dSRodney W. Grimes 
235289f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(prev_object);
2353df8bae1dSRodney W. Grimes 	return (TRUE);
2354df8bae1dSRodney W. Grimes }
2355df8bae1dSRodney W. Grimes 
23567a5a6352SMatthew Dillon void
2357180bcaa4SKonstantin Belousov vm_object_set_writeable_dirty_(vm_object_t object)
23587a5a6352SMatthew Dillon {
235967d0e293SJeff Roberson 	atomic_add_int(&object->generation, 1);
23607a5a6352SMatthew Dillon }
23617a5a6352SMatthew Dillon 
2362c23c555bSKonstantin Belousov bool
2363c23c555bSKonstantin Belousov vm_object_mightbedirty_(vm_object_t object)
2364c23c555bSKonstantin Belousov {
2365c23c555bSKonstantin Belousov 	return (object->generation != object->cleangeneration);
2366c23c555bSKonstantin Belousov }
2367c23c555bSKonstantin Belousov 
236803462509SAlan Cox /*
236903462509SAlan Cox  *	vm_object_unwire:
237003462509SAlan Cox  *
237103462509SAlan Cox  *	For each page offset within the specified range of the given object,
237203462509SAlan Cox  *	find the highest-level page in the shadow chain and unwire it.  A page
237303462509SAlan Cox  *	must exist at every page offset, and the highest-level page must be
237403462509SAlan Cox  *	wired.
237503462509SAlan Cox  */
237603462509SAlan Cox void
237703462509SAlan Cox vm_object_unwire(vm_object_t object, vm_ooffset_t offset, vm_size_t length,
237803462509SAlan Cox     uint8_t queue)
237903462509SAlan Cox {
238020e4afbfSKonstantin Belousov 	vm_object_t tobject, t1object;
238103462509SAlan Cox 	vm_page_t m, tm;
238203462509SAlan Cox 	vm_pindex_t end_pindex, pindex, tpindex;
238303462509SAlan Cox 	int depth, locked_depth;
238403462509SAlan Cox 
238503462509SAlan Cox 	KASSERT((offset & PAGE_MASK) == 0,
238603462509SAlan Cox 	    ("vm_object_unwire: offset is not page aligned"));
238703462509SAlan Cox 	KASSERT((length & PAGE_MASK) == 0,
238803462509SAlan Cox 	    ("vm_object_unwire: length is not a multiple of PAGE_SIZE"));
238903462509SAlan Cox 	/* The wired count of a fictitious page never changes. */
239003462509SAlan Cox 	if ((object->flags & OBJ_FICTITIOUS) != 0)
239103462509SAlan Cox 		return;
239203462509SAlan Cox 	pindex = OFF_TO_IDX(offset);
239303462509SAlan Cox 	end_pindex = pindex + atop(length);
239420e4afbfSKonstantin Belousov again:
239503462509SAlan Cox 	locked_depth = 1;
239603462509SAlan Cox 	VM_OBJECT_RLOCK(object);
239703462509SAlan Cox 	m = vm_page_find_least(object, pindex);
239803462509SAlan Cox 	while (pindex < end_pindex) {
239903462509SAlan Cox 		if (m == NULL || pindex < m->pindex) {
240003462509SAlan Cox 			/*
240103462509SAlan Cox 			 * The first object in the shadow chain doesn't
240203462509SAlan Cox 			 * contain a page at the current index.  Therefore,
240303462509SAlan Cox 			 * the page must exist in a backing object.
240403462509SAlan Cox 			 */
240503462509SAlan Cox 			tobject = object;
240603462509SAlan Cox 			tpindex = pindex;
240703462509SAlan Cox 			depth = 0;
240803462509SAlan Cox 			do {
240903462509SAlan Cox 				tpindex +=
241003462509SAlan Cox 				    OFF_TO_IDX(tobject->backing_object_offset);
241103462509SAlan Cox 				tobject = tobject->backing_object;
241203462509SAlan Cox 				KASSERT(tobject != NULL,
241303462509SAlan Cox 				    ("vm_object_unwire: missing page"));
241403462509SAlan Cox 				if ((tobject->flags & OBJ_FICTITIOUS) != 0)
241503462509SAlan Cox 					goto next_page;
241603462509SAlan Cox 				depth++;
241703462509SAlan Cox 				if (depth == locked_depth) {
241803462509SAlan Cox 					locked_depth++;
241903462509SAlan Cox 					VM_OBJECT_RLOCK(tobject);
242003462509SAlan Cox 				}
242103462509SAlan Cox 			} while ((tm = vm_page_lookup(tobject, tpindex)) ==
242203462509SAlan Cox 			    NULL);
242303462509SAlan Cox 		} else {
242403462509SAlan Cox 			tm = m;
242503462509SAlan Cox 			m = TAILQ_NEXT(m, listq);
242603462509SAlan Cox 		}
242763e97555SJeff Roberson 		if (vm_page_trysbusy(tm) == 0) {
242887e93ea6SMark Johnston 			for (tobject = object; locked_depth >= 1;
242920e4afbfSKonstantin Belousov 			    locked_depth--) {
243020e4afbfSKonstantin Belousov 				t1object = tobject->backing_object;
243187e93ea6SMark Johnston 				if (tm->object != tobject)
243220e4afbfSKonstantin Belousov 					VM_OBJECT_RUNLOCK(tobject);
243320e4afbfSKonstantin Belousov 				tobject = t1object;
243420e4afbfSKonstantin Belousov 			}
243587b64663SMark Johnston 			tobject = tm->object;
243687b64663SMark Johnston 			if (!vm_page_busy_sleep(tm, "unwbo",
243787b64663SMark Johnston 			    VM_ALLOC_IGN_SBUSY))
243887b64663SMark Johnston 				VM_OBJECT_RUNLOCK(tobject);
243920e4afbfSKonstantin Belousov 			goto again;
244020e4afbfSKonstantin Belousov 		}
244103462509SAlan Cox 		vm_page_unwire(tm, queue);
244263e97555SJeff Roberson 		vm_page_sunbusy(tm);
244303462509SAlan Cox next_page:
244403462509SAlan Cox 		pindex++;
244503462509SAlan Cox 	}
244603462509SAlan Cox 	/* Release the accumulated object locks. */
244720e4afbfSKonstantin Belousov 	for (tobject = object; locked_depth >= 1; locked_depth--) {
244820e4afbfSKonstantin Belousov 		t1object = tobject->backing_object;
244920e4afbfSKonstantin Belousov 		VM_OBJECT_RUNLOCK(tobject);
245020e4afbfSKonstantin Belousov 		tobject = t1object;
245103462509SAlan Cox 	}
245203462509SAlan Cox }
245303462509SAlan Cox 
24540951bd36SEric van Gyzen /*
24550951bd36SEric van Gyzen  * Return the vnode for the given object, or NULL if none exists.
24560951bd36SEric van Gyzen  * For tmpfs objects, the function may return NULL if there is
24570951bd36SEric van Gyzen  * no vnode allocated at the time of the call.
24580951bd36SEric van Gyzen  */
245963e4c6cdSEric van Gyzen struct vnode *
246063e4c6cdSEric van Gyzen vm_object_vnode(vm_object_t object)
246163e4c6cdSEric van Gyzen {
24620951bd36SEric van Gyzen 	struct vnode *vp;
246363e4c6cdSEric van Gyzen 
246463e4c6cdSEric van Gyzen 	VM_OBJECT_ASSERT_LOCKED(object);
2465a7c198a2SKonstantin Belousov 	vm_pager_getvp(object, &vp, NULL);
24660951bd36SEric van Gyzen 	return (vp);
246763e4c6cdSEric van Gyzen }
246863e4c6cdSEric van Gyzen 
2469205be21dSJeff Roberson /*
2470205be21dSJeff Roberson  * Busy the vm object.  This prevents new pages belonging to the object from
2471205be21dSJeff Roberson  * becoming busy.  Existing pages persist as busy.  Callers are responsible
2472205be21dSJeff Roberson  * for checking page state before proceeding.
2473205be21dSJeff Roberson  */
2474205be21dSJeff Roberson void
2475205be21dSJeff Roberson vm_object_busy(vm_object_t obj)
2476205be21dSJeff Roberson {
2477205be21dSJeff Roberson 
2478205be21dSJeff Roberson 	VM_OBJECT_ASSERT_LOCKED(obj);
2479205be21dSJeff Roberson 
2480c99d0c58SMark Johnston 	blockcount_acquire(&obj->busy, 1);
2481205be21dSJeff Roberson 	/* The fence is required to order loads of page busy. */
2482205be21dSJeff Roberson 	atomic_thread_fence_acq_rel();
2483205be21dSJeff Roberson }
2484205be21dSJeff Roberson 
2485205be21dSJeff Roberson void
2486205be21dSJeff Roberson vm_object_unbusy(vm_object_t obj)
2487205be21dSJeff Roberson {
2488205be21dSJeff Roberson 
2489c99d0c58SMark Johnston 	blockcount_release(&obj->busy, 1);
2490205be21dSJeff Roberson }
2491205be21dSJeff Roberson 
2492205be21dSJeff Roberson void
2493205be21dSJeff Roberson vm_object_busy_wait(vm_object_t obj, const char *wmesg)
2494205be21dSJeff Roberson {
2495205be21dSJeff Roberson 
2496205be21dSJeff Roberson 	VM_OBJECT_ASSERT_UNLOCKED(obj);
2497205be21dSJeff Roberson 
2498c99d0c58SMark Johnston 	(void)blockcount_sleep(&obj->busy, NULL, wmesg, PVM);
2499205be21dSJeff Roberson }
2500205be21dSJeff Roberson 
2501cd37afd8SRick Macklem /*
2502cd37afd8SRick Macklem  * This function aims to determine if the object is mapped,
2503cd37afd8SRick Macklem  * specifically, if it is referenced by a vm_map_entry.  Because
2504cd37afd8SRick Macklem  * objects occasionally acquire transient references that do not
2505cd37afd8SRick Macklem  * represent a mapping, the method used here is inexact.  However, it
2506cd37afd8SRick Macklem  * has very low overhead and is good enough for the advisory
2507cd37afd8SRick Macklem  * vm.vmtotal sysctl.
2508cd37afd8SRick Macklem  */
2509cd37afd8SRick Macklem bool
2510cd37afd8SRick Macklem vm_object_is_active(vm_object_t obj)
2511cd37afd8SRick Macklem {
2512cd37afd8SRick Macklem 
2513cd37afd8SRick Macklem 	return (obj->ref_count > obj->shadow_count);
2514cd37afd8SRick Macklem }
2515cd37afd8SRick Macklem 
2516ff87ae35SJohn Baldwin static int
251742812cccSKonstantin Belousov vm_object_list_handler(struct sysctl_req *req, bool swap_only)
2518ff87ae35SJohn Baldwin {
25190ecee546SKonstantin Belousov 	struct kinfo_vmobject *kvo;
2520ff87ae35SJohn Baldwin 	char *fullpath, *freepath;
2521ff87ae35SJohn Baldwin 	struct vnode *vp;
2522ff87ae35SJohn Baldwin 	struct vattr va;
2523ff87ae35SJohn Baldwin 	vm_object_t obj;
2524ff87ae35SJohn Baldwin 	vm_page_t m;
2525ecfbddf0SKonstantin Belousov 	u_long sp;
2526ff87ae35SJohn Baldwin 	int count, error;
2527ff87ae35SJohn Baldwin 
2528ff87ae35SJohn Baldwin 	if (req->oldptr == NULL) {
2529ff87ae35SJohn Baldwin 		/*
2530ff87ae35SJohn Baldwin 		 * If an old buffer has not been provided, generate an
2531ff87ae35SJohn Baldwin 		 * estimate of the space needed for a subsequent call.
2532ff87ae35SJohn Baldwin 		 */
2533ff87ae35SJohn Baldwin 		mtx_lock(&vm_object_list_mtx);
2534ff87ae35SJohn Baldwin 		count = 0;
2535ff87ae35SJohn Baldwin 		TAILQ_FOREACH(obj, &vm_object_list, object_list) {
2536ff87ae35SJohn Baldwin 			if (obj->type == OBJT_DEAD)
2537ff87ae35SJohn Baldwin 				continue;
2538ff87ae35SJohn Baldwin 			count++;
2539ff87ae35SJohn Baldwin 		}
2540ff87ae35SJohn Baldwin 		mtx_unlock(&vm_object_list_mtx);
2541ff87ae35SJohn Baldwin 		return (SYSCTL_OUT(req, NULL, sizeof(struct kinfo_vmobject) *
2542ff87ae35SJohn Baldwin 		    count * 11 / 10));
2543ff87ae35SJohn Baldwin 	}
2544ff87ae35SJohn Baldwin 
25450ecee546SKonstantin Belousov 	kvo = malloc(sizeof(*kvo), M_TEMP, M_WAITOK);
2546ff87ae35SJohn Baldwin 	error = 0;
2547ff87ae35SJohn Baldwin 
2548ff87ae35SJohn Baldwin 	/*
2549ff87ae35SJohn Baldwin 	 * VM objects are type stable and are never removed from the
2550ff87ae35SJohn Baldwin 	 * list once added.  This allows us to safely read obj->object_list
2551ff87ae35SJohn Baldwin 	 * after reacquiring the VM object lock.
2552ff87ae35SJohn Baldwin 	 */
2553ff87ae35SJohn Baldwin 	mtx_lock(&vm_object_list_mtx);
2554ff87ae35SJohn Baldwin 	TAILQ_FOREACH(obj, &vm_object_list, object_list) {
255542812cccSKonstantin Belousov 		if (obj->type == OBJT_DEAD ||
255642812cccSKonstantin Belousov 		    (swap_only && (obj->flags & (OBJ_ANON | OBJ_SWAP)) == 0))
2557ff87ae35SJohn Baldwin 			continue;
2558ff87ae35SJohn Baldwin 		VM_OBJECT_RLOCK(obj);
255942812cccSKonstantin Belousov 		if (obj->type == OBJT_DEAD ||
256042812cccSKonstantin Belousov 		    (swap_only && (obj->flags & (OBJ_ANON | OBJ_SWAP)) == 0)) {
2561ff87ae35SJohn Baldwin 			VM_OBJECT_RUNLOCK(obj);
2562ff87ae35SJohn Baldwin 			continue;
2563ff87ae35SJohn Baldwin 		}
2564ff87ae35SJohn Baldwin 		mtx_unlock(&vm_object_list_mtx);
25650ecee546SKonstantin Belousov 		kvo->kvo_size = ptoa(obj->size);
25660ecee546SKonstantin Belousov 		kvo->kvo_resident = obj->resident_page_count;
25670ecee546SKonstantin Belousov 		kvo->kvo_ref_count = obj->ref_count;
25680ecee546SKonstantin Belousov 		kvo->kvo_shadow_count = obj->shadow_count;
25690ecee546SKonstantin Belousov 		kvo->kvo_memattr = obj->memattr;
25700ecee546SKonstantin Belousov 		kvo->kvo_active = 0;
25710ecee546SKonstantin Belousov 		kvo->kvo_inactive = 0;
25727738118eSKonstantin Belousov 		if (!swap_only) {
2573ff87ae35SJohn Baldwin 			TAILQ_FOREACH(m, &obj->memq, listq) {
2574ff87ae35SJohn Baldwin 				/*
2575ff87ae35SJohn Baldwin 				 * A page may belong to the object but be
2576ff87ae35SJohn Baldwin 				 * dequeued and set to PQ_NONE while the
2577ff87ae35SJohn Baldwin 				 * object lock is not held.  This makes the
2578ff87ae35SJohn Baldwin 				 * reads of m->queue below racy, and we do not
2579ff87ae35SJohn Baldwin 				 * count pages set to PQ_NONE.  However, this
2580ff87ae35SJohn Baldwin 				 * sysctl is only meant to give an
2581ff87ae35SJohn Baldwin 				 * approximation of the system anyway.
2582ff87ae35SJohn Baldwin 				 */
25835cff1f4dSMark Johnston 				if (m->a.queue == PQ_ACTIVE)
25840ecee546SKonstantin Belousov 					kvo->kvo_active++;
25855cff1f4dSMark Johnston 				else if (m->a.queue == PQ_INACTIVE)
25860ecee546SKonstantin Belousov 					kvo->kvo_inactive++;
2587ff87ae35SJohn Baldwin 			}
25887738118eSKonstantin Belousov 		}
2589ff87ae35SJohn Baldwin 
25900ecee546SKonstantin Belousov 		kvo->kvo_vn_fileid = 0;
25910ecee546SKonstantin Belousov 		kvo->kvo_vn_fsid = 0;
25920ecee546SKonstantin Belousov 		kvo->kvo_vn_fsid_freebsd11 = 0;
2593ff87ae35SJohn Baldwin 		freepath = NULL;
2594ff87ae35SJohn Baldwin 		fullpath = "";
25957738118eSKonstantin Belousov 		vp = NULL;
25967738118eSKonstantin Belousov 		kvo->kvo_type = vm_object_kvme_type(obj, swap_only ? NULL : &vp);
2597ecfbddf0SKonstantin Belousov 		if (vp != NULL) {
2598ff87ae35SJohn Baldwin 			vref(vp);
2599ecfbddf0SKonstantin Belousov 		} else if ((obj->flags & OBJ_ANON) != 0) {
2600ecfbddf0SKonstantin Belousov 			MPASS(kvo->kvo_type == KVME_TYPE_DEFAULT ||
2601ecfbddf0SKonstantin Belousov 			    kvo->kvo_type == KVME_TYPE_SWAP);
2602ecfbddf0SKonstantin Belousov 			kvo->kvo_me = (uintptr_t)obj;
2603ecfbddf0SKonstantin Belousov 			/* tmpfs objs are reported as vnodes */
2604ecfbddf0SKonstantin Belousov 			kvo->kvo_backing_obj = (uintptr_t)obj->backing_object;
2605ecfbddf0SKonstantin Belousov 			sp = swap_pager_swapped_pages(obj);
2606ecfbddf0SKonstantin Belousov 			kvo->kvo_swapped = sp > UINT32_MAX ? UINT32_MAX : sp;
2607ecfbddf0SKonstantin Belousov 		}
2608ff87ae35SJohn Baldwin 		VM_OBJECT_RUNLOCK(obj);
2609ff87ae35SJohn Baldwin 		if (vp != NULL) {
2610feabaaf9SMateusz Guzik 			vn_fullpath(vp, &fullpath, &freepath);
2611ff87ae35SJohn Baldwin 			vn_lock(vp, LK_SHARED | LK_RETRY);
2612ff87ae35SJohn Baldwin 			if (VOP_GETATTR(vp, &va, curthread->td_ucred) == 0) {
26130ecee546SKonstantin Belousov 				kvo->kvo_vn_fileid = va.va_fileid;
26140ecee546SKonstantin Belousov 				kvo->kvo_vn_fsid = va.va_fsid;
26150ecee546SKonstantin Belousov 				kvo->kvo_vn_fsid_freebsd11 = va.va_fsid;
261669921123SKonstantin Belousov 								/* truncate */
2617ff87ae35SJohn Baldwin 			}
2618ff87ae35SJohn Baldwin 			vput(vp);
2619ff87ae35SJohn Baldwin 		}
2620ff87ae35SJohn Baldwin 
26210ecee546SKonstantin Belousov 		strlcpy(kvo->kvo_path, fullpath, sizeof(kvo->kvo_path));
2622ff87ae35SJohn Baldwin 		if (freepath != NULL)
2623ff87ae35SJohn Baldwin 			free(freepath, M_TEMP);
2624ff87ae35SJohn Baldwin 
2625ff87ae35SJohn Baldwin 		/* Pack record size down */
26260ecee546SKonstantin Belousov 		kvo->kvo_structsize = offsetof(struct kinfo_vmobject, kvo_path)
26270ecee546SKonstantin Belousov 		    + strlen(kvo->kvo_path) + 1;
26280ecee546SKonstantin Belousov 		kvo->kvo_structsize = roundup(kvo->kvo_structsize,
2629ff87ae35SJohn Baldwin 		    sizeof(uint64_t));
26300ecee546SKonstantin Belousov 		error = SYSCTL_OUT(req, kvo, kvo->kvo_structsize);
2631350fc36bSKonstantin Belousov 		maybe_yield();
2632ff87ae35SJohn Baldwin 		mtx_lock(&vm_object_list_mtx);
2633ff87ae35SJohn Baldwin 		if (error)
2634ff87ae35SJohn Baldwin 			break;
2635ff87ae35SJohn Baldwin 	}
2636ff87ae35SJohn Baldwin 	mtx_unlock(&vm_object_list_mtx);
26370ecee546SKonstantin Belousov 	free(kvo, M_TEMP);
2638ff87ae35SJohn Baldwin 	return (error);
2639ff87ae35SJohn Baldwin }
26401b610624SKonstantin Belousov 
26411b610624SKonstantin Belousov static int
26421b610624SKonstantin Belousov sysctl_vm_object_list(SYSCTL_HANDLER_ARGS)
26431b610624SKonstantin Belousov {
264442812cccSKonstantin Belousov 	return (vm_object_list_handler(req, false));
26451b610624SKonstantin Belousov }
26461b610624SKonstantin Belousov 
2647ff87ae35SJohn Baldwin SYSCTL_PROC(_vm, OID_AUTO, objects, CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_SKIP |
2648ff87ae35SJohn Baldwin     CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_object_list, "S,kinfo_vmobject",
2649ff87ae35SJohn Baldwin     "List of VM objects");
2650ff87ae35SJohn Baldwin 
265142812cccSKonstantin Belousov static int
265242812cccSKonstantin Belousov sysctl_vm_object_list_swap(SYSCTL_HANDLER_ARGS)
265342812cccSKonstantin Belousov {
265442812cccSKonstantin Belousov 	return (vm_object_list_handler(req, true));
265542812cccSKonstantin Belousov }
265642812cccSKonstantin Belousov 
265742812cccSKonstantin Belousov /*
265842812cccSKonstantin Belousov  * This sysctl returns list of the anonymous or swap objects. Intent
265942812cccSKonstantin Belousov  * is to provide stripped optimized list useful to analyze swap use.
266042812cccSKonstantin Belousov  * Since technically non-swap (default) objects participate in the
266142812cccSKonstantin Belousov  * shadow chains, and are converted to swap type as needed by swap
266242812cccSKonstantin Belousov  * pager, we must report them.
266342812cccSKonstantin Belousov  */
266442812cccSKonstantin Belousov SYSCTL_PROC(_vm, OID_AUTO, swap_objects,
266542812cccSKonstantin Belousov     CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_SKIP | CTLFLAG_MPSAFE, NULL, 0,
266642812cccSKonstantin Belousov     sysctl_vm_object_list_swap, "S,kinfo_vmobject",
266742812cccSKonstantin Belousov     "List of swap VM objects");
266842812cccSKonstantin Belousov 
2669c7c34a24SBruce Evans #include "opt_ddb.h"
2670c3cb3e12SDavid Greenman #ifdef DDB
2671c7c34a24SBruce Evans #include <sys/kernel.h>
2672c7c34a24SBruce Evans 
2673ce9edcf5SPoul-Henning Kamp #include <sys/cons.h>
2674c7c34a24SBruce Evans 
2675c7c34a24SBruce Evans #include <ddb/ddb.h>
2676c7c34a24SBruce Evans 
2677cac597e4SBruce Evans static int
26781b40f8c0SMatthew Dillon _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry)
2679a1f6d91cSDavid Greenman {
2680a1f6d91cSDavid Greenman 	vm_map_t tmpm;
2681a1f6d91cSDavid Greenman 	vm_map_entry_t tmpe;
2682a1f6d91cSDavid Greenman 	vm_object_t obj;
2683a1f6d91cSDavid Greenman 
2684a1f6d91cSDavid Greenman 	if (map == 0)
2685a1f6d91cSDavid Greenman 		return 0;
2686a1f6d91cSDavid Greenman 
2687a1f6d91cSDavid Greenman 	if (entry == 0) {
26882288078cSDoug Moore 		VM_MAP_ENTRY_FOREACH(tmpe, map) {
2689a1f6d91cSDavid Greenman 			if (_vm_object_in_map(map, object, tmpe)) {
2690a1f6d91cSDavid Greenman 				return 1;
2691a1f6d91cSDavid Greenman 			}
2692a1f6d91cSDavid Greenman 		}
26939fdfe602SMatthew Dillon 	} else if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
26949fdfe602SMatthew Dillon 		tmpm = entry->object.sub_map;
26952288078cSDoug Moore 		VM_MAP_ENTRY_FOREACH(tmpe, tmpm) {
2696a1f6d91cSDavid Greenman 			if (_vm_object_in_map(tmpm, object, tmpe)) {
2697a1f6d91cSDavid Greenman 				return 1;
2698a1f6d91cSDavid Greenman 			}
2699a1f6d91cSDavid Greenman 		}
27008aef1712SMatthew Dillon 	} else if ((obj = entry->object.vm_object) != NULL) {
270124a1cce3SDavid Greenman 		for (; obj; obj = obj->backing_object)
2702a1f6d91cSDavid Greenman 			if (obj == object) {
2703a1f6d91cSDavid Greenman 				return 1;
2704a1f6d91cSDavid Greenman 			}
2705a1f6d91cSDavid Greenman 	}
2706a1f6d91cSDavid Greenman 	return 0;
2707a1f6d91cSDavid Greenman }
2708a1f6d91cSDavid Greenman 
2709cac597e4SBruce Evans static int
27101b40f8c0SMatthew Dillon vm_object_in_map(vm_object_t object)
2711a1f6d91cSDavid Greenman {
2712a1f6d91cSDavid Greenman 	struct proc *p;
27131005a129SJohn Baldwin 
271460517fd1SJohn Baldwin 	/* sx_slock(&allproc_lock); */
2715f67af5c9SXin LI 	FOREACH_PROC_IN_SYSTEM(p) {
2716a1f6d91cSDavid Greenman 		if (!p->p_vmspace /* || (p->p_flag & (P_SYSTEM|P_WEXIT)) */)
2717a1f6d91cSDavid Greenman 			continue;
2718553629ebSJake Burkholder 		if (_vm_object_in_map(&p->p_vmspace->vm_map, object, 0)) {
271960517fd1SJohn Baldwin 			/* sx_sunlock(&allproc_lock); */
2720a1f6d91cSDavid Greenman 			return 1;
2721a1f6d91cSDavid Greenman 		}
2722553629ebSJake Burkholder 	}
272360517fd1SJohn Baldwin 	/* sx_sunlock(&allproc_lock); */
2724a1f6d91cSDavid Greenman 	if (_vm_object_in_map(kernel_map, object, 0))
2725a1f6d91cSDavid Greenman 		return 1;
2726a1f6d91cSDavid Greenman 	return 0;
2727a1f6d91cSDavid Greenman }
2728a1f6d91cSDavid Greenman 
2729c7c34a24SBruce Evans DB_SHOW_COMMAND(vmochk, vm_object_check)
2730f708ef1bSPoul-Henning Kamp {
2731a1f6d91cSDavid Greenman 	vm_object_t object;
2732a1f6d91cSDavid Greenman 
2733a1f6d91cSDavid Greenman 	/*
2734a1f6d91cSDavid Greenman 	 * make sure that internal objs are in a map somewhere
2735a1f6d91cSDavid Greenman 	 * and none have zero ref counts.
2736a1f6d91cSDavid Greenman 	 */
2737cc64b484SAlfred Perlstein 	TAILQ_FOREACH(object, &vm_object_list, object_list) {
273832362449SKonstantin Belousov 		if ((object->flags & OBJ_ANON) != 0) {
2739a1f6d91cSDavid Greenman 			if (object->ref_count == 0) {
27403efc015bSPeter Wemm 				db_printf("vmochk: internal obj has zero ref count: %ld\n",
27413efc015bSPeter Wemm 					(long)object->size);
2742a1f6d91cSDavid Greenman 			}
2743a1f6d91cSDavid Greenman 			if (!vm_object_in_map(object)) {
2744fc62ef1fSBruce Evans 				db_printf(
2745fc62ef1fSBruce Evans 			"vmochk: internal obj is not in a map: "
2746fc62ef1fSBruce Evans 			"ref: %d, size: %lu: 0x%lx, backing_object: %p\n",
2747fc62ef1fSBruce Evans 				    object->ref_count, (u_long)object->size,
2748fc62ef1fSBruce Evans 				    (u_long)object->size,
2749fc62ef1fSBruce Evans 				    (void *)object->backing_object);
2750a1f6d91cSDavid Greenman 			}
2751a1f6d91cSDavid Greenman 		}
27528cc8c586SEric van Gyzen 		if (db_pager_quit)
27538cc8c586SEric van Gyzen 			return;
2754a1f6d91cSDavid Greenman 	}
2755a1f6d91cSDavid Greenman }
2756a1f6d91cSDavid Greenman 
275726f9a767SRodney W. Grimes /*
2758df8bae1dSRodney W. Grimes  *	vm_object_print:	[ debug ]
2759df8bae1dSRodney W. Grimes  */
2760c7c34a24SBruce Evans DB_SHOW_COMMAND(object, vm_object_print_static)
2761df8bae1dSRodney W. Grimes {
2762c7c34a24SBruce Evans 	/* XXX convert args. */
2763c7c34a24SBruce Evans 	vm_object_t object = (vm_object_t)addr;
2764c7c34a24SBruce Evans 	boolean_t full = have_addr;
2765c7c34a24SBruce Evans 
2766d031cff1SMatthew Dillon 	vm_page_t p;
2767df8bae1dSRodney W. Grimes 
2768c7c34a24SBruce Evans 	/* XXX count is an (unused) arg.  Avoid shadowing it. */
2769c7c34a24SBruce Evans #define	count	was_count
2770c7c34a24SBruce Evans 
2771d031cff1SMatthew Dillon 	int count;
2772df8bae1dSRodney W. Grimes 
2773df8bae1dSRodney W. Grimes 	if (object == NULL)
2774df8bae1dSRodney W. Grimes 		return;
2775df8bae1dSRodney W. Grimes 
2776eb95adefSBruce Evans 	db_iprintf(
2777ef694c1aSEdward Tomasz Napierala 	    "Object %p: type=%d, size=0x%jx, res=%d, ref=%d, flags=0x%x ruid %d charge %jx\n",
2778e47cd172SMaxime Henrion 	    object, (int)object->type, (uintmax_t)object->size,
27793364c323SKonstantin Belousov 	    object->resident_page_count, object->ref_count, object->flags,
2780ef694c1aSEdward Tomasz Napierala 	    object->cred ? object->cred->cr_ruid : -1, (uintmax_t)object->charge);
2781e47cd172SMaxime Henrion 	db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%jx\n",
27821c7c3c6aSMatthew Dillon 	    object->shadow_count,
2783eb95adefSBruce Evans 	    object->backing_object ? object->backing_object->ref_count : 0,
2784e47cd172SMaxime Henrion 	    object->backing_object, (uintmax_t)object->backing_object_offset);
2785df8bae1dSRodney W. Grimes 
2786df8bae1dSRodney W. Grimes 	if (!full)
2787df8bae1dSRodney W. Grimes 		return;
2788df8bae1dSRodney W. Grimes 
2789c7c34a24SBruce Evans 	db_indent += 2;
2790df8bae1dSRodney W. Grimes 	count = 0;
2791fc2ffbe6SPoul-Henning Kamp 	TAILQ_FOREACH(p, &object->memq, listq) {
2792df8bae1dSRodney W. Grimes 		if (count == 0)
2793c7c34a24SBruce Evans 			db_iprintf("memory:=");
2794df8bae1dSRodney W. Grimes 		else if (count == 6) {
2795c7c34a24SBruce Evans 			db_printf("\n");
2796c7c34a24SBruce Evans 			db_iprintf(" ...");
2797df8bae1dSRodney W. Grimes 			count = 0;
2798df8bae1dSRodney W. Grimes 		} else
2799c7c34a24SBruce Evans 			db_printf(",");
2800df8bae1dSRodney W. Grimes 		count++;
2801df8bae1dSRodney W. Grimes 
2802e47cd172SMaxime Henrion 		db_printf("(off=0x%jx,page=0x%jx)",
2803e47cd172SMaxime Henrion 		    (uintmax_t)p->pindex, (uintmax_t)VM_PAGE_TO_PHYS(p));
28048cc8c586SEric van Gyzen 
28058cc8c586SEric van Gyzen 		if (db_pager_quit)
28068cc8c586SEric van Gyzen 			break;
2807df8bae1dSRodney W. Grimes 	}
2808df8bae1dSRodney W. Grimes 	if (count != 0)
2809c7c34a24SBruce Evans 		db_printf("\n");
2810c7c34a24SBruce Evans 	db_indent -= 2;
2811df8bae1dSRodney W. Grimes }
28125070c7f8SJohn Dyson 
2813c7c34a24SBruce Evans /* XXX. */
2814c7c34a24SBruce Evans #undef count
2815c7c34a24SBruce Evans 
2816c7c34a24SBruce Evans /* XXX need this non-static entry for calling from vm_map_print. */
28175070c7f8SJohn Dyson void
28181b40f8c0SMatthew Dillon vm_object_print(
28191b40f8c0SMatthew Dillon         /* db_expr_t */ long addr,
28201b40f8c0SMatthew Dillon 	boolean_t have_addr,
28211b40f8c0SMatthew Dillon 	/* db_expr_t */ long count,
28221b40f8c0SMatthew Dillon 	char *modif)
2823c7c34a24SBruce Evans {
2824c7c34a24SBruce Evans 	vm_object_print_static(addr, have_addr, count, modif);
2825c7c34a24SBruce Evans }
2826c7c34a24SBruce Evans 
2827c7c34a24SBruce Evans DB_SHOW_COMMAND(vmopag, vm_object_print_pages)
28285070c7f8SJohn Dyson {
28295070c7f8SJohn Dyson 	vm_object_t object;
2830bb2ac86fSKonstantin Belousov 	vm_pindex_t fidx;
2831bb2ac86fSKonstantin Belousov 	vm_paddr_t pa;
2832bb2ac86fSKonstantin Belousov 	vm_page_t m, prev_m;
2833*18048b6eSStephen J. Kiernan 	int rcount;
2834cc64b484SAlfred Perlstein 
2835cc64b484SAlfred Perlstein 	TAILQ_FOREACH(object, &vm_object_list, object_list) {
2836fc62ef1fSBruce Evans 		db_printf("new object: %p\n", (void *)object);
2837*18048b6eSStephen J. Kiernan 		if (db_pager_quit)
28385070c7f8SJohn Dyson 			return;
2839*18048b6eSStephen J. Kiernan 
28405070c7f8SJohn Dyson 		rcount = 0;
28415070c7f8SJohn Dyson 		fidx = 0;
2842bb2ac86fSKonstantin Belousov 		pa = -1;
2843bb2ac86fSKonstantin Belousov 		TAILQ_FOREACH(m, &object->memq, listq) {
2844bb2ac86fSKonstantin Belousov 			if (m->pindex > 128)
2845bb2ac86fSKonstantin Belousov 				break;
2846bb2ac86fSKonstantin Belousov 			if ((prev_m = TAILQ_PREV(m, pglist, listq)) != NULL &&
2847bb2ac86fSKonstantin Belousov 			    prev_m->pindex + 1 != m->pindex) {
28485070c7f8SJohn Dyson 				if (rcount) {
28493efc015bSPeter Wemm 					db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
28503efc015bSPeter Wemm 						(long)fidx, rcount, (long)pa);
2851*18048b6eSStephen J. Kiernan 					if (db_pager_quit)
28525070c7f8SJohn Dyson 						return;
28535070c7f8SJohn Dyson 					rcount = 0;
28545070c7f8SJohn Dyson 				}
28555070c7f8SJohn Dyson 			}
28565070c7f8SJohn Dyson 			if (rcount &&
28575070c7f8SJohn Dyson 				(VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) {
28585070c7f8SJohn Dyson 				++rcount;
28595070c7f8SJohn Dyson 				continue;
28605070c7f8SJohn Dyson 			}
28615070c7f8SJohn Dyson 			if (rcount) {
28622446e4f0SAlan Cox 				db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
28633efc015bSPeter Wemm 					(long)fidx, rcount, (long)pa);
2864*18048b6eSStephen J. Kiernan 				if (db_pager_quit)
28655070c7f8SJohn Dyson 					return;
28665070c7f8SJohn Dyson 			}
2867bb2ac86fSKonstantin Belousov 			fidx = m->pindex;
28685070c7f8SJohn Dyson 			pa = VM_PAGE_TO_PHYS(m);
28695070c7f8SJohn Dyson 			rcount = 1;
28705070c7f8SJohn Dyson 		}
28715070c7f8SJohn Dyson 		if (rcount) {
28723efc015bSPeter Wemm 			db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
28733efc015bSPeter Wemm 				(long)fidx, rcount, (long)pa);
2874*18048b6eSStephen J. Kiernan 			if (db_pager_quit)
28755070c7f8SJohn Dyson 				return;
28765070c7f8SJohn Dyson 		}
28775070c7f8SJohn Dyson 	}
28785070c7f8SJohn Dyson }
2879c3cb3e12SDavid Greenman #endif /* DDB */
2880