xref: /freebsd/sys/vm/vm_object.c (revision d500a85e640d1cd270747c12e17c511b53864436)
1 /*-
2  * SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU)
3  *
4  * Copyright (c) 1991, 1993
5  *	The Regents of the University of California.  All rights reserved.
6  *
7  * This code is derived from software contributed to Berkeley by
8  * The Mach Operating System project at Carnegie-Mellon University.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 3. Neither the name of the University nor the names of its contributors
19  *    may be used to endorse or promote products derived from this software
20  *    without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  *	from: @(#)vm_object.c	8.5 (Berkeley) 3/22/94
35  *
36  *
37  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
38  * All rights reserved.
39  *
40  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
41  *
42  * Permission to use, copy, modify and distribute this software and
43  * its documentation is hereby granted, provided that both the copyright
44  * notice and this permission notice appear in all copies of the
45  * software, derivative works or modified versions, and any portions
46  * thereof, and that both notices appear in supporting documentation.
47  *
48  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
49  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
50  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
51  *
52  * Carnegie Mellon requests users of this software to return to
53  *
54  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
55  *  School of Computer Science
56  *  Carnegie Mellon University
57  *  Pittsburgh PA 15213-3890
58  *
59  * any improvements or extensions that they make and grant Carnegie the
60  * rights to redistribute these changes.
61  */
62 
63 /*
64  *	Virtual memory object module.
65  */
66 
67 #include <sys/cdefs.h>
68 __FBSDID("$FreeBSD$");
69 
70 #include "opt_vm.h"
71 
72 #include <sys/param.h>
73 #include <sys/systm.h>
74 #include <sys/blockcount.h>
75 #include <sys/cpuset.h>
76 #include <sys/limits.h>
77 #include <sys/lock.h>
78 #include <sys/mman.h>
79 #include <sys/mount.h>
80 #include <sys/kernel.h>
81 #include <sys/pctrie.h>
82 #include <sys/sysctl.h>
83 #include <sys/mutex.h>
84 #include <sys/proc.h>		/* for curproc, pageproc */
85 #include <sys/refcount.h>
86 #include <sys/socket.h>
87 #include <sys/resourcevar.h>
88 #include <sys/refcount.h>
89 #include <sys/rwlock.h>
90 #include <sys/user.h>
91 #include <sys/vnode.h>
92 #include <sys/vmmeter.h>
93 #include <sys/sx.h>
94 
95 #include <vm/vm.h>
96 #include <vm/vm_param.h>
97 #include <vm/pmap.h>
98 #include <vm/vm_map.h>
99 #include <vm/vm_object.h>
100 #include <vm/vm_page.h>
101 #include <vm/vm_pageout.h>
102 #include <vm/vm_pager.h>
103 #include <vm/vm_phys.h>
104 #include <vm/vm_pagequeue.h>
105 #include <vm/swap_pager.h>
106 #include <vm/vm_kern.h>
107 #include <vm/vm_extern.h>
108 #include <vm/vm_radix.h>
109 #include <vm/vm_reserv.h>
110 #include <vm/uma.h>
111 
112 static int old_msync;
113 SYSCTL_INT(_vm, OID_AUTO, old_msync, CTLFLAG_RW, &old_msync, 0,
114     "Use old (insecure) msync behavior");
115 
116 static int	vm_object_page_collect_flush(vm_object_t object, vm_page_t p,
117 		    int pagerflags, int flags, boolean_t *allclean,
118 		    boolean_t *eio);
119 static boolean_t vm_object_page_remove_write(vm_page_t p, int flags,
120 		    boolean_t *allclean);
121 static void	vm_object_backing_remove(vm_object_t object);
122 
123 /*
124  *	Virtual memory objects maintain the actual data
125  *	associated with allocated virtual memory.  A given
126  *	page of memory exists within exactly one object.
127  *
128  *	An object is only deallocated when all "references"
129  *	are given up.  Only one "reference" to a given
130  *	region of an object should be writeable.
131  *
132  *	Associated with each object is a list of all resident
133  *	memory pages belonging to that object; this list is
134  *	maintained by the "vm_page" module, and locked by the object's
135  *	lock.
136  *
137  *	Each object also records a "pager" routine which is
138  *	used to retrieve (and store) pages to the proper backing
139  *	storage.  In addition, objects may be backed by other
140  *	objects from which they were virtual-copied.
141  *
142  *	The only items within the object structure which are
143  *	modified after time of creation are:
144  *		reference count		locked by object's lock
145  *		pager routine		locked by object's lock
146  *
147  */
148 
149 struct object_q vm_object_list;
150 struct mtx vm_object_list_mtx;	/* lock for object list and count */
151 
152 struct vm_object kernel_object_store;
153 
154 static SYSCTL_NODE(_vm_stats, OID_AUTO, object, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
155     "VM object stats");
156 
157 static COUNTER_U64_DEFINE_EARLY(object_collapses);
158 SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, collapses, CTLFLAG_RD,
159     &object_collapses,
160     "VM object collapses");
161 
162 static COUNTER_U64_DEFINE_EARLY(object_bypasses);
163 SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, bypasses, CTLFLAG_RD,
164     &object_bypasses,
165     "VM object bypasses");
166 
167 static COUNTER_U64_DEFINE_EARLY(object_collapse_waits);
168 SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, collapse_waits, CTLFLAG_RD,
169     &object_collapse_waits,
170     "Number of sleeps for collapse");
171 
172 static uma_zone_t obj_zone;
173 
174 static int vm_object_zinit(void *mem, int size, int flags);
175 
176 #ifdef INVARIANTS
177 static void vm_object_zdtor(void *mem, int size, void *arg);
178 
179 static void
180 vm_object_zdtor(void *mem, int size, void *arg)
181 {
182 	vm_object_t object;
183 
184 	object = (vm_object_t)mem;
185 	KASSERT(object->ref_count == 0,
186 	    ("object %p ref_count = %d", object, object->ref_count));
187 	KASSERT(TAILQ_EMPTY(&object->memq),
188 	    ("object %p has resident pages in its memq", object));
189 	KASSERT(vm_radix_is_empty(&object->rtree),
190 	    ("object %p has resident pages in its trie", object));
191 #if VM_NRESERVLEVEL > 0
192 	KASSERT(LIST_EMPTY(&object->rvq),
193 	    ("object %p has reservations",
194 	    object));
195 #endif
196 	KASSERT(!vm_object_busied(object),
197 	    ("object %p busy = %d", object, blockcount_read(&object->busy)));
198 	KASSERT(object->resident_page_count == 0,
199 	    ("object %p resident_page_count = %d",
200 	    object, object->resident_page_count));
201 	KASSERT(object->shadow_count == 0,
202 	    ("object %p shadow_count = %d",
203 	    object, object->shadow_count));
204 	KASSERT(object->type == OBJT_DEAD,
205 	    ("object %p has non-dead type %d",
206 	    object, object->type));
207 }
208 #endif
209 
210 static int
211 vm_object_zinit(void *mem, int size, int flags)
212 {
213 	vm_object_t object;
214 
215 	object = (vm_object_t)mem;
216 	rw_init_flags(&object->lock, "vm object", RW_DUPOK | RW_NEW);
217 
218 	/* These are true for any object that has been freed */
219 	object->type = OBJT_DEAD;
220 	vm_radix_init(&object->rtree);
221 	refcount_init(&object->ref_count, 0);
222 	blockcount_init(&object->paging_in_progress);
223 	blockcount_init(&object->busy);
224 	object->resident_page_count = 0;
225 	object->shadow_count = 0;
226 	object->flags = OBJ_DEAD;
227 
228 	mtx_lock(&vm_object_list_mtx);
229 	TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);
230 	mtx_unlock(&vm_object_list_mtx);
231 	return (0);
232 }
233 
234 static void
235 _vm_object_allocate(objtype_t type, vm_pindex_t size, u_short flags,
236     vm_object_t object, void *handle)
237 {
238 
239 	TAILQ_INIT(&object->memq);
240 	LIST_INIT(&object->shadow_head);
241 
242 	object->type = type;
243 	if (type == OBJT_SWAP)
244 		pctrie_init(&object->un_pager.swp.swp_blks);
245 
246 	/*
247 	 * Ensure that swap_pager_swapoff() iteration over object_list
248 	 * sees up to date type and pctrie head if it observed
249 	 * non-dead object.
250 	 */
251 	atomic_thread_fence_rel();
252 
253 	object->pg_color = 0;
254 	object->flags = flags;
255 	object->size = size;
256 	object->domain.dr_policy = NULL;
257 	object->generation = 1;
258 	object->cleangeneration = 1;
259 	refcount_init(&object->ref_count, 1);
260 	object->memattr = VM_MEMATTR_DEFAULT;
261 	object->cred = NULL;
262 	object->charge = 0;
263 	object->handle = handle;
264 	object->backing_object = NULL;
265 	object->backing_object_offset = (vm_ooffset_t) 0;
266 #if VM_NRESERVLEVEL > 0
267 	LIST_INIT(&object->rvq);
268 #endif
269 	umtx_shm_object_init(object);
270 }
271 
272 /*
273  *	vm_object_init:
274  *
275  *	Initialize the VM objects module.
276  */
277 void
278 vm_object_init(void)
279 {
280 	TAILQ_INIT(&vm_object_list);
281 	mtx_init(&vm_object_list_mtx, "vm object_list", NULL, MTX_DEF);
282 
283 	rw_init(&kernel_object->lock, "kernel vm object");
284 	_vm_object_allocate(OBJT_PHYS, atop(VM_MAX_KERNEL_ADDRESS -
285 	    VM_MIN_KERNEL_ADDRESS), OBJ_UNMANAGED, kernel_object, NULL);
286 #if VM_NRESERVLEVEL > 0
287 	kernel_object->flags |= OBJ_COLORED;
288 	kernel_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS);
289 #endif
290 	kernel_object->un_pager.phys.ops = &default_phys_pg_ops;
291 
292 	/*
293 	 * The lock portion of struct vm_object must be type stable due
294 	 * to vm_pageout_fallback_object_lock locking a vm object
295 	 * without holding any references to it.
296 	 *
297 	 * paging_in_progress is valid always.  Lockless references to
298 	 * the objects may acquire pip and then check OBJ_DEAD.
299 	 */
300 	obj_zone = uma_zcreate("VM OBJECT", sizeof (struct vm_object), NULL,
301 #ifdef INVARIANTS
302 	    vm_object_zdtor,
303 #else
304 	    NULL,
305 #endif
306 	    vm_object_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
307 
308 	vm_radix_zinit();
309 }
310 
311 void
312 vm_object_clear_flag(vm_object_t object, u_short bits)
313 {
314 
315 	VM_OBJECT_ASSERT_WLOCKED(object);
316 	object->flags &= ~bits;
317 }
318 
319 /*
320  *	Sets the default memory attribute for the specified object.  Pages
321  *	that are allocated to this object are by default assigned this memory
322  *	attribute.
323  *
324  *	Presently, this function must be called before any pages are allocated
325  *	to the object.  In the future, this requirement may be relaxed for
326  *	"default" and "swap" objects.
327  */
328 int
329 vm_object_set_memattr(vm_object_t object, vm_memattr_t memattr)
330 {
331 
332 	VM_OBJECT_ASSERT_WLOCKED(object);
333 	switch (object->type) {
334 	case OBJT_DEFAULT:
335 	case OBJT_DEVICE:
336 	case OBJT_MGTDEVICE:
337 	case OBJT_PHYS:
338 	case OBJT_SG:
339 	case OBJT_SWAP:
340 	case OBJT_VNODE:
341 		if (!TAILQ_EMPTY(&object->memq))
342 			return (KERN_FAILURE);
343 		break;
344 	case OBJT_DEAD:
345 		return (KERN_INVALID_ARGUMENT);
346 	default:
347 		panic("vm_object_set_memattr: object %p is of undefined type",
348 		    object);
349 	}
350 	object->memattr = memattr;
351 	return (KERN_SUCCESS);
352 }
353 
354 void
355 vm_object_pip_add(vm_object_t object, short i)
356 {
357 
358 	if (i > 0)
359 		blockcount_acquire(&object->paging_in_progress, i);
360 }
361 
362 void
363 vm_object_pip_wakeup(vm_object_t object)
364 {
365 
366 	vm_object_pip_wakeupn(object, 1);
367 }
368 
369 void
370 vm_object_pip_wakeupn(vm_object_t object, short i)
371 {
372 
373 	if (i > 0)
374 		blockcount_release(&object->paging_in_progress, i);
375 }
376 
377 /*
378  * Atomically drop the object lock and wait for pip to drain.  This protects
379  * from sleep/wakeup races due to identity changes.  The lock is not re-acquired
380  * on return.
381  */
382 static void
383 vm_object_pip_sleep(vm_object_t object, const char *waitid)
384 {
385 
386 	(void)blockcount_sleep(&object->paging_in_progress, &object->lock,
387 	    waitid, PVM | PDROP);
388 }
389 
390 void
391 vm_object_pip_wait(vm_object_t object, const char *waitid)
392 {
393 
394 	VM_OBJECT_ASSERT_WLOCKED(object);
395 
396 	blockcount_wait(&object->paging_in_progress, &object->lock, waitid,
397 	    PVM);
398 }
399 
400 void
401 vm_object_pip_wait_unlocked(vm_object_t object, const char *waitid)
402 {
403 
404 	VM_OBJECT_ASSERT_UNLOCKED(object);
405 
406 	blockcount_wait(&object->paging_in_progress, NULL, waitid, PVM);
407 }
408 
409 /*
410  *	vm_object_allocate:
411  *
412  *	Returns a new object with the given size.
413  */
414 vm_object_t
415 vm_object_allocate(objtype_t type, vm_pindex_t size)
416 {
417 	vm_object_t object;
418 	u_short flags;
419 
420 	switch (type) {
421 	case OBJT_DEAD:
422 		panic("vm_object_allocate: can't create OBJT_DEAD");
423 	case OBJT_DEFAULT:
424 	case OBJT_SWAP:
425 		flags = OBJ_COLORED;
426 		break;
427 	case OBJT_DEVICE:
428 	case OBJT_SG:
429 		flags = OBJ_FICTITIOUS | OBJ_UNMANAGED;
430 		break;
431 	case OBJT_MGTDEVICE:
432 		flags = OBJ_FICTITIOUS;
433 		break;
434 	case OBJT_PHYS:
435 		flags = OBJ_UNMANAGED;
436 		break;
437 	case OBJT_VNODE:
438 		flags = 0;
439 		break;
440 	default:
441 		panic("vm_object_allocate: type %d is undefined", type);
442 	}
443 	object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);
444 	_vm_object_allocate(type, size, flags, object, NULL);
445 
446 	return (object);
447 }
448 
449 /*
450  *	vm_object_allocate_anon:
451  *
452  *	Returns a new default object of the given size and marked as
453  *	anonymous memory for special split/collapse handling.  Color
454  *	to be initialized by the caller.
455  */
456 vm_object_t
457 vm_object_allocate_anon(vm_pindex_t size, vm_object_t backing_object,
458     struct ucred *cred, vm_size_t charge)
459 {
460 	vm_object_t handle, object;
461 
462 	if (backing_object == NULL)
463 		handle = NULL;
464 	else if ((backing_object->flags & OBJ_ANON) != 0)
465 		handle = backing_object->handle;
466 	else
467 		handle = backing_object;
468 	object = uma_zalloc(obj_zone, M_WAITOK);
469 	_vm_object_allocate(OBJT_DEFAULT, size, OBJ_ANON | OBJ_ONEMAPPING,
470 	    object, handle);
471 	object->cred = cred;
472 	object->charge = cred != NULL ? charge : 0;
473 	return (object);
474 }
475 
476 static void
477 vm_object_reference_vnode(vm_object_t object)
478 {
479 	u_int old;
480 
481 	/*
482 	 * vnode objects need the lock for the first reference
483 	 * to serialize with vnode_object_deallocate().
484 	 */
485 	if (!refcount_acquire_if_gt(&object->ref_count, 0)) {
486 		VM_OBJECT_RLOCK(object);
487 		old = refcount_acquire(&object->ref_count);
488 		if (object->type == OBJT_VNODE && old == 0)
489 			vref(object->handle);
490 		VM_OBJECT_RUNLOCK(object);
491 	}
492 }
493 
494 /*
495  *	vm_object_reference:
496  *
497  *	Acquires a reference to the given object.
498  */
499 void
500 vm_object_reference(vm_object_t object)
501 {
502 
503 	if (object == NULL)
504 		return;
505 
506 	if (object->type == OBJT_VNODE)
507 		vm_object_reference_vnode(object);
508 	else
509 		refcount_acquire(&object->ref_count);
510 	KASSERT((object->flags & OBJ_DEAD) == 0,
511 	    ("vm_object_reference: Referenced dead object."));
512 }
513 
514 /*
515  *	vm_object_reference_locked:
516  *
517  *	Gets another reference to the given object.
518  *
519  *	The object must be locked.
520  */
521 void
522 vm_object_reference_locked(vm_object_t object)
523 {
524 	u_int old;
525 
526 	VM_OBJECT_ASSERT_LOCKED(object);
527 	old = refcount_acquire(&object->ref_count);
528 	if (object->type == OBJT_VNODE && old == 0)
529 		vref(object->handle);
530 	KASSERT((object->flags & OBJ_DEAD) == 0,
531 	    ("vm_object_reference: Referenced dead object."));
532 }
533 
534 /*
535  * Handle deallocating an object of type OBJT_VNODE.
536  */
537 static void
538 vm_object_deallocate_vnode(vm_object_t object)
539 {
540 	struct vnode *vp = (struct vnode *) object->handle;
541 	bool last;
542 
543 	KASSERT(object->type == OBJT_VNODE,
544 	    ("vm_object_deallocate_vnode: not a vnode object"));
545 	KASSERT(vp != NULL, ("vm_object_deallocate_vnode: missing vp"));
546 
547 	/* Object lock to protect handle lookup. */
548 	last = refcount_release(&object->ref_count);
549 	VM_OBJECT_RUNLOCK(object);
550 
551 	if (!last)
552 		return;
553 
554 	if (!umtx_shm_vnobj_persistent)
555 		umtx_shm_object_terminated(object);
556 
557 	/* vrele may need the vnode lock. */
558 	vrele(vp);
559 }
560 
561 /*
562  * We dropped a reference on an object and discovered that it had a
563  * single remaining shadow.  This is a sibling of the reference we
564  * dropped.  Attempt to collapse the sibling and backing object.
565  */
566 static vm_object_t
567 vm_object_deallocate_anon(vm_object_t backing_object)
568 {
569 	vm_object_t object;
570 
571 	/* Fetch the final shadow.  */
572 	object = LIST_FIRST(&backing_object->shadow_head);
573 	KASSERT(object != NULL && backing_object->shadow_count == 1,
574 	    ("vm_object_anon_deallocate: ref_count: %d, shadow_count: %d",
575 	    backing_object->ref_count, backing_object->shadow_count));
576 	KASSERT((object->flags & (OBJ_TMPFS_NODE | OBJ_ANON)) == OBJ_ANON,
577 	    ("invalid shadow object %p", object));
578 
579 	if (!VM_OBJECT_TRYWLOCK(object)) {
580 		/*
581 		 * Prevent object from disappearing since we do not have a
582 		 * reference.
583 		 */
584 		vm_object_pip_add(object, 1);
585 		VM_OBJECT_WUNLOCK(backing_object);
586 		VM_OBJECT_WLOCK(object);
587 		vm_object_pip_wakeup(object);
588 	} else
589 		VM_OBJECT_WUNLOCK(backing_object);
590 
591 	/*
592 	 * Check for a collapse/terminate race with the last reference holder.
593 	 */
594 	if ((object->flags & (OBJ_DEAD | OBJ_COLLAPSING)) != 0 ||
595 	    !refcount_acquire_if_not_zero(&object->ref_count)) {
596 		VM_OBJECT_WUNLOCK(object);
597 		return (NULL);
598 	}
599 	backing_object = object->backing_object;
600 	if (backing_object != NULL && (backing_object->flags & OBJ_ANON) != 0)
601 		vm_object_collapse(object);
602 	VM_OBJECT_WUNLOCK(object);
603 
604 	return (object);
605 }
606 
607 /*
608  *	vm_object_deallocate:
609  *
610  *	Release a reference to the specified object,
611  *	gained either through a vm_object_allocate
612  *	or a vm_object_reference call.  When all references
613  *	are gone, storage associated with this object
614  *	may be relinquished.
615  *
616  *	No object may be locked.
617  */
618 void
619 vm_object_deallocate(vm_object_t object)
620 {
621 	vm_object_t temp;
622 	bool released;
623 
624 	while (object != NULL) {
625 		/*
626 		 * If the reference count goes to 0 we start calling
627 		 * vm_object_terminate() on the object chain.  A ref count
628 		 * of 1 may be a special case depending on the shadow count
629 		 * being 0 or 1.  These cases require a write lock on the
630 		 * object.
631 		 */
632 		if ((object->flags & OBJ_ANON) == 0)
633 			released = refcount_release_if_gt(&object->ref_count, 1);
634 		else
635 			released = refcount_release_if_gt(&object->ref_count, 2);
636 		if (released)
637 			return;
638 
639 		if (object->type == OBJT_VNODE) {
640 			VM_OBJECT_RLOCK(object);
641 			if (object->type == OBJT_VNODE) {
642 				vm_object_deallocate_vnode(object);
643 				return;
644 			}
645 			VM_OBJECT_RUNLOCK(object);
646 		}
647 
648 		VM_OBJECT_WLOCK(object);
649 		KASSERT(object->ref_count > 0,
650 		    ("vm_object_deallocate: object deallocated too many times: %d",
651 		    object->type));
652 
653 		/*
654 		 * If this is not the final reference to an anonymous
655 		 * object we may need to collapse the shadow chain.
656 		 */
657 		if (!refcount_release(&object->ref_count)) {
658 			if (object->ref_count > 1 ||
659 			    object->shadow_count == 0) {
660 				if ((object->flags & OBJ_ANON) != 0 &&
661 				    object->ref_count == 1)
662 					vm_object_set_flag(object,
663 					    OBJ_ONEMAPPING);
664 				VM_OBJECT_WUNLOCK(object);
665 				return;
666 			}
667 
668 			/* Handle collapsing last ref on anonymous objects. */
669 			object = vm_object_deallocate_anon(object);
670 			continue;
671 		}
672 
673 		/*
674 		 * Handle the final reference to an object.  We restart
675 		 * the loop with the backing object to avoid recursion.
676 		 */
677 		umtx_shm_object_terminated(object);
678 		temp = object->backing_object;
679 		if (temp != NULL) {
680 			KASSERT((object->flags & OBJ_TMPFS_NODE) == 0,
681 			    ("shadowed tmpfs v_object 2 %p", object));
682 			vm_object_backing_remove(object);
683 		}
684 
685 		KASSERT((object->flags & OBJ_DEAD) == 0,
686 		    ("vm_object_deallocate: Terminating dead object."));
687 		vm_object_set_flag(object, OBJ_DEAD);
688 		vm_object_terminate(object);
689 		object = temp;
690 	}
691 }
692 
693 /*
694  *	vm_object_destroy removes the object from the global object list
695  *      and frees the space for the object.
696  */
697 void
698 vm_object_destroy(vm_object_t object)
699 {
700 
701 	/*
702 	 * Release the allocation charge.
703 	 */
704 	if (object->cred != NULL) {
705 		swap_release_by_cred(object->charge, object->cred);
706 		object->charge = 0;
707 		crfree(object->cred);
708 		object->cred = NULL;
709 	}
710 
711 	/*
712 	 * Free the space for the object.
713 	 */
714 	uma_zfree(obj_zone, object);
715 }
716 
717 static void
718 vm_object_backing_remove_locked(vm_object_t object)
719 {
720 	vm_object_t backing_object;
721 
722 	backing_object = object->backing_object;
723 	VM_OBJECT_ASSERT_WLOCKED(object);
724 	VM_OBJECT_ASSERT_WLOCKED(backing_object);
725 
726 	KASSERT((object->flags & OBJ_COLLAPSING) == 0,
727 	    ("vm_object_backing_remove: Removing collapsing object."));
728 
729 	if ((object->flags & OBJ_SHADOWLIST) != 0) {
730 		LIST_REMOVE(object, shadow_list);
731 		backing_object->shadow_count--;
732 		object->flags &= ~OBJ_SHADOWLIST;
733 	}
734 	object->backing_object = NULL;
735 }
736 
737 static void
738 vm_object_backing_remove(vm_object_t object)
739 {
740 	vm_object_t backing_object;
741 
742 	VM_OBJECT_ASSERT_WLOCKED(object);
743 
744 	if ((object->flags & OBJ_SHADOWLIST) != 0) {
745 		backing_object = object->backing_object;
746 		VM_OBJECT_WLOCK(backing_object);
747 		vm_object_backing_remove_locked(object);
748 		VM_OBJECT_WUNLOCK(backing_object);
749 	} else
750 		object->backing_object = NULL;
751 }
752 
753 static void
754 vm_object_backing_insert_locked(vm_object_t object, vm_object_t backing_object)
755 {
756 
757 	VM_OBJECT_ASSERT_WLOCKED(object);
758 
759 	if ((backing_object->flags & OBJ_ANON) != 0) {
760 		VM_OBJECT_ASSERT_WLOCKED(backing_object);
761 		LIST_INSERT_HEAD(&backing_object->shadow_head, object,
762 		    shadow_list);
763 		backing_object->shadow_count++;
764 		object->flags |= OBJ_SHADOWLIST;
765 	}
766 	object->backing_object = backing_object;
767 }
768 
769 static void
770 vm_object_backing_insert(vm_object_t object, vm_object_t backing_object)
771 {
772 
773 	VM_OBJECT_ASSERT_WLOCKED(object);
774 
775 	if ((backing_object->flags & OBJ_ANON) != 0) {
776 		VM_OBJECT_WLOCK(backing_object);
777 		vm_object_backing_insert_locked(object, backing_object);
778 		VM_OBJECT_WUNLOCK(backing_object);
779 	} else
780 		object->backing_object = backing_object;
781 }
782 
783 /*
784  * Insert an object into a backing_object's shadow list with an additional
785  * reference to the backing_object added.
786  */
787 static void
788 vm_object_backing_insert_ref(vm_object_t object, vm_object_t backing_object)
789 {
790 
791 	VM_OBJECT_ASSERT_WLOCKED(object);
792 
793 	if ((backing_object->flags & OBJ_ANON) != 0) {
794 		VM_OBJECT_WLOCK(backing_object);
795 		KASSERT((backing_object->flags & OBJ_DEAD) == 0,
796 		    ("shadowing dead anonymous object"));
797 		vm_object_reference_locked(backing_object);
798 		vm_object_backing_insert_locked(object, backing_object);
799 		vm_object_clear_flag(backing_object, OBJ_ONEMAPPING);
800 		VM_OBJECT_WUNLOCK(backing_object);
801 	} else {
802 		vm_object_reference(backing_object);
803 		object->backing_object = backing_object;
804 	}
805 }
806 
807 /*
808  * Transfer a backing reference from backing_object to object.
809  */
810 static void
811 vm_object_backing_transfer(vm_object_t object, vm_object_t backing_object)
812 {
813 	vm_object_t new_backing_object;
814 
815 	/*
816 	 * Note that the reference to backing_object->backing_object
817 	 * moves from within backing_object to within object.
818 	 */
819 	vm_object_backing_remove_locked(object);
820 	new_backing_object = backing_object->backing_object;
821 	if (new_backing_object == NULL)
822 		return;
823 	if ((new_backing_object->flags & OBJ_ANON) != 0) {
824 		VM_OBJECT_WLOCK(new_backing_object);
825 		vm_object_backing_remove_locked(backing_object);
826 		vm_object_backing_insert_locked(object, new_backing_object);
827 		VM_OBJECT_WUNLOCK(new_backing_object);
828 	} else {
829 		object->backing_object = new_backing_object;
830 		backing_object->backing_object = NULL;
831 	}
832 }
833 
834 /*
835  * Wait for a concurrent collapse to settle.
836  */
837 static void
838 vm_object_collapse_wait(vm_object_t object)
839 {
840 
841 	VM_OBJECT_ASSERT_WLOCKED(object);
842 
843 	while ((object->flags & OBJ_COLLAPSING) != 0) {
844 		vm_object_pip_wait(object, "vmcolwait");
845 		counter_u64_add(object_collapse_waits, 1);
846 	}
847 }
848 
849 /*
850  * Waits for a backing object to clear a pending collapse and returns
851  * it locked if it is an ANON object.
852  */
853 static vm_object_t
854 vm_object_backing_collapse_wait(vm_object_t object)
855 {
856 	vm_object_t backing_object;
857 
858 	VM_OBJECT_ASSERT_WLOCKED(object);
859 
860 	for (;;) {
861 		backing_object = object->backing_object;
862 		if (backing_object == NULL ||
863 		    (backing_object->flags & OBJ_ANON) == 0)
864 			return (NULL);
865 		VM_OBJECT_WLOCK(backing_object);
866 		if ((backing_object->flags & (OBJ_DEAD | OBJ_COLLAPSING)) == 0)
867 			break;
868 		VM_OBJECT_WUNLOCK(object);
869 		vm_object_pip_sleep(backing_object, "vmbckwait");
870 		counter_u64_add(object_collapse_waits, 1);
871 		VM_OBJECT_WLOCK(object);
872 	}
873 	return (backing_object);
874 }
875 
876 /*
877  *	vm_object_terminate_pages removes any remaining pageable pages
878  *	from the object and resets the object to an empty state.
879  */
880 static void
881 vm_object_terminate_pages(vm_object_t object)
882 {
883 	vm_page_t p, p_next;
884 
885 	VM_OBJECT_ASSERT_WLOCKED(object);
886 
887 	/*
888 	 * Free any remaining pageable pages.  This also removes them from the
889 	 * paging queues.  However, don't free wired pages, just remove them
890 	 * from the object.  Rather than incrementally removing each page from
891 	 * the object, the page and object are reset to any empty state.
892 	 */
893 	TAILQ_FOREACH_SAFE(p, &object->memq, listq, p_next) {
894 		vm_page_assert_unbusied(p);
895 		KASSERT(p->object == object &&
896 		    (p->ref_count & VPRC_OBJREF) != 0,
897 		    ("vm_object_terminate_pages: page %p is inconsistent", p));
898 
899 		p->object = NULL;
900 		if (vm_page_drop(p, VPRC_OBJREF) == VPRC_OBJREF) {
901 			VM_CNT_INC(v_pfree);
902 			vm_page_free(p);
903 		}
904 	}
905 
906 	/*
907 	 * If the object contained any pages, then reset it to an empty state.
908 	 * None of the object's fields, including "resident_page_count", were
909 	 * modified by the preceding loop.
910 	 */
911 	if (object->resident_page_count != 0) {
912 		vm_radix_reclaim_allnodes(&object->rtree);
913 		TAILQ_INIT(&object->memq);
914 		object->resident_page_count = 0;
915 		if (object->type == OBJT_VNODE)
916 			vdrop(object->handle);
917 	}
918 }
919 
920 /*
921  *	vm_object_terminate actually destroys the specified object, freeing
922  *	up all previously used resources.
923  *
924  *	The object must be locked.
925  *	This routine may block.
926  */
927 void
928 vm_object_terminate(vm_object_t object)
929 {
930 
931 	VM_OBJECT_ASSERT_WLOCKED(object);
932 	KASSERT((object->flags & OBJ_DEAD) != 0,
933 	    ("terminating non-dead obj %p", object));
934 	KASSERT((object->flags & OBJ_COLLAPSING) == 0,
935 	    ("terminating collapsing obj %p", object));
936 	KASSERT(object->backing_object == NULL,
937 	    ("terminating shadow obj %p", object));
938 
939 	/*
940 	 * Wait for the pageout daemon and other current users to be
941 	 * done with the object.  Note that new paging_in_progress
942 	 * users can come after this wait, but they must check
943 	 * OBJ_DEAD flag set (without unlocking the object), and avoid
944 	 * the object being terminated.
945 	 */
946 	vm_object_pip_wait(object, "objtrm");
947 
948 	KASSERT(object->ref_count == 0,
949 	    ("vm_object_terminate: object with references, ref_count=%d",
950 	    object->ref_count));
951 
952 	if ((object->flags & OBJ_PG_DTOR) == 0)
953 		vm_object_terminate_pages(object);
954 
955 #if VM_NRESERVLEVEL > 0
956 	if (__predict_false(!LIST_EMPTY(&object->rvq)))
957 		vm_reserv_break_all(object);
958 #endif
959 
960 	KASSERT(object->cred == NULL || object->type == OBJT_DEFAULT ||
961 	    object->type == OBJT_SWAP,
962 	    ("%s: non-swap obj %p has cred", __func__, object));
963 
964 	/*
965 	 * Let the pager know object is dead.
966 	 */
967 	vm_pager_deallocate(object);
968 	VM_OBJECT_WUNLOCK(object);
969 
970 	vm_object_destroy(object);
971 }
972 
973 /*
974  * Make the page read-only so that we can clear the object flags.  However, if
975  * this is a nosync mmap then the object is likely to stay dirty so do not
976  * mess with the page and do not clear the object flags.  Returns TRUE if the
977  * page should be flushed, and FALSE otherwise.
978  */
979 static boolean_t
980 vm_object_page_remove_write(vm_page_t p, int flags, boolean_t *allclean)
981 {
982 
983 	vm_page_assert_busied(p);
984 
985 	/*
986 	 * If we have been asked to skip nosync pages and this is a
987 	 * nosync page, skip it.  Note that the object flags were not
988 	 * cleared in this case so we do not have to set them.
989 	 */
990 	if ((flags & OBJPC_NOSYNC) != 0 && (p->a.flags & PGA_NOSYNC) != 0) {
991 		*allclean = FALSE;
992 		return (FALSE);
993 	} else {
994 		pmap_remove_write(p);
995 		return (p->dirty != 0);
996 	}
997 }
998 
999 /*
1000  *	vm_object_page_clean
1001  *
1002  *	Clean all dirty pages in the specified range of object.  Leaves page
1003  * 	on whatever queue it is currently on.   If NOSYNC is set then do not
1004  *	write out pages with PGA_NOSYNC set (originally comes from MAP_NOSYNC),
1005  *	leaving the object dirty.
1006  *
1007  *	For swap objects backing tmpfs regular files, do not flush anything,
1008  *	but remove write protection on the mapped pages to update mtime through
1009  *	mmaped writes.
1010  *
1011  *	When stuffing pages asynchronously, allow clustering.  XXX we need a
1012  *	synchronous clustering mode implementation.
1013  *
1014  *	Odd semantics: if start == end, we clean everything.
1015  *
1016  *	The object must be locked.
1017  *
1018  *	Returns FALSE if some page from the range was not written, as
1019  *	reported by the pager, and TRUE otherwise.
1020  */
1021 boolean_t
1022 vm_object_page_clean(vm_object_t object, vm_ooffset_t start, vm_ooffset_t end,
1023     int flags)
1024 {
1025 	vm_page_t np, p;
1026 	vm_pindex_t pi, tend, tstart;
1027 	int curgeneration, n, pagerflags;
1028 	boolean_t eio, res, allclean;
1029 
1030 	VM_OBJECT_ASSERT_WLOCKED(object);
1031 
1032 	if (!vm_object_mightbedirty(object) || object->resident_page_count == 0)
1033 		return (TRUE);
1034 
1035 	pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) != 0 ?
1036 	    VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK;
1037 	pagerflags |= (flags & OBJPC_INVAL) != 0 ? VM_PAGER_PUT_INVAL : 0;
1038 
1039 	tstart = OFF_TO_IDX(start);
1040 	tend = (end == 0) ? object->size : OFF_TO_IDX(end + PAGE_MASK);
1041 	allclean = tstart == 0 && tend >= object->size;
1042 	res = TRUE;
1043 
1044 rescan:
1045 	curgeneration = object->generation;
1046 
1047 	for (p = vm_page_find_least(object, tstart); p != NULL; p = np) {
1048 		pi = p->pindex;
1049 		if (pi >= tend)
1050 			break;
1051 		np = TAILQ_NEXT(p, listq);
1052 		if (vm_page_none_valid(p))
1053 			continue;
1054 		if (vm_page_busy_acquire(p, VM_ALLOC_WAITFAIL) == 0) {
1055 			if (object->generation != curgeneration &&
1056 			    (flags & OBJPC_SYNC) != 0)
1057 				goto rescan;
1058 			np = vm_page_find_least(object, pi);
1059 			continue;
1060 		}
1061 		if (!vm_object_page_remove_write(p, flags, &allclean)) {
1062 			vm_page_xunbusy(p);
1063 			continue;
1064 		}
1065 		if (object->type == OBJT_VNODE) {
1066 			n = vm_object_page_collect_flush(object, p, pagerflags,
1067 			    flags, &allclean, &eio);
1068 			if (eio) {
1069 				res = FALSE;
1070 				allclean = FALSE;
1071 			}
1072 			if (object->generation != curgeneration &&
1073 			    (flags & OBJPC_SYNC) != 0)
1074 				goto rescan;
1075 
1076 			/*
1077 			 * If the VOP_PUTPAGES() did a truncated write, so
1078 			 * that even the first page of the run is not fully
1079 			 * written, vm_pageout_flush() returns 0 as the run
1080 			 * length.  Since the condition that caused truncated
1081 			 * write may be permanent, e.g. exhausted free space,
1082 			 * accepting n == 0 would cause an infinite loop.
1083 			 *
1084 			 * Forwarding the iterator leaves the unwritten page
1085 			 * behind, but there is not much we can do there if
1086 			 * filesystem refuses to write it.
1087 			 */
1088 			if (n == 0) {
1089 				n = 1;
1090 				allclean = FALSE;
1091 			}
1092 		} else {
1093 			n = 1;
1094 			vm_page_xunbusy(p);
1095 		}
1096 		np = vm_page_find_least(object, pi + n);
1097 	}
1098 #if 0
1099 	VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC) ? MNT_WAIT : 0);
1100 #endif
1101 
1102 	/*
1103 	 * Leave updating cleangeneration for tmpfs objects to tmpfs
1104 	 * scan.  It needs to update mtime, which happens for other
1105 	 * filesystems during page writeouts.
1106 	 */
1107 	if (allclean && object->type == OBJT_VNODE)
1108 		object->cleangeneration = curgeneration;
1109 	return (res);
1110 }
1111 
1112 static int
1113 vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags,
1114     int flags, boolean_t *allclean, boolean_t *eio)
1115 {
1116 	vm_page_t ma[vm_pageout_page_count], p_first, tp;
1117 	int count, i, mreq, runlen;
1118 
1119 	vm_page_lock_assert(p, MA_NOTOWNED);
1120 	vm_page_assert_xbusied(p);
1121 	VM_OBJECT_ASSERT_WLOCKED(object);
1122 
1123 	count = 1;
1124 	mreq = 0;
1125 
1126 	for (tp = p; count < vm_pageout_page_count; count++) {
1127 		tp = vm_page_next(tp);
1128 		if (tp == NULL || vm_page_tryxbusy(tp) == 0)
1129 			break;
1130 		if (!vm_object_page_remove_write(tp, flags, allclean)) {
1131 			vm_page_xunbusy(tp);
1132 			break;
1133 		}
1134 	}
1135 
1136 	for (p_first = p; count < vm_pageout_page_count; count++) {
1137 		tp = vm_page_prev(p_first);
1138 		if (tp == NULL || vm_page_tryxbusy(tp) == 0)
1139 			break;
1140 		if (!vm_object_page_remove_write(tp, flags, allclean)) {
1141 			vm_page_xunbusy(tp);
1142 			break;
1143 		}
1144 		p_first = tp;
1145 		mreq++;
1146 	}
1147 
1148 	for (tp = p_first, i = 0; i < count; tp = TAILQ_NEXT(tp, listq), i++)
1149 		ma[i] = tp;
1150 
1151 	vm_pageout_flush(ma, count, pagerflags, mreq, &runlen, eio);
1152 	return (runlen);
1153 }
1154 
1155 /*
1156  * Note that there is absolutely no sense in writing out
1157  * anonymous objects, so we track down the vnode object
1158  * to write out.
1159  * We invalidate (remove) all pages from the address space
1160  * for semantic correctness.
1161  *
1162  * If the backing object is a device object with unmanaged pages, then any
1163  * mappings to the specified range of pages must be removed before this
1164  * function is called.
1165  *
1166  * Note: certain anonymous maps, such as MAP_NOSYNC maps,
1167  * may start out with a NULL object.
1168  */
1169 boolean_t
1170 vm_object_sync(vm_object_t object, vm_ooffset_t offset, vm_size_t size,
1171     boolean_t syncio, boolean_t invalidate)
1172 {
1173 	vm_object_t backing_object;
1174 	struct vnode *vp;
1175 	struct mount *mp;
1176 	int error, flags, fsync_after;
1177 	boolean_t res;
1178 
1179 	if (object == NULL)
1180 		return (TRUE);
1181 	res = TRUE;
1182 	error = 0;
1183 	VM_OBJECT_WLOCK(object);
1184 	while ((backing_object = object->backing_object) != NULL) {
1185 		VM_OBJECT_WLOCK(backing_object);
1186 		offset += object->backing_object_offset;
1187 		VM_OBJECT_WUNLOCK(object);
1188 		object = backing_object;
1189 		if (object->size < OFF_TO_IDX(offset + size))
1190 			size = IDX_TO_OFF(object->size) - offset;
1191 	}
1192 	/*
1193 	 * Flush pages if writing is allowed, invalidate them
1194 	 * if invalidation requested.  Pages undergoing I/O
1195 	 * will be ignored by vm_object_page_remove().
1196 	 *
1197 	 * We cannot lock the vnode and then wait for paging
1198 	 * to complete without deadlocking against vm_fault.
1199 	 * Instead we simply call vm_object_page_remove() and
1200 	 * allow it to block internally on a page-by-page
1201 	 * basis when it encounters pages undergoing async
1202 	 * I/O.
1203 	 */
1204 	if (object->type == OBJT_VNODE &&
1205 	    vm_object_mightbedirty(object) != 0 &&
1206 	    ((vp = object->handle)->v_vflag & VV_NOSYNC) == 0) {
1207 		VM_OBJECT_WUNLOCK(object);
1208 		(void) vn_start_write(vp, &mp, V_WAIT);
1209 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1210 		if (syncio && !invalidate && offset == 0 &&
1211 		    atop(size) == object->size) {
1212 			/*
1213 			 * If syncing the whole mapping of the file,
1214 			 * it is faster to schedule all the writes in
1215 			 * async mode, also allowing the clustering,
1216 			 * and then wait for i/o to complete.
1217 			 */
1218 			flags = 0;
1219 			fsync_after = TRUE;
1220 		} else {
1221 			flags = (syncio || invalidate) ? OBJPC_SYNC : 0;
1222 			flags |= invalidate ? (OBJPC_SYNC | OBJPC_INVAL) : 0;
1223 			fsync_after = FALSE;
1224 		}
1225 		VM_OBJECT_WLOCK(object);
1226 		res = vm_object_page_clean(object, offset, offset + size,
1227 		    flags);
1228 		VM_OBJECT_WUNLOCK(object);
1229 		if (fsync_after)
1230 			error = VOP_FSYNC(vp, MNT_WAIT, curthread);
1231 		VOP_UNLOCK(vp);
1232 		vn_finished_write(mp);
1233 		if (error != 0)
1234 			res = FALSE;
1235 		VM_OBJECT_WLOCK(object);
1236 	}
1237 	if ((object->type == OBJT_VNODE ||
1238 	     object->type == OBJT_DEVICE) && invalidate) {
1239 		if (object->type == OBJT_DEVICE)
1240 			/*
1241 			 * The option OBJPR_NOTMAPPED must be passed here
1242 			 * because vm_object_page_remove() cannot remove
1243 			 * unmanaged mappings.
1244 			 */
1245 			flags = OBJPR_NOTMAPPED;
1246 		else if (old_msync)
1247 			flags = 0;
1248 		else
1249 			flags = OBJPR_CLEANONLY;
1250 		vm_object_page_remove(object, OFF_TO_IDX(offset),
1251 		    OFF_TO_IDX(offset + size + PAGE_MASK), flags);
1252 	}
1253 	VM_OBJECT_WUNLOCK(object);
1254 	return (res);
1255 }
1256 
1257 /*
1258  * Determine whether the given advice can be applied to the object.  Advice is
1259  * not applied to unmanaged pages since they never belong to page queues, and
1260  * since MADV_FREE is destructive, it can apply only to anonymous pages that
1261  * have been mapped at most once.
1262  */
1263 static bool
1264 vm_object_advice_applies(vm_object_t object, int advice)
1265 {
1266 
1267 	if ((object->flags & OBJ_UNMANAGED) != 0)
1268 		return (false);
1269 	if (advice != MADV_FREE)
1270 		return (true);
1271 	return ((object->flags & (OBJ_ONEMAPPING | OBJ_ANON)) ==
1272 	    (OBJ_ONEMAPPING | OBJ_ANON));
1273 }
1274 
1275 static void
1276 vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex,
1277     vm_size_t size)
1278 {
1279 
1280 	if (advice == MADV_FREE && object->type == OBJT_SWAP)
1281 		swap_pager_freespace(object, pindex, size);
1282 }
1283 
1284 /*
1285  *	vm_object_madvise:
1286  *
1287  *	Implements the madvise function at the object/page level.
1288  *
1289  *	MADV_WILLNEED	(any object)
1290  *
1291  *	    Activate the specified pages if they are resident.
1292  *
1293  *	MADV_DONTNEED	(any object)
1294  *
1295  *	    Deactivate the specified pages if they are resident.
1296  *
1297  *	MADV_FREE	(OBJT_DEFAULT/OBJT_SWAP objects,
1298  *			 OBJ_ONEMAPPING only)
1299  *
1300  *	    Deactivate and clean the specified pages if they are
1301  *	    resident.  This permits the process to reuse the pages
1302  *	    without faulting or the kernel to reclaim the pages
1303  *	    without I/O.
1304  */
1305 void
1306 vm_object_madvise(vm_object_t object, vm_pindex_t pindex, vm_pindex_t end,
1307     int advice)
1308 {
1309 	vm_pindex_t tpindex;
1310 	vm_object_t backing_object, tobject;
1311 	vm_page_t m, tm;
1312 
1313 	if (object == NULL)
1314 		return;
1315 
1316 relookup:
1317 	VM_OBJECT_WLOCK(object);
1318 	if (!vm_object_advice_applies(object, advice)) {
1319 		VM_OBJECT_WUNLOCK(object);
1320 		return;
1321 	}
1322 	for (m = vm_page_find_least(object, pindex); pindex < end; pindex++) {
1323 		tobject = object;
1324 
1325 		/*
1326 		 * If the next page isn't resident in the top-level object, we
1327 		 * need to search the shadow chain.  When applying MADV_FREE, we
1328 		 * take care to release any swap space used to store
1329 		 * non-resident pages.
1330 		 */
1331 		if (m == NULL || pindex < m->pindex) {
1332 			/*
1333 			 * Optimize a common case: if the top-level object has
1334 			 * no backing object, we can skip over the non-resident
1335 			 * range in constant time.
1336 			 */
1337 			if (object->backing_object == NULL) {
1338 				tpindex = (m != NULL && m->pindex < end) ?
1339 				    m->pindex : end;
1340 				vm_object_madvise_freespace(object, advice,
1341 				    pindex, tpindex - pindex);
1342 				if ((pindex = tpindex) == end)
1343 					break;
1344 				goto next_page;
1345 			}
1346 
1347 			tpindex = pindex;
1348 			do {
1349 				vm_object_madvise_freespace(tobject, advice,
1350 				    tpindex, 1);
1351 				/*
1352 				 * Prepare to search the next object in the
1353 				 * chain.
1354 				 */
1355 				backing_object = tobject->backing_object;
1356 				if (backing_object == NULL)
1357 					goto next_pindex;
1358 				VM_OBJECT_WLOCK(backing_object);
1359 				tpindex +=
1360 				    OFF_TO_IDX(tobject->backing_object_offset);
1361 				if (tobject != object)
1362 					VM_OBJECT_WUNLOCK(tobject);
1363 				tobject = backing_object;
1364 				if (!vm_object_advice_applies(tobject, advice))
1365 					goto next_pindex;
1366 			} while ((tm = vm_page_lookup(tobject, tpindex)) ==
1367 			    NULL);
1368 		} else {
1369 next_page:
1370 			tm = m;
1371 			m = TAILQ_NEXT(m, listq);
1372 		}
1373 
1374 		/*
1375 		 * If the page is not in a normal state, skip it.  The page
1376 		 * can not be invalidated while the object lock is held.
1377 		 */
1378 		if (!vm_page_all_valid(tm) || vm_page_wired(tm))
1379 			goto next_pindex;
1380 		KASSERT((tm->flags & PG_FICTITIOUS) == 0,
1381 		    ("vm_object_madvise: page %p is fictitious", tm));
1382 		KASSERT((tm->oflags & VPO_UNMANAGED) == 0,
1383 		    ("vm_object_madvise: page %p is not managed", tm));
1384 		if (vm_page_tryxbusy(tm) == 0) {
1385 			if (object != tobject)
1386 				VM_OBJECT_WUNLOCK(object);
1387 			if (advice == MADV_WILLNEED) {
1388 				/*
1389 				 * Reference the page before unlocking and
1390 				 * sleeping so that the page daemon is less
1391 				 * likely to reclaim it.
1392 				 */
1393 				vm_page_aflag_set(tm, PGA_REFERENCED);
1394 			}
1395 			vm_page_busy_sleep(tm, "madvpo", false);
1396   			goto relookup;
1397 		}
1398 		vm_page_advise(tm, advice);
1399 		vm_page_xunbusy(tm);
1400 		vm_object_madvise_freespace(tobject, advice, tm->pindex, 1);
1401 next_pindex:
1402 		if (tobject != object)
1403 			VM_OBJECT_WUNLOCK(tobject);
1404 	}
1405 	VM_OBJECT_WUNLOCK(object);
1406 }
1407 
1408 /*
1409  *	vm_object_shadow:
1410  *
1411  *	Create a new object which is backed by the
1412  *	specified existing object range.  The source
1413  *	object reference is deallocated.
1414  *
1415  *	The new object and offset into that object
1416  *	are returned in the source parameters.
1417  */
1418 void
1419 vm_object_shadow(vm_object_t *object, vm_ooffset_t *offset, vm_size_t length,
1420     struct ucred *cred, bool shared)
1421 {
1422 	vm_object_t source;
1423 	vm_object_t result;
1424 
1425 	source = *object;
1426 
1427 	/*
1428 	 * Don't create the new object if the old object isn't shared.
1429 	 *
1430 	 * If we hold the only reference we can guarantee that it won't
1431 	 * increase while we have the map locked.  Otherwise the race is
1432 	 * harmless and we will end up with an extra shadow object that
1433 	 * will be collapsed later.
1434 	 */
1435 	if (source != NULL && source->ref_count == 1 &&
1436 	    (source->flags & OBJ_ANON) != 0)
1437 		return;
1438 
1439 	/*
1440 	 * Allocate a new object with the given length.
1441 	 */
1442 	result = vm_object_allocate_anon(atop(length), source, cred, length);
1443 
1444 	/*
1445 	 * Store the offset into the source object, and fix up the offset into
1446 	 * the new object.
1447 	 */
1448 	result->backing_object_offset = *offset;
1449 
1450 	if (shared || source != NULL) {
1451 		VM_OBJECT_WLOCK(result);
1452 
1453 		/*
1454 		 * The new object shadows the source object, adding a
1455 		 * reference to it.  Our caller changes his reference
1456 		 * to point to the new object, removing a reference to
1457 		 * the source object.  Net result: no change of
1458 		 * reference count, unless the caller needs to add one
1459 		 * more reference due to forking a shared map entry.
1460 		 */
1461 		if (shared) {
1462 			vm_object_reference_locked(result);
1463 			vm_object_clear_flag(result, OBJ_ONEMAPPING);
1464 		}
1465 
1466 		/*
1467 		 * Try to optimize the result object's page color when
1468 		 * shadowing in order to maintain page coloring
1469 		 * consistency in the combined shadowed object.
1470 		 */
1471 		if (source != NULL) {
1472 			vm_object_backing_insert(result, source);
1473 			result->domain = source->domain;
1474 #if VM_NRESERVLEVEL > 0
1475 			result->flags |= source->flags & OBJ_COLORED;
1476 			result->pg_color = (source->pg_color +
1477 			    OFF_TO_IDX(*offset)) & ((1 << (VM_NFREEORDER -
1478 			    1)) - 1);
1479 #endif
1480 		}
1481 		VM_OBJECT_WUNLOCK(result);
1482 	}
1483 
1484 	/*
1485 	 * Return the new things
1486 	 */
1487 	*offset = 0;
1488 	*object = result;
1489 }
1490 
1491 /*
1492  *	vm_object_split:
1493  *
1494  * Split the pages in a map entry into a new object.  This affords
1495  * easier removal of unused pages, and keeps object inheritance from
1496  * being a negative impact on memory usage.
1497  */
1498 void
1499 vm_object_split(vm_map_entry_t entry)
1500 {
1501 	vm_page_t m, m_busy, m_next;
1502 	vm_object_t orig_object, new_object, backing_object;
1503 	vm_pindex_t idx, offidxstart;
1504 	vm_size_t size;
1505 
1506 	orig_object = entry->object.vm_object;
1507 	KASSERT((orig_object->flags & OBJ_ONEMAPPING) != 0,
1508 	    ("vm_object_split:  Splitting object with multiple mappings."));
1509 	if ((orig_object->flags & OBJ_ANON) == 0)
1510 		return;
1511 	if (orig_object->ref_count <= 1)
1512 		return;
1513 	VM_OBJECT_WUNLOCK(orig_object);
1514 
1515 	offidxstart = OFF_TO_IDX(entry->offset);
1516 	size = atop(entry->end - entry->start);
1517 
1518 	/*
1519 	 * If swap_pager_copy() is later called, it will convert new_object
1520 	 * into a swap object.
1521 	 */
1522 	new_object = vm_object_allocate_anon(size, orig_object,
1523 	    orig_object->cred, ptoa(size));
1524 
1525 	/*
1526 	 * We must wait for the orig_object to complete any in-progress
1527 	 * collapse so that the swap blocks are stable below.  The
1528 	 * additional reference on backing_object by new object will
1529 	 * prevent further collapse operations until split completes.
1530 	 */
1531 	VM_OBJECT_WLOCK(orig_object);
1532 	vm_object_collapse_wait(orig_object);
1533 
1534 	/*
1535 	 * At this point, the new object is still private, so the order in
1536 	 * which the original and new objects are locked does not matter.
1537 	 */
1538 	VM_OBJECT_WLOCK(new_object);
1539 	new_object->domain = orig_object->domain;
1540 	backing_object = orig_object->backing_object;
1541 	if (backing_object != NULL) {
1542 		vm_object_backing_insert_ref(new_object, backing_object);
1543 		new_object->backing_object_offset =
1544 		    orig_object->backing_object_offset + entry->offset;
1545 	}
1546 	if (orig_object->cred != NULL) {
1547 		crhold(orig_object->cred);
1548 		KASSERT(orig_object->charge >= ptoa(size),
1549 		    ("orig_object->charge < 0"));
1550 		orig_object->charge -= ptoa(size);
1551 	}
1552 
1553 	/*
1554 	 * Mark the split operation so that swap_pager_getpages() knows
1555 	 * that the object is in transition.
1556 	 */
1557 	vm_object_set_flag(orig_object, OBJ_SPLIT);
1558 	m_busy = NULL;
1559 #ifdef INVARIANTS
1560 	idx = 0;
1561 #endif
1562 retry:
1563 	m = vm_page_find_least(orig_object, offidxstart);
1564 	KASSERT(m == NULL || idx <= m->pindex - offidxstart,
1565 	    ("%s: object %p was repopulated", __func__, orig_object));
1566 	for (; m != NULL && (idx = m->pindex - offidxstart) < size;
1567 	    m = m_next) {
1568 		m_next = TAILQ_NEXT(m, listq);
1569 
1570 		/*
1571 		 * We must wait for pending I/O to complete before we can
1572 		 * rename the page.
1573 		 *
1574 		 * We do not have to VM_PROT_NONE the page as mappings should
1575 		 * not be changed by this operation.
1576 		 */
1577 		if (vm_page_tryxbusy(m) == 0) {
1578 			VM_OBJECT_WUNLOCK(new_object);
1579 			vm_page_sleep_if_busy(m, "spltwt");
1580 			VM_OBJECT_WLOCK(new_object);
1581 			goto retry;
1582 		}
1583 
1584 		/*
1585 		 * The page was left invalid.  Likely placed there by
1586 		 * an incomplete fault.  Just remove and ignore.
1587 		 */
1588 		if (vm_page_none_valid(m)) {
1589 			if (vm_page_remove(m))
1590 				vm_page_free(m);
1591 			continue;
1592 		}
1593 
1594 		/* vm_page_rename() will dirty the page. */
1595 		if (vm_page_rename(m, new_object, idx)) {
1596 			vm_page_xunbusy(m);
1597 			VM_OBJECT_WUNLOCK(new_object);
1598 			VM_OBJECT_WUNLOCK(orig_object);
1599 			vm_radix_wait();
1600 			VM_OBJECT_WLOCK(orig_object);
1601 			VM_OBJECT_WLOCK(new_object);
1602 			goto retry;
1603 		}
1604 
1605 #if VM_NRESERVLEVEL > 0
1606 		/*
1607 		 * If some of the reservation's allocated pages remain with
1608 		 * the original object, then transferring the reservation to
1609 		 * the new object is neither particularly beneficial nor
1610 		 * particularly harmful as compared to leaving the reservation
1611 		 * with the original object.  If, however, all of the
1612 		 * reservation's allocated pages are transferred to the new
1613 		 * object, then transferring the reservation is typically
1614 		 * beneficial.  Determining which of these two cases applies
1615 		 * would be more costly than unconditionally renaming the
1616 		 * reservation.
1617 		 */
1618 		vm_reserv_rename(m, new_object, orig_object, offidxstart);
1619 #endif
1620 
1621 		/*
1622 		 * orig_object's type may change while sleeping, so keep track
1623 		 * of the beginning of the busied range.
1624 		 */
1625 		if (orig_object->type != OBJT_SWAP)
1626 			vm_page_xunbusy(m);
1627 		else if (m_busy == NULL)
1628 			m_busy = m;
1629 	}
1630 	if (orig_object->type == OBJT_SWAP) {
1631 		/*
1632 		 * swap_pager_copy() can sleep, in which case the orig_object's
1633 		 * and new_object's locks are released and reacquired.
1634 		 */
1635 		swap_pager_copy(orig_object, new_object, offidxstart, 0);
1636 		if (m_busy != NULL)
1637 			TAILQ_FOREACH_FROM(m_busy, &new_object->memq, listq)
1638 				vm_page_xunbusy(m_busy);
1639 	}
1640 	vm_object_clear_flag(orig_object, OBJ_SPLIT);
1641 	VM_OBJECT_WUNLOCK(orig_object);
1642 	VM_OBJECT_WUNLOCK(new_object);
1643 	entry->object.vm_object = new_object;
1644 	entry->offset = 0LL;
1645 	vm_object_deallocate(orig_object);
1646 	VM_OBJECT_WLOCK(new_object);
1647 }
1648 
1649 static vm_page_t
1650 vm_object_collapse_scan_wait(vm_object_t object, vm_page_t p)
1651 {
1652 	vm_object_t backing_object;
1653 
1654 	VM_OBJECT_ASSERT_WLOCKED(object);
1655 	backing_object = object->backing_object;
1656 	VM_OBJECT_ASSERT_WLOCKED(backing_object);
1657 
1658 	KASSERT(p == NULL || p->object == object || p->object == backing_object,
1659 	    ("invalid ownership %p %p %p", p, object, backing_object));
1660 	/* The page is only NULL when rename fails. */
1661 	if (p == NULL) {
1662 		VM_OBJECT_WUNLOCK(object);
1663 		VM_OBJECT_WUNLOCK(backing_object);
1664 		vm_radix_wait();
1665 	} else {
1666 		if (p->object == object)
1667 			VM_OBJECT_WUNLOCK(backing_object);
1668 		else
1669 			VM_OBJECT_WUNLOCK(object);
1670 		vm_page_busy_sleep(p, "vmocol", false);
1671 	}
1672 	VM_OBJECT_WLOCK(object);
1673 	VM_OBJECT_WLOCK(backing_object);
1674 	return (TAILQ_FIRST(&backing_object->memq));
1675 }
1676 
1677 static bool
1678 vm_object_scan_all_shadowed(vm_object_t object)
1679 {
1680 	vm_object_t backing_object;
1681 	vm_page_t p, pp;
1682 	vm_pindex_t backing_offset_index, new_pindex, pi, ps;
1683 
1684 	VM_OBJECT_ASSERT_WLOCKED(object);
1685 	VM_OBJECT_ASSERT_WLOCKED(object->backing_object);
1686 
1687 	backing_object = object->backing_object;
1688 
1689 	if ((backing_object->flags & OBJ_ANON) == 0)
1690 		return (false);
1691 
1692 	pi = backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
1693 	p = vm_page_find_least(backing_object, pi);
1694 	ps = swap_pager_find_least(backing_object, pi);
1695 
1696 	/*
1697 	 * Only check pages inside the parent object's range and
1698 	 * inside the parent object's mapping of the backing object.
1699 	 */
1700 	for (;; pi++) {
1701 		if (p != NULL && p->pindex < pi)
1702 			p = TAILQ_NEXT(p, listq);
1703 		if (ps < pi)
1704 			ps = swap_pager_find_least(backing_object, pi);
1705 		if (p == NULL && ps >= backing_object->size)
1706 			break;
1707 		else if (p == NULL)
1708 			pi = ps;
1709 		else
1710 			pi = MIN(p->pindex, ps);
1711 
1712 		new_pindex = pi - backing_offset_index;
1713 		if (new_pindex >= object->size)
1714 			break;
1715 
1716 		if (p != NULL) {
1717 			/*
1718 			 * If the backing object page is busy a
1719 			 * grandparent or older page may still be
1720 			 * undergoing CoW.  It is not safe to collapse
1721 			 * the backing object until it is quiesced.
1722 			 */
1723 			if (vm_page_tryxbusy(p) == 0)
1724 				return (false);
1725 
1726 			/*
1727 			 * We raced with the fault handler that left
1728 			 * newly allocated invalid page on the object
1729 			 * queue and retried.
1730 			 */
1731 			if (!vm_page_all_valid(p))
1732 				goto unbusy_ret;
1733 		}
1734 
1735 		/*
1736 		 * See if the parent has the page or if the parent's object
1737 		 * pager has the page.  If the parent has the page but the page
1738 		 * is not valid, the parent's object pager must have the page.
1739 		 *
1740 		 * If this fails, the parent does not completely shadow the
1741 		 * object and we might as well give up now.
1742 		 */
1743 		pp = vm_page_lookup(object, new_pindex);
1744 
1745 		/*
1746 		 * The valid check here is stable due to object lock
1747 		 * being required to clear valid and initiate paging.
1748 		 * Busy of p disallows fault handler to validate pp.
1749 		 */
1750 		if ((pp == NULL || vm_page_none_valid(pp)) &&
1751 		    !vm_pager_has_page(object, new_pindex, NULL, NULL))
1752 			goto unbusy_ret;
1753 		if (p != NULL)
1754 			vm_page_xunbusy(p);
1755 	}
1756 	return (true);
1757 
1758 unbusy_ret:
1759 	if (p != NULL)
1760 		vm_page_xunbusy(p);
1761 	return (false);
1762 }
1763 
1764 static void
1765 vm_object_collapse_scan(vm_object_t object)
1766 {
1767 	vm_object_t backing_object;
1768 	vm_page_t next, p, pp;
1769 	vm_pindex_t backing_offset_index, new_pindex;
1770 
1771 	VM_OBJECT_ASSERT_WLOCKED(object);
1772 	VM_OBJECT_ASSERT_WLOCKED(object->backing_object);
1773 
1774 	backing_object = object->backing_object;
1775 	backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
1776 
1777 	/*
1778 	 * Our scan
1779 	 */
1780 	for (p = TAILQ_FIRST(&backing_object->memq); p != NULL; p = next) {
1781 		next = TAILQ_NEXT(p, listq);
1782 		new_pindex = p->pindex - backing_offset_index;
1783 
1784 		/*
1785 		 * Check for busy page
1786 		 */
1787 		if (vm_page_tryxbusy(p) == 0) {
1788 			next = vm_object_collapse_scan_wait(object, p);
1789 			continue;
1790 		}
1791 
1792 		KASSERT(object->backing_object == backing_object,
1793 		    ("vm_object_collapse_scan: backing object mismatch %p != %p",
1794 		    object->backing_object, backing_object));
1795 		KASSERT(p->object == backing_object,
1796 		    ("vm_object_collapse_scan: object mismatch %p != %p",
1797 		    p->object, backing_object));
1798 
1799 		if (p->pindex < backing_offset_index ||
1800 		    new_pindex >= object->size) {
1801 			if (backing_object->type == OBJT_SWAP)
1802 				swap_pager_freespace(backing_object, p->pindex,
1803 				    1);
1804 
1805 			KASSERT(!pmap_page_is_mapped(p),
1806 			    ("freeing mapped page %p", p));
1807 			if (vm_page_remove(p))
1808 				vm_page_free(p);
1809 			continue;
1810 		}
1811 
1812 		if (!vm_page_all_valid(p)) {
1813 			KASSERT(!pmap_page_is_mapped(p),
1814 			    ("freeing mapped page %p", p));
1815 			if (vm_page_remove(p))
1816 				vm_page_free(p);
1817 			continue;
1818 		}
1819 
1820 		pp = vm_page_lookup(object, new_pindex);
1821 		if (pp != NULL && vm_page_tryxbusy(pp) == 0) {
1822 			vm_page_xunbusy(p);
1823 			/*
1824 			 * The page in the parent is busy and possibly not
1825 			 * (yet) valid.  Until its state is finalized by the
1826 			 * busy bit owner, we can't tell whether it shadows the
1827 			 * original page.
1828 			 */
1829 			next = vm_object_collapse_scan_wait(object, pp);
1830 			continue;
1831 		}
1832 
1833 		if (pp != NULL && vm_page_none_valid(pp)) {
1834 			/*
1835 			 * The page was invalid in the parent.  Likely placed
1836 			 * there by an incomplete fault.  Just remove and
1837 			 * ignore.  p can replace it.
1838 			 */
1839 			if (vm_page_remove(pp))
1840 				vm_page_free(pp);
1841 			pp = NULL;
1842 		}
1843 
1844 		if (pp != NULL || vm_pager_has_page(object, new_pindex, NULL,
1845 			NULL)) {
1846 			/*
1847 			 * The page already exists in the parent OR swap exists
1848 			 * for this location in the parent.  Leave the parent's
1849 			 * page alone.  Destroy the original page from the
1850 			 * backing object.
1851 			 */
1852 			if (backing_object->type == OBJT_SWAP)
1853 				swap_pager_freespace(backing_object, p->pindex,
1854 				    1);
1855 			KASSERT(!pmap_page_is_mapped(p),
1856 			    ("freeing mapped page %p", p));
1857 			if (vm_page_remove(p))
1858 				vm_page_free(p);
1859 			if (pp != NULL)
1860 				vm_page_xunbusy(pp);
1861 			continue;
1862 		}
1863 
1864 		/*
1865 		 * Page does not exist in parent, rename the page from the
1866 		 * backing object to the main object.
1867 		 *
1868 		 * If the page was mapped to a process, it can remain mapped
1869 		 * through the rename.  vm_page_rename() will dirty the page.
1870 		 */
1871 		if (vm_page_rename(p, object, new_pindex)) {
1872 			vm_page_xunbusy(p);
1873 			next = vm_object_collapse_scan_wait(object, NULL);
1874 			continue;
1875 		}
1876 
1877 		/* Use the old pindex to free the right page. */
1878 		if (backing_object->type == OBJT_SWAP)
1879 			swap_pager_freespace(backing_object,
1880 			    new_pindex + backing_offset_index, 1);
1881 
1882 #if VM_NRESERVLEVEL > 0
1883 		/*
1884 		 * Rename the reservation.
1885 		 */
1886 		vm_reserv_rename(p, object, backing_object,
1887 		    backing_offset_index);
1888 #endif
1889 		vm_page_xunbusy(p);
1890 	}
1891 	return;
1892 }
1893 
1894 /*
1895  *	vm_object_collapse:
1896  *
1897  *	Collapse an object with the object backing it.
1898  *	Pages in the backing object are moved into the
1899  *	parent, and the backing object is deallocated.
1900  */
1901 void
1902 vm_object_collapse(vm_object_t object)
1903 {
1904 	vm_object_t backing_object, new_backing_object;
1905 
1906 	VM_OBJECT_ASSERT_WLOCKED(object);
1907 
1908 	while (TRUE) {
1909 		KASSERT((object->flags & (OBJ_DEAD | OBJ_ANON)) == OBJ_ANON,
1910 		    ("collapsing invalid object"));
1911 
1912 		/*
1913 		 * Wait for the backing_object to finish any pending
1914 		 * collapse so that the caller sees the shortest possible
1915 		 * shadow chain.
1916 		 */
1917 		backing_object = vm_object_backing_collapse_wait(object);
1918 		if (backing_object == NULL)
1919 			return;
1920 
1921 		KASSERT(object->ref_count > 0 &&
1922 		    object->ref_count > object->shadow_count,
1923 		    ("collapse with invalid ref %d or shadow %d count.",
1924 		    object->ref_count, object->shadow_count));
1925 		KASSERT((backing_object->flags &
1926 		    (OBJ_COLLAPSING | OBJ_DEAD)) == 0,
1927 		    ("vm_object_collapse: Backing object already collapsing."));
1928 		KASSERT((object->flags & (OBJ_COLLAPSING | OBJ_DEAD)) == 0,
1929 		    ("vm_object_collapse: object is already collapsing."));
1930 
1931 		/*
1932 		 * We know that we can either collapse the backing object if
1933 		 * the parent is the only reference to it, or (perhaps) have
1934 		 * the parent bypass the object if the parent happens to shadow
1935 		 * all the resident pages in the entire backing object.
1936 		 */
1937 		if (backing_object->ref_count == 1) {
1938 			KASSERT(backing_object->shadow_count == 1,
1939 			    ("vm_object_collapse: shadow_count: %d",
1940 			    backing_object->shadow_count));
1941 			vm_object_pip_add(object, 1);
1942 			vm_object_set_flag(object, OBJ_COLLAPSING);
1943 			vm_object_pip_add(backing_object, 1);
1944 			vm_object_set_flag(backing_object, OBJ_DEAD);
1945 
1946 			/*
1947 			 * If there is exactly one reference to the backing
1948 			 * object, we can collapse it into the parent.
1949 			 */
1950 			vm_object_collapse_scan(object);
1951 
1952 #if VM_NRESERVLEVEL > 0
1953 			/*
1954 			 * Break any reservations from backing_object.
1955 			 */
1956 			if (__predict_false(!LIST_EMPTY(&backing_object->rvq)))
1957 				vm_reserv_break_all(backing_object);
1958 #endif
1959 
1960 			/*
1961 			 * Move the pager from backing_object to object.
1962 			 */
1963 			if (backing_object->type == OBJT_SWAP) {
1964 				/*
1965 				 * swap_pager_copy() can sleep, in which case
1966 				 * the backing_object's and object's locks are
1967 				 * released and reacquired.
1968 				 * Since swap_pager_copy() is being asked to
1969 				 * destroy backing_object, it will change the
1970 				 * type to OBJT_DEFAULT.
1971 				 */
1972 				swap_pager_copy(
1973 				    backing_object,
1974 				    object,
1975 				    OFF_TO_IDX(object->backing_object_offset), TRUE);
1976 			}
1977 
1978 			/*
1979 			 * Object now shadows whatever backing_object did.
1980 			 */
1981 			vm_object_clear_flag(object, OBJ_COLLAPSING);
1982 			vm_object_backing_transfer(object, backing_object);
1983 			object->backing_object_offset +=
1984 			    backing_object->backing_object_offset;
1985 			VM_OBJECT_WUNLOCK(object);
1986 			vm_object_pip_wakeup(object);
1987 
1988 			/*
1989 			 * Discard backing_object.
1990 			 *
1991 			 * Since the backing object has no pages, no pager left,
1992 			 * and no object references within it, all that is
1993 			 * necessary is to dispose of it.
1994 			 */
1995 			KASSERT(backing_object->ref_count == 1, (
1996 "backing_object %p was somehow re-referenced during collapse!",
1997 			    backing_object));
1998 			vm_object_pip_wakeup(backing_object);
1999 			(void)refcount_release(&backing_object->ref_count);
2000 			vm_object_terminate(backing_object);
2001 			counter_u64_add(object_collapses, 1);
2002 			VM_OBJECT_WLOCK(object);
2003 		} else {
2004 			/*
2005 			 * If we do not entirely shadow the backing object,
2006 			 * there is nothing we can do so we give up.
2007 			 *
2008 			 * The object lock and backing_object lock must not
2009 			 * be dropped during this sequence.
2010 			 */
2011 			if (!vm_object_scan_all_shadowed(object)) {
2012 				VM_OBJECT_WUNLOCK(backing_object);
2013 				break;
2014 			}
2015 
2016 			/*
2017 			 * Make the parent shadow the next object in the
2018 			 * chain.  Deallocating backing_object will not remove
2019 			 * it, since its reference count is at least 2.
2020 			 */
2021 			vm_object_backing_remove_locked(object);
2022 			new_backing_object = backing_object->backing_object;
2023 			if (new_backing_object != NULL) {
2024 				vm_object_backing_insert_ref(object,
2025 				    new_backing_object);
2026 				object->backing_object_offset +=
2027 				    backing_object->backing_object_offset;
2028 			}
2029 
2030 			/*
2031 			 * Drop the reference count on backing_object. Since
2032 			 * its ref_count was at least 2, it will not vanish.
2033 			 */
2034 			(void)refcount_release(&backing_object->ref_count);
2035 			KASSERT(backing_object->ref_count >= 1, (
2036 "backing_object %p was somehow dereferenced during collapse!",
2037 			    backing_object));
2038 			VM_OBJECT_WUNLOCK(backing_object);
2039 			counter_u64_add(object_bypasses, 1);
2040 		}
2041 
2042 		/*
2043 		 * Try again with this object's new backing object.
2044 		 */
2045 	}
2046 }
2047 
2048 /*
2049  *	vm_object_page_remove:
2050  *
2051  *	For the given object, either frees or invalidates each of the
2052  *	specified pages.  In general, a page is freed.  However, if a page is
2053  *	wired for any reason other than the existence of a managed, wired
2054  *	mapping, then it may be invalidated but not removed from the object.
2055  *	Pages are specified by the given range ["start", "end") and the option
2056  *	OBJPR_CLEANONLY.  As a special case, if "end" is zero, then the range
2057  *	extends from "start" to the end of the object.  If the option
2058  *	OBJPR_CLEANONLY is specified, then only the non-dirty pages within the
2059  *	specified range are affected.  If the option OBJPR_NOTMAPPED is
2060  *	specified, then the pages within the specified range must have no
2061  *	mappings.  Otherwise, if this option is not specified, any mappings to
2062  *	the specified pages are removed before the pages are freed or
2063  *	invalidated.
2064  *
2065  *	In general, this operation should only be performed on objects that
2066  *	contain managed pages.  There are, however, two exceptions.  First, it
2067  *	is performed on the kernel and kmem objects by vm_map_entry_delete().
2068  *	Second, it is used by msync(..., MS_INVALIDATE) to invalidate device-
2069  *	backed pages.  In both of these cases, the option OBJPR_CLEANONLY must
2070  *	not be specified and the option OBJPR_NOTMAPPED must be specified.
2071  *
2072  *	The object must be locked.
2073  */
2074 void
2075 vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
2076     int options)
2077 {
2078 	vm_page_t p, next;
2079 
2080 	VM_OBJECT_ASSERT_WLOCKED(object);
2081 	KASSERT((object->flags & OBJ_UNMANAGED) == 0 ||
2082 	    (options & (OBJPR_CLEANONLY | OBJPR_NOTMAPPED)) == OBJPR_NOTMAPPED,
2083 	    ("vm_object_page_remove: illegal options for object %p", object));
2084 	if (object->resident_page_count == 0)
2085 		return;
2086 	vm_object_pip_add(object, 1);
2087 again:
2088 	p = vm_page_find_least(object, start);
2089 
2090 	/*
2091 	 * Here, the variable "p" is either (1) the page with the least pindex
2092 	 * greater than or equal to the parameter "start" or (2) NULL.
2093 	 */
2094 	for (; p != NULL && (p->pindex < end || end == 0); p = next) {
2095 		next = TAILQ_NEXT(p, listq);
2096 
2097 		/*
2098 		 * If the page is wired for any reason besides the existence
2099 		 * of managed, wired mappings, then it cannot be freed.  For
2100 		 * example, fictitious pages, which represent device memory,
2101 		 * are inherently wired and cannot be freed.  They can,
2102 		 * however, be invalidated if the option OBJPR_CLEANONLY is
2103 		 * not specified.
2104 		 */
2105 		if (vm_page_tryxbusy(p) == 0) {
2106 			vm_page_sleep_if_busy(p, "vmopar");
2107 			goto again;
2108 		}
2109 		if (vm_page_wired(p)) {
2110 wired:
2111 			if ((options & OBJPR_NOTMAPPED) == 0 &&
2112 			    object->ref_count != 0)
2113 				pmap_remove_all(p);
2114 			if ((options & OBJPR_CLEANONLY) == 0) {
2115 				vm_page_invalid(p);
2116 				vm_page_undirty(p);
2117 			}
2118 			vm_page_xunbusy(p);
2119 			continue;
2120 		}
2121 		KASSERT((p->flags & PG_FICTITIOUS) == 0,
2122 		    ("vm_object_page_remove: page %p is fictitious", p));
2123 		if ((options & OBJPR_CLEANONLY) != 0 &&
2124 		    !vm_page_none_valid(p)) {
2125 			if ((options & OBJPR_NOTMAPPED) == 0 &&
2126 			    object->ref_count != 0 &&
2127 			    !vm_page_try_remove_write(p))
2128 				goto wired;
2129 			if (p->dirty != 0) {
2130 				vm_page_xunbusy(p);
2131 				continue;
2132 			}
2133 		}
2134 		if ((options & OBJPR_NOTMAPPED) == 0 &&
2135 		    object->ref_count != 0 && !vm_page_try_remove_all(p))
2136 			goto wired;
2137 		vm_page_free(p);
2138 	}
2139 	vm_object_pip_wakeup(object);
2140 
2141 	if (object->type == OBJT_SWAP) {
2142 		if (end == 0)
2143 			end = object->size;
2144 		swap_pager_freespace(object, start, end - start);
2145 	}
2146 }
2147 
2148 /*
2149  *	vm_object_page_noreuse:
2150  *
2151  *	For the given object, attempt to move the specified pages to
2152  *	the head of the inactive queue.  This bypasses regular LRU
2153  *	operation and allows the pages to be reused quickly under memory
2154  *	pressure.  If a page is wired for any reason, then it will not
2155  *	be queued.  Pages are specified by the range ["start", "end").
2156  *	As a special case, if "end" is zero, then the range extends from
2157  *	"start" to the end of the object.
2158  *
2159  *	This operation should only be performed on objects that
2160  *	contain non-fictitious, managed pages.
2161  *
2162  *	The object must be locked.
2163  */
2164 void
2165 vm_object_page_noreuse(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
2166 {
2167 	vm_page_t p, next;
2168 
2169 	VM_OBJECT_ASSERT_LOCKED(object);
2170 	KASSERT((object->flags & (OBJ_FICTITIOUS | OBJ_UNMANAGED)) == 0,
2171 	    ("vm_object_page_noreuse: illegal object %p", object));
2172 	if (object->resident_page_count == 0)
2173 		return;
2174 	p = vm_page_find_least(object, start);
2175 
2176 	/*
2177 	 * Here, the variable "p" is either (1) the page with the least pindex
2178 	 * greater than or equal to the parameter "start" or (2) NULL.
2179 	 */
2180 	for (; p != NULL && (p->pindex < end || end == 0); p = next) {
2181 		next = TAILQ_NEXT(p, listq);
2182 		vm_page_deactivate_noreuse(p);
2183 	}
2184 }
2185 
2186 /*
2187  *	Populate the specified range of the object with valid pages.  Returns
2188  *	TRUE if the range is successfully populated and FALSE otherwise.
2189  *
2190  *	Note: This function should be optimized to pass a larger array of
2191  *	pages to vm_pager_get_pages() before it is applied to a non-
2192  *	OBJT_DEVICE object.
2193  *
2194  *	The object must be locked.
2195  */
2196 boolean_t
2197 vm_object_populate(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
2198 {
2199 	vm_page_t m;
2200 	vm_pindex_t pindex;
2201 	int rv;
2202 
2203 	VM_OBJECT_ASSERT_WLOCKED(object);
2204 	for (pindex = start; pindex < end; pindex++) {
2205 		rv = vm_page_grab_valid(&m, object, pindex, VM_ALLOC_NORMAL);
2206 		if (rv != VM_PAGER_OK)
2207 			break;
2208 
2209 		/*
2210 		 * Keep "m" busy because a subsequent iteration may unlock
2211 		 * the object.
2212 		 */
2213 	}
2214 	if (pindex > start) {
2215 		m = vm_page_lookup(object, start);
2216 		while (m != NULL && m->pindex < pindex) {
2217 			vm_page_xunbusy(m);
2218 			m = TAILQ_NEXT(m, listq);
2219 		}
2220 	}
2221 	return (pindex == end);
2222 }
2223 
2224 /*
2225  *	Routine:	vm_object_coalesce
2226  *	Function:	Coalesces two objects backing up adjoining
2227  *			regions of memory into a single object.
2228  *
2229  *	returns TRUE if objects were combined.
2230  *
2231  *	NOTE:	Only works at the moment if the second object is NULL -
2232  *		if it's not, which object do we lock first?
2233  *
2234  *	Parameters:
2235  *		prev_object	First object to coalesce
2236  *		prev_offset	Offset into prev_object
2237  *		prev_size	Size of reference to prev_object
2238  *		next_size	Size of reference to the second object
2239  *		reserved	Indicator that extension region has
2240  *				swap accounted for
2241  *
2242  *	Conditions:
2243  *	The object must *not* be locked.
2244  */
2245 boolean_t
2246 vm_object_coalesce(vm_object_t prev_object, vm_ooffset_t prev_offset,
2247     vm_size_t prev_size, vm_size_t next_size, boolean_t reserved)
2248 {
2249 	vm_pindex_t next_pindex;
2250 
2251 	if (prev_object == NULL)
2252 		return (TRUE);
2253 	if ((prev_object->flags & OBJ_ANON) == 0)
2254 		return (FALSE);
2255 
2256 	VM_OBJECT_WLOCK(prev_object);
2257 	/*
2258 	 * Try to collapse the object first.
2259 	 */
2260 	vm_object_collapse(prev_object);
2261 
2262 	/*
2263 	 * Can't coalesce if: . more than one reference . paged out . shadows
2264 	 * another object . has a copy elsewhere (any of which mean that the
2265 	 * pages not mapped to prev_entry may be in use anyway)
2266 	 */
2267 	if (prev_object->backing_object != NULL) {
2268 		VM_OBJECT_WUNLOCK(prev_object);
2269 		return (FALSE);
2270 	}
2271 
2272 	prev_size >>= PAGE_SHIFT;
2273 	next_size >>= PAGE_SHIFT;
2274 	next_pindex = OFF_TO_IDX(prev_offset) + prev_size;
2275 
2276 	if (prev_object->ref_count > 1 &&
2277 	    prev_object->size != next_pindex &&
2278 	    (prev_object->flags & OBJ_ONEMAPPING) == 0) {
2279 		VM_OBJECT_WUNLOCK(prev_object);
2280 		return (FALSE);
2281 	}
2282 
2283 	/*
2284 	 * Account for the charge.
2285 	 */
2286 	if (prev_object->cred != NULL) {
2287 		/*
2288 		 * If prev_object was charged, then this mapping,
2289 		 * although not charged now, may become writable
2290 		 * later. Non-NULL cred in the object would prevent
2291 		 * swap reservation during enabling of the write
2292 		 * access, so reserve swap now. Failed reservation
2293 		 * cause allocation of the separate object for the map
2294 		 * entry, and swap reservation for this entry is
2295 		 * managed in appropriate time.
2296 		 */
2297 		if (!reserved && !swap_reserve_by_cred(ptoa(next_size),
2298 		    prev_object->cred)) {
2299 			VM_OBJECT_WUNLOCK(prev_object);
2300 			return (FALSE);
2301 		}
2302 		prev_object->charge += ptoa(next_size);
2303 	}
2304 
2305 	/*
2306 	 * Remove any pages that may still be in the object from a previous
2307 	 * deallocation.
2308 	 */
2309 	if (next_pindex < prev_object->size) {
2310 		vm_object_page_remove(prev_object, next_pindex, next_pindex +
2311 		    next_size, 0);
2312 #if 0
2313 		if (prev_object->cred != NULL) {
2314 			KASSERT(prev_object->charge >=
2315 			    ptoa(prev_object->size - next_pindex),
2316 			    ("object %p overcharged 1 %jx %jx", prev_object,
2317 				(uintmax_t)next_pindex, (uintmax_t)next_size));
2318 			prev_object->charge -= ptoa(prev_object->size -
2319 			    next_pindex);
2320 		}
2321 #endif
2322 	}
2323 
2324 	/*
2325 	 * Extend the object if necessary.
2326 	 */
2327 	if (next_pindex + next_size > prev_object->size)
2328 		prev_object->size = next_pindex + next_size;
2329 
2330 	VM_OBJECT_WUNLOCK(prev_object);
2331 	return (TRUE);
2332 }
2333 
2334 void
2335 vm_object_set_writeable_dirty(vm_object_t object)
2336 {
2337 
2338 	/* Only set for vnodes & tmpfs */
2339 	if (object->type != OBJT_VNODE &&
2340 	    (object->flags & OBJ_TMPFS_NODE) == 0)
2341 		return;
2342 	atomic_add_int(&object->generation, 1);
2343 }
2344 
2345 /*
2346  *	vm_object_unwire:
2347  *
2348  *	For each page offset within the specified range of the given object,
2349  *	find the highest-level page in the shadow chain and unwire it.  A page
2350  *	must exist at every page offset, and the highest-level page must be
2351  *	wired.
2352  */
2353 void
2354 vm_object_unwire(vm_object_t object, vm_ooffset_t offset, vm_size_t length,
2355     uint8_t queue)
2356 {
2357 	vm_object_t tobject, t1object;
2358 	vm_page_t m, tm;
2359 	vm_pindex_t end_pindex, pindex, tpindex;
2360 	int depth, locked_depth;
2361 
2362 	KASSERT((offset & PAGE_MASK) == 0,
2363 	    ("vm_object_unwire: offset is not page aligned"));
2364 	KASSERT((length & PAGE_MASK) == 0,
2365 	    ("vm_object_unwire: length is not a multiple of PAGE_SIZE"));
2366 	/* The wired count of a fictitious page never changes. */
2367 	if ((object->flags & OBJ_FICTITIOUS) != 0)
2368 		return;
2369 	pindex = OFF_TO_IDX(offset);
2370 	end_pindex = pindex + atop(length);
2371 again:
2372 	locked_depth = 1;
2373 	VM_OBJECT_RLOCK(object);
2374 	m = vm_page_find_least(object, pindex);
2375 	while (pindex < end_pindex) {
2376 		if (m == NULL || pindex < m->pindex) {
2377 			/*
2378 			 * The first object in the shadow chain doesn't
2379 			 * contain a page at the current index.  Therefore,
2380 			 * the page must exist in a backing object.
2381 			 */
2382 			tobject = object;
2383 			tpindex = pindex;
2384 			depth = 0;
2385 			do {
2386 				tpindex +=
2387 				    OFF_TO_IDX(tobject->backing_object_offset);
2388 				tobject = tobject->backing_object;
2389 				KASSERT(tobject != NULL,
2390 				    ("vm_object_unwire: missing page"));
2391 				if ((tobject->flags & OBJ_FICTITIOUS) != 0)
2392 					goto next_page;
2393 				depth++;
2394 				if (depth == locked_depth) {
2395 					locked_depth++;
2396 					VM_OBJECT_RLOCK(tobject);
2397 				}
2398 			} while ((tm = vm_page_lookup(tobject, tpindex)) ==
2399 			    NULL);
2400 		} else {
2401 			tm = m;
2402 			m = TAILQ_NEXT(m, listq);
2403 		}
2404 		if (vm_page_trysbusy(tm) == 0) {
2405 			for (tobject = object; locked_depth >= 1;
2406 			    locked_depth--) {
2407 				t1object = tobject->backing_object;
2408 				if (tm->object != tobject)
2409 					VM_OBJECT_RUNLOCK(tobject);
2410 				tobject = t1object;
2411 			}
2412 			vm_page_busy_sleep(tm, "unwbo", true);
2413 			goto again;
2414 		}
2415 		vm_page_unwire(tm, queue);
2416 		vm_page_sunbusy(tm);
2417 next_page:
2418 		pindex++;
2419 	}
2420 	/* Release the accumulated object locks. */
2421 	for (tobject = object; locked_depth >= 1; locked_depth--) {
2422 		t1object = tobject->backing_object;
2423 		VM_OBJECT_RUNLOCK(tobject);
2424 		tobject = t1object;
2425 	}
2426 }
2427 
2428 /*
2429  * Return the vnode for the given object, or NULL if none exists.
2430  * For tmpfs objects, the function may return NULL if there is
2431  * no vnode allocated at the time of the call.
2432  */
2433 struct vnode *
2434 vm_object_vnode(vm_object_t object)
2435 {
2436 	struct vnode *vp;
2437 
2438 	VM_OBJECT_ASSERT_LOCKED(object);
2439 	if (object->type == OBJT_VNODE) {
2440 		vp = object->handle;
2441 		KASSERT(vp != NULL, ("%s: OBJT_VNODE has no vnode", __func__));
2442 	} else if (object->type == OBJT_SWAP &&
2443 	    (object->flags & OBJ_TMPFS) != 0) {
2444 		vp = object->un_pager.swp.swp_tmpfs;
2445 		KASSERT(vp != NULL, ("%s: OBJT_TMPFS has no vnode", __func__));
2446 	} else {
2447 		vp = NULL;
2448 	}
2449 	return (vp);
2450 }
2451 
2452 /*
2453  * Busy the vm object.  This prevents new pages belonging to the object from
2454  * becoming busy.  Existing pages persist as busy.  Callers are responsible
2455  * for checking page state before proceeding.
2456  */
2457 void
2458 vm_object_busy(vm_object_t obj)
2459 {
2460 
2461 	VM_OBJECT_ASSERT_LOCKED(obj);
2462 
2463 	blockcount_acquire(&obj->busy, 1);
2464 	/* The fence is required to order loads of page busy. */
2465 	atomic_thread_fence_acq_rel();
2466 }
2467 
2468 void
2469 vm_object_unbusy(vm_object_t obj)
2470 {
2471 
2472 	blockcount_release(&obj->busy, 1);
2473 }
2474 
2475 void
2476 vm_object_busy_wait(vm_object_t obj, const char *wmesg)
2477 {
2478 
2479 	VM_OBJECT_ASSERT_UNLOCKED(obj);
2480 
2481 	(void)blockcount_sleep(&obj->busy, NULL, wmesg, PVM);
2482 }
2483 
2484 /*
2485  * Return the kvme type of the given object.
2486  * If vpp is not NULL, set it to the object's vm_object_vnode() or NULL.
2487  */
2488 int
2489 vm_object_kvme_type(vm_object_t object, struct vnode **vpp)
2490 {
2491 
2492 	VM_OBJECT_ASSERT_LOCKED(object);
2493 	if (vpp != NULL)
2494 		*vpp = vm_object_vnode(object);
2495 	switch (object->type) {
2496 	case OBJT_DEFAULT:
2497 		return (KVME_TYPE_DEFAULT);
2498 	case OBJT_VNODE:
2499 		return (KVME_TYPE_VNODE);
2500 	case OBJT_SWAP:
2501 		if ((object->flags & OBJ_TMPFS_NODE) != 0)
2502 			return (KVME_TYPE_VNODE);
2503 		return (KVME_TYPE_SWAP);
2504 	case OBJT_DEVICE:
2505 		return (KVME_TYPE_DEVICE);
2506 	case OBJT_PHYS:
2507 		return (KVME_TYPE_PHYS);
2508 	case OBJT_DEAD:
2509 		return (KVME_TYPE_DEAD);
2510 	case OBJT_SG:
2511 		return (KVME_TYPE_SG);
2512 	case OBJT_MGTDEVICE:
2513 		return (KVME_TYPE_MGTDEVICE);
2514 	default:
2515 		return (KVME_TYPE_UNKNOWN);
2516 	}
2517 }
2518 
2519 static int
2520 sysctl_vm_object_list(SYSCTL_HANDLER_ARGS)
2521 {
2522 	struct kinfo_vmobject *kvo;
2523 	char *fullpath, *freepath;
2524 	struct vnode *vp;
2525 	struct vattr va;
2526 	vm_object_t obj;
2527 	vm_page_t m;
2528 	u_long sp;
2529 	int count, error;
2530 
2531 	if (req->oldptr == NULL) {
2532 		/*
2533 		 * If an old buffer has not been provided, generate an
2534 		 * estimate of the space needed for a subsequent call.
2535 		 */
2536 		mtx_lock(&vm_object_list_mtx);
2537 		count = 0;
2538 		TAILQ_FOREACH(obj, &vm_object_list, object_list) {
2539 			if (obj->type == OBJT_DEAD)
2540 				continue;
2541 			count++;
2542 		}
2543 		mtx_unlock(&vm_object_list_mtx);
2544 		return (SYSCTL_OUT(req, NULL, sizeof(struct kinfo_vmobject) *
2545 		    count * 11 / 10));
2546 	}
2547 
2548 	kvo = malloc(sizeof(*kvo), M_TEMP, M_WAITOK);
2549 	error = 0;
2550 
2551 	/*
2552 	 * VM objects are type stable and are never removed from the
2553 	 * list once added.  This allows us to safely read obj->object_list
2554 	 * after reacquiring the VM object lock.
2555 	 */
2556 	mtx_lock(&vm_object_list_mtx);
2557 	TAILQ_FOREACH(obj, &vm_object_list, object_list) {
2558 		if (obj->type == OBJT_DEAD)
2559 			continue;
2560 		VM_OBJECT_RLOCK(obj);
2561 		if (obj->type == OBJT_DEAD) {
2562 			VM_OBJECT_RUNLOCK(obj);
2563 			continue;
2564 		}
2565 		mtx_unlock(&vm_object_list_mtx);
2566 		kvo->kvo_size = ptoa(obj->size);
2567 		kvo->kvo_resident = obj->resident_page_count;
2568 		kvo->kvo_ref_count = obj->ref_count;
2569 		kvo->kvo_shadow_count = obj->shadow_count;
2570 		kvo->kvo_memattr = obj->memattr;
2571 		kvo->kvo_active = 0;
2572 		kvo->kvo_inactive = 0;
2573 		TAILQ_FOREACH(m, &obj->memq, listq) {
2574 			/*
2575 			 * A page may belong to the object but be
2576 			 * dequeued and set to PQ_NONE while the
2577 			 * object lock is not held.  This makes the
2578 			 * reads of m->queue below racy, and we do not
2579 			 * count pages set to PQ_NONE.  However, this
2580 			 * sysctl is only meant to give an
2581 			 * approximation of the system anyway.
2582 			 */
2583 			if (m->a.queue == PQ_ACTIVE)
2584 				kvo->kvo_active++;
2585 			else if (m->a.queue == PQ_INACTIVE)
2586 				kvo->kvo_inactive++;
2587 		}
2588 
2589 		kvo->kvo_vn_fileid = 0;
2590 		kvo->kvo_vn_fsid = 0;
2591 		kvo->kvo_vn_fsid_freebsd11 = 0;
2592 		freepath = NULL;
2593 		fullpath = "";
2594 		kvo->kvo_type = vm_object_kvme_type(obj, &vp);
2595 		if (vp != NULL) {
2596 			vref(vp);
2597 		} else if ((obj->flags & OBJ_ANON) != 0) {
2598 			MPASS(kvo->kvo_type == KVME_TYPE_DEFAULT ||
2599 			    kvo->kvo_type == KVME_TYPE_SWAP);
2600 			kvo->kvo_me = (uintptr_t)obj;
2601 			/* tmpfs objs are reported as vnodes */
2602 			kvo->kvo_backing_obj = (uintptr_t)obj->backing_object;
2603 			sp = swap_pager_swapped_pages(obj);
2604 			kvo->kvo_swapped = sp > UINT32_MAX ? UINT32_MAX : sp;
2605 		}
2606 		VM_OBJECT_RUNLOCK(obj);
2607 		if (vp != NULL) {
2608 			vn_fullpath(vp, &fullpath, &freepath);
2609 			vn_lock(vp, LK_SHARED | LK_RETRY);
2610 			if (VOP_GETATTR(vp, &va, curthread->td_ucred) == 0) {
2611 				kvo->kvo_vn_fileid = va.va_fileid;
2612 				kvo->kvo_vn_fsid = va.va_fsid;
2613 				kvo->kvo_vn_fsid_freebsd11 = va.va_fsid;
2614 								/* truncate */
2615 			}
2616 			vput(vp);
2617 		}
2618 
2619 		strlcpy(kvo->kvo_path, fullpath, sizeof(kvo->kvo_path));
2620 		if (freepath != NULL)
2621 			free(freepath, M_TEMP);
2622 
2623 		/* Pack record size down */
2624 		kvo->kvo_structsize = offsetof(struct kinfo_vmobject, kvo_path)
2625 		    + strlen(kvo->kvo_path) + 1;
2626 		kvo->kvo_structsize = roundup(kvo->kvo_structsize,
2627 		    sizeof(uint64_t));
2628 		error = SYSCTL_OUT(req, kvo, kvo->kvo_structsize);
2629 		mtx_lock(&vm_object_list_mtx);
2630 		if (error)
2631 			break;
2632 	}
2633 	mtx_unlock(&vm_object_list_mtx);
2634 	free(kvo, M_TEMP);
2635 	return (error);
2636 }
2637 SYSCTL_PROC(_vm, OID_AUTO, objects, CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_SKIP |
2638     CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_object_list, "S,kinfo_vmobject",
2639     "List of VM objects");
2640 
2641 #include "opt_ddb.h"
2642 #ifdef DDB
2643 #include <sys/kernel.h>
2644 
2645 #include <sys/cons.h>
2646 
2647 #include <ddb/ddb.h>
2648 
2649 static int
2650 _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry)
2651 {
2652 	vm_map_t tmpm;
2653 	vm_map_entry_t tmpe;
2654 	vm_object_t obj;
2655 
2656 	if (map == 0)
2657 		return 0;
2658 
2659 	if (entry == 0) {
2660 		VM_MAP_ENTRY_FOREACH(tmpe, map) {
2661 			if (_vm_object_in_map(map, object, tmpe)) {
2662 				return 1;
2663 			}
2664 		}
2665 	} else if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
2666 		tmpm = entry->object.sub_map;
2667 		VM_MAP_ENTRY_FOREACH(tmpe, tmpm) {
2668 			if (_vm_object_in_map(tmpm, object, tmpe)) {
2669 				return 1;
2670 			}
2671 		}
2672 	} else if ((obj = entry->object.vm_object) != NULL) {
2673 		for (; obj; obj = obj->backing_object)
2674 			if (obj == object) {
2675 				return 1;
2676 			}
2677 	}
2678 	return 0;
2679 }
2680 
2681 static int
2682 vm_object_in_map(vm_object_t object)
2683 {
2684 	struct proc *p;
2685 
2686 	/* sx_slock(&allproc_lock); */
2687 	FOREACH_PROC_IN_SYSTEM(p) {
2688 		if (!p->p_vmspace /* || (p->p_flag & (P_SYSTEM|P_WEXIT)) */)
2689 			continue;
2690 		if (_vm_object_in_map(&p->p_vmspace->vm_map, object, 0)) {
2691 			/* sx_sunlock(&allproc_lock); */
2692 			return 1;
2693 		}
2694 	}
2695 	/* sx_sunlock(&allproc_lock); */
2696 	if (_vm_object_in_map(kernel_map, object, 0))
2697 		return 1;
2698 	return 0;
2699 }
2700 
2701 DB_SHOW_COMMAND(vmochk, vm_object_check)
2702 {
2703 	vm_object_t object;
2704 
2705 	/*
2706 	 * make sure that internal objs are in a map somewhere
2707 	 * and none have zero ref counts.
2708 	 */
2709 	TAILQ_FOREACH(object, &vm_object_list, object_list) {
2710 		if ((object->flags & OBJ_ANON) != 0) {
2711 			if (object->ref_count == 0) {
2712 				db_printf("vmochk: internal obj has zero ref count: %ld\n",
2713 					(long)object->size);
2714 			}
2715 			if (!vm_object_in_map(object)) {
2716 				db_printf(
2717 			"vmochk: internal obj is not in a map: "
2718 			"ref: %d, size: %lu: 0x%lx, backing_object: %p\n",
2719 				    object->ref_count, (u_long)object->size,
2720 				    (u_long)object->size,
2721 				    (void *)object->backing_object);
2722 			}
2723 		}
2724 		if (db_pager_quit)
2725 			return;
2726 	}
2727 }
2728 
2729 /*
2730  *	vm_object_print:	[ debug ]
2731  */
2732 DB_SHOW_COMMAND(object, vm_object_print_static)
2733 {
2734 	/* XXX convert args. */
2735 	vm_object_t object = (vm_object_t)addr;
2736 	boolean_t full = have_addr;
2737 
2738 	vm_page_t p;
2739 
2740 	/* XXX count is an (unused) arg.  Avoid shadowing it. */
2741 #define	count	was_count
2742 
2743 	int count;
2744 
2745 	if (object == NULL)
2746 		return;
2747 
2748 	db_iprintf(
2749 	    "Object %p: type=%d, size=0x%jx, res=%d, ref=%d, flags=0x%x ruid %d charge %jx\n",
2750 	    object, (int)object->type, (uintmax_t)object->size,
2751 	    object->resident_page_count, object->ref_count, object->flags,
2752 	    object->cred ? object->cred->cr_ruid : -1, (uintmax_t)object->charge);
2753 	db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%jx\n",
2754 	    object->shadow_count,
2755 	    object->backing_object ? object->backing_object->ref_count : 0,
2756 	    object->backing_object, (uintmax_t)object->backing_object_offset);
2757 
2758 	if (!full)
2759 		return;
2760 
2761 	db_indent += 2;
2762 	count = 0;
2763 	TAILQ_FOREACH(p, &object->memq, listq) {
2764 		if (count == 0)
2765 			db_iprintf("memory:=");
2766 		else if (count == 6) {
2767 			db_printf("\n");
2768 			db_iprintf(" ...");
2769 			count = 0;
2770 		} else
2771 			db_printf(",");
2772 		count++;
2773 
2774 		db_printf("(off=0x%jx,page=0x%jx)",
2775 		    (uintmax_t)p->pindex, (uintmax_t)VM_PAGE_TO_PHYS(p));
2776 
2777 		if (db_pager_quit)
2778 			break;
2779 	}
2780 	if (count != 0)
2781 		db_printf("\n");
2782 	db_indent -= 2;
2783 }
2784 
2785 /* XXX. */
2786 #undef count
2787 
2788 /* XXX need this non-static entry for calling from vm_map_print. */
2789 void
2790 vm_object_print(
2791         /* db_expr_t */ long addr,
2792 	boolean_t have_addr,
2793 	/* db_expr_t */ long count,
2794 	char *modif)
2795 {
2796 	vm_object_print_static(addr, have_addr, count, modif);
2797 }
2798 
2799 DB_SHOW_COMMAND(vmopag, vm_object_print_pages)
2800 {
2801 	vm_object_t object;
2802 	vm_pindex_t fidx;
2803 	vm_paddr_t pa;
2804 	vm_page_t m, prev_m;
2805 	int rcount, nl, c;
2806 
2807 	nl = 0;
2808 	TAILQ_FOREACH(object, &vm_object_list, object_list) {
2809 		db_printf("new object: %p\n", (void *)object);
2810 		if (nl > 18) {
2811 			c = cngetc();
2812 			if (c != ' ')
2813 				return;
2814 			nl = 0;
2815 		}
2816 		nl++;
2817 		rcount = 0;
2818 		fidx = 0;
2819 		pa = -1;
2820 		TAILQ_FOREACH(m, &object->memq, listq) {
2821 			if (m->pindex > 128)
2822 				break;
2823 			if ((prev_m = TAILQ_PREV(m, pglist, listq)) != NULL &&
2824 			    prev_m->pindex + 1 != m->pindex) {
2825 				if (rcount) {
2826 					db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2827 						(long)fidx, rcount, (long)pa);
2828 					if (nl > 18) {
2829 						c = cngetc();
2830 						if (c != ' ')
2831 							return;
2832 						nl = 0;
2833 					}
2834 					nl++;
2835 					rcount = 0;
2836 				}
2837 			}
2838 			if (rcount &&
2839 				(VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) {
2840 				++rcount;
2841 				continue;
2842 			}
2843 			if (rcount) {
2844 				db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2845 					(long)fidx, rcount, (long)pa);
2846 				if (nl > 18) {
2847 					c = cngetc();
2848 					if (c != ' ')
2849 						return;
2850 					nl = 0;
2851 				}
2852 				nl++;
2853 			}
2854 			fidx = m->pindex;
2855 			pa = VM_PAGE_TO_PHYS(m);
2856 			rcount = 1;
2857 		}
2858 		if (rcount) {
2859 			db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2860 				(long)fidx, rcount, (long)pa);
2861 			if (nl > 18) {
2862 				c = cngetc();
2863 				if (c != ' ')
2864 					return;
2865 				nl = 0;
2866 			}
2867 			nl++;
2868 		}
2869 	}
2870 }
2871 #endif /* DDB */
2872