xref: /freebsd/sys/vm/vm_object.c (revision 6966ac055c3b7a39266fb982493330df7a097997)
1 /*-
2  * SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU)
3  *
4  * Copyright (c) 1991, 1993
5  *	The Regents of the University of California.  All rights reserved.
6  *
7  * This code is derived from software contributed to Berkeley by
8  * The Mach Operating System project at Carnegie-Mellon University.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 3. Neither the name of the University nor the names of its contributors
19  *    may be used to endorse or promote products derived from this software
20  *    without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  *	from: @(#)vm_object.c	8.5 (Berkeley) 3/22/94
35  *
36  *
37  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
38  * All rights reserved.
39  *
40  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
41  *
42  * Permission to use, copy, modify and distribute this software and
43  * its documentation is hereby granted, provided that both the copyright
44  * notice and this permission notice appear in all copies of the
45  * software, derivative works or modified versions, and any portions
46  * thereof, and that both notices appear in supporting documentation.
47  *
48  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
49  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
50  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
51  *
52  * Carnegie Mellon requests users of this software to return to
53  *
54  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
55  *  School of Computer Science
56  *  Carnegie Mellon University
57  *  Pittsburgh PA 15213-3890
58  *
59  * any improvements or extensions that they make and grant Carnegie the
60  * rights to redistribute these changes.
61  */
62 
63 /*
64  *	Virtual memory object module.
65  */
66 
67 #include <sys/cdefs.h>
68 __FBSDID("$FreeBSD$");
69 
70 #include "opt_vm.h"
71 
72 #include <sys/param.h>
73 #include <sys/systm.h>
74 #include <sys/cpuset.h>
75 #include <sys/lock.h>
76 #include <sys/mman.h>
77 #include <sys/mount.h>
78 #include <sys/kernel.h>
79 #include <sys/pctrie.h>
80 #include <sys/sysctl.h>
81 #include <sys/mutex.h>
82 #include <sys/proc.h>		/* for curproc, pageproc */
83 #include <sys/refcount.h>
84 #include <sys/socket.h>
85 #include <sys/resourcevar.h>
86 #include <sys/refcount.h>
87 #include <sys/rwlock.h>
88 #include <sys/user.h>
89 #include <sys/vnode.h>
90 #include <sys/vmmeter.h>
91 #include <sys/sx.h>
92 
93 #include <vm/vm.h>
94 #include <vm/vm_param.h>
95 #include <vm/pmap.h>
96 #include <vm/vm_map.h>
97 #include <vm/vm_object.h>
98 #include <vm/vm_page.h>
99 #include <vm/vm_pageout.h>
100 #include <vm/vm_pager.h>
101 #include <vm/vm_phys.h>
102 #include <vm/vm_pagequeue.h>
103 #include <vm/swap_pager.h>
104 #include <vm/vm_kern.h>
105 #include <vm/vm_extern.h>
106 #include <vm/vm_radix.h>
107 #include <vm/vm_reserv.h>
108 #include <vm/uma.h>
109 
110 static int old_msync;
111 SYSCTL_INT(_vm, OID_AUTO, old_msync, CTLFLAG_RW, &old_msync, 0,
112     "Use old (insecure) msync behavior");
113 
114 static int	vm_object_page_collect_flush(vm_object_t object, vm_page_t p,
115 		    int pagerflags, int flags, boolean_t *allclean,
116 		    boolean_t *eio);
117 static boolean_t vm_object_page_remove_write(vm_page_t p, int flags,
118 		    boolean_t *allclean);
119 static void	vm_object_backing_remove(vm_object_t object);
120 
121 /*
122  *	Virtual memory objects maintain the actual data
123  *	associated with allocated virtual memory.  A given
124  *	page of memory exists within exactly one object.
125  *
126  *	An object is only deallocated when all "references"
127  *	are given up.  Only one "reference" to a given
128  *	region of an object should be writeable.
129  *
130  *	Associated with each object is a list of all resident
131  *	memory pages belonging to that object; this list is
132  *	maintained by the "vm_page" module, and locked by the object's
133  *	lock.
134  *
135  *	Each object also records a "pager" routine which is
136  *	used to retrieve (and store) pages to the proper backing
137  *	storage.  In addition, objects may be backed by other
138  *	objects from which they were virtual-copied.
139  *
140  *	The only items within the object structure which are
141  *	modified after time of creation are:
142  *		reference count		locked by object's lock
143  *		pager routine		locked by object's lock
144  *
145  */
146 
147 struct object_q vm_object_list;
148 struct mtx vm_object_list_mtx;	/* lock for object list and count */
149 
150 struct vm_object kernel_object_store;
151 
152 static SYSCTL_NODE(_vm_stats, OID_AUTO, object, CTLFLAG_RD, 0,
153     "VM object stats");
154 
155 static counter_u64_t object_collapses = EARLY_COUNTER;
156 SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, collapses, CTLFLAG_RD,
157     &object_collapses,
158     "VM object collapses");
159 
160 static counter_u64_t object_bypasses = EARLY_COUNTER;
161 SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, bypasses, CTLFLAG_RD,
162     &object_bypasses,
163     "VM object bypasses");
164 
165 static counter_u64_t object_collapse_waits = EARLY_COUNTER;
166 SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, collapse_waits, CTLFLAG_RD,
167     &object_collapse_waits,
168     "Number of sleeps for collapse");
169 
170 static void
171 counter_startup(void)
172 {
173 
174 	object_collapses = counter_u64_alloc(M_WAITOK);
175 	object_bypasses = counter_u64_alloc(M_WAITOK);
176 	object_collapse_waits = counter_u64_alloc(M_WAITOK);
177 }
178 SYSINIT(object_counters, SI_SUB_CPU, SI_ORDER_ANY, counter_startup, NULL);
179 
180 static uma_zone_t obj_zone;
181 
182 static int vm_object_zinit(void *mem, int size, int flags);
183 
184 #ifdef INVARIANTS
185 static void vm_object_zdtor(void *mem, int size, void *arg);
186 
187 static void
188 vm_object_zdtor(void *mem, int size, void *arg)
189 {
190 	vm_object_t object;
191 
192 	object = (vm_object_t)mem;
193 	KASSERT(object->ref_count == 0,
194 	    ("object %p ref_count = %d", object, object->ref_count));
195 	KASSERT(TAILQ_EMPTY(&object->memq),
196 	    ("object %p has resident pages in its memq", object));
197 	KASSERT(vm_radix_is_empty(&object->rtree),
198 	    ("object %p has resident pages in its trie", object));
199 #if VM_NRESERVLEVEL > 0
200 	KASSERT(LIST_EMPTY(&object->rvq),
201 	    ("object %p has reservations",
202 	    object));
203 #endif
204 	KASSERT(REFCOUNT_COUNT(object->paging_in_progress) == 0,
205 	    ("object %p paging_in_progress = %d",
206 	    object, REFCOUNT_COUNT(object->paging_in_progress)));
207 	KASSERT(object->busy == 0,
208 	    ("object %p busy = %d",
209 	    object, object->busy));
210 	KASSERT(object->resident_page_count == 0,
211 	    ("object %p resident_page_count = %d",
212 	    object, object->resident_page_count));
213 	KASSERT(object->shadow_count == 0,
214 	    ("object %p shadow_count = %d",
215 	    object, object->shadow_count));
216 	KASSERT(object->type == OBJT_DEAD,
217 	    ("object %p has non-dead type %d",
218 	    object, object->type));
219 }
220 #endif
221 
222 static int
223 vm_object_zinit(void *mem, int size, int flags)
224 {
225 	vm_object_t object;
226 
227 	object = (vm_object_t)mem;
228 	rw_init_flags(&object->lock, "vm object", RW_DUPOK | RW_NEW);
229 
230 	/* These are true for any object that has been freed */
231 	object->type = OBJT_DEAD;
232 	vm_radix_init(&object->rtree);
233 	refcount_init(&object->ref_count, 0);
234 	refcount_init(&object->paging_in_progress, 0);
235 	refcount_init(&object->busy, 0);
236 	object->resident_page_count = 0;
237 	object->shadow_count = 0;
238 	object->flags = OBJ_DEAD;
239 
240 	mtx_lock(&vm_object_list_mtx);
241 	TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);
242 	mtx_unlock(&vm_object_list_mtx);
243 	return (0);
244 }
245 
246 static void
247 _vm_object_allocate(objtype_t type, vm_pindex_t size, u_short flags,
248     vm_object_t object, void *handle)
249 {
250 
251 	TAILQ_INIT(&object->memq);
252 	LIST_INIT(&object->shadow_head);
253 
254 	object->type = type;
255 	if (type == OBJT_SWAP)
256 		pctrie_init(&object->un_pager.swp.swp_blks);
257 
258 	/*
259 	 * Ensure that swap_pager_swapoff() iteration over object_list
260 	 * sees up to date type and pctrie head if it observed
261 	 * non-dead object.
262 	 */
263 	atomic_thread_fence_rel();
264 
265 	object->pg_color = 0;
266 	object->flags = flags;
267 	object->size = size;
268 	object->domain.dr_policy = NULL;
269 	object->generation = 1;
270 	object->cleangeneration = 1;
271 	refcount_init(&object->ref_count, 1);
272 	object->memattr = VM_MEMATTR_DEFAULT;
273 	object->cred = NULL;
274 	object->charge = 0;
275 	object->handle = handle;
276 	object->backing_object = NULL;
277 	object->backing_object_offset = (vm_ooffset_t) 0;
278 #if VM_NRESERVLEVEL > 0
279 	LIST_INIT(&object->rvq);
280 #endif
281 	umtx_shm_object_init(object);
282 }
283 
284 /*
285  *	vm_object_init:
286  *
287  *	Initialize the VM objects module.
288  */
289 void
290 vm_object_init(void)
291 {
292 	TAILQ_INIT(&vm_object_list);
293 	mtx_init(&vm_object_list_mtx, "vm object_list", NULL, MTX_DEF);
294 
295 	rw_init(&kernel_object->lock, "kernel vm object");
296 	_vm_object_allocate(OBJT_PHYS, atop(VM_MAX_KERNEL_ADDRESS -
297 	    VM_MIN_KERNEL_ADDRESS), OBJ_UNMANAGED, kernel_object, NULL);
298 #if VM_NRESERVLEVEL > 0
299 	kernel_object->flags |= OBJ_COLORED;
300 	kernel_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS);
301 #endif
302 
303 	/*
304 	 * The lock portion of struct vm_object must be type stable due
305 	 * to vm_pageout_fallback_object_lock locking a vm object
306 	 * without holding any references to it.
307 	 */
308 	obj_zone = uma_zcreate("VM OBJECT", sizeof (struct vm_object), NULL,
309 #ifdef INVARIANTS
310 	    vm_object_zdtor,
311 #else
312 	    NULL,
313 #endif
314 	    vm_object_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
315 
316 	vm_radix_zinit();
317 }
318 
319 void
320 vm_object_clear_flag(vm_object_t object, u_short bits)
321 {
322 
323 	VM_OBJECT_ASSERT_WLOCKED(object);
324 	object->flags &= ~bits;
325 }
326 
327 /*
328  *	Sets the default memory attribute for the specified object.  Pages
329  *	that are allocated to this object are by default assigned this memory
330  *	attribute.
331  *
332  *	Presently, this function must be called before any pages are allocated
333  *	to the object.  In the future, this requirement may be relaxed for
334  *	"default" and "swap" objects.
335  */
336 int
337 vm_object_set_memattr(vm_object_t object, vm_memattr_t memattr)
338 {
339 
340 	VM_OBJECT_ASSERT_WLOCKED(object);
341 	switch (object->type) {
342 	case OBJT_DEFAULT:
343 	case OBJT_DEVICE:
344 	case OBJT_MGTDEVICE:
345 	case OBJT_PHYS:
346 	case OBJT_SG:
347 	case OBJT_SWAP:
348 	case OBJT_VNODE:
349 		if (!TAILQ_EMPTY(&object->memq))
350 			return (KERN_FAILURE);
351 		break;
352 	case OBJT_DEAD:
353 		return (KERN_INVALID_ARGUMENT);
354 	default:
355 		panic("vm_object_set_memattr: object %p is of undefined type",
356 		    object);
357 	}
358 	object->memattr = memattr;
359 	return (KERN_SUCCESS);
360 }
361 
362 void
363 vm_object_pip_add(vm_object_t object, short i)
364 {
365 
366 	refcount_acquiren(&object->paging_in_progress, i);
367 }
368 
369 void
370 vm_object_pip_wakeup(vm_object_t object)
371 {
372 
373 	refcount_release(&object->paging_in_progress);
374 }
375 
376 void
377 vm_object_pip_wakeupn(vm_object_t object, short i)
378 {
379 
380 	refcount_releasen(&object->paging_in_progress, i);
381 }
382 
383 /*
384  * Atomically drop the interlock and wait for pip to drain.  This protects
385  * from sleep/wakeup races due to identity changes.  The lock is not
386  * re-acquired on return.
387  */
388 static void
389 vm_object_pip_sleep(vm_object_t object, char *waitid)
390 {
391 
392 	refcount_sleep_interlock(&object->paging_in_progress,
393 	    &object->lock, waitid, PVM);
394 }
395 
396 void
397 vm_object_pip_wait(vm_object_t object, char *waitid)
398 {
399 
400 	VM_OBJECT_ASSERT_WLOCKED(object);
401 
402 	while (REFCOUNT_COUNT(object->paging_in_progress) > 0) {
403 		vm_object_pip_sleep(object, waitid);
404 		VM_OBJECT_WLOCK(object);
405 	}
406 }
407 
408 void
409 vm_object_pip_wait_unlocked(vm_object_t object, char *waitid)
410 {
411 
412 	VM_OBJECT_ASSERT_UNLOCKED(object);
413 
414 	while (REFCOUNT_COUNT(object->paging_in_progress) > 0)
415 		refcount_wait(&object->paging_in_progress, waitid, PVM);
416 }
417 
418 /*
419  *	vm_object_allocate:
420  *
421  *	Returns a new object with the given size.
422  */
423 vm_object_t
424 vm_object_allocate(objtype_t type, vm_pindex_t size)
425 {
426 	vm_object_t object;
427 	u_short flags;
428 
429 	switch (type) {
430 	case OBJT_DEAD:
431 		panic("vm_object_allocate: can't create OBJT_DEAD");
432 	case OBJT_DEFAULT:
433 	case OBJT_SWAP:
434 		flags = OBJ_COLORED;
435 		break;
436 	case OBJT_DEVICE:
437 	case OBJT_SG:
438 		flags = OBJ_FICTITIOUS | OBJ_UNMANAGED;
439 		break;
440 	case OBJT_MGTDEVICE:
441 		flags = OBJ_FICTITIOUS;
442 		break;
443 	case OBJT_PHYS:
444 		flags = OBJ_UNMANAGED;
445 		break;
446 	case OBJT_VNODE:
447 		flags = 0;
448 		break;
449 	default:
450 		panic("vm_object_allocate: type %d is undefined", type);
451 	}
452 	object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);
453 	_vm_object_allocate(type, size, flags, object, NULL);
454 
455 	return (object);
456 }
457 
458 /*
459  *	vm_object_allocate_anon:
460  *
461  *	Returns a new default object of the given size and marked as
462  *	anonymous memory for special split/collapse handling.  Color
463  *	to be initialized by the caller.
464  */
465 vm_object_t
466 vm_object_allocate_anon(vm_pindex_t size, vm_object_t backing_object,
467     struct ucred *cred, vm_size_t charge)
468 {
469 	vm_object_t handle, object;
470 
471 	if (backing_object == NULL)
472 		handle = NULL;
473 	else if ((backing_object->flags & OBJ_ANON) != 0)
474 		handle = backing_object->handle;
475 	else
476 		handle = backing_object;
477 	object = uma_zalloc(obj_zone, M_WAITOK);
478 	_vm_object_allocate(OBJT_DEFAULT, size, OBJ_ANON | OBJ_ONEMAPPING,
479 	    object, handle);
480 	object->cred = cred;
481 	object->charge = cred != NULL ? charge : 0;
482 	return (object);
483 }
484 
485 static void
486 vm_object_reference_vnode(vm_object_t object)
487 {
488 	struct vnode *vp;
489 	u_int old;
490 
491 	/*
492 	 * vnode objects need the lock for the first reference
493 	 * to serialize with vnode_object_deallocate().
494 	 */
495 	if (!refcount_acquire_if_gt(&object->ref_count, 0)) {
496 		VM_OBJECT_RLOCK(object);
497 		old = refcount_acquire(&object->ref_count);
498 		if (object->type == OBJT_VNODE && old == 0) {
499 			vp = object->handle;
500 			vref(vp);
501 		}
502 		VM_OBJECT_RUNLOCK(object);
503 	}
504 }
505 
506 /*
507  *	vm_object_reference:
508  *
509  *	Acquires a reference to the given object.
510  */
511 void
512 vm_object_reference(vm_object_t object)
513 {
514 
515 	if (object == NULL)
516 		return;
517 
518 	if (object->type == OBJT_VNODE)
519 		vm_object_reference_vnode(object);
520 	else
521 		refcount_acquire(&object->ref_count);
522 	KASSERT((object->flags & OBJ_DEAD) == 0,
523 	    ("vm_object_reference: Referenced dead object."));
524 }
525 
526 /*
527  *	vm_object_reference_locked:
528  *
529  *	Gets another reference to the given object.
530  *
531  *	The object must be locked.
532  */
533 void
534 vm_object_reference_locked(vm_object_t object)
535 {
536 	struct vnode *vp;
537 	u_int old;
538 
539 	VM_OBJECT_ASSERT_LOCKED(object);
540 	old = refcount_acquire(&object->ref_count);
541 	if (object->type == OBJT_VNODE && old == 0) {
542 		vp = object->handle; vref(vp); }
543 	KASSERT((object->flags & OBJ_DEAD) == 0,
544 	    ("vm_object_reference: Referenced dead object."));
545 }
546 
547 /*
548  * Handle deallocating an object of type OBJT_VNODE.
549  */
550 static void
551 vm_object_deallocate_vnode(vm_object_t object)
552 {
553 	struct vnode *vp = (struct vnode *) object->handle;
554 	bool last;
555 
556 	KASSERT(object->type == OBJT_VNODE,
557 	    ("vm_object_deallocate_vnode: not a vnode object"));
558 	KASSERT(vp != NULL, ("vm_object_deallocate_vnode: missing vp"));
559 
560 	/* Object lock to protect handle lookup. */
561 	last = refcount_release(&object->ref_count);
562 	VM_OBJECT_RUNLOCK(object);
563 
564 	if (!last)
565 		return;
566 
567 	if (!umtx_shm_vnobj_persistent)
568 		umtx_shm_object_terminated(object);
569 
570 	/* vrele may need the vnode lock. */
571 	vrele(vp);
572 }
573 
574 
575 /*
576  * We dropped a reference on an object and discovered that it had a
577  * single remaining shadow.  This is a sibling of the reference we
578  * dropped.  Attempt to collapse the sibling and backing object.
579  */
580 static vm_object_t
581 vm_object_deallocate_anon(vm_object_t backing_object)
582 {
583 	vm_object_t object;
584 
585 	/* Fetch the final shadow.  */
586 	object = LIST_FIRST(&backing_object->shadow_head);
587 	KASSERT(object != NULL && backing_object->shadow_count == 1,
588 	    ("vm_object_anon_deallocate: ref_count: %d, shadow_count: %d",
589 	    backing_object->ref_count, backing_object->shadow_count));
590 	KASSERT((object->flags & (OBJ_TMPFS_NODE | OBJ_ANON)) == OBJ_ANON,
591 	    ("invalid shadow object %p", object));
592 
593 	if (!VM_OBJECT_TRYWLOCK(object)) {
594 		/*
595 		 * Prevent object from disappearing since we do not have a
596 		 * reference.
597 		 */
598 		vm_object_pip_add(object, 1);
599 		VM_OBJECT_WUNLOCK(backing_object);
600 		VM_OBJECT_WLOCK(object);
601 		vm_object_pip_wakeup(object);
602 	} else
603 		VM_OBJECT_WUNLOCK(backing_object);
604 
605 	/*
606 	 * Check for a collapse/terminate race with the last reference holder.
607 	 */
608 	if ((object->flags & (OBJ_DEAD | OBJ_COLLAPSING)) != 0 ||
609 	    !refcount_acquire_if_not_zero(&object->ref_count)) {
610 		VM_OBJECT_WUNLOCK(object);
611 		return (NULL);
612 	}
613 	backing_object = object->backing_object;
614 	if (backing_object != NULL && (backing_object->flags & OBJ_ANON) != 0)
615 		vm_object_collapse(object);
616 	VM_OBJECT_WUNLOCK(object);
617 
618 	return (object);
619 }
620 
621 /*
622  *	vm_object_deallocate:
623  *
624  *	Release a reference to the specified object,
625  *	gained either through a vm_object_allocate
626  *	or a vm_object_reference call.  When all references
627  *	are gone, storage associated with this object
628  *	may be relinquished.
629  *
630  *	No object may be locked.
631  */
632 void
633 vm_object_deallocate(vm_object_t object)
634 {
635 	vm_object_t temp;
636 	bool released;
637 
638 	while (object != NULL) {
639 		/*
640 		 * If the reference count goes to 0 we start calling
641 		 * vm_object_terminate() on the object chain.  A ref count
642 		 * of 1 may be a special case depending on the shadow count
643 		 * being 0 or 1.  These cases require a write lock on the
644 		 * object.
645 		 */
646 		if ((object->flags & OBJ_ANON) == 0)
647 			released = refcount_release_if_gt(&object->ref_count, 1);
648 		else
649 			released = refcount_release_if_gt(&object->ref_count, 2);
650 		if (released)
651 			return;
652 
653 		if (object->type == OBJT_VNODE) {
654 			VM_OBJECT_RLOCK(object);
655 			if (object->type == OBJT_VNODE) {
656 				vm_object_deallocate_vnode(object);
657 				return;
658 			}
659 			VM_OBJECT_RUNLOCK(object);
660 		}
661 
662 		VM_OBJECT_WLOCK(object);
663 		KASSERT(object->ref_count > 0,
664 		    ("vm_object_deallocate: object deallocated too many times: %d",
665 		    object->type));
666 
667 		/*
668 		 * If this is not the final reference to an anonymous
669 		 * object we may need to collapse the shadow chain.
670 		 */
671 		if (!refcount_release(&object->ref_count)) {
672 			if (object->ref_count > 1 ||
673 			    object->shadow_count == 0) {
674 				if ((object->flags & OBJ_ANON) != 0 &&
675 				    object->ref_count == 1)
676 					vm_object_set_flag(object,
677 					    OBJ_ONEMAPPING);
678 				VM_OBJECT_WUNLOCK(object);
679 				return;
680 			}
681 
682 			/* Handle collapsing last ref on anonymous objects. */
683 			object = vm_object_deallocate_anon(object);
684 			continue;
685 		}
686 
687 		/*
688 		 * Handle the final reference to an object.  We restart
689 		 * the loop with the backing object to avoid recursion.
690 		 */
691 		umtx_shm_object_terminated(object);
692 		temp = object->backing_object;
693 		if (temp != NULL) {
694 			KASSERT((object->flags & OBJ_TMPFS_NODE) == 0,
695 			    ("shadowed tmpfs v_object 2 %p", object));
696 			vm_object_backing_remove(object);
697 		}
698 
699 		KASSERT((object->flags & OBJ_DEAD) == 0,
700 		    ("vm_object_deallocate: Terminating dead object."));
701 		vm_object_set_flag(object, OBJ_DEAD);
702 		vm_object_terminate(object);
703 		object = temp;
704 	}
705 }
706 
707 /*
708  *	vm_object_destroy removes the object from the global object list
709  *      and frees the space for the object.
710  */
711 void
712 vm_object_destroy(vm_object_t object)
713 {
714 
715 	/*
716 	 * Release the allocation charge.
717 	 */
718 	if (object->cred != NULL) {
719 		swap_release_by_cred(object->charge, object->cred);
720 		object->charge = 0;
721 		crfree(object->cred);
722 		object->cred = NULL;
723 	}
724 
725 	/*
726 	 * Free the space for the object.
727 	 */
728 	uma_zfree(obj_zone, object);
729 }
730 
731 static void
732 vm_object_backing_remove_locked(vm_object_t object)
733 {
734 	vm_object_t backing_object;
735 
736 	backing_object = object->backing_object;
737 	VM_OBJECT_ASSERT_WLOCKED(object);
738 	VM_OBJECT_ASSERT_WLOCKED(backing_object);
739 
740 	KASSERT((object->flags & OBJ_COLLAPSING) == 0,
741 	    ("vm_object_backing_remove: Removing collapsing object."));
742 
743 	if ((object->flags & OBJ_SHADOWLIST) != 0) {
744 		LIST_REMOVE(object, shadow_list);
745 		backing_object->shadow_count--;
746 		object->flags &= ~OBJ_SHADOWLIST;
747 	}
748 	object->backing_object = NULL;
749 }
750 
751 static void
752 vm_object_backing_remove(vm_object_t object)
753 {
754 	vm_object_t backing_object;
755 
756 	VM_OBJECT_ASSERT_WLOCKED(object);
757 
758 	if ((object->flags & OBJ_SHADOWLIST) != 0) {
759 		backing_object = object->backing_object;
760 		VM_OBJECT_WLOCK(backing_object);
761 		vm_object_backing_remove_locked(object);
762 		VM_OBJECT_WUNLOCK(backing_object);
763 	} else
764 		object->backing_object = NULL;
765 }
766 
767 static void
768 vm_object_backing_insert_locked(vm_object_t object, vm_object_t backing_object)
769 {
770 
771 	VM_OBJECT_ASSERT_WLOCKED(object);
772 
773 	if ((backing_object->flags & OBJ_ANON) != 0) {
774 		VM_OBJECT_ASSERT_WLOCKED(backing_object);
775 		LIST_INSERT_HEAD(&backing_object->shadow_head, object,
776 		    shadow_list);
777 		backing_object->shadow_count++;
778 		object->flags |= OBJ_SHADOWLIST;
779 	}
780 	object->backing_object = backing_object;
781 }
782 
783 static void
784 vm_object_backing_insert(vm_object_t object, vm_object_t backing_object)
785 {
786 
787 	VM_OBJECT_ASSERT_WLOCKED(object);
788 
789 	if ((backing_object->flags & OBJ_ANON) != 0) {
790 		VM_OBJECT_WLOCK(backing_object);
791 		vm_object_backing_insert_locked(object, backing_object);
792 		VM_OBJECT_WUNLOCK(backing_object);
793 	} else
794 		object->backing_object = backing_object;
795 }
796 
797 /*
798  * Insert an object into a backing_object's shadow list with an additional
799  * reference to the backing_object added.
800  */
801 static void
802 vm_object_backing_insert_ref(vm_object_t object, vm_object_t backing_object)
803 {
804 
805 	VM_OBJECT_ASSERT_WLOCKED(object);
806 
807 	if ((backing_object->flags & OBJ_ANON) != 0) {
808 		VM_OBJECT_WLOCK(backing_object);
809 		KASSERT((backing_object->flags & OBJ_DEAD) == 0,
810 		    ("shadowing dead anonymous object"));
811 		vm_object_reference_locked(backing_object);
812 		vm_object_backing_insert_locked(object, backing_object);
813 		vm_object_clear_flag(backing_object, OBJ_ONEMAPPING);
814 		VM_OBJECT_WUNLOCK(backing_object);
815 	} else {
816 		vm_object_reference(backing_object);
817 		object->backing_object = backing_object;
818 	}
819 }
820 
821 /*
822  * Transfer a backing reference from backing_object to object.
823  */
824 static void
825 vm_object_backing_transfer(vm_object_t object, vm_object_t backing_object)
826 {
827 	vm_object_t new_backing_object;
828 
829 	/*
830 	 * Note that the reference to backing_object->backing_object
831 	 * moves from within backing_object to within object.
832 	 */
833 	vm_object_backing_remove_locked(object);
834 	new_backing_object = backing_object->backing_object;
835 	if (new_backing_object == NULL)
836 		return;
837 	if ((new_backing_object->flags & OBJ_ANON) != 0) {
838 		VM_OBJECT_WLOCK(new_backing_object);
839 		vm_object_backing_remove_locked(backing_object);
840 		vm_object_backing_insert_locked(object, new_backing_object);
841 		VM_OBJECT_WUNLOCK(new_backing_object);
842 	} else {
843 		object->backing_object = new_backing_object;
844 		backing_object->backing_object = NULL;
845 	}
846 }
847 
848 /*
849  * Wait for a concurrent collapse to settle.
850  */
851 static void
852 vm_object_collapse_wait(vm_object_t object)
853 {
854 
855 	VM_OBJECT_ASSERT_WLOCKED(object);
856 
857 	while ((object->flags & OBJ_COLLAPSING) != 0) {
858 		vm_object_pip_wait(object, "vmcolwait");
859 		counter_u64_add(object_collapse_waits, 1);
860 	}
861 }
862 
863 /*
864  * Waits for a backing object to clear a pending collapse and returns
865  * it locked if it is an ANON object.
866  */
867 static vm_object_t
868 vm_object_backing_collapse_wait(vm_object_t object)
869 {
870 	vm_object_t backing_object;
871 
872 	VM_OBJECT_ASSERT_WLOCKED(object);
873 
874 	for (;;) {
875 		backing_object = object->backing_object;
876 		if (backing_object == NULL ||
877 		    (backing_object->flags & OBJ_ANON) == 0)
878 			return (NULL);
879 		VM_OBJECT_WLOCK(backing_object);
880 		if ((backing_object->flags & (OBJ_DEAD | OBJ_COLLAPSING)) == 0)
881 			break;
882 		VM_OBJECT_WUNLOCK(object);
883 		vm_object_pip_sleep(backing_object, "vmbckwait");
884 		counter_u64_add(object_collapse_waits, 1);
885 		VM_OBJECT_WLOCK(object);
886 	}
887 	return (backing_object);
888 }
889 
890 /*
891  *	vm_object_terminate_pages removes any remaining pageable pages
892  *	from the object and resets the object to an empty state.
893  */
894 static void
895 vm_object_terminate_pages(vm_object_t object)
896 {
897 	vm_page_t p, p_next;
898 
899 	VM_OBJECT_ASSERT_WLOCKED(object);
900 
901 	/*
902 	 * Free any remaining pageable pages.  This also removes them from the
903 	 * paging queues.  However, don't free wired pages, just remove them
904 	 * from the object.  Rather than incrementally removing each page from
905 	 * the object, the page and object are reset to any empty state.
906 	 */
907 	TAILQ_FOREACH_SAFE(p, &object->memq, listq, p_next) {
908 		vm_page_assert_unbusied(p);
909 		KASSERT(p->object == object &&
910 		    (p->ref_count & VPRC_OBJREF) != 0,
911 		    ("vm_object_terminate_pages: page %p is inconsistent", p));
912 
913 		p->object = NULL;
914 		if (vm_page_drop(p, VPRC_OBJREF) == VPRC_OBJREF) {
915 			VM_CNT_INC(v_pfree);
916 			vm_page_free(p);
917 		}
918 	}
919 
920 	/*
921 	 * If the object contained any pages, then reset it to an empty state.
922 	 * None of the object's fields, including "resident_page_count", were
923 	 * modified by the preceding loop.
924 	 */
925 	if (object->resident_page_count != 0) {
926 		vm_radix_reclaim_allnodes(&object->rtree);
927 		TAILQ_INIT(&object->memq);
928 		object->resident_page_count = 0;
929 		if (object->type == OBJT_VNODE)
930 			vdrop(object->handle);
931 	}
932 }
933 
934 /*
935  *	vm_object_terminate actually destroys the specified object, freeing
936  *	up all previously used resources.
937  *
938  *	The object must be locked.
939  *	This routine may block.
940  */
941 void
942 vm_object_terminate(vm_object_t object)
943 {
944 
945 	VM_OBJECT_ASSERT_WLOCKED(object);
946 	KASSERT((object->flags & OBJ_DEAD) != 0,
947 	    ("terminating non-dead obj %p", object));
948 	KASSERT((object->flags & OBJ_COLLAPSING) == 0,
949 	    ("terminating collapsing obj %p", object));
950 	KASSERT(object->backing_object == NULL,
951 	    ("terminating shadow obj %p", object));
952 
953 	/*
954 	 * wait for the pageout daemon to be done with the object
955 	 */
956 	vm_object_pip_wait(object, "objtrm");
957 
958 	KASSERT(!REFCOUNT_COUNT(object->paging_in_progress),
959 	    ("vm_object_terminate: pageout in progress"));
960 
961 	KASSERT(object->ref_count == 0,
962 	    ("vm_object_terminate: object with references, ref_count=%d",
963 	    object->ref_count));
964 
965 	if ((object->flags & OBJ_PG_DTOR) == 0)
966 		vm_object_terminate_pages(object);
967 
968 #if VM_NRESERVLEVEL > 0
969 	if (__predict_false(!LIST_EMPTY(&object->rvq)))
970 		vm_reserv_break_all(object);
971 #endif
972 
973 	KASSERT(object->cred == NULL || object->type == OBJT_DEFAULT ||
974 	    object->type == OBJT_SWAP,
975 	    ("%s: non-swap obj %p has cred", __func__, object));
976 
977 	/*
978 	 * Let the pager know object is dead.
979 	 */
980 	vm_pager_deallocate(object);
981 	VM_OBJECT_WUNLOCK(object);
982 
983 	vm_object_destroy(object);
984 }
985 
986 /*
987  * Make the page read-only so that we can clear the object flags.  However, if
988  * this is a nosync mmap then the object is likely to stay dirty so do not
989  * mess with the page and do not clear the object flags.  Returns TRUE if the
990  * page should be flushed, and FALSE otherwise.
991  */
992 static boolean_t
993 vm_object_page_remove_write(vm_page_t p, int flags, boolean_t *allclean)
994 {
995 
996 	vm_page_assert_busied(p);
997 
998 	/*
999 	 * If we have been asked to skip nosync pages and this is a
1000 	 * nosync page, skip it.  Note that the object flags were not
1001 	 * cleared in this case so we do not have to set them.
1002 	 */
1003 	if ((flags & OBJPC_NOSYNC) != 0 && (p->a.flags & PGA_NOSYNC) != 0) {
1004 		*allclean = FALSE;
1005 		return (FALSE);
1006 	} else {
1007 		pmap_remove_write(p);
1008 		return (p->dirty != 0);
1009 	}
1010 }
1011 
1012 /*
1013  *	vm_object_page_clean
1014  *
1015  *	Clean all dirty pages in the specified range of object.  Leaves page
1016  * 	on whatever queue it is currently on.   If NOSYNC is set then do not
1017  *	write out pages with PGA_NOSYNC set (originally comes from MAP_NOSYNC),
1018  *	leaving the object dirty.
1019  *
1020  *	When stuffing pages asynchronously, allow clustering.  XXX we need a
1021  *	synchronous clustering mode implementation.
1022  *
1023  *	Odd semantics: if start == end, we clean everything.
1024  *
1025  *	The object must be locked.
1026  *
1027  *	Returns FALSE if some page from the range was not written, as
1028  *	reported by the pager, and TRUE otherwise.
1029  */
1030 boolean_t
1031 vm_object_page_clean(vm_object_t object, vm_ooffset_t start, vm_ooffset_t end,
1032     int flags)
1033 {
1034 	vm_page_t np, p;
1035 	vm_pindex_t pi, tend, tstart;
1036 	int curgeneration, n, pagerflags;
1037 	boolean_t eio, res, allclean;
1038 
1039 	VM_OBJECT_ASSERT_WLOCKED(object);
1040 
1041 	if (object->type != OBJT_VNODE || !vm_object_mightbedirty(object) ||
1042 	    object->resident_page_count == 0)
1043 		return (TRUE);
1044 
1045 	pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) != 0 ?
1046 	    VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK;
1047 	pagerflags |= (flags & OBJPC_INVAL) != 0 ? VM_PAGER_PUT_INVAL : 0;
1048 
1049 	tstart = OFF_TO_IDX(start);
1050 	tend = (end == 0) ? object->size : OFF_TO_IDX(end + PAGE_MASK);
1051 	allclean = tstart == 0 && tend >= object->size;
1052 	res = TRUE;
1053 
1054 rescan:
1055 	curgeneration = object->generation;
1056 
1057 	for (p = vm_page_find_least(object, tstart); p != NULL; p = np) {
1058 		pi = p->pindex;
1059 		if (pi >= tend)
1060 			break;
1061 		np = TAILQ_NEXT(p, listq);
1062 		if (vm_page_none_valid(p))
1063 			continue;
1064 		if (vm_page_busy_acquire(p, VM_ALLOC_WAITFAIL) == 0) {
1065 			if (object->generation != curgeneration &&
1066 			    (flags & OBJPC_SYNC) != 0)
1067 				goto rescan;
1068 			np = vm_page_find_least(object, pi);
1069 			continue;
1070 		}
1071 		if (!vm_object_page_remove_write(p, flags, &allclean)) {
1072 			vm_page_xunbusy(p);
1073 			continue;
1074 		}
1075 
1076 		n = vm_object_page_collect_flush(object, p, pagerflags,
1077 		    flags, &allclean, &eio);
1078 		if (eio) {
1079 			res = FALSE;
1080 			allclean = FALSE;
1081 		}
1082 		if (object->generation != curgeneration &&
1083 		    (flags & OBJPC_SYNC) != 0)
1084 			goto rescan;
1085 
1086 		/*
1087 		 * If the VOP_PUTPAGES() did a truncated write, so
1088 		 * that even the first page of the run is not fully
1089 		 * written, vm_pageout_flush() returns 0 as the run
1090 		 * length.  Since the condition that caused truncated
1091 		 * write may be permanent, e.g. exhausted free space,
1092 		 * accepting n == 0 would cause an infinite loop.
1093 		 *
1094 		 * Forwarding the iterator leaves the unwritten page
1095 		 * behind, but there is not much we can do there if
1096 		 * filesystem refuses to write it.
1097 		 */
1098 		if (n == 0) {
1099 			n = 1;
1100 			allclean = FALSE;
1101 		}
1102 		np = vm_page_find_least(object, pi + n);
1103 	}
1104 #if 0
1105 	VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC) ? MNT_WAIT : 0);
1106 #endif
1107 
1108 	if (allclean)
1109 		object->cleangeneration = curgeneration;
1110 	return (res);
1111 }
1112 
1113 static int
1114 vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags,
1115     int flags, boolean_t *allclean, boolean_t *eio)
1116 {
1117 	vm_page_t ma[vm_pageout_page_count], p_first, tp;
1118 	int count, i, mreq, runlen;
1119 
1120 	vm_page_lock_assert(p, MA_NOTOWNED);
1121 	vm_page_assert_xbusied(p);
1122 	VM_OBJECT_ASSERT_WLOCKED(object);
1123 
1124 	count = 1;
1125 	mreq = 0;
1126 
1127 	for (tp = p; count < vm_pageout_page_count; count++) {
1128 		tp = vm_page_next(tp);
1129 		if (tp == NULL || vm_page_tryxbusy(tp) == 0)
1130 			break;
1131 		if (!vm_object_page_remove_write(tp, flags, allclean)) {
1132 			vm_page_xunbusy(tp);
1133 			break;
1134 		}
1135 	}
1136 
1137 	for (p_first = p; count < vm_pageout_page_count; count++) {
1138 		tp = vm_page_prev(p_first);
1139 		if (tp == NULL || vm_page_tryxbusy(tp) == 0)
1140 			break;
1141 		if (!vm_object_page_remove_write(tp, flags, allclean)) {
1142 			vm_page_xunbusy(tp);
1143 			break;
1144 		}
1145 		p_first = tp;
1146 		mreq++;
1147 	}
1148 
1149 	for (tp = p_first, i = 0; i < count; tp = TAILQ_NEXT(tp, listq), i++)
1150 		ma[i] = tp;
1151 
1152 	vm_pageout_flush(ma, count, pagerflags, mreq, &runlen, eio);
1153 	return (runlen);
1154 }
1155 
1156 /*
1157  * Note that there is absolutely no sense in writing out
1158  * anonymous objects, so we track down the vnode object
1159  * to write out.
1160  * We invalidate (remove) all pages from the address space
1161  * for semantic correctness.
1162  *
1163  * If the backing object is a device object with unmanaged pages, then any
1164  * mappings to the specified range of pages must be removed before this
1165  * function is called.
1166  *
1167  * Note: certain anonymous maps, such as MAP_NOSYNC maps,
1168  * may start out with a NULL object.
1169  */
1170 boolean_t
1171 vm_object_sync(vm_object_t object, vm_ooffset_t offset, vm_size_t size,
1172     boolean_t syncio, boolean_t invalidate)
1173 {
1174 	vm_object_t backing_object;
1175 	struct vnode *vp;
1176 	struct mount *mp;
1177 	int error, flags, fsync_after;
1178 	boolean_t res;
1179 
1180 	if (object == NULL)
1181 		return (TRUE);
1182 	res = TRUE;
1183 	error = 0;
1184 	VM_OBJECT_WLOCK(object);
1185 	while ((backing_object = object->backing_object) != NULL) {
1186 		VM_OBJECT_WLOCK(backing_object);
1187 		offset += object->backing_object_offset;
1188 		VM_OBJECT_WUNLOCK(object);
1189 		object = backing_object;
1190 		if (object->size < OFF_TO_IDX(offset + size))
1191 			size = IDX_TO_OFF(object->size) - offset;
1192 	}
1193 	/*
1194 	 * Flush pages if writing is allowed, invalidate them
1195 	 * if invalidation requested.  Pages undergoing I/O
1196 	 * will be ignored by vm_object_page_remove().
1197 	 *
1198 	 * We cannot lock the vnode and then wait for paging
1199 	 * to complete without deadlocking against vm_fault.
1200 	 * Instead we simply call vm_object_page_remove() and
1201 	 * allow it to block internally on a page-by-page
1202 	 * basis when it encounters pages undergoing async
1203 	 * I/O.
1204 	 */
1205 	if (object->type == OBJT_VNODE &&
1206 	    vm_object_mightbedirty(object) != 0 &&
1207 	    ((vp = object->handle)->v_vflag & VV_NOSYNC) == 0) {
1208 		VM_OBJECT_WUNLOCK(object);
1209 		(void) vn_start_write(vp, &mp, V_WAIT);
1210 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1211 		if (syncio && !invalidate && offset == 0 &&
1212 		    atop(size) == object->size) {
1213 			/*
1214 			 * If syncing the whole mapping of the file,
1215 			 * it is faster to schedule all the writes in
1216 			 * async mode, also allowing the clustering,
1217 			 * and then wait for i/o to complete.
1218 			 */
1219 			flags = 0;
1220 			fsync_after = TRUE;
1221 		} else {
1222 			flags = (syncio || invalidate) ? OBJPC_SYNC : 0;
1223 			flags |= invalidate ? (OBJPC_SYNC | OBJPC_INVAL) : 0;
1224 			fsync_after = FALSE;
1225 		}
1226 		VM_OBJECT_WLOCK(object);
1227 		res = vm_object_page_clean(object, offset, offset + size,
1228 		    flags);
1229 		VM_OBJECT_WUNLOCK(object);
1230 		if (fsync_after)
1231 			error = VOP_FSYNC(vp, MNT_WAIT, curthread);
1232 		VOP_UNLOCK(vp);
1233 		vn_finished_write(mp);
1234 		if (error != 0)
1235 			res = FALSE;
1236 		VM_OBJECT_WLOCK(object);
1237 	}
1238 	if ((object->type == OBJT_VNODE ||
1239 	     object->type == OBJT_DEVICE) && invalidate) {
1240 		if (object->type == OBJT_DEVICE)
1241 			/*
1242 			 * The option OBJPR_NOTMAPPED must be passed here
1243 			 * because vm_object_page_remove() cannot remove
1244 			 * unmanaged mappings.
1245 			 */
1246 			flags = OBJPR_NOTMAPPED;
1247 		else if (old_msync)
1248 			flags = 0;
1249 		else
1250 			flags = OBJPR_CLEANONLY;
1251 		vm_object_page_remove(object, OFF_TO_IDX(offset),
1252 		    OFF_TO_IDX(offset + size + PAGE_MASK), flags);
1253 	}
1254 	VM_OBJECT_WUNLOCK(object);
1255 	return (res);
1256 }
1257 
1258 /*
1259  * Determine whether the given advice can be applied to the object.  Advice is
1260  * not applied to unmanaged pages since they never belong to page queues, and
1261  * since MADV_FREE is destructive, it can apply only to anonymous pages that
1262  * have been mapped at most once.
1263  */
1264 static bool
1265 vm_object_advice_applies(vm_object_t object, int advice)
1266 {
1267 
1268 	if ((object->flags & OBJ_UNMANAGED) != 0)
1269 		return (false);
1270 	if (advice != MADV_FREE)
1271 		return (true);
1272 	return ((object->flags & (OBJ_ONEMAPPING | OBJ_ANON)) ==
1273 	    (OBJ_ONEMAPPING | OBJ_ANON));
1274 }
1275 
1276 static void
1277 vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex,
1278     vm_size_t size)
1279 {
1280 
1281 	if (advice == MADV_FREE && object->type == OBJT_SWAP)
1282 		swap_pager_freespace(object, pindex, size);
1283 }
1284 
1285 /*
1286  *	vm_object_madvise:
1287  *
1288  *	Implements the madvise function at the object/page level.
1289  *
1290  *	MADV_WILLNEED	(any object)
1291  *
1292  *	    Activate the specified pages if they are resident.
1293  *
1294  *	MADV_DONTNEED	(any object)
1295  *
1296  *	    Deactivate the specified pages if they are resident.
1297  *
1298  *	MADV_FREE	(OBJT_DEFAULT/OBJT_SWAP objects,
1299  *			 OBJ_ONEMAPPING only)
1300  *
1301  *	    Deactivate and clean the specified pages if they are
1302  *	    resident.  This permits the process to reuse the pages
1303  *	    without faulting or the kernel to reclaim the pages
1304  *	    without I/O.
1305  */
1306 void
1307 vm_object_madvise(vm_object_t object, vm_pindex_t pindex, vm_pindex_t end,
1308     int advice)
1309 {
1310 	vm_pindex_t tpindex;
1311 	vm_object_t backing_object, tobject;
1312 	vm_page_t m, tm;
1313 
1314 	if (object == NULL)
1315 		return;
1316 
1317 relookup:
1318 	VM_OBJECT_WLOCK(object);
1319 	if (!vm_object_advice_applies(object, advice)) {
1320 		VM_OBJECT_WUNLOCK(object);
1321 		return;
1322 	}
1323 	for (m = vm_page_find_least(object, pindex); pindex < end; pindex++) {
1324 		tobject = object;
1325 
1326 		/*
1327 		 * If the next page isn't resident in the top-level object, we
1328 		 * need to search the shadow chain.  When applying MADV_FREE, we
1329 		 * take care to release any swap space used to store
1330 		 * non-resident pages.
1331 		 */
1332 		if (m == NULL || pindex < m->pindex) {
1333 			/*
1334 			 * Optimize a common case: if the top-level object has
1335 			 * no backing object, we can skip over the non-resident
1336 			 * range in constant time.
1337 			 */
1338 			if (object->backing_object == NULL) {
1339 				tpindex = (m != NULL && m->pindex < end) ?
1340 				    m->pindex : end;
1341 				vm_object_madvise_freespace(object, advice,
1342 				    pindex, tpindex - pindex);
1343 				if ((pindex = tpindex) == end)
1344 					break;
1345 				goto next_page;
1346 			}
1347 
1348 			tpindex = pindex;
1349 			do {
1350 				vm_object_madvise_freespace(tobject, advice,
1351 				    tpindex, 1);
1352 				/*
1353 				 * Prepare to search the next object in the
1354 				 * chain.
1355 				 */
1356 				backing_object = tobject->backing_object;
1357 				if (backing_object == NULL)
1358 					goto next_pindex;
1359 				VM_OBJECT_WLOCK(backing_object);
1360 				tpindex +=
1361 				    OFF_TO_IDX(tobject->backing_object_offset);
1362 				if (tobject != object)
1363 					VM_OBJECT_WUNLOCK(tobject);
1364 				tobject = backing_object;
1365 				if (!vm_object_advice_applies(tobject, advice))
1366 					goto next_pindex;
1367 			} while ((tm = vm_page_lookup(tobject, tpindex)) ==
1368 			    NULL);
1369 		} else {
1370 next_page:
1371 			tm = m;
1372 			m = TAILQ_NEXT(m, listq);
1373 		}
1374 
1375 		/*
1376 		 * If the page is not in a normal state, skip it.  The page
1377 		 * can not be invalidated while the object lock is held.
1378 		 */
1379 		if (!vm_page_all_valid(tm) || vm_page_wired(tm))
1380 			goto next_pindex;
1381 		KASSERT((tm->flags & PG_FICTITIOUS) == 0,
1382 		    ("vm_object_madvise: page %p is fictitious", tm));
1383 		KASSERT((tm->oflags & VPO_UNMANAGED) == 0,
1384 		    ("vm_object_madvise: page %p is not managed", tm));
1385 		if (vm_page_tryxbusy(tm) == 0) {
1386 			if (object != tobject)
1387 				VM_OBJECT_WUNLOCK(object);
1388 			if (advice == MADV_WILLNEED) {
1389 				/*
1390 				 * Reference the page before unlocking and
1391 				 * sleeping so that the page daemon is less
1392 				 * likely to reclaim it.
1393 				 */
1394 				vm_page_aflag_set(tm, PGA_REFERENCED);
1395 			}
1396 			vm_page_busy_sleep(tm, "madvpo", false);
1397   			goto relookup;
1398 		}
1399 		vm_page_advise(tm, advice);
1400 		vm_page_xunbusy(tm);
1401 		vm_object_madvise_freespace(tobject, advice, tm->pindex, 1);
1402 next_pindex:
1403 		if (tobject != object)
1404 			VM_OBJECT_WUNLOCK(tobject);
1405 	}
1406 	VM_OBJECT_WUNLOCK(object);
1407 }
1408 
1409 /*
1410  *	vm_object_shadow:
1411  *
1412  *	Create a new object which is backed by the
1413  *	specified existing object range.  The source
1414  *	object reference is deallocated.
1415  *
1416  *	The new object and offset into that object
1417  *	are returned in the source parameters.
1418  */
1419 void
1420 vm_object_shadow(vm_object_t *object, vm_ooffset_t *offset, vm_size_t length,
1421     struct ucred *cred, bool shared)
1422 {
1423 	vm_object_t source;
1424 	vm_object_t result;
1425 
1426 	source = *object;
1427 
1428 	/*
1429 	 * Don't create the new object if the old object isn't shared.
1430 	 *
1431 	 * If we hold the only reference we can guarantee that it won't
1432 	 * increase while we have the map locked.  Otherwise the race is
1433 	 * harmless and we will end up with an extra shadow object that
1434 	 * will be collapsed later.
1435 	 */
1436 	if (source != NULL && source->ref_count == 1 &&
1437 	    (source->flags & OBJ_ANON) != 0)
1438 		return;
1439 
1440 	/*
1441 	 * Allocate a new object with the given length.
1442 	 */
1443 	result = vm_object_allocate_anon(atop(length), source, cred, length);
1444 
1445 	/*
1446 	 * Store the offset into the source object, and fix up the offset into
1447 	 * the new object.
1448 	 */
1449 	result->backing_object_offset = *offset;
1450 
1451 	if (shared || source != NULL) {
1452 		VM_OBJECT_WLOCK(result);
1453 
1454 		/*
1455 		 * The new object shadows the source object, adding a
1456 		 * reference to it.  Our caller changes his reference
1457 		 * to point to the new object, removing a reference to
1458 		 * the source object.  Net result: no change of
1459 		 * reference count, unless the caller needs to add one
1460 		 * more reference due to forking a shared map entry.
1461 		 */
1462 		if (shared) {
1463 			vm_object_reference_locked(result);
1464 			vm_object_clear_flag(result, OBJ_ONEMAPPING);
1465 		}
1466 
1467 		/*
1468 		 * Try to optimize the result object's page color when
1469 		 * shadowing in order to maintain page coloring
1470 		 * consistency in the combined shadowed object.
1471 		 */
1472 		if (source != NULL) {
1473 			vm_object_backing_insert(result, source);
1474 			result->domain = source->domain;
1475 #if VM_NRESERVLEVEL > 0
1476 			result->flags |= source->flags & OBJ_COLORED;
1477 			result->pg_color = (source->pg_color +
1478 			    OFF_TO_IDX(*offset)) & ((1 << (VM_NFREEORDER -
1479 			    1)) - 1);
1480 #endif
1481 		}
1482 		VM_OBJECT_WUNLOCK(result);
1483 	}
1484 
1485 	/*
1486 	 * Return the new things
1487 	 */
1488 	*offset = 0;
1489 	*object = result;
1490 }
1491 
1492 /*
1493  *	vm_object_split:
1494  *
1495  * Split the pages in a map entry into a new object.  This affords
1496  * easier removal of unused pages, and keeps object inheritance from
1497  * being a negative impact on memory usage.
1498  */
1499 void
1500 vm_object_split(vm_map_entry_t entry)
1501 {
1502 	vm_page_t m, m_next;
1503 	vm_object_t orig_object, new_object, backing_object;
1504 	vm_pindex_t idx, offidxstart;
1505 	vm_size_t size;
1506 
1507 	orig_object = entry->object.vm_object;
1508 	KASSERT((orig_object->flags & OBJ_ONEMAPPING) != 0,
1509 	    ("vm_object_split:  Splitting object with multiple mappings."));
1510 	if ((orig_object->flags & OBJ_ANON) == 0)
1511 		return;
1512 	if (orig_object->ref_count <= 1)
1513 		return;
1514 	VM_OBJECT_WUNLOCK(orig_object);
1515 
1516 	offidxstart = OFF_TO_IDX(entry->offset);
1517 	size = atop(entry->end - entry->start);
1518 
1519 	/*
1520 	 * If swap_pager_copy() is later called, it will convert new_object
1521 	 * into a swap object.
1522 	 */
1523 	new_object = vm_object_allocate_anon(size, orig_object,
1524 	    orig_object->cred, ptoa(size));
1525 
1526 	/*
1527 	 * We must wait for the orig_object to complete any in-progress
1528 	 * collapse so that the swap blocks are stable below.  The
1529 	 * additional reference on backing_object by new object will
1530 	 * prevent further collapse operations until split completes.
1531 	 */
1532 	VM_OBJECT_WLOCK(orig_object);
1533 	vm_object_collapse_wait(orig_object);
1534 
1535 	/*
1536 	 * At this point, the new object is still private, so the order in
1537 	 * which the original and new objects are locked does not matter.
1538 	 */
1539 	VM_OBJECT_WLOCK(new_object);
1540 	new_object->domain = orig_object->domain;
1541 	backing_object = orig_object->backing_object;
1542 	if (backing_object != NULL) {
1543 		vm_object_backing_insert_ref(new_object, backing_object);
1544 		new_object->backing_object_offset =
1545 		    orig_object->backing_object_offset + entry->offset;
1546 	}
1547 	if (orig_object->cred != NULL) {
1548 		crhold(orig_object->cred);
1549 		KASSERT(orig_object->charge >= ptoa(size),
1550 		    ("orig_object->charge < 0"));
1551 		orig_object->charge -= ptoa(size);
1552 	}
1553 
1554 	/*
1555 	 * Mark the split operation so that swap_pager_getpages() knows
1556 	 * that the object is in transition.
1557 	 */
1558 	vm_object_set_flag(orig_object, OBJ_SPLIT);
1559 retry:
1560 	m = vm_page_find_least(orig_object, offidxstart);
1561 	for (; m != NULL && (idx = m->pindex - offidxstart) < size;
1562 	    m = m_next) {
1563 		m_next = TAILQ_NEXT(m, listq);
1564 
1565 		/*
1566 		 * We must wait for pending I/O to complete before we can
1567 		 * rename the page.
1568 		 *
1569 		 * We do not have to VM_PROT_NONE the page as mappings should
1570 		 * not be changed by this operation.
1571 		 */
1572 		if (vm_page_tryxbusy(m) == 0) {
1573 			VM_OBJECT_WUNLOCK(new_object);
1574 			vm_page_sleep_if_busy(m, "spltwt");
1575 			VM_OBJECT_WLOCK(new_object);
1576 			goto retry;
1577 		}
1578 
1579 		/*
1580 		 * The page was left invalid.  Likely placed there by
1581 		 * an incomplete fault.  Just remove and ignore.
1582 		 */
1583 		if (vm_page_none_valid(m)) {
1584 			if (vm_page_remove(m))
1585 				vm_page_free(m);
1586 			continue;
1587 		}
1588 
1589 		/* vm_page_rename() will dirty the page. */
1590 		if (vm_page_rename(m, new_object, idx)) {
1591 			vm_page_xunbusy(m);
1592 			VM_OBJECT_WUNLOCK(new_object);
1593 			VM_OBJECT_WUNLOCK(orig_object);
1594 			vm_radix_wait();
1595 			VM_OBJECT_WLOCK(orig_object);
1596 			VM_OBJECT_WLOCK(new_object);
1597 			goto retry;
1598 		}
1599 
1600 #if VM_NRESERVLEVEL > 0
1601 		/*
1602 		 * If some of the reservation's allocated pages remain with
1603 		 * the original object, then transferring the reservation to
1604 		 * the new object is neither particularly beneficial nor
1605 		 * particularly harmful as compared to leaving the reservation
1606 		 * with the original object.  If, however, all of the
1607 		 * reservation's allocated pages are transferred to the new
1608 		 * object, then transferring the reservation is typically
1609 		 * beneficial.  Determining which of these two cases applies
1610 		 * would be more costly than unconditionally renaming the
1611 		 * reservation.
1612 		 */
1613 		vm_reserv_rename(m, new_object, orig_object, offidxstart);
1614 #endif
1615 		if (orig_object->type != OBJT_SWAP)
1616 			vm_page_xunbusy(m);
1617 	}
1618 	if (orig_object->type == OBJT_SWAP) {
1619 		/*
1620 		 * swap_pager_copy() can sleep, in which case the orig_object's
1621 		 * and new_object's locks are released and reacquired.
1622 		 */
1623 		swap_pager_copy(orig_object, new_object, offidxstart, 0);
1624 		TAILQ_FOREACH(m, &new_object->memq, listq)
1625 			vm_page_xunbusy(m);
1626 	}
1627 	vm_object_clear_flag(orig_object, OBJ_SPLIT);
1628 	VM_OBJECT_WUNLOCK(orig_object);
1629 	VM_OBJECT_WUNLOCK(new_object);
1630 	entry->object.vm_object = new_object;
1631 	entry->offset = 0LL;
1632 	vm_object_deallocate(orig_object);
1633 	VM_OBJECT_WLOCK(new_object);
1634 }
1635 
1636 static vm_page_t
1637 vm_object_collapse_scan_wait(vm_object_t object, vm_page_t p)
1638 {
1639 	vm_object_t backing_object;
1640 
1641 	VM_OBJECT_ASSERT_WLOCKED(object);
1642 	backing_object = object->backing_object;
1643 	VM_OBJECT_ASSERT_WLOCKED(backing_object);
1644 
1645 	KASSERT(p == NULL || p->object == object || p->object == backing_object,
1646 	    ("invalid ownership %p %p %p", p, object, backing_object));
1647 	/* The page is only NULL when rename fails. */
1648 	if (p == NULL) {
1649 		VM_OBJECT_WUNLOCK(object);
1650 		VM_OBJECT_WUNLOCK(backing_object);
1651 		vm_radix_wait();
1652 	} else {
1653 		if (p->object == object)
1654 			VM_OBJECT_WUNLOCK(backing_object);
1655 		else
1656 			VM_OBJECT_WUNLOCK(object);
1657 		vm_page_busy_sleep(p, "vmocol", false);
1658 	}
1659 	VM_OBJECT_WLOCK(object);
1660 	VM_OBJECT_WLOCK(backing_object);
1661 	return (TAILQ_FIRST(&backing_object->memq));
1662 }
1663 
1664 static bool
1665 vm_object_scan_all_shadowed(vm_object_t object)
1666 {
1667 	vm_object_t backing_object;
1668 	vm_page_t p, pp;
1669 	vm_pindex_t backing_offset_index, new_pindex, pi, ps;
1670 
1671 	VM_OBJECT_ASSERT_WLOCKED(object);
1672 	VM_OBJECT_ASSERT_WLOCKED(object->backing_object);
1673 
1674 	backing_object = object->backing_object;
1675 
1676 	if ((backing_object->flags & OBJ_ANON) == 0)
1677 		return (false);
1678 
1679 	pi = backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
1680 	p = vm_page_find_least(backing_object, pi);
1681 	ps = swap_pager_find_least(backing_object, pi);
1682 
1683 	/*
1684 	 * Only check pages inside the parent object's range and
1685 	 * inside the parent object's mapping of the backing object.
1686 	 */
1687 	for (;; pi++) {
1688 		if (p != NULL && p->pindex < pi)
1689 			p = TAILQ_NEXT(p, listq);
1690 		if (ps < pi)
1691 			ps = swap_pager_find_least(backing_object, pi);
1692 		if (p == NULL && ps >= backing_object->size)
1693 			break;
1694 		else if (p == NULL)
1695 			pi = ps;
1696 		else
1697 			pi = MIN(p->pindex, ps);
1698 
1699 		new_pindex = pi - backing_offset_index;
1700 		if (new_pindex >= object->size)
1701 			break;
1702 
1703 		/*
1704 		 * If the backing object page is busy a grandparent or older
1705 		 * page may still be undergoing CoW.  It is not safe to
1706 		 * collapse the backing object until it is quiesced.
1707 		 */
1708 		if (p != NULL && vm_page_busied(p))
1709 			return (false);
1710 
1711 		/*
1712 		 * See if the parent has the page or if the parent's object
1713 		 * pager has the page.  If the parent has the page but the page
1714 		 * is not valid, the parent's object pager must have the page.
1715 		 *
1716 		 * If this fails, the parent does not completely shadow the
1717 		 * object and we might as well give up now.
1718 		 */
1719 		pp = vm_page_lookup(object, new_pindex);
1720 		/*
1721 		 * The valid check here is stable due to object lock being
1722 		 * required to clear valid and initiate paging.
1723 		 */
1724 		if ((pp == NULL || vm_page_none_valid(pp)) &&
1725 		    !vm_pager_has_page(object, new_pindex, NULL, NULL))
1726 			return (false);
1727 	}
1728 	return (true);
1729 }
1730 
1731 static void
1732 vm_object_collapse_scan(vm_object_t object)
1733 {
1734 	vm_object_t backing_object;
1735 	vm_page_t next, p, pp;
1736 	vm_pindex_t backing_offset_index, new_pindex;
1737 
1738 	VM_OBJECT_ASSERT_WLOCKED(object);
1739 	VM_OBJECT_ASSERT_WLOCKED(object->backing_object);
1740 
1741 	backing_object = object->backing_object;
1742 	backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
1743 
1744 	/*
1745 	 * Our scan
1746 	 */
1747 	for (p = TAILQ_FIRST(&backing_object->memq); p != NULL; p = next) {
1748 		next = TAILQ_NEXT(p, listq);
1749 		new_pindex = p->pindex - backing_offset_index;
1750 
1751 		/*
1752 		 * Check for busy page
1753 		 */
1754 		if (vm_page_tryxbusy(p) == 0) {
1755 			next = vm_object_collapse_scan_wait(object, p);
1756 			continue;
1757 		}
1758 
1759 		KASSERT(object->backing_object == backing_object,
1760 		    ("vm_object_collapse_scan: backing object mismatch %p != %p",
1761 		    object->backing_object, backing_object));
1762 		KASSERT(p->object == backing_object,
1763 		    ("vm_object_collapse_scan: object mismatch %p != %p",
1764 		    p->object, backing_object));
1765 
1766 		if (p->pindex < backing_offset_index ||
1767 		    new_pindex >= object->size) {
1768 			if (backing_object->type == OBJT_SWAP)
1769 				swap_pager_freespace(backing_object, p->pindex,
1770 				    1);
1771 
1772 			KASSERT(!pmap_page_is_mapped(p),
1773 			    ("freeing mapped page %p", p));
1774 			if (vm_page_remove(p))
1775 				vm_page_free(p);
1776 			continue;
1777 		}
1778 
1779 		pp = vm_page_lookup(object, new_pindex);
1780 		if (pp != NULL && vm_page_tryxbusy(pp) == 0) {
1781 			vm_page_xunbusy(p);
1782 			/*
1783 			 * The page in the parent is busy and possibly not
1784 			 * (yet) valid.  Until its state is finalized by the
1785 			 * busy bit owner, we can't tell whether it shadows the
1786 			 * original page.
1787 			 */
1788 			next = vm_object_collapse_scan_wait(object, pp);
1789 			continue;
1790 		}
1791 
1792 		if (pp != NULL && vm_page_none_valid(pp)) {
1793 			/*
1794 			 * The page was invalid in the parent.  Likely placed
1795 			 * there by an incomplete fault.  Just remove and
1796 			 * ignore.  p can replace it.
1797 			 */
1798 			if (vm_page_remove(pp))
1799 				vm_page_free(pp);
1800 			pp = NULL;
1801 		}
1802 
1803 		if (pp != NULL || vm_pager_has_page(object, new_pindex, NULL,
1804 			NULL)) {
1805 			/*
1806 			 * The page already exists in the parent OR swap exists
1807 			 * for this location in the parent.  Leave the parent's
1808 			 * page alone.  Destroy the original page from the
1809 			 * backing object.
1810 			 */
1811 			if (backing_object->type == OBJT_SWAP)
1812 				swap_pager_freespace(backing_object, p->pindex,
1813 				    1);
1814 			KASSERT(!pmap_page_is_mapped(p),
1815 			    ("freeing mapped page %p", p));
1816 			if (vm_page_remove(p))
1817 				vm_page_free(p);
1818 			if (pp != NULL)
1819 				vm_page_xunbusy(pp);
1820 			continue;
1821 		}
1822 
1823 		/*
1824 		 * Page does not exist in parent, rename the page from the
1825 		 * backing object to the main object.
1826 		 *
1827 		 * If the page was mapped to a process, it can remain mapped
1828 		 * through the rename.  vm_page_rename() will dirty the page.
1829 		 */
1830 		if (vm_page_rename(p, object, new_pindex)) {
1831 			vm_page_xunbusy(p);
1832 			next = vm_object_collapse_scan_wait(object, NULL);
1833 			continue;
1834 		}
1835 
1836 		/* Use the old pindex to free the right page. */
1837 		if (backing_object->type == OBJT_SWAP)
1838 			swap_pager_freespace(backing_object,
1839 			    new_pindex + backing_offset_index, 1);
1840 
1841 #if VM_NRESERVLEVEL > 0
1842 		/*
1843 		 * Rename the reservation.
1844 		 */
1845 		vm_reserv_rename(p, object, backing_object,
1846 		    backing_offset_index);
1847 #endif
1848 		vm_page_xunbusy(p);
1849 	}
1850 	return;
1851 }
1852 
1853 /*
1854  *	vm_object_collapse:
1855  *
1856  *	Collapse an object with the object backing it.
1857  *	Pages in the backing object are moved into the
1858  *	parent, and the backing object is deallocated.
1859  */
1860 void
1861 vm_object_collapse(vm_object_t object)
1862 {
1863 	vm_object_t backing_object, new_backing_object;
1864 
1865 	VM_OBJECT_ASSERT_WLOCKED(object);
1866 
1867 	while (TRUE) {
1868 		KASSERT((object->flags & (OBJ_DEAD | OBJ_ANON)) == OBJ_ANON,
1869 		    ("collapsing invalid object"));
1870 
1871 		/*
1872 		 * Wait for the backing_object to finish any pending
1873 		 * collapse so that the caller sees the shortest possible
1874 		 * shadow chain.
1875 		 */
1876 		backing_object = vm_object_backing_collapse_wait(object);
1877 		if (backing_object == NULL)
1878 			return;
1879 
1880 		KASSERT(object->ref_count > 0 &&
1881 		    object->ref_count > object->shadow_count,
1882 		    ("collapse with invalid ref %d or shadow %d count.",
1883 		    object->ref_count, object->shadow_count));
1884 		KASSERT((backing_object->flags &
1885 		    (OBJ_COLLAPSING | OBJ_DEAD)) == 0,
1886 		    ("vm_object_collapse: Backing object already collapsing."));
1887 		KASSERT((object->flags & (OBJ_COLLAPSING | OBJ_DEAD)) == 0,
1888 		    ("vm_object_collapse: object is already collapsing."));
1889 
1890 		/*
1891 		 * We know that we can either collapse the backing object if
1892 		 * the parent is the only reference to it, or (perhaps) have
1893 		 * the parent bypass the object if the parent happens to shadow
1894 		 * all the resident pages in the entire backing object.
1895 		 */
1896 		if (backing_object->ref_count == 1) {
1897 			KASSERT(backing_object->shadow_count == 1,
1898 			    ("vm_object_collapse: shadow_count: %d",
1899 			    backing_object->shadow_count));
1900 			vm_object_pip_add(object, 1);
1901 			vm_object_set_flag(object, OBJ_COLLAPSING);
1902 			vm_object_pip_add(backing_object, 1);
1903 			vm_object_set_flag(backing_object, OBJ_DEAD);
1904 
1905 			/*
1906 			 * If there is exactly one reference to the backing
1907 			 * object, we can collapse it into the parent.
1908 			 */
1909 			vm_object_collapse_scan(object);
1910 
1911 #if VM_NRESERVLEVEL > 0
1912 			/*
1913 			 * Break any reservations from backing_object.
1914 			 */
1915 			if (__predict_false(!LIST_EMPTY(&backing_object->rvq)))
1916 				vm_reserv_break_all(backing_object);
1917 #endif
1918 
1919 			/*
1920 			 * Move the pager from backing_object to object.
1921 			 */
1922 			if (backing_object->type == OBJT_SWAP) {
1923 				/*
1924 				 * swap_pager_copy() can sleep, in which case
1925 				 * the backing_object's and object's locks are
1926 				 * released and reacquired.
1927 				 * Since swap_pager_copy() is being asked to
1928 				 * destroy backing_object, it will change the
1929 				 * type to OBJT_DEFAULT.
1930 				 */
1931 				swap_pager_copy(
1932 				    backing_object,
1933 				    object,
1934 				    OFF_TO_IDX(object->backing_object_offset), TRUE);
1935 			}
1936 
1937 			/*
1938 			 * Object now shadows whatever backing_object did.
1939 			 */
1940 			vm_object_clear_flag(object, OBJ_COLLAPSING);
1941 			vm_object_backing_transfer(object, backing_object);
1942 			object->backing_object_offset +=
1943 			    backing_object->backing_object_offset;
1944 			VM_OBJECT_WUNLOCK(object);
1945 			vm_object_pip_wakeup(object);
1946 
1947 			/*
1948 			 * Discard backing_object.
1949 			 *
1950 			 * Since the backing object has no pages, no pager left,
1951 			 * and no object references within it, all that is
1952 			 * necessary is to dispose of it.
1953 			 */
1954 			KASSERT(backing_object->ref_count == 1, (
1955 "backing_object %p was somehow re-referenced during collapse!",
1956 			    backing_object));
1957 			vm_object_pip_wakeup(backing_object);
1958 			(void)refcount_release(&backing_object->ref_count);
1959 			vm_object_terminate(backing_object);
1960 			counter_u64_add(object_collapses, 1);
1961 			VM_OBJECT_WLOCK(object);
1962 		} else {
1963 			/*
1964 			 * If we do not entirely shadow the backing object,
1965 			 * there is nothing we can do so we give up.
1966 			 *
1967 			 * The object lock and backing_object lock must not
1968 			 * be dropped during this sequence.
1969 			 */
1970 			if (!vm_object_scan_all_shadowed(object)) {
1971 				VM_OBJECT_WUNLOCK(backing_object);
1972 				break;
1973 			}
1974 
1975 			/*
1976 			 * Make the parent shadow the next object in the
1977 			 * chain.  Deallocating backing_object will not remove
1978 			 * it, since its reference count is at least 2.
1979 			 */
1980 			vm_object_backing_remove_locked(object);
1981 			new_backing_object = backing_object->backing_object;
1982 			if (new_backing_object != NULL) {
1983 				vm_object_backing_insert_ref(object,
1984 				    new_backing_object);
1985 				object->backing_object_offset +=
1986 				    backing_object->backing_object_offset;
1987 			}
1988 
1989 			/*
1990 			 * Drop the reference count on backing_object. Since
1991 			 * its ref_count was at least 2, it will not vanish.
1992 			 */
1993 			(void)refcount_release(&backing_object->ref_count);
1994 			KASSERT(backing_object->ref_count >= 1, (
1995 "backing_object %p was somehow dereferenced during collapse!",
1996 			    backing_object));
1997 			VM_OBJECT_WUNLOCK(backing_object);
1998 			counter_u64_add(object_bypasses, 1);
1999 		}
2000 
2001 		/*
2002 		 * Try again with this object's new backing object.
2003 		 */
2004 	}
2005 }
2006 
2007 /*
2008  *	vm_object_page_remove:
2009  *
2010  *	For the given object, either frees or invalidates each of the
2011  *	specified pages.  In general, a page is freed.  However, if a page is
2012  *	wired for any reason other than the existence of a managed, wired
2013  *	mapping, then it may be invalidated but not removed from the object.
2014  *	Pages are specified by the given range ["start", "end") and the option
2015  *	OBJPR_CLEANONLY.  As a special case, if "end" is zero, then the range
2016  *	extends from "start" to the end of the object.  If the option
2017  *	OBJPR_CLEANONLY is specified, then only the non-dirty pages within the
2018  *	specified range are affected.  If the option OBJPR_NOTMAPPED is
2019  *	specified, then the pages within the specified range must have no
2020  *	mappings.  Otherwise, if this option is not specified, any mappings to
2021  *	the specified pages are removed before the pages are freed or
2022  *	invalidated.
2023  *
2024  *	In general, this operation should only be performed on objects that
2025  *	contain managed pages.  There are, however, two exceptions.  First, it
2026  *	is performed on the kernel and kmem objects by vm_map_entry_delete().
2027  *	Second, it is used by msync(..., MS_INVALIDATE) to invalidate device-
2028  *	backed pages.  In both of these cases, the option OBJPR_CLEANONLY must
2029  *	not be specified and the option OBJPR_NOTMAPPED must be specified.
2030  *
2031  *	The object must be locked.
2032  */
2033 void
2034 vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
2035     int options)
2036 {
2037 	vm_page_t p, next;
2038 
2039 	VM_OBJECT_ASSERT_WLOCKED(object);
2040 	KASSERT((object->flags & OBJ_UNMANAGED) == 0 ||
2041 	    (options & (OBJPR_CLEANONLY | OBJPR_NOTMAPPED)) == OBJPR_NOTMAPPED,
2042 	    ("vm_object_page_remove: illegal options for object %p", object));
2043 	if (object->resident_page_count == 0)
2044 		return;
2045 	vm_object_pip_add(object, 1);
2046 again:
2047 	p = vm_page_find_least(object, start);
2048 
2049 	/*
2050 	 * Here, the variable "p" is either (1) the page with the least pindex
2051 	 * greater than or equal to the parameter "start" or (2) NULL.
2052 	 */
2053 	for (; p != NULL && (p->pindex < end || end == 0); p = next) {
2054 		next = TAILQ_NEXT(p, listq);
2055 
2056 		/*
2057 		 * If the page is wired for any reason besides the existence
2058 		 * of managed, wired mappings, then it cannot be freed.  For
2059 		 * example, fictitious pages, which represent device memory,
2060 		 * are inherently wired and cannot be freed.  They can,
2061 		 * however, be invalidated if the option OBJPR_CLEANONLY is
2062 		 * not specified.
2063 		 */
2064 		if (vm_page_tryxbusy(p) == 0) {
2065 			vm_page_sleep_if_busy(p, "vmopar");
2066 			goto again;
2067 		}
2068 		if (vm_page_wired(p)) {
2069 wired:
2070 			if ((options & OBJPR_NOTMAPPED) == 0 &&
2071 			    object->ref_count != 0)
2072 				pmap_remove_all(p);
2073 			if ((options & OBJPR_CLEANONLY) == 0) {
2074 				vm_page_invalid(p);
2075 				vm_page_undirty(p);
2076 			}
2077 			vm_page_xunbusy(p);
2078 			continue;
2079 		}
2080 		KASSERT((p->flags & PG_FICTITIOUS) == 0,
2081 		    ("vm_object_page_remove: page %p is fictitious", p));
2082 		if ((options & OBJPR_CLEANONLY) != 0 &&
2083 		    !vm_page_none_valid(p)) {
2084 			if ((options & OBJPR_NOTMAPPED) == 0 &&
2085 			    object->ref_count != 0 &&
2086 			    !vm_page_try_remove_write(p))
2087 				goto wired;
2088 			if (p->dirty != 0) {
2089 				vm_page_xunbusy(p);
2090 				continue;
2091 			}
2092 		}
2093 		if ((options & OBJPR_NOTMAPPED) == 0 &&
2094 		    object->ref_count != 0 && !vm_page_try_remove_all(p))
2095 			goto wired;
2096 		vm_page_free(p);
2097 	}
2098 	vm_object_pip_wakeup(object);
2099 }
2100 
2101 /*
2102  *	vm_object_page_noreuse:
2103  *
2104  *	For the given object, attempt to move the specified pages to
2105  *	the head of the inactive queue.  This bypasses regular LRU
2106  *	operation and allows the pages to be reused quickly under memory
2107  *	pressure.  If a page is wired for any reason, then it will not
2108  *	be queued.  Pages are specified by the range ["start", "end").
2109  *	As a special case, if "end" is zero, then the range extends from
2110  *	"start" to the end of the object.
2111  *
2112  *	This operation should only be performed on objects that
2113  *	contain non-fictitious, managed pages.
2114  *
2115  *	The object must be locked.
2116  */
2117 void
2118 vm_object_page_noreuse(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
2119 {
2120 	vm_page_t p, next;
2121 
2122 	VM_OBJECT_ASSERT_LOCKED(object);
2123 	KASSERT((object->flags & (OBJ_FICTITIOUS | OBJ_UNMANAGED)) == 0,
2124 	    ("vm_object_page_noreuse: illegal object %p", object));
2125 	if (object->resident_page_count == 0)
2126 		return;
2127 	p = vm_page_find_least(object, start);
2128 
2129 	/*
2130 	 * Here, the variable "p" is either (1) the page with the least pindex
2131 	 * greater than or equal to the parameter "start" or (2) NULL.
2132 	 */
2133 	for (; p != NULL && (p->pindex < end || end == 0); p = next) {
2134 		next = TAILQ_NEXT(p, listq);
2135 		vm_page_deactivate_noreuse(p);
2136 	}
2137 }
2138 
2139 /*
2140  *	Populate the specified range of the object with valid pages.  Returns
2141  *	TRUE if the range is successfully populated and FALSE otherwise.
2142  *
2143  *	Note: This function should be optimized to pass a larger array of
2144  *	pages to vm_pager_get_pages() before it is applied to a non-
2145  *	OBJT_DEVICE object.
2146  *
2147  *	The object must be locked.
2148  */
2149 boolean_t
2150 vm_object_populate(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
2151 {
2152 	vm_page_t m;
2153 	vm_pindex_t pindex;
2154 	int rv;
2155 
2156 	VM_OBJECT_ASSERT_WLOCKED(object);
2157 	for (pindex = start; pindex < end; pindex++) {
2158 		rv = vm_page_grab_valid(&m, object, pindex, VM_ALLOC_NORMAL);
2159 		if (rv != VM_PAGER_OK)
2160 			break;
2161 
2162 		/*
2163 		 * Keep "m" busy because a subsequent iteration may unlock
2164 		 * the object.
2165 		 */
2166 	}
2167 	if (pindex > start) {
2168 		m = vm_page_lookup(object, start);
2169 		while (m != NULL && m->pindex < pindex) {
2170 			vm_page_xunbusy(m);
2171 			m = TAILQ_NEXT(m, listq);
2172 		}
2173 	}
2174 	return (pindex == end);
2175 }
2176 
2177 /*
2178  *	Routine:	vm_object_coalesce
2179  *	Function:	Coalesces two objects backing up adjoining
2180  *			regions of memory into a single object.
2181  *
2182  *	returns TRUE if objects were combined.
2183  *
2184  *	NOTE:	Only works at the moment if the second object is NULL -
2185  *		if it's not, which object do we lock first?
2186  *
2187  *	Parameters:
2188  *		prev_object	First object to coalesce
2189  *		prev_offset	Offset into prev_object
2190  *		prev_size	Size of reference to prev_object
2191  *		next_size	Size of reference to the second object
2192  *		reserved	Indicator that extension region has
2193  *				swap accounted for
2194  *
2195  *	Conditions:
2196  *	The object must *not* be locked.
2197  */
2198 boolean_t
2199 vm_object_coalesce(vm_object_t prev_object, vm_ooffset_t prev_offset,
2200     vm_size_t prev_size, vm_size_t next_size, boolean_t reserved)
2201 {
2202 	vm_pindex_t next_pindex;
2203 
2204 	if (prev_object == NULL)
2205 		return (TRUE);
2206 	if ((prev_object->flags & OBJ_ANON) == 0)
2207 		return (FALSE);
2208 
2209 	VM_OBJECT_WLOCK(prev_object);
2210 	/*
2211 	 * Try to collapse the object first.
2212 	 */
2213 	vm_object_collapse(prev_object);
2214 
2215 	/*
2216 	 * Can't coalesce if: . more than one reference . paged out . shadows
2217 	 * another object . has a copy elsewhere (any of which mean that the
2218 	 * pages not mapped to prev_entry may be in use anyway)
2219 	 */
2220 	if (prev_object->backing_object != NULL) {
2221 		VM_OBJECT_WUNLOCK(prev_object);
2222 		return (FALSE);
2223 	}
2224 
2225 	prev_size >>= PAGE_SHIFT;
2226 	next_size >>= PAGE_SHIFT;
2227 	next_pindex = OFF_TO_IDX(prev_offset) + prev_size;
2228 
2229 	if (prev_object->ref_count > 1 &&
2230 	    prev_object->size != next_pindex &&
2231 	    (prev_object->flags & OBJ_ONEMAPPING) == 0) {
2232 		VM_OBJECT_WUNLOCK(prev_object);
2233 		return (FALSE);
2234 	}
2235 
2236 	/*
2237 	 * Account for the charge.
2238 	 */
2239 	if (prev_object->cred != NULL) {
2240 
2241 		/*
2242 		 * If prev_object was charged, then this mapping,
2243 		 * although not charged now, may become writable
2244 		 * later. Non-NULL cred in the object would prevent
2245 		 * swap reservation during enabling of the write
2246 		 * access, so reserve swap now. Failed reservation
2247 		 * cause allocation of the separate object for the map
2248 		 * entry, and swap reservation for this entry is
2249 		 * managed in appropriate time.
2250 		 */
2251 		if (!reserved && !swap_reserve_by_cred(ptoa(next_size),
2252 		    prev_object->cred)) {
2253 			VM_OBJECT_WUNLOCK(prev_object);
2254 			return (FALSE);
2255 		}
2256 		prev_object->charge += ptoa(next_size);
2257 	}
2258 
2259 	/*
2260 	 * Remove any pages that may still be in the object from a previous
2261 	 * deallocation.
2262 	 */
2263 	if (next_pindex < prev_object->size) {
2264 		vm_object_page_remove(prev_object, next_pindex, next_pindex +
2265 		    next_size, 0);
2266 		if (prev_object->type == OBJT_SWAP)
2267 			swap_pager_freespace(prev_object,
2268 					     next_pindex, next_size);
2269 #if 0
2270 		if (prev_object->cred != NULL) {
2271 			KASSERT(prev_object->charge >=
2272 			    ptoa(prev_object->size - next_pindex),
2273 			    ("object %p overcharged 1 %jx %jx", prev_object,
2274 				(uintmax_t)next_pindex, (uintmax_t)next_size));
2275 			prev_object->charge -= ptoa(prev_object->size -
2276 			    next_pindex);
2277 		}
2278 #endif
2279 	}
2280 
2281 	/*
2282 	 * Extend the object if necessary.
2283 	 */
2284 	if (next_pindex + next_size > prev_object->size)
2285 		prev_object->size = next_pindex + next_size;
2286 
2287 	VM_OBJECT_WUNLOCK(prev_object);
2288 	return (TRUE);
2289 }
2290 
2291 void
2292 vm_object_set_writeable_dirty(vm_object_t object)
2293 {
2294 
2295 	/* Only set for vnodes & tmpfs */
2296 	if (object->type != OBJT_VNODE &&
2297 	    (object->flags & OBJ_TMPFS_NODE) == 0)
2298 		return;
2299 	atomic_add_int(&object->generation, 1);
2300 }
2301 
2302 /*
2303  *	vm_object_unwire:
2304  *
2305  *	For each page offset within the specified range of the given object,
2306  *	find the highest-level page in the shadow chain and unwire it.  A page
2307  *	must exist at every page offset, and the highest-level page must be
2308  *	wired.
2309  */
2310 void
2311 vm_object_unwire(vm_object_t object, vm_ooffset_t offset, vm_size_t length,
2312     uint8_t queue)
2313 {
2314 	vm_object_t tobject, t1object;
2315 	vm_page_t m, tm;
2316 	vm_pindex_t end_pindex, pindex, tpindex;
2317 	int depth, locked_depth;
2318 
2319 	KASSERT((offset & PAGE_MASK) == 0,
2320 	    ("vm_object_unwire: offset is not page aligned"));
2321 	KASSERT((length & PAGE_MASK) == 0,
2322 	    ("vm_object_unwire: length is not a multiple of PAGE_SIZE"));
2323 	/* The wired count of a fictitious page never changes. */
2324 	if ((object->flags & OBJ_FICTITIOUS) != 0)
2325 		return;
2326 	pindex = OFF_TO_IDX(offset);
2327 	end_pindex = pindex + atop(length);
2328 again:
2329 	locked_depth = 1;
2330 	VM_OBJECT_RLOCK(object);
2331 	m = vm_page_find_least(object, pindex);
2332 	while (pindex < end_pindex) {
2333 		if (m == NULL || pindex < m->pindex) {
2334 			/*
2335 			 * The first object in the shadow chain doesn't
2336 			 * contain a page at the current index.  Therefore,
2337 			 * the page must exist in a backing object.
2338 			 */
2339 			tobject = object;
2340 			tpindex = pindex;
2341 			depth = 0;
2342 			do {
2343 				tpindex +=
2344 				    OFF_TO_IDX(tobject->backing_object_offset);
2345 				tobject = tobject->backing_object;
2346 				KASSERT(tobject != NULL,
2347 				    ("vm_object_unwire: missing page"));
2348 				if ((tobject->flags & OBJ_FICTITIOUS) != 0)
2349 					goto next_page;
2350 				depth++;
2351 				if (depth == locked_depth) {
2352 					locked_depth++;
2353 					VM_OBJECT_RLOCK(tobject);
2354 				}
2355 			} while ((tm = vm_page_lookup(tobject, tpindex)) ==
2356 			    NULL);
2357 		} else {
2358 			tm = m;
2359 			m = TAILQ_NEXT(m, listq);
2360 		}
2361 		if (vm_page_trysbusy(tm) == 0) {
2362 			for (tobject = object; locked_depth >= 1;
2363 			    locked_depth--) {
2364 				t1object = tobject->backing_object;
2365 				if (tm->object != tobject)
2366 					VM_OBJECT_RUNLOCK(tobject);
2367 				tobject = t1object;
2368 			}
2369 			vm_page_busy_sleep(tm, "unwbo", true);
2370 			goto again;
2371 		}
2372 		vm_page_unwire(tm, queue);
2373 		vm_page_sunbusy(tm);
2374 next_page:
2375 		pindex++;
2376 	}
2377 	/* Release the accumulated object locks. */
2378 	for (tobject = object; locked_depth >= 1; locked_depth--) {
2379 		t1object = tobject->backing_object;
2380 		VM_OBJECT_RUNLOCK(tobject);
2381 		tobject = t1object;
2382 	}
2383 }
2384 
2385 /*
2386  * Return the vnode for the given object, or NULL if none exists.
2387  * For tmpfs objects, the function may return NULL if there is
2388  * no vnode allocated at the time of the call.
2389  */
2390 struct vnode *
2391 vm_object_vnode(vm_object_t object)
2392 {
2393 	struct vnode *vp;
2394 
2395 	VM_OBJECT_ASSERT_LOCKED(object);
2396 	if (object->type == OBJT_VNODE) {
2397 		vp = object->handle;
2398 		KASSERT(vp != NULL, ("%s: OBJT_VNODE has no vnode", __func__));
2399 	} else if (object->type == OBJT_SWAP &&
2400 	    (object->flags & OBJ_TMPFS) != 0) {
2401 		vp = object->un_pager.swp.swp_tmpfs;
2402 		KASSERT(vp != NULL, ("%s: OBJT_TMPFS has no vnode", __func__));
2403 	} else {
2404 		vp = NULL;
2405 	}
2406 	return (vp);
2407 }
2408 
2409 
2410 /*
2411  * Busy the vm object.  This prevents new pages belonging to the object from
2412  * becoming busy.  Existing pages persist as busy.  Callers are responsible
2413  * for checking page state before proceeding.
2414  */
2415 void
2416 vm_object_busy(vm_object_t obj)
2417 {
2418 
2419 	VM_OBJECT_ASSERT_LOCKED(obj);
2420 
2421 	refcount_acquire(&obj->busy);
2422 	/* The fence is required to order loads of page busy. */
2423 	atomic_thread_fence_acq_rel();
2424 }
2425 
2426 void
2427 vm_object_unbusy(vm_object_t obj)
2428 {
2429 
2430 
2431 	refcount_release(&obj->busy);
2432 }
2433 
2434 void
2435 vm_object_busy_wait(vm_object_t obj, const char *wmesg)
2436 {
2437 
2438 	VM_OBJECT_ASSERT_UNLOCKED(obj);
2439 
2440 	if (obj->busy)
2441 		refcount_sleep(&obj->busy, wmesg, PVM);
2442 }
2443 
2444 /*
2445  * Return the kvme type of the given object.
2446  * If vpp is not NULL, set it to the object's vm_object_vnode() or NULL.
2447  */
2448 int
2449 vm_object_kvme_type(vm_object_t object, struct vnode **vpp)
2450 {
2451 
2452 	VM_OBJECT_ASSERT_LOCKED(object);
2453 	if (vpp != NULL)
2454 		*vpp = vm_object_vnode(object);
2455 	switch (object->type) {
2456 	case OBJT_DEFAULT:
2457 		return (KVME_TYPE_DEFAULT);
2458 	case OBJT_VNODE:
2459 		return (KVME_TYPE_VNODE);
2460 	case OBJT_SWAP:
2461 		if ((object->flags & OBJ_TMPFS_NODE) != 0)
2462 			return (KVME_TYPE_VNODE);
2463 		return (KVME_TYPE_SWAP);
2464 	case OBJT_DEVICE:
2465 		return (KVME_TYPE_DEVICE);
2466 	case OBJT_PHYS:
2467 		return (KVME_TYPE_PHYS);
2468 	case OBJT_DEAD:
2469 		return (KVME_TYPE_DEAD);
2470 	case OBJT_SG:
2471 		return (KVME_TYPE_SG);
2472 	case OBJT_MGTDEVICE:
2473 		return (KVME_TYPE_MGTDEVICE);
2474 	default:
2475 		return (KVME_TYPE_UNKNOWN);
2476 	}
2477 }
2478 
2479 static int
2480 sysctl_vm_object_list(SYSCTL_HANDLER_ARGS)
2481 {
2482 	struct kinfo_vmobject *kvo;
2483 	char *fullpath, *freepath;
2484 	struct vnode *vp;
2485 	struct vattr va;
2486 	vm_object_t obj;
2487 	vm_page_t m;
2488 	int count, error;
2489 
2490 	if (req->oldptr == NULL) {
2491 		/*
2492 		 * If an old buffer has not been provided, generate an
2493 		 * estimate of the space needed for a subsequent call.
2494 		 */
2495 		mtx_lock(&vm_object_list_mtx);
2496 		count = 0;
2497 		TAILQ_FOREACH(obj, &vm_object_list, object_list) {
2498 			if (obj->type == OBJT_DEAD)
2499 				continue;
2500 			count++;
2501 		}
2502 		mtx_unlock(&vm_object_list_mtx);
2503 		return (SYSCTL_OUT(req, NULL, sizeof(struct kinfo_vmobject) *
2504 		    count * 11 / 10));
2505 	}
2506 
2507 	kvo = malloc(sizeof(*kvo), M_TEMP, M_WAITOK);
2508 	error = 0;
2509 
2510 	/*
2511 	 * VM objects are type stable and are never removed from the
2512 	 * list once added.  This allows us to safely read obj->object_list
2513 	 * after reacquiring the VM object lock.
2514 	 */
2515 	mtx_lock(&vm_object_list_mtx);
2516 	TAILQ_FOREACH(obj, &vm_object_list, object_list) {
2517 		if (obj->type == OBJT_DEAD)
2518 			continue;
2519 		VM_OBJECT_RLOCK(obj);
2520 		if (obj->type == OBJT_DEAD) {
2521 			VM_OBJECT_RUNLOCK(obj);
2522 			continue;
2523 		}
2524 		mtx_unlock(&vm_object_list_mtx);
2525 		kvo->kvo_size = ptoa(obj->size);
2526 		kvo->kvo_resident = obj->resident_page_count;
2527 		kvo->kvo_ref_count = obj->ref_count;
2528 		kvo->kvo_shadow_count = obj->shadow_count;
2529 		kvo->kvo_memattr = obj->memattr;
2530 		kvo->kvo_active = 0;
2531 		kvo->kvo_inactive = 0;
2532 		TAILQ_FOREACH(m, &obj->memq, listq) {
2533 			/*
2534 			 * A page may belong to the object but be
2535 			 * dequeued and set to PQ_NONE while the
2536 			 * object lock is not held.  This makes the
2537 			 * reads of m->queue below racy, and we do not
2538 			 * count pages set to PQ_NONE.  However, this
2539 			 * sysctl is only meant to give an
2540 			 * approximation of the system anyway.
2541 			 */
2542 			if (m->a.queue == PQ_ACTIVE)
2543 				kvo->kvo_active++;
2544 			else if (m->a.queue == PQ_INACTIVE)
2545 				kvo->kvo_inactive++;
2546 		}
2547 
2548 		kvo->kvo_vn_fileid = 0;
2549 		kvo->kvo_vn_fsid = 0;
2550 		kvo->kvo_vn_fsid_freebsd11 = 0;
2551 		freepath = NULL;
2552 		fullpath = "";
2553 		kvo->kvo_type = vm_object_kvme_type(obj, &vp);
2554 		if (vp != NULL)
2555 			vref(vp);
2556 		VM_OBJECT_RUNLOCK(obj);
2557 		if (vp != NULL) {
2558 			vn_fullpath(curthread, vp, &fullpath, &freepath);
2559 			vn_lock(vp, LK_SHARED | LK_RETRY);
2560 			if (VOP_GETATTR(vp, &va, curthread->td_ucred) == 0) {
2561 				kvo->kvo_vn_fileid = va.va_fileid;
2562 				kvo->kvo_vn_fsid = va.va_fsid;
2563 				kvo->kvo_vn_fsid_freebsd11 = va.va_fsid;
2564 								/* truncate */
2565 			}
2566 			vput(vp);
2567 		}
2568 
2569 		strlcpy(kvo->kvo_path, fullpath, sizeof(kvo->kvo_path));
2570 		if (freepath != NULL)
2571 			free(freepath, M_TEMP);
2572 
2573 		/* Pack record size down */
2574 		kvo->kvo_structsize = offsetof(struct kinfo_vmobject, kvo_path)
2575 		    + strlen(kvo->kvo_path) + 1;
2576 		kvo->kvo_structsize = roundup(kvo->kvo_structsize,
2577 		    sizeof(uint64_t));
2578 		error = SYSCTL_OUT(req, kvo, kvo->kvo_structsize);
2579 		mtx_lock(&vm_object_list_mtx);
2580 		if (error)
2581 			break;
2582 	}
2583 	mtx_unlock(&vm_object_list_mtx);
2584 	free(kvo, M_TEMP);
2585 	return (error);
2586 }
2587 SYSCTL_PROC(_vm, OID_AUTO, objects, CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_SKIP |
2588     CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_object_list, "S,kinfo_vmobject",
2589     "List of VM objects");
2590 
2591 #include "opt_ddb.h"
2592 #ifdef DDB
2593 #include <sys/kernel.h>
2594 
2595 #include <sys/cons.h>
2596 
2597 #include <ddb/ddb.h>
2598 
2599 static int
2600 _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry)
2601 {
2602 	vm_map_t tmpm;
2603 	vm_map_entry_t tmpe;
2604 	vm_object_t obj;
2605 
2606 	if (map == 0)
2607 		return 0;
2608 
2609 	if (entry == 0) {
2610 		VM_MAP_ENTRY_FOREACH(tmpe, map) {
2611 			if (_vm_object_in_map(map, object, tmpe)) {
2612 				return 1;
2613 			}
2614 		}
2615 	} else if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
2616 		tmpm = entry->object.sub_map;
2617 		VM_MAP_ENTRY_FOREACH(tmpe, tmpm) {
2618 			if (_vm_object_in_map(tmpm, object, tmpe)) {
2619 				return 1;
2620 			}
2621 		}
2622 	} else if ((obj = entry->object.vm_object) != NULL) {
2623 		for (; obj; obj = obj->backing_object)
2624 			if (obj == object) {
2625 				return 1;
2626 			}
2627 	}
2628 	return 0;
2629 }
2630 
2631 static int
2632 vm_object_in_map(vm_object_t object)
2633 {
2634 	struct proc *p;
2635 
2636 	/* sx_slock(&allproc_lock); */
2637 	FOREACH_PROC_IN_SYSTEM(p) {
2638 		if (!p->p_vmspace /* || (p->p_flag & (P_SYSTEM|P_WEXIT)) */)
2639 			continue;
2640 		if (_vm_object_in_map(&p->p_vmspace->vm_map, object, 0)) {
2641 			/* sx_sunlock(&allproc_lock); */
2642 			return 1;
2643 		}
2644 	}
2645 	/* sx_sunlock(&allproc_lock); */
2646 	if (_vm_object_in_map(kernel_map, object, 0))
2647 		return 1;
2648 	return 0;
2649 }
2650 
2651 DB_SHOW_COMMAND(vmochk, vm_object_check)
2652 {
2653 	vm_object_t object;
2654 
2655 	/*
2656 	 * make sure that internal objs are in a map somewhere
2657 	 * and none have zero ref counts.
2658 	 */
2659 	TAILQ_FOREACH(object, &vm_object_list, object_list) {
2660 		if ((object->flags & OBJ_ANON) != 0) {
2661 			if (object->ref_count == 0) {
2662 				db_printf("vmochk: internal obj has zero ref count: %ld\n",
2663 					(long)object->size);
2664 			}
2665 			if (!vm_object_in_map(object)) {
2666 				db_printf(
2667 			"vmochk: internal obj is not in a map: "
2668 			"ref: %d, size: %lu: 0x%lx, backing_object: %p\n",
2669 				    object->ref_count, (u_long)object->size,
2670 				    (u_long)object->size,
2671 				    (void *)object->backing_object);
2672 			}
2673 		}
2674 	}
2675 }
2676 
2677 /*
2678  *	vm_object_print:	[ debug ]
2679  */
2680 DB_SHOW_COMMAND(object, vm_object_print_static)
2681 {
2682 	/* XXX convert args. */
2683 	vm_object_t object = (vm_object_t)addr;
2684 	boolean_t full = have_addr;
2685 
2686 	vm_page_t p;
2687 
2688 	/* XXX count is an (unused) arg.  Avoid shadowing it. */
2689 #define	count	was_count
2690 
2691 	int count;
2692 
2693 	if (object == NULL)
2694 		return;
2695 
2696 	db_iprintf(
2697 	    "Object %p: type=%d, size=0x%jx, res=%d, ref=%d, flags=0x%x ruid %d charge %jx\n",
2698 	    object, (int)object->type, (uintmax_t)object->size,
2699 	    object->resident_page_count, object->ref_count, object->flags,
2700 	    object->cred ? object->cred->cr_ruid : -1, (uintmax_t)object->charge);
2701 	db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%jx\n",
2702 	    object->shadow_count,
2703 	    object->backing_object ? object->backing_object->ref_count : 0,
2704 	    object->backing_object, (uintmax_t)object->backing_object_offset);
2705 
2706 	if (!full)
2707 		return;
2708 
2709 	db_indent += 2;
2710 	count = 0;
2711 	TAILQ_FOREACH(p, &object->memq, listq) {
2712 		if (count == 0)
2713 			db_iprintf("memory:=");
2714 		else if (count == 6) {
2715 			db_printf("\n");
2716 			db_iprintf(" ...");
2717 			count = 0;
2718 		} else
2719 			db_printf(",");
2720 		count++;
2721 
2722 		db_printf("(off=0x%jx,page=0x%jx)",
2723 		    (uintmax_t)p->pindex, (uintmax_t)VM_PAGE_TO_PHYS(p));
2724 	}
2725 	if (count != 0)
2726 		db_printf("\n");
2727 	db_indent -= 2;
2728 }
2729 
2730 /* XXX. */
2731 #undef count
2732 
2733 /* XXX need this non-static entry for calling from vm_map_print. */
2734 void
2735 vm_object_print(
2736         /* db_expr_t */ long addr,
2737 	boolean_t have_addr,
2738 	/* db_expr_t */ long count,
2739 	char *modif)
2740 {
2741 	vm_object_print_static(addr, have_addr, count, modif);
2742 }
2743 
2744 DB_SHOW_COMMAND(vmopag, vm_object_print_pages)
2745 {
2746 	vm_object_t object;
2747 	vm_pindex_t fidx;
2748 	vm_paddr_t pa;
2749 	vm_page_t m, prev_m;
2750 	int rcount, nl, c;
2751 
2752 	nl = 0;
2753 	TAILQ_FOREACH(object, &vm_object_list, object_list) {
2754 		db_printf("new object: %p\n", (void *)object);
2755 		if (nl > 18) {
2756 			c = cngetc();
2757 			if (c != ' ')
2758 				return;
2759 			nl = 0;
2760 		}
2761 		nl++;
2762 		rcount = 0;
2763 		fidx = 0;
2764 		pa = -1;
2765 		TAILQ_FOREACH(m, &object->memq, listq) {
2766 			if (m->pindex > 128)
2767 				break;
2768 			if ((prev_m = TAILQ_PREV(m, pglist, listq)) != NULL &&
2769 			    prev_m->pindex + 1 != m->pindex) {
2770 				if (rcount) {
2771 					db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2772 						(long)fidx, rcount, (long)pa);
2773 					if (nl > 18) {
2774 						c = cngetc();
2775 						if (c != ' ')
2776 							return;
2777 						nl = 0;
2778 					}
2779 					nl++;
2780 					rcount = 0;
2781 				}
2782 			}
2783 			if (rcount &&
2784 				(VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) {
2785 				++rcount;
2786 				continue;
2787 			}
2788 			if (rcount) {
2789 				db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2790 					(long)fidx, rcount, (long)pa);
2791 				if (nl > 18) {
2792 					c = cngetc();
2793 					if (c != ' ')
2794 						return;
2795 					nl = 0;
2796 				}
2797 				nl++;
2798 			}
2799 			fidx = m->pindex;
2800 			pa = VM_PAGE_TO_PHYS(m);
2801 			rcount = 1;
2802 		}
2803 		if (rcount) {
2804 			db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2805 				(long)fidx, rcount, (long)pa);
2806 			if (nl > 18) {
2807 				c = cngetc();
2808 				if (c != ' ')
2809 					return;
2810 				nl = 0;
2811 			}
2812 			nl++;
2813 		}
2814 	}
2815 }
2816 #endif /* DDB */
2817