xref: /freebsd/sys/vm/vm_object.c (revision e4f6a1bfa31a2299612bbf9dae402bb38d38b1df)
1 /*-
2  * SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU)
3  *
4  * Copyright (c) 1991, 1993
5  *	The Regents of the University of California.  All rights reserved.
6  *
7  * This code is derived from software contributed to Berkeley by
8  * The Mach Operating System project at Carnegie-Mellon University.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 3. Neither the name of the University nor the names of its contributors
19  *    may be used to endorse or promote products derived from this software
20  *    without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  *	from: @(#)vm_object.c	8.5 (Berkeley) 3/22/94
35  *
36  *
37  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
38  * All rights reserved.
39  *
40  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
41  *
42  * Permission to use, copy, modify and distribute this software and
43  * its documentation is hereby granted, provided that both the copyright
44  * notice and this permission notice appear in all copies of the
45  * software, derivative works or modified versions, and any portions
46  * thereof, and that both notices appear in supporting documentation.
47  *
48  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
49  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
50  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
51  *
52  * Carnegie Mellon requests users of this software to return to
53  *
54  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
55  *  School of Computer Science
56  *  Carnegie Mellon University
57  *  Pittsburgh PA 15213-3890
58  *
59  * any improvements or extensions that they make and grant Carnegie the
60  * rights to redistribute these changes.
61  */
62 
63 /*
64  *	Virtual memory object module.
65  */
66 
67 #include <sys/cdefs.h>
68 __FBSDID("$FreeBSD$");
69 
70 #include "opt_vm.h"
71 
72 #include <sys/param.h>
73 #include <sys/systm.h>
74 #include <sys/lock.h>
75 #include <sys/mman.h>
76 #include <sys/mount.h>
77 #include <sys/kernel.h>
78 #include <sys/pctrie.h>
79 #include <sys/sysctl.h>
80 #include <sys/mutex.h>
81 #include <sys/proc.h>		/* for curproc, pageproc */
82 #include <sys/socket.h>
83 #include <sys/resourcevar.h>
84 #include <sys/rwlock.h>
85 #include <sys/user.h>
86 #include <sys/vnode.h>
87 #include <sys/vmmeter.h>
88 #include <sys/sx.h>
89 
90 #include <vm/vm.h>
91 #include <vm/vm_param.h>
92 #include <vm/pmap.h>
93 #include <vm/vm_map.h>
94 #include <vm/vm_object.h>
95 #include <vm/vm_page.h>
96 #include <vm/vm_pageout.h>
97 #include <vm/vm_pager.h>
98 #include <vm/swap_pager.h>
99 #include <vm/vm_kern.h>
100 #include <vm/vm_extern.h>
101 #include <vm/vm_radix.h>
102 #include <vm/vm_reserv.h>
103 #include <vm/uma.h>
104 
105 static int old_msync;
106 SYSCTL_INT(_vm, OID_AUTO, old_msync, CTLFLAG_RW, &old_msync, 0,
107     "Use old (insecure) msync behavior");
108 
109 static int	vm_object_page_collect_flush(vm_object_t object, vm_page_t p,
110 		    int pagerflags, int flags, boolean_t *clearobjflags,
111 		    boolean_t *eio);
112 static boolean_t vm_object_page_remove_write(vm_page_t p, int flags,
113 		    boolean_t *clearobjflags);
114 static void	vm_object_qcollapse(vm_object_t object);
115 static void	vm_object_vndeallocate(vm_object_t object);
116 
117 /*
118  *	Virtual memory objects maintain the actual data
119  *	associated with allocated virtual memory.  A given
120  *	page of memory exists within exactly one object.
121  *
122  *	An object is only deallocated when all "references"
123  *	are given up.  Only one "reference" to a given
124  *	region of an object should be writeable.
125  *
126  *	Associated with each object is a list of all resident
127  *	memory pages belonging to that object; this list is
128  *	maintained by the "vm_page" module, and locked by the object's
129  *	lock.
130  *
131  *	Each object also records a "pager" routine which is
132  *	used to retrieve (and store) pages to the proper backing
133  *	storage.  In addition, objects may be backed by other
134  *	objects from which they were virtual-copied.
135  *
136  *	The only items within the object structure which are
137  *	modified after time of creation are:
138  *		reference count		locked by object's lock
139  *		pager routine		locked by object's lock
140  *
141  */
142 
143 struct object_q vm_object_list;
144 struct mtx vm_object_list_mtx;	/* lock for object list and count */
145 
146 struct vm_object kernel_object_store;
147 
148 static SYSCTL_NODE(_vm_stats, OID_AUTO, object, CTLFLAG_RD, 0,
149     "VM object stats");
150 
151 static long object_collapses;
152 SYSCTL_LONG(_vm_stats_object, OID_AUTO, collapses, CTLFLAG_RD,
153     &object_collapses, 0, "VM object collapses");
154 
155 static long object_bypasses;
156 SYSCTL_LONG(_vm_stats_object, OID_AUTO, bypasses, CTLFLAG_RD,
157     &object_bypasses, 0, "VM object bypasses");
158 
159 static uma_zone_t obj_zone;
160 
161 static int vm_object_zinit(void *mem, int size, int flags);
162 
163 #ifdef INVARIANTS
164 static void vm_object_zdtor(void *mem, int size, void *arg);
165 
166 static void
167 vm_object_zdtor(void *mem, int size, void *arg)
168 {
169 	vm_object_t object;
170 
171 	object = (vm_object_t)mem;
172 	KASSERT(object->ref_count == 0,
173 	    ("object %p ref_count = %d", object, object->ref_count));
174 	KASSERT(TAILQ_EMPTY(&object->memq),
175 	    ("object %p has resident pages in its memq", object));
176 	KASSERT(vm_radix_is_empty(&object->rtree),
177 	    ("object %p has resident pages in its trie", object));
178 #if VM_NRESERVLEVEL > 0
179 	KASSERT(LIST_EMPTY(&object->rvq),
180 	    ("object %p has reservations",
181 	    object));
182 #endif
183 	KASSERT(object->paging_in_progress == 0,
184 	    ("object %p paging_in_progress = %d",
185 	    object, object->paging_in_progress));
186 	KASSERT(object->resident_page_count == 0,
187 	    ("object %p resident_page_count = %d",
188 	    object, object->resident_page_count));
189 	KASSERT(object->shadow_count == 0,
190 	    ("object %p shadow_count = %d",
191 	    object, object->shadow_count));
192 	KASSERT(object->type == OBJT_DEAD,
193 	    ("object %p has non-dead type %d",
194 	    object, object->type));
195 }
196 #endif
197 
198 static int
199 vm_object_zinit(void *mem, int size, int flags)
200 {
201 	vm_object_t object;
202 
203 	object = (vm_object_t)mem;
204 	rw_init_flags(&object->lock, "vm object", RW_DUPOK | RW_NEW);
205 
206 	/* These are true for any object that has been freed */
207 	object->type = OBJT_DEAD;
208 	object->ref_count = 0;
209 	vm_radix_init(&object->rtree);
210 	object->paging_in_progress = 0;
211 	object->resident_page_count = 0;
212 	object->shadow_count = 0;
213 	object->flags = OBJ_DEAD;
214 
215 	mtx_lock(&vm_object_list_mtx);
216 	TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);
217 	mtx_unlock(&vm_object_list_mtx);
218 	return (0);
219 }
220 
221 static void
222 _vm_object_allocate(objtype_t type, vm_pindex_t size, vm_object_t object)
223 {
224 
225 	TAILQ_INIT(&object->memq);
226 	LIST_INIT(&object->shadow_head);
227 
228 	object->type = type;
229 	if (type == OBJT_SWAP)
230 		pctrie_init(&object->un_pager.swp.swp_blks);
231 
232 	/*
233 	 * Ensure that swap_pager_swapoff() iteration over object_list
234 	 * sees up to date type and pctrie head if it observed
235 	 * non-dead object.
236 	 */
237 	atomic_thread_fence_rel();
238 
239 	switch (type) {
240 	case OBJT_DEAD:
241 		panic("_vm_object_allocate: can't create OBJT_DEAD");
242 	case OBJT_DEFAULT:
243 	case OBJT_SWAP:
244 		object->flags = OBJ_ONEMAPPING;
245 		break;
246 	case OBJT_DEVICE:
247 	case OBJT_SG:
248 		object->flags = OBJ_FICTITIOUS | OBJ_UNMANAGED;
249 		break;
250 	case OBJT_MGTDEVICE:
251 		object->flags = OBJ_FICTITIOUS;
252 		break;
253 	case OBJT_PHYS:
254 		object->flags = OBJ_UNMANAGED;
255 		break;
256 	case OBJT_VNODE:
257 		object->flags = 0;
258 		break;
259 	default:
260 		panic("_vm_object_allocate: type %d is undefined", type);
261 	}
262 	object->size = size;
263 	object->generation = 1;
264 	object->ref_count = 1;
265 	object->memattr = VM_MEMATTR_DEFAULT;
266 	object->cred = NULL;
267 	object->charge = 0;
268 	object->handle = NULL;
269 	object->backing_object = NULL;
270 	object->backing_object_offset = (vm_ooffset_t) 0;
271 #if VM_NRESERVLEVEL > 0
272 	LIST_INIT(&object->rvq);
273 #endif
274 	umtx_shm_object_init(object);
275 }
276 
277 /*
278  *	vm_object_init:
279  *
280  *	Initialize the VM objects module.
281  */
282 void
283 vm_object_init(void)
284 {
285 	TAILQ_INIT(&vm_object_list);
286 	mtx_init(&vm_object_list_mtx, "vm object_list", NULL, MTX_DEF);
287 
288 	rw_init(&kernel_object->lock, "kernel vm object");
289 	_vm_object_allocate(OBJT_PHYS, atop(VM_MAX_KERNEL_ADDRESS -
290 	    VM_MIN_KERNEL_ADDRESS), kernel_object);
291 #if VM_NRESERVLEVEL > 0
292 	kernel_object->flags |= OBJ_COLORED;
293 	kernel_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS);
294 #endif
295 
296 	/*
297 	 * The lock portion of struct vm_object must be type stable due
298 	 * to vm_pageout_fallback_object_lock locking a vm object
299 	 * without holding any references to it.
300 	 */
301 	obj_zone = uma_zcreate("VM OBJECT", sizeof (struct vm_object), NULL,
302 #ifdef INVARIANTS
303 	    vm_object_zdtor,
304 #else
305 	    NULL,
306 #endif
307 	    vm_object_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
308 
309 	vm_radix_zinit();
310 }
311 
312 void
313 vm_object_clear_flag(vm_object_t object, u_short bits)
314 {
315 
316 	VM_OBJECT_ASSERT_WLOCKED(object);
317 	object->flags &= ~bits;
318 }
319 
320 /*
321  *	Sets the default memory attribute for the specified object.  Pages
322  *	that are allocated to this object are by default assigned this memory
323  *	attribute.
324  *
325  *	Presently, this function must be called before any pages are allocated
326  *	to the object.  In the future, this requirement may be relaxed for
327  *	"default" and "swap" objects.
328  */
329 int
330 vm_object_set_memattr(vm_object_t object, vm_memattr_t memattr)
331 {
332 
333 	VM_OBJECT_ASSERT_WLOCKED(object);
334 	switch (object->type) {
335 	case OBJT_DEFAULT:
336 	case OBJT_DEVICE:
337 	case OBJT_MGTDEVICE:
338 	case OBJT_PHYS:
339 	case OBJT_SG:
340 	case OBJT_SWAP:
341 	case OBJT_VNODE:
342 		if (!TAILQ_EMPTY(&object->memq))
343 			return (KERN_FAILURE);
344 		break;
345 	case OBJT_DEAD:
346 		return (KERN_INVALID_ARGUMENT);
347 	default:
348 		panic("vm_object_set_memattr: object %p is of undefined type",
349 		    object);
350 	}
351 	object->memattr = memattr;
352 	return (KERN_SUCCESS);
353 }
354 
355 void
356 vm_object_pip_add(vm_object_t object, short i)
357 {
358 
359 	VM_OBJECT_ASSERT_WLOCKED(object);
360 	object->paging_in_progress += i;
361 }
362 
363 void
364 vm_object_pip_subtract(vm_object_t object, short i)
365 {
366 
367 	VM_OBJECT_ASSERT_WLOCKED(object);
368 	object->paging_in_progress -= i;
369 }
370 
371 void
372 vm_object_pip_wakeup(vm_object_t object)
373 {
374 
375 	VM_OBJECT_ASSERT_WLOCKED(object);
376 	object->paging_in_progress--;
377 	if ((object->flags & OBJ_PIPWNT) && object->paging_in_progress == 0) {
378 		vm_object_clear_flag(object, OBJ_PIPWNT);
379 		wakeup(object);
380 	}
381 }
382 
383 void
384 vm_object_pip_wakeupn(vm_object_t object, short i)
385 {
386 
387 	VM_OBJECT_ASSERT_WLOCKED(object);
388 	if (i)
389 		object->paging_in_progress -= i;
390 	if ((object->flags & OBJ_PIPWNT) && object->paging_in_progress == 0) {
391 		vm_object_clear_flag(object, OBJ_PIPWNT);
392 		wakeup(object);
393 	}
394 }
395 
396 void
397 vm_object_pip_wait(vm_object_t object, char *waitid)
398 {
399 
400 	VM_OBJECT_ASSERT_WLOCKED(object);
401 	while (object->paging_in_progress) {
402 		object->flags |= OBJ_PIPWNT;
403 		VM_OBJECT_SLEEP(object, object, PVM, waitid, 0);
404 	}
405 }
406 
407 /*
408  *	vm_object_allocate:
409  *
410  *	Returns a new object with the given size.
411  */
412 vm_object_t
413 vm_object_allocate(objtype_t type, vm_pindex_t size)
414 {
415 	vm_object_t object;
416 
417 	object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);
418 	_vm_object_allocate(type, size, object);
419 	return (object);
420 }
421 
422 
423 /*
424  *	vm_object_reference:
425  *
426  *	Gets another reference to the given object.  Note: OBJ_DEAD
427  *	objects can be referenced during final cleaning.
428  */
429 void
430 vm_object_reference(vm_object_t object)
431 {
432 	if (object == NULL)
433 		return;
434 	VM_OBJECT_WLOCK(object);
435 	vm_object_reference_locked(object);
436 	VM_OBJECT_WUNLOCK(object);
437 }
438 
439 /*
440  *	vm_object_reference_locked:
441  *
442  *	Gets another reference to the given object.
443  *
444  *	The object must be locked.
445  */
446 void
447 vm_object_reference_locked(vm_object_t object)
448 {
449 	struct vnode *vp;
450 
451 	VM_OBJECT_ASSERT_WLOCKED(object);
452 	object->ref_count++;
453 	if (object->type == OBJT_VNODE) {
454 		vp = object->handle;
455 		vref(vp);
456 	}
457 }
458 
459 /*
460  * Handle deallocating an object of type OBJT_VNODE.
461  */
462 static void
463 vm_object_vndeallocate(vm_object_t object)
464 {
465 	struct vnode *vp = (struct vnode *) object->handle;
466 
467 	VM_OBJECT_ASSERT_WLOCKED(object);
468 	KASSERT(object->type == OBJT_VNODE,
469 	    ("vm_object_vndeallocate: not a vnode object"));
470 	KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp"));
471 #ifdef INVARIANTS
472 	if (object->ref_count == 0) {
473 		vn_printf(vp, "vm_object_vndeallocate ");
474 		panic("vm_object_vndeallocate: bad object reference count");
475 	}
476 #endif
477 
478 	if (!umtx_shm_vnobj_persistent && object->ref_count == 1)
479 		umtx_shm_object_terminated(object);
480 
481 	/*
482 	 * The test for text of vp vnode does not need a bypass to
483 	 * reach right VV_TEXT there, since it is obtained from
484 	 * object->handle.
485 	 */
486 	if (object->ref_count > 1 || (vp->v_vflag & VV_TEXT) == 0) {
487 		object->ref_count--;
488 		VM_OBJECT_WUNLOCK(object);
489 		/* vrele may need the vnode lock. */
490 		vrele(vp);
491 	} else {
492 		vhold(vp);
493 		VM_OBJECT_WUNLOCK(object);
494 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
495 		vdrop(vp);
496 		VM_OBJECT_WLOCK(object);
497 		object->ref_count--;
498 		if (object->type == OBJT_DEAD) {
499 			VM_OBJECT_WUNLOCK(object);
500 			VOP_UNLOCK(vp, 0);
501 		} else {
502 			if (object->ref_count == 0)
503 				VOP_UNSET_TEXT(vp);
504 			VM_OBJECT_WUNLOCK(object);
505 			vput(vp);
506 		}
507 	}
508 }
509 
510 /*
511  *	vm_object_deallocate:
512  *
513  *	Release a reference to the specified object,
514  *	gained either through a vm_object_allocate
515  *	or a vm_object_reference call.  When all references
516  *	are gone, storage associated with this object
517  *	may be relinquished.
518  *
519  *	No object may be locked.
520  */
521 void
522 vm_object_deallocate(vm_object_t object)
523 {
524 	vm_object_t temp;
525 	struct vnode *vp;
526 
527 	while (object != NULL) {
528 		VM_OBJECT_WLOCK(object);
529 		if (object->type == OBJT_VNODE) {
530 			vm_object_vndeallocate(object);
531 			return;
532 		}
533 
534 		KASSERT(object->ref_count != 0,
535 			("vm_object_deallocate: object deallocated too many times: %d", object->type));
536 
537 		/*
538 		 * If the reference count goes to 0 we start calling
539 		 * vm_object_terminate() on the object chain.
540 		 * A ref count of 1 may be a special case depending on the
541 		 * shadow count being 0 or 1.
542 		 */
543 		object->ref_count--;
544 		if (object->ref_count > 1) {
545 			VM_OBJECT_WUNLOCK(object);
546 			return;
547 		} else if (object->ref_count == 1) {
548 			if (object->type == OBJT_SWAP &&
549 			    (object->flags & OBJ_TMPFS) != 0) {
550 				vp = object->un_pager.swp.swp_tmpfs;
551 				vhold(vp);
552 				VM_OBJECT_WUNLOCK(object);
553 				vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
554 				VM_OBJECT_WLOCK(object);
555 				if (object->type == OBJT_DEAD ||
556 				    object->ref_count != 1) {
557 					VM_OBJECT_WUNLOCK(object);
558 					VOP_UNLOCK(vp, 0);
559 					vdrop(vp);
560 					return;
561 				}
562 				if ((object->flags & OBJ_TMPFS) != 0)
563 					VOP_UNSET_TEXT(vp);
564 				VOP_UNLOCK(vp, 0);
565 				vdrop(vp);
566 			}
567 			if (object->shadow_count == 0 &&
568 			    object->handle == NULL &&
569 			    (object->type == OBJT_DEFAULT ||
570 			    (object->type == OBJT_SWAP &&
571 			    (object->flags & OBJ_TMPFS_NODE) == 0))) {
572 				vm_object_set_flag(object, OBJ_ONEMAPPING);
573 			} else if ((object->shadow_count == 1) &&
574 			    (object->handle == NULL) &&
575 			    (object->type == OBJT_DEFAULT ||
576 			     object->type == OBJT_SWAP)) {
577 				vm_object_t robject;
578 
579 				robject = LIST_FIRST(&object->shadow_head);
580 				KASSERT(robject != NULL,
581 				    ("vm_object_deallocate: ref_count: %d, shadow_count: %d",
582 					 object->ref_count,
583 					 object->shadow_count));
584 				KASSERT((robject->flags & OBJ_TMPFS_NODE) == 0,
585 				    ("shadowed tmpfs v_object %p", object));
586 				if (!VM_OBJECT_TRYWLOCK(robject)) {
587 					/*
588 					 * Avoid a potential deadlock.
589 					 */
590 					object->ref_count++;
591 					VM_OBJECT_WUNLOCK(object);
592 					/*
593 					 * More likely than not the thread
594 					 * holding robject's lock has lower
595 					 * priority than the current thread.
596 					 * Let the lower priority thread run.
597 					 */
598 					pause("vmo_de", 1);
599 					continue;
600 				}
601 				/*
602 				 * Collapse object into its shadow unless its
603 				 * shadow is dead.  In that case, object will
604 				 * be deallocated by the thread that is
605 				 * deallocating its shadow.
606 				 */
607 				if ((robject->flags & OBJ_DEAD) == 0 &&
608 				    (robject->handle == NULL) &&
609 				    (robject->type == OBJT_DEFAULT ||
610 				     robject->type == OBJT_SWAP)) {
611 
612 					robject->ref_count++;
613 retry:
614 					if (robject->paging_in_progress) {
615 						VM_OBJECT_WUNLOCK(object);
616 						vm_object_pip_wait(robject,
617 						    "objde1");
618 						temp = robject->backing_object;
619 						if (object == temp) {
620 							VM_OBJECT_WLOCK(object);
621 							goto retry;
622 						}
623 					} else if (object->paging_in_progress) {
624 						VM_OBJECT_WUNLOCK(robject);
625 						object->flags |= OBJ_PIPWNT;
626 						VM_OBJECT_SLEEP(object, object,
627 						    PDROP | PVM, "objde2", 0);
628 						VM_OBJECT_WLOCK(robject);
629 						temp = robject->backing_object;
630 						if (object == temp) {
631 							VM_OBJECT_WLOCK(object);
632 							goto retry;
633 						}
634 					} else
635 						VM_OBJECT_WUNLOCK(object);
636 
637 					if (robject->ref_count == 1) {
638 						robject->ref_count--;
639 						object = robject;
640 						goto doterm;
641 					}
642 					object = robject;
643 					vm_object_collapse(object);
644 					VM_OBJECT_WUNLOCK(object);
645 					continue;
646 				}
647 				VM_OBJECT_WUNLOCK(robject);
648 			}
649 			VM_OBJECT_WUNLOCK(object);
650 			return;
651 		}
652 doterm:
653 		umtx_shm_object_terminated(object);
654 		temp = object->backing_object;
655 		if (temp != NULL) {
656 			KASSERT((object->flags & OBJ_TMPFS_NODE) == 0,
657 			    ("shadowed tmpfs v_object 2 %p", object));
658 			VM_OBJECT_WLOCK(temp);
659 			LIST_REMOVE(object, shadow_list);
660 			temp->shadow_count--;
661 			VM_OBJECT_WUNLOCK(temp);
662 			object->backing_object = NULL;
663 		}
664 		/*
665 		 * Don't double-terminate, we could be in a termination
666 		 * recursion due to the terminate having to sync data
667 		 * to disk.
668 		 */
669 		if ((object->flags & OBJ_DEAD) == 0)
670 			vm_object_terminate(object);
671 		else
672 			VM_OBJECT_WUNLOCK(object);
673 		object = temp;
674 	}
675 }
676 
677 /*
678  *	vm_object_destroy removes the object from the global object list
679  *      and frees the space for the object.
680  */
681 void
682 vm_object_destroy(vm_object_t object)
683 {
684 
685 	/*
686 	 * Release the allocation charge.
687 	 */
688 	if (object->cred != NULL) {
689 		swap_release_by_cred(object->charge, object->cred);
690 		object->charge = 0;
691 		crfree(object->cred);
692 		object->cred = NULL;
693 	}
694 
695 	/*
696 	 * Free the space for the object.
697 	 */
698 	uma_zfree(obj_zone, object);
699 }
700 
701 /*
702  *	vm_object_terminate_pages removes any remaining pageable pages
703  *	from the object and resets the object to an empty state.
704  */
705 static void
706 vm_object_terminate_pages(vm_object_t object)
707 {
708 	vm_page_t p, p_next;
709 	struct mtx *mtx, *mtx1;
710 	struct vm_pagequeue *pq, *pq1;
711 	int dequeued;
712 
713 	VM_OBJECT_ASSERT_WLOCKED(object);
714 
715 	mtx = NULL;
716 	pq = NULL;
717 
718 	/*
719 	 * Free any remaining pageable pages.  This also removes them from the
720 	 * paging queues.  However, don't free wired pages, just remove them
721 	 * from the object.  Rather than incrementally removing each page from
722 	 * the object, the page and object are reset to any empty state.
723 	 */
724 	TAILQ_FOREACH_SAFE(p, &object->memq, listq, p_next) {
725 		vm_page_assert_unbusied(p);
726 		if ((object->flags & OBJ_UNMANAGED) == 0) {
727 			/*
728 			 * vm_page_free_prep() only needs the page
729 			 * lock for managed pages.
730 			 */
731 			mtx1 = vm_page_lockptr(p);
732 			if (mtx1 != mtx) {
733 				if (mtx != NULL)
734 					mtx_unlock(mtx);
735 				if (pq != NULL) {
736 					vm_pagequeue_cnt_add(pq, dequeued);
737 					vm_pagequeue_unlock(pq);
738 					pq = NULL;
739 				}
740 				mtx = mtx1;
741 				mtx_lock(mtx);
742 			}
743 		}
744 		p->object = NULL;
745 		if (p->wire_count != 0)
746 			goto unlist;
747 		VM_CNT_INC(v_pfree);
748 		p->flags &= ~PG_ZERO;
749 		if (p->queue != PQ_NONE) {
750 			KASSERT(p->queue < PQ_COUNT, ("vm_object_terminate: "
751 			    "page %p is not queued", p));
752 			pq1 = vm_page_pagequeue(p);
753 			if (pq != pq1) {
754 				if (pq != NULL) {
755 					vm_pagequeue_cnt_add(pq, dequeued);
756 					vm_pagequeue_unlock(pq);
757 				}
758 				pq = pq1;
759 				vm_pagequeue_lock(pq);
760 				dequeued = 0;
761 			}
762 			p->queue = PQ_NONE;
763 			TAILQ_REMOVE(&pq->pq_pl, p, plinks.q);
764 			dequeued--;
765 		}
766 		if (vm_page_free_prep(p, true))
767 			continue;
768 unlist:
769 		TAILQ_REMOVE(&object->memq, p, listq);
770 	}
771 	if (pq != NULL) {
772 		vm_pagequeue_cnt_add(pq, dequeued);
773 		vm_pagequeue_unlock(pq);
774 	}
775 	if (mtx != NULL)
776 		mtx_unlock(mtx);
777 
778 	vm_page_free_phys_pglist(&object->memq);
779 
780 	/*
781 	 * If the object contained any pages, then reset it to an empty state.
782 	 * None of the object's fields, including "resident_page_count", were
783 	 * modified by the preceding loop.
784 	 */
785 	if (object->resident_page_count != 0) {
786 		vm_radix_reclaim_allnodes(&object->rtree);
787 		TAILQ_INIT(&object->memq);
788 		object->resident_page_count = 0;
789 		if (object->type == OBJT_VNODE)
790 			vdrop(object->handle);
791 	}
792 }
793 
794 /*
795  *	vm_object_terminate actually destroys the specified object, freeing
796  *	up all previously used resources.
797  *
798  *	The object must be locked.
799  *	This routine may block.
800  */
801 void
802 vm_object_terminate(vm_object_t object)
803 {
804 
805 	VM_OBJECT_ASSERT_WLOCKED(object);
806 
807 	/*
808 	 * Make sure no one uses us.
809 	 */
810 	vm_object_set_flag(object, OBJ_DEAD);
811 
812 	/*
813 	 * wait for the pageout daemon to be done with the object
814 	 */
815 	vm_object_pip_wait(object, "objtrm");
816 
817 	KASSERT(!object->paging_in_progress,
818 		("vm_object_terminate: pageout in progress"));
819 
820 	/*
821 	 * Clean and free the pages, as appropriate. All references to the
822 	 * object are gone, so we don't need to lock it.
823 	 */
824 	if (object->type == OBJT_VNODE) {
825 		struct vnode *vp = (struct vnode *)object->handle;
826 
827 		/*
828 		 * Clean pages and flush buffers.
829 		 */
830 		vm_object_page_clean(object, 0, 0, OBJPC_SYNC);
831 		VM_OBJECT_WUNLOCK(object);
832 
833 		vinvalbuf(vp, V_SAVE, 0, 0);
834 
835 		BO_LOCK(&vp->v_bufobj);
836 		vp->v_bufobj.bo_flag |= BO_DEAD;
837 		BO_UNLOCK(&vp->v_bufobj);
838 
839 		VM_OBJECT_WLOCK(object);
840 	}
841 
842 	KASSERT(object->ref_count == 0,
843 		("vm_object_terminate: object with references, ref_count=%d",
844 		object->ref_count));
845 
846 	if ((object->flags & OBJ_PG_DTOR) == 0)
847 		vm_object_terminate_pages(object);
848 
849 #if VM_NRESERVLEVEL > 0
850 	if (__predict_false(!LIST_EMPTY(&object->rvq)))
851 		vm_reserv_break_all(object);
852 #endif
853 
854 	KASSERT(object->cred == NULL || object->type == OBJT_DEFAULT ||
855 	    object->type == OBJT_SWAP,
856 	    ("%s: non-swap obj %p has cred", __func__, object));
857 
858 	/*
859 	 * Let the pager know object is dead.
860 	 */
861 	vm_pager_deallocate(object);
862 	VM_OBJECT_WUNLOCK(object);
863 
864 	vm_object_destroy(object);
865 }
866 
867 /*
868  * Make the page read-only so that we can clear the object flags.  However, if
869  * this is a nosync mmap then the object is likely to stay dirty so do not
870  * mess with the page and do not clear the object flags.  Returns TRUE if the
871  * page should be flushed, and FALSE otherwise.
872  */
873 static boolean_t
874 vm_object_page_remove_write(vm_page_t p, int flags, boolean_t *clearobjflags)
875 {
876 
877 	/*
878 	 * If we have been asked to skip nosync pages and this is a
879 	 * nosync page, skip it.  Note that the object flags were not
880 	 * cleared in this case so we do not have to set them.
881 	 */
882 	if ((flags & OBJPC_NOSYNC) != 0 && (p->oflags & VPO_NOSYNC) != 0) {
883 		*clearobjflags = FALSE;
884 		return (FALSE);
885 	} else {
886 		pmap_remove_write(p);
887 		return (p->dirty != 0);
888 	}
889 }
890 
891 /*
892  *	vm_object_page_clean
893  *
894  *	Clean all dirty pages in the specified range of object.  Leaves page
895  * 	on whatever queue it is currently on.   If NOSYNC is set then do not
896  *	write out pages with VPO_NOSYNC set (originally comes from MAP_NOSYNC),
897  *	leaving the object dirty.
898  *
899  *	When stuffing pages asynchronously, allow clustering.  XXX we need a
900  *	synchronous clustering mode implementation.
901  *
902  *	Odd semantics: if start == end, we clean everything.
903  *
904  *	The object must be locked.
905  *
906  *	Returns FALSE if some page from the range was not written, as
907  *	reported by the pager, and TRUE otherwise.
908  */
909 boolean_t
910 vm_object_page_clean(vm_object_t object, vm_ooffset_t start, vm_ooffset_t end,
911     int flags)
912 {
913 	vm_page_t np, p;
914 	vm_pindex_t pi, tend, tstart;
915 	int curgeneration, n, pagerflags;
916 	boolean_t clearobjflags, eio, res;
917 
918 	VM_OBJECT_ASSERT_WLOCKED(object);
919 
920 	/*
921 	 * The OBJ_MIGHTBEDIRTY flag is only set for OBJT_VNODE
922 	 * objects.  The check below prevents the function from
923 	 * operating on non-vnode objects.
924 	 */
925 	if ((object->flags & OBJ_MIGHTBEDIRTY) == 0 ||
926 	    object->resident_page_count == 0)
927 		return (TRUE);
928 
929 	pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) != 0 ?
930 	    VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK;
931 	pagerflags |= (flags & OBJPC_INVAL) != 0 ? VM_PAGER_PUT_INVAL : 0;
932 
933 	tstart = OFF_TO_IDX(start);
934 	tend = (end == 0) ? object->size : OFF_TO_IDX(end + PAGE_MASK);
935 	clearobjflags = tstart == 0 && tend >= object->size;
936 	res = TRUE;
937 
938 rescan:
939 	curgeneration = object->generation;
940 
941 	for (p = vm_page_find_least(object, tstart); p != NULL; p = np) {
942 		pi = p->pindex;
943 		if (pi >= tend)
944 			break;
945 		np = TAILQ_NEXT(p, listq);
946 		if (p->valid == 0)
947 			continue;
948 		if (vm_page_sleep_if_busy(p, "vpcwai")) {
949 			if (object->generation != curgeneration) {
950 				if ((flags & OBJPC_SYNC) != 0)
951 					goto rescan;
952 				else
953 					clearobjflags = FALSE;
954 			}
955 			np = vm_page_find_least(object, pi);
956 			continue;
957 		}
958 		if (!vm_object_page_remove_write(p, flags, &clearobjflags))
959 			continue;
960 
961 		n = vm_object_page_collect_flush(object, p, pagerflags,
962 		    flags, &clearobjflags, &eio);
963 		if (eio) {
964 			res = FALSE;
965 			clearobjflags = FALSE;
966 		}
967 		if (object->generation != curgeneration) {
968 			if ((flags & OBJPC_SYNC) != 0)
969 				goto rescan;
970 			else
971 				clearobjflags = FALSE;
972 		}
973 
974 		/*
975 		 * If the VOP_PUTPAGES() did a truncated write, so
976 		 * that even the first page of the run is not fully
977 		 * written, vm_pageout_flush() returns 0 as the run
978 		 * length.  Since the condition that caused truncated
979 		 * write may be permanent, e.g. exhausted free space,
980 		 * accepting n == 0 would cause an infinite loop.
981 		 *
982 		 * Forwarding the iterator leaves the unwritten page
983 		 * behind, but there is not much we can do there if
984 		 * filesystem refuses to write it.
985 		 */
986 		if (n == 0) {
987 			n = 1;
988 			clearobjflags = FALSE;
989 		}
990 		np = vm_page_find_least(object, pi + n);
991 	}
992 #if 0
993 	VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC) ? MNT_WAIT : 0);
994 #endif
995 
996 	if (clearobjflags)
997 		vm_object_clear_flag(object, OBJ_MIGHTBEDIRTY);
998 	return (res);
999 }
1000 
1001 static int
1002 vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags,
1003     int flags, boolean_t *clearobjflags, boolean_t *eio)
1004 {
1005 	vm_page_t ma[vm_pageout_page_count], p_first, tp;
1006 	int count, i, mreq, runlen;
1007 
1008 	vm_page_lock_assert(p, MA_NOTOWNED);
1009 	VM_OBJECT_ASSERT_WLOCKED(object);
1010 
1011 	count = 1;
1012 	mreq = 0;
1013 
1014 	for (tp = p; count < vm_pageout_page_count; count++) {
1015 		tp = vm_page_next(tp);
1016 		if (tp == NULL || vm_page_busied(tp))
1017 			break;
1018 		if (!vm_object_page_remove_write(tp, flags, clearobjflags))
1019 			break;
1020 	}
1021 
1022 	for (p_first = p; count < vm_pageout_page_count; count++) {
1023 		tp = vm_page_prev(p_first);
1024 		if (tp == NULL || vm_page_busied(tp))
1025 			break;
1026 		if (!vm_object_page_remove_write(tp, flags, clearobjflags))
1027 			break;
1028 		p_first = tp;
1029 		mreq++;
1030 	}
1031 
1032 	for (tp = p_first, i = 0; i < count; tp = TAILQ_NEXT(tp, listq), i++)
1033 		ma[i] = tp;
1034 
1035 	vm_pageout_flush(ma, count, pagerflags, mreq, &runlen, eio);
1036 	return (runlen);
1037 }
1038 
1039 /*
1040  * Note that there is absolutely no sense in writing out
1041  * anonymous objects, so we track down the vnode object
1042  * to write out.
1043  * We invalidate (remove) all pages from the address space
1044  * for semantic correctness.
1045  *
1046  * If the backing object is a device object with unmanaged pages, then any
1047  * mappings to the specified range of pages must be removed before this
1048  * function is called.
1049  *
1050  * Note: certain anonymous maps, such as MAP_NOSYNC maps,
1051  * may start out with a NULL object.
1052  */
1053 boolean_t
1054 vm_object_sync(vm_object_t object, vm_ooffset_t offset, vm_size_t size,
1055     boolean_t syncio, boolean_t invalidate)
1056 {
1057 	vm_object_t backing_object;
1058 	struct vnode *vp;
1059 	struct mount *mp;
1060 	int error, flags, fsync_after;
1061 	boolean_t res;
1062 
1063 	if (object == NULL)
1064 		return (TRUE);
1065 	res = TRUE;
1066 	error = 0;
1067 	VM_OBJECT_WLOCK(object);
1068 	while ((backing_object = object->backing_object) != NULL) {
1069 		VM_OBJECT_WLOCK(backing_object);
1070 		offset += object->backing_object_offset;
1071 		VM_OBJECT_WUNLOCK(object);
1072 		object = backing_object;
1073 		if (object->size < OFF_TO_IDX(offset + size))
1074 			size = IDX_TO_OFF(object->size) - offset;
1075 	}
1076 	/*
1077 	 * Flush pages if writing is allowed, invalidate them
1078 	 * if invalidation requested.  Pages undergoing I/O
1079 	 * will be ignored by vm_object_page_remove().
1080 	 *
1081 	 * We cannot lock the vnode and then wait for paging
1082 	 * to complete without deadlocking against vm_fault.
1083 	 * Instead we simply call vm_object_page_remove() and
1084 	 * allow it to block internally on a page-by-page
1085 	 * basis when it encounters pages undergoing async
1086 	 * I/O.
1087 	 */
1088 	if (object->type == OBJT_VNODE &&
1089 	    (object->flags & OBJ_MIGHTBEDIRTY) != 0 &&
1090 	    ((vp = object->handle)->v_vflag & VV_NOSYNC) == 0) {
1091 		VM_OBJECT_WUNLOCK(object);
1092 		(void) vn_start_write(vp, &mp, V_WAIT);
1093 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1094 		if (syncio && !invalidate && offset == 0 &&
1095 		    atop(size) == object->size) {
1096 			/*
1097 			 * If syncing the whole mapping of the file,
1098 			 * it is faster to schedule all the writes in
1099 			 * async mode, also allowing the clustering,
1100 			 * and then wait for i/o to complete.
1101 			 */
1102 			flags = 0;
1103 			fsync_after = TRUE;
1104 		} else {
1105 			flags = (syncio || invalidate) ? OBJPC_SYNC : 0;
1106 			flags |= invalidate ? (OBJPC_SYNC | OBJPC_INVAL) : 0;
1107 			fsync_after = FALSE;
1108 		}
1109 		VM_OBJECT_WLOCK(object);
1110 		res = vm_object_page_clean(object, offset, offset + size,
1111 		    flags);
1112 		VM_OBJECT_WUNLOCK(object);
1113 		if (fsync_after)
1114 			error = VOP_FSYNC(vp, MNT_WAIT, curthread);
1115 		VOP_UNLOCK(vp, 0);
1116 		vn_finished_write(mp);
1117 		if (error != 0)
1118 			res = FALSE;
1119 		VM_OBJECT_WLOCK(object);
1120 	}
1121 	if ((object->type == OBJT_VNODE ||
1122 	     object->type == OBJT_DEVICE) && invalidate) {
1123 		if (object->type == OBJT_DEVICE)
1124 			/*
1125 			 * The option OBJPR_NOTMAPPED must be passed here
1126 			 * because vm_object_page_remove() cannot remove
1127 			 * unmanaged mappings.
1128 			 */
1129 			flags = OBJPR_NOTMAPPED;
1130 		else if (old_msync)
1131 			flags = 0;
1132 		else
1133 			flags = OBJPR_CLEANONLY;
1134 		vm_object_page_remove(object, OFF_TO_IDX(offset),
1135 		    OFF_TO_IDX(offset + size + PAGE_MASK), flags);
1136 	}
1137 	VM_OBJECT_WUNLOCK(object);
1138 	return (res);
1139 }
1140 
1141 /*
1142  * Determine whether the given advice can be applied to the object.  Advice is
1143  * not applied to unmanaged pages since they never belong to page queues, and
1144  * since MADV_FREE is destructive, it can apply only to anonymous pages that
1145  * have been mapped at most once.
1146  */
1147 static bool
1148 vm_object_advice_applies(vm_object_t object, int advice)
1149 {
1150 
1151 	if ((object->flags & OBJ_UNMANAGED) != 0)
1152 		return (false);
1153 	if (advice != MADV_FREE)
1154 		return (true);
1155 	return ((object->type == OBJT_DEFAULT || object->type == OBJT_SWAP) &&
1156 	    (object->flags & OBJ_ONEMAPPING) != 0);
1157 }
1158 
1159 static void
1160 vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex,
1161     vm_size_t size)
1162 {
1163 
1164 	if (advice == MADV_FREE && object->type == OBJT_SWAP)
1165 		swap_pager_freespace(object, pindex, size);
1166 }
1167 
1168 /*
1169  *	vm_object_madvise:
1170  *
1171  *	Implements the madvise function at the object/page level.
1172  *
1173  *	MADV_WILLNEED	(any object)
1174  *
1175  *	    Activate the specified pages if they are resident.
1176  *
1177  *	MADV_DONTNEED	(any object)
1178  *
1179  *	    Deactivate the specified pages if they are resident.
1180  *
1181  *	MADV_FREE	(OBJT_DEFAULT/OBJT_SWAP objects,
1182  *			 OBJ_ONEMAPPING only)
1183  *
1184  *	    Deactivate and clean the specified pages if they are
1185  *	    resident.  This permits the process to reuse the pages
1186  *	    without faulting or the kernel to reclaim the pages
1187  *	    without I/O.
1188  */
1189 void
1190 vm_object_madvise(vm_object_t object, vm_pindex_t pindex, vm_pindex_t end,
1191     int advice)
1192 {
1193 	vm_pindex_t tpindex;
1194 	vm_object_t backing_object, tobject;
1195 	vm_page_t m, tm;
1196 
1197 	if (object == NULL)
1198 		return;
1199 
1200 relookup:
1201 	VM_OBJECT_WLOCK(object);
1202 	if (!vm_object_advice_applies(object, advice)) {
1203 		VM_OBJECT_WUNLOCK(object);
1204 		return;
1205 	}
1206 	for (m = vm_page_find_least(object, pindex); pindex < end; pindex++) {
1207 		tobject = object;
1208 
1209 		/*
1210 		 * If the next page isn't resident in the top-level object, we
1211 		 * need to search the shadow chain.  When applying MADV_FREE, we
1212 		 * take care to release any swap space used to store
1213 		 * non-resident pages.
1214 		 */
1215 		if (m == NULL || pindex < m->pindex) {
1216 			/*
1217 			 * Optimize a common case: if the top-level object has
1218 			 * no backing object, we can skip over the non-resident
1219 			 * range in constant time.
1220 			 */
1221 			if (object->backing_object == NULL) {
1222 				tpindex = (m != NULL && m->pindex < end) ?
1223 				    m->pindex : end;
1224 				vm_object_madvise_freespace(object, advice,
1225 				    pindex, tpindex - pindex);
1226 				if ((pindex = tpindex) == end)
1227 					break;
1228 				goto next_page;
1229 			}
1230 
1231 			tpindex = pindex;
1232 			do {
1233 				vm_object_madvise_freespace(tobject, advice,
1234 				    tpindex, 1);
1235 				/*
1236 				 * Prepare to search the next object in the
1237 				 * chain.
1238 				 */
1239 				backing_object = tobject->backing_object;
1240 				if (backing_object == NULL)
1241 					goto next_pindex;
1242 				VM_OBJECT_WLOCK(backing_object);
1243 				tpindex +=
1244 				    OFF_TO_IDX(tobject->backing_object_offset);
1245 				if (tobject != object)
1246 					VM_OBJECT_WUNLOCK(tobject);
1247 				tobject = backing_object;
1248 				if (!vm_object_advice_applies(tobject, advice))
1249 					goto next_pindex;
1250 			} while ((tm = vm_page_lookup(tobject, tpindex)) ==
1251 			    NULL);
1252 		} else {
1253 next_page:
1254 			tm = m;
1255 			m = TAILQ_NEXT(m, listq);
1256 		}
1257 
1258 		/*
1259 		 * If the page is not in a normal state, skip it.
1260 		 */
1261 		if (tm->valid != VM_PAGE_BITS_ALL)
1262 			goto next_pindex;
1263 		vm_page_lock(tm);
1264 		if (tm->hold_count != 0 || tm->wire_count != 0) {
1265 			vm_page_unlock(tm);
1266 			goto next_pindex;
1267 		}
1268 		KASSERT((tm->flags & PG_FICTITIOUS) == 0,
1269 		    ("vm_object_madvise: page %p is fictitious", tm));
1270 		KASSERT((tm->oflags & VPO_UNMANAGED) == 0,
1271 		    ("vm_object_madvise: page %p is not managed", tm));
1272 		if (vm_page_busied(tm)) {
1273 			if (object != tobject)
1274 				VM_OBJECT_WUNLOCK(tobject);
1275 			VM_OBJECT_WUNLOCK(object);
1276 			if (advice == MADV_WILLNEED) {
1277 				/*
1278 				 * Reference the page before unlocking and
1279 				 * sleeping so that the page daemon is less
1280 				 * likely to reclaim it.
1281 				 */
1282 				vm_page_aflag_set(tm, PGA_REFERENCED);
1283 			}
1284 			vm_page_busy_sleep(tm, "madvpo", false);
1285   			goto relookup;
1286 		}
1287 		vm_page_advise(tm, advice);
1288 		vm_page_unlock(tm);
1289 		vm_object_madvise_freespace(tobject, advice, tm->pindex, 1);
1290 next_pindex:
1291 		if (tobject != object)
1292 			VM_OBJECT_WUNLOCK(tobject);
1293 	}
1294 	VM_OBJECT_WUNLOCK(object);
1295 }
1296 
1297 /*
1298  *	vm_object_shadow:
1299  *
1300  *	Create a new object which is backed by the
1301  *	specified existing object range.  The source
1302  *	object reference is deallocated.
1303  *
1304  *	The new object and offset into that object
1305  *	are returned in the source parameters.
1306  */
1307 void
1308 vm_object_shadow(
1309 	vm_object_t *object,	/* IN/OUT */
1310 	vm_ooffset_t *offset,	/* IN/OUT */
1311 	vm_size_t length)
1312 {
1313 	vm_object_t source;
1314 	vm_object_t result;
1315 
1316 	source = *object;
1317 
1318 	/*
1319 	 * Don't create the new object if the old object isn't shared.
1320 	 */
1321 	if (source != NULL) {
1322 		VM_OBJECT_WLOCK(source);
1323 		if (source->ref_count == 1 &&
1324 		    source->handle == NULL &&
1325 		    (source->type == OBJT_DEFAULT ||
1326 		     source->type == OBJT_SWAP)) {
1327 			VM_OBJECT_WUNLOCK(source);
1328 			return;
1329 		}
1330 		VM_OBJECT_WUNLOCK(source);
1331 	}
1332 
1333 	/*
1334 	 * Allocate a new object with the given length.
1335 	 */
1336 	result = vm_object_allocate(OBJT_DEFAULT, atop(length));
1337 
1338 	/*
1339 	 * The new object shadows the source object, adding a reference to it.
1340 	 * Our caller changes his reference to point to the new object,
1341 	 * removing a reference to the source object.  Net result: no change
1342 	 * of reference count.
1343 	 *
1344 	 * Try to optimize the result object's page color when shadowing
1345 	 * in order to maintain page coloring consistency in the combined
1346 	 * shadowed object.
1347 	 */
1348 	result->backing_object = source;
1349 	/*
1350 	 * Store the offset into the source object, and fix up the offset into
1351 	 * the new object.
1352 	 */
1353 	result->backing_object_offset = *offset;
1354 	if (source != NULL) {
1355 		VM_OBJECT_WLOCK(source);
1356 		LIST_INSERT_HEAD(&source->shadow_head, result, shadow_list);
1357 		source->shadow_count++;
1358 #if VM_NRESERVLEVEL > 0
1359 		result->flags |= source->flags & OBJ_COLORED;
1360 		result->pg_color = (source->pg_color + OFF_TO_IDX(*offset)) &
1361 		    ((1 << (VM_NFREEORDER - 1)) - 1);
1362 #endif
1363 		VM_OBJECT_WUNLOCK(source);
1364 	}
1365 
1366 
1367 	/*
1368 	 * Return the new things
1369 	 */
1370 	*offset = 0;
1371 	*object = result;
1372 }
1373 
1374 /*
1375  *	vm_object_split:
1376  *
1377  * Split the pages in a map entry into a new object.  This affords
1378  * easier removal of unused pages, and keeps object inheritance from
1379  * being a negative impact on memory usage.
1380  */
1381 void
1382 vm_object_split(vm_map_entry_t entry)
1383 {
1384 	vm_page_t m, m_next;
1385 	vm_object_t orig_object, new_object, source;
1386 	vm_pindex_t idx, offidxstart;
1387 	vm_size_t size;
1388 
1389 	orig_object = entry->object.vm_object;
1390 	if (orig_object->type != OBJT_DEFAULT && orig_object->type != OBJT_SWAP)
1391 		return;
1392 	if (orig_object->ref_count <= 1)
1393 		return;
1394 	VM_OBJECT_WUNLOCK(orig_object);
1395 
1396 	offidxstart = OFF_TO_IDX(entry->offset);
1397 	size = atop(entry->end - entry->start);
1398 
1399 	/*
1400 	 * If swap_pager_copy() is later called, it will convert new_object
1401 	 * into a swap object.
1402 	 */
1403 	new_object = vm_object_allocate(OBJT_DEFAULT, size);
1404 
1405 	/*
1406 	 * At this point, the new object is still private, so the order in
1407 	 * which the original and new objects are locked does not matter.
1408 	 */
1409 	VM_OBJECT_WLOCK(new_object);
1410 	VM_OBJECT_WLOCK(orig_object);
1411 	source = orig_object->backing_object;
1412 	if (source != NULL) {
1413 		VM_OBJECT_WLOCK(source);
1414 		if ((source->flags & OBJ_DEAD) != 0) {
1415 			VM_OBJECT_WUNLOCK(source);
1416 			VM_OBJECT_WUNLOCK(orig_object);
1417 			VM_OBJECT_WUNLOCK(new_object);
1418 			vm_object_deallocate(new_object);
1419 			VM_OBJECT_WLOCK(orig_object);
1420 			return;
1421 		}
1422 		LIST_INSERT_HEAD(&source->shadow_head,
1423 				  new_object, shadow_list);
1424 		source->shadow_count++;
1425 		vm_object_reference_locked(source);	/* for new_object */
1426 		vm_object_clear_flag(source, OBJ_ONEMAPPING);
1427 		VM_OBJECT_WUNLOCK(source);
1428 		new_object->backing_object_offset =
1429 			orig_object->backing_object_offset + entry->offset;
1430 		new_object->backing_object = source;
1431 	}
1432 	if (orig_object->cred != NULL) {
1433 		new_object->cred = orig_object->cred;
1434 		crhold(orig_object->cred);
1435 		new_object->charge = ptoa(size);
1436 		KASSERT(orig_object->charge >= ptoa(size),
1437 		    ("orig_object->charge < 0"));
1438 		orig_object->charge -= ptoa(size);
1439 	}
1440 retry:
1441 	m = vm_page_find_least(orig_object, offidxstart);
1442 	for (; m != NULL && (idx = m->pindex - offidxstart) < size;
1443 	    m = m_next) {
1444 		m_next = TAILQ_NEXT(m, listq);
1445 
1446 		/*
1447 		 * We must wait for pending I/O to complete before we can
1448 		 * rename the page.
1449 		 *
1450 		 * We do not have to VM_PROT_NONE the page as mappings should
1451 		 * not be changed by this operation.
1452 		 */
1453 		if (vm_page_busied(m)) {
1454 			VM_OBJECT_WUNLOCK(new_object);
1455 			vm_page_lock(m);
1456 			VM_OBJECT_WUNLOCK(orig_object);
1457 			vm_page_busy_sleep(m, "spltwt", false);
1458 			VM_OBJECT_WLOCK(orig_object);
1459 			VM_OBJECT_WLOCK(new_object);
1460 			goto retry;
1461 		}
1462 
1463 		/* vm_page_rename() will dirty the page. */
1464 		if (vm_page_rename(m, new_object, idx)) {
1465 			VM_OBJECT_WUNLOCK(new_object);
1466 			VM_OBJECT_WUNLOCK(orig_object);
1467 			vm_radix_wait();
1468 			VM_OBJECT_WLOCK(orig_object);
1469 			VM_OBJECT_WLOCK(new_object);
1470 			goto retry;
1471 		}
1472 #if VM_NRESERVLEVEL > 0
1473 		/*
1474 		 * If some of the reservation's allocated pages remain with
1475 		 * the original object, then transferring the reservation to
1476 		 * the new object is neither particularly beneficial nor
1477 		 * particularly harmful as compared to leaving the reservation
1478 		 * with the original object.  If, however, all of the
1479 		 * reservation's allocated pages are transferred to the new
1480 		 * object, then transferring the reservation is typically
1481 		 * beneficial.  Determining which of these two cases applies
1482 		 * would be more costly than unconditionally renaming the
1483 		 * reservation.
1484 		 */
1485 		vm_reserv_rename(m, new_object, orig_object, offidxstart);
1486 #endif
1487 		if (orig_object->type == OBJT_SWAP)
1488 			vm_page_xbusy(m);
1489 	}
1490 	if (orig_object->type == OBJT_SWAP) {
1491 		/*
1492 		 * swap_pager_copy() can sleep, in which case the orig_object's
1493 		 * and new_object's locks are released and reacquired.
1494 		 */
1495 		swap_pager_copy(orig_object, new_object, offidxstart, 0);
1496 		TAILQ_FOREACH(m, &new_object->memq, listq)
1497 			vm_page_xunbusy(m);
1498 	}
1499 	VM_OBJECT_WUNLOCK(orig_object);
1500 	VM_OBJECT_WUNLOCK(new_object);
1501 	entry->object.vm_object = new_object;
1502 	entry->offset = 0LL;
1503 	vm_object_deallocate(orig_object);
1504 	VM_OBJECT_WLOCK(new_object);
1505 }
1506 
1507 #define	OBSC_COLLAPSE_NOWAIT	0x0002
1508 #define	OBSC_COLLAPSE_WAIT	0x0004
1509 
1510 static vm_page_t
1511 vm_object_collapse_scan_wait(vm_object_t object, vm_page_t p, vm_page_t next,
1512     int op)
1513 {
1514 	vm_object_t backing_object;
1515 
1516 	VM_OBJECT_ASSERT_WLOCKED(object);
1517 	backing_object = object->backing_object;
1518 	VM_OBJECT_ASSERT_WLOCKED(backing_object);
1519 
1520 	KASSERT(p == NULL || vm_page_busied(p), ("unbusy page %p", p));
1521 	KASSERT(p == NULL || p->object == object || p->object == backing_object,
1522 	    ("invalid ownership %p %p %p", p, object, backing_object));
1523 	if ((op & OBSC_COLLAPSE_NOWAIT) != 0)
1524 		return (next);
1525 	if (p != NULL)
1526 		vm_page_lock(p);
1527 	VM_OBJECT_WUNLOCK(object);
1528 	VM_OBJECT_WUNLOCK(backing_object);
1529 	/* The page is only NULL when rename fails. */
1530 	if (p == NULL)
1531 		vm_radix_wait();
1532 	else
1533 		vm_page_busy_sleep(p, "vmocol", false);
1534 	VM_OBJECT_WLOCK(object);
1535 	VM_OBJECT_WLOCK(backing_object);
1536 	return (TAILQ_FIRST(&backing_object->memq));
1537 }
1538 
1539 static bool
1540 vm_object_scan_all_shadowed(vm_object_t object)
1541 {
1542 	vm_object_t backing_object;
1543 	vm_page_t p, pp;
1544 	vm_pindex_t backing_offset_index, new_pindex, pi, ps;
1545 
1546 	VM_OBJECT_ASSERT_WLOCKED(object);
1547 	VM_OBJECT_ASSERT_WLOCKED(object->backing_object);
1548 
1549 	backing_object = object->backing_object;
1550 
1551 	if (backing_object->type != OBJT_DEFAULT &&
1552 	    backing_object->type != OBJT_SWAP)
1553 		return (false);
1554 
1555 	pi = backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
1556 	p = vm_page_find_least(backing_object, pi);
1557 	ps = swap_pager_find_least(backing_object, pi);
1558 
1559 	/*
1560 	 * Only check pages inside the parent object's range and
1561 	 * inside the parent object's mapping of the backing object.
1562 	 */
1563 	for (;; pi++) {
1564 		if (p != NULL && p->pindex < pi)
1565 			p = TAILQ_NEXT(p, listq);
1566 		if (ps < pi)
1567 			ps = swap_pager_find_least(backing_object, pi);
1568 		if (p == NULL && ps >= backing_object->size)
1569 			break;
1570 		else if (p == NULL)
1571 			pi = ps;
1572 		else
1573 			pi = MIN(p->pindex, ps);
1574 
1575 		new_pindex = pi - backing_offset_index;
1576 		if (new_pindex >= object->size)
1577 			break;
1578 
1579 		/*
1580 		 * See if the parent has the page or if the parent's object
1581 		 * pager has the page.  If the parent has the page but the page
1582 		 * is not valid, the parent's object pager must have the page.
1583 		 *
1584 		 * If this fails, the parent does not completely shadow the
1585 		 * object and we might as well give up now.
1586 		 */
1587 		pp = vm_page_lookup(object, new_pindex);
1588 		if ((pp == NULL || pp->valid == 0) &&
1589 		    !vm_pager_has_page(object, new_pindex, NULL, NULL))
1590 			return (false);
1591 	}
1592 	return (true);
1593 }
1594 
1595 static bool
1596 vm_object_collapse_scan(vm_object_t object, int op)
1597 {
1598 	vm_object_t backing_object;
1599 	vm_page_t next, p, pp;
1600 	vm_pindex_t backing_offset_index, new_pindex;
1601 
1602 	VM_OBJECT_ASSERT_WLOCKED(object);
1603 	VM_OBJECT_ASSERT_WLOCKED(object->backing_object);
1604 
1605 	backing_object = object->backing_object;
1606 	backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
1607 
1608 	/*
1609 	 * Initial conditions
1610 	 */
1611 	if ((op & OBSC_COLLAPSE_WAIT) != 0)
1612 		vm_object_set_flag(backing_object, OBJ_DEAD);
1613 
1614 	/*
1615 	 * Our scan
1616 	 */
1617 	for (p = TAILQ_FIRST(&backing_object->memq); p != NULL; p = next) {
1618 		next = TAILQ_NEXT(p, listq);
1619 		new_pindex = p->pindex - backing_offset_index;
1620 
1621 		/*
1622 		 * Check for busy page
1623 		 */
1624 		if (vm_page_busied(p)) {
1625 			next = vm_object_collapse_scan_wait(object, p, next, op);
1626 			continue;
1627 		}
1628 
1629 		KASSERT(p->object == backing_object,
1630 		    ("vm_object_collapse_scan: object mismatch"));
1631 
1632 		if (p->pindex < backing_offset_index ||
1633 		    new_pindex >= object->size) {
1634 			if (backing_object->type == OBJT_SWAP)
1635 				swap_pager_freespace(backing_object, p->pindex,
1636 				    1);
1637 
1638 			/*
1639 			 * Page is out of the parent object's range, we can
1640 			 * simply destroy it.
1641 			 */
1642 			vm_page_lock(p);
1643 			KASSERT(!pmap_page_is_mapped(p),
1644 			    ("freeing mapped page %p", p));
1645 			if (p->wire_count == 0)
1646 				vm_page_free(p);
1647 			else
1648 				vm_page_remove(p);
1649 			vm_page_unlock(p);
1650 			continue;
1651 		}
1652 
1653 		pp = vm_page_lookup(object, new_pindex);
1654 		if (pp != NULL && vm_page_busied(pp)) {
1655 			/*
1656 			 * The page in the parent is busy and possibly not
1657 			 * (yet) valid.  Until its state is finalized by the
1658 			 * busy bit owner, we can't tell whether it shadows the
1659 			 * original page.  Therefore, we must either skip it
1660 			 * and the original (backing_object) page or wait for
1661 			 * its state to be finalized.
1662 			 *
1663 			 * This is due to a race with vm_fault() where we must
1664 			 * unbusy the original (backing_obj) page before we can
1665 			 * (re)lock the parent.  Hence we can get here.
1666 			 */
1667 			next = vm_object_collapse_scan_wait(object, pp, next,
1668 			    op);
1669 			continue;
1670 		}
1671 
1672 		KASSERT(pp == NULL || pp->valid != 0,
1673 		    ("unbusy invalid page %p", pp));
1674 
1675 		if (pp != NULL || vm_pager_has_page(object, new_pindex, NULL,
1676 			NULL)) {
1677 			/*
1678 			 * The page already exists in the parent OR swap exists
1679 			 * for this location in the parent.  Leave the parent's
1680 			 * page alone.  Destroy the original page from the
1681 			 * backing object.
1682 			 */
1683 			if (backing_object->type == OBJT_SWAP)
1684 				swap_pager_freespace(backing_object, p->pindex,
1685 				    1);
1686 			vm_page_lock(p);
1687 			KASSERT(!pmap_page_is_mapped(p),
1688 			    ("freeing mapped page %p", p));
1689 			if (p->wire_count == 0)
1690 				vm_page_free(p);
1691 			else
1692 				vm_page_remove(p);
1693 			vm_page_unlock(p);
1694 			continue;
1695 		}
1696 
1697 		/*
1698 		 * Page does not exist in parent, rename the page from the
1699 		 * backing object to the main object.
1700 		 *
1701 		 * If the page was mapped to a process, it can remain mapped
1702 		 * through the rename.  vm_page_rename() will dirty the page.
1703 		 */
1704 		if (vm_page_rename(p, object, new_pindex)) {
1705 			next = vm_object_collapse_scan_wait(object, NULL, next,
1706 			    op);
1707 			continue;
1708 		}
1709 
1710 		/* Use the old pindex to free the right page. */
1711 		if (backing_object->type == OBJT_SWAP)
1712 			swap_pager_freespace(backing_object,
1713 			    new_pindex + backing_offset_index, 1);
1714 
1715 #if VM_NRESERVLEVEL > 0
1716 		/*
1717 		 * Rename the reservation.
1718 		 */
1719 		vm_reserv_rename(p, object, backing_object,
1720 		    backing_offset_index);
1721 #endif
1722 	}
1723 	return (true);
1724 }
1725 
1726 
1727 /*
1728  * this version of collapse allows the operation to occur earlier and
1729  * when paging_in_progress is true for an object...  This is not a complete
1730  * operation, but should plug 99.9% of the rest of the leaks.
1731  */
1732 static void
1733 vm_object_qcollapse(vm_object_t object)
1734 {
1735 	vm_object_t backing_object = object->backing_object;
1736 
1737 	VM_OBJECT_ASSERT_WLOCKED(object);
1738 	VM_OBJECT_ASSERT_WLOCKED(backing_object);
1739 
1740 	if (backing_object->ref_count != 1)
1741 		return;
1742 
1743 	vm_object_collapse_scan(object, OBSC_COLLAPSE_NOWAIT);
1744 }
1745 
1746 /*
1747  *	vm_object_collapse:
1748  *
1749  *	Collapse an object with the object backing it.
1750  *	Pages in the backing object are moved into the
1751  *	parent, and the backing object is deallocated.
1752  */
1753 void
1754 vm_object_collapse(vm_object_t object)
1755 {
1756 	vm_object_t backing_object, new_backing_object;
1757 
1758 	VM_OBJECT_ASSERT_WLOCKED(object);
1759 
1760 	while (TRUE) {
1761 		/*
1762 		 * Verify that the conditions are right for collapse:
1763 		 *
1764 		 * The object exists and the backing object exists.
1765 		 */
1766 		if ((backing_object = object->backing_object) == NULL)
1767 			break;
1768 
1769 		/*
1770 		 * we check the backing object first, because it is most likely
1771 		 * not collapsable.
1772 		 */
1773 		VM_OBJECT_WLOCK(backing_object);
1774 		if (backing_object->handle != NULL ||
1775 		    (backing_object->type != OBJT_DEFAULT &&
1776 		     backing_object->type != OBJT_SWAP) ||
1777 		    (backing_object->flags & OBJ_DEAD) ||
1778 		    object->handle != NULL ||
1779 		    (object->type != OBJT_DEFAULT &&
1780 		     object->type != OBJT_SWAP) ||
1781 		    (object->flags & OBJ_DEAD)) {
1782 			VM_OBJECT_WUNLOCK(backing_object);
1783 			break;
1784 		}
1785 
1786 		if (object->paging_in_progress != 0 ||
1787 		    backing_object->paging_in_progress != 0) {
1788 			vm_object_qcollapse(object);
1789 			VM_OBJECT_WUNLOCK(backing_object);
1790 			break;
1791 		}
1792 
1793 		/*
1794 		 * We know that we can either collapse the backing object (if
1795 		 * the parent is the only reference to it) or (perhaps) have
1796 		 * the parent bypass the object if the parent happens to shadow
1797 		 * all the resident pages in the entire backing object.
1798 		 *
1799 		 * This is ignoring pager-backed pages such as swap pages.
1800 		 * vm_object_collapse_scan fails the shadowing test in this
1801 		 * case.
1802 		 */
1803 		if (backing_object->ref_count == 1) {
1804 			vm_object_pip_add(object, 1);
1805 			vm_object_pip_add(backing_object, 1);
1806 
1807 			/*
1808 			 * If there is exactly one reference to the backing
1809 			 * object, we can collapse it into the parent.
1810 			 */
1811 			vm_object_collapse_scan(object, OBSC_COLLAPSE_WAIT);
1812 
1813 #if VM_NRESERVLEVEL > 0
1814 			/*
1815 			 * Break any reservations from backing_object.
1816 			 */
1817 			if (__predict_false(!LIST_EMPTY(&backing_object->rvq)))
1818 				vm_reserv_break_all(backing_object);
1819 #endif
1820 
1821 			/*
1822 			 * Move the pager from backing_object to object.
1823 			 */
1824 			if (backing_object->type == OBJT_SWAP) {
1825 				/*
1826 				 * swap_pager_copy() can sleep, in which case
1827 				 * the backing_object's and object's locks are
1828 				 * released and reacquired.
1829 				 * Since swap_pager_copy() is being asked to
1830 				 * destroy the source, it will change the
1831 				 * backing_object's type to OBJT_DEFAULT.
1832 				 */
1833 				swap_pager_copy(
1834 				    backing_object,
1835 				    object,
1836 				    OFF_TO_IDX(object->backing_object_offset), TRUE);
1837 			}
1838 			/*
1839 			 * Object now shadows whatever backing_object did.
1840 			 * Note that the reference to
1841 			 * backing_object->backing_object moves from within
1842 			 * backing_object to within object.
1843 			 */
1844 			LIST_REMOVE(object, shadow_list);
1845 			backing_object->shadow_count--;
1846 			if (backing_object->backing_object) {
1847 				VM_OBJECT_WLOCK(backing_object->backing_object);
1848 				LIST_REMOVE(backing_object, shadow_list);
1849 				LIST_INSERT_HEAD(
1850 				    &backing_object->backing_object->shadow_head,
1851 				    object, shadow_list);
1852 				/*
1853 				 * The shadow_count has not changed.
1854 				 */
1855 				VM_OBJECT_WUNLOCK(backing_object->backing_object);
1856 			}
1857 			object->backing_object = backing_object->backing_object;
1858 			object->backing_object_offset +=
1859 			    backing_object->backing_object_offset;
1860 
1861 			/*
1862 			 * Discard backing_object.
1863 			 *
1864 			 * Since the backing object has no pages, no pager left,
1865 			 * and no object references within it, all that is
1866 			 * necessary is to dispose of it.
1867 			 */
1868 			KASSERT(backing_object->ref_count == 1, (
1869 "backing_object %p was somehow re-referenced during collapse!",
1870 			    backing_object));
1871 			vm_object_pip_wakeup(backing_object);
1872 			backing_object->type = OBJT_DEAD;
1873 			backing_object->ref_count = 0;
1874 			VM_OBJECT_WUNLOCK(backing_object);
1875 			vm_object_destroy(backing_object);
1876 
1877 			vm_object_pip_wakeup(object);
1878 			object_collapses++;
1879 		} else {
1880 			/*
1881 			 * If we do not entirely shadow the backing object,
1882 			 * there is nothing we can do so we give up.
1883 			 */
1884 			if (object->resident_page_count != object->size &&
1885 			    !vm_object_scan_all_shadowed(object)) {
1886 				VM_OBJECT_WUNLOCK(backing_object);
1887 				break;
1888 			}
1889 
1890 			/*
1891 			 * Make the parent shadow the next object in the
1892 			 * chain.  Deallocating backing_object will not remove
1893 			 * it, since its reference count is at least 2.
1894 			 */
1895 			LIST_REMOVE(object, shadow_list);
1896 			backing_object->shadow_count--;
1897 
1898 			new_backing_object = backing_object->backing_object;
1899 			if ((object->backing_object = new_backing_object) != NULL) {
1900 				VM_OBJECT_WLOCK(new_backing_object);
1901 				LIST_INSERT_HEAD(
1902 				    &new_backing_object->shadow_head,
1903 				    object,
1904 				    shadow_list
1905 				);
1906 				new_backing_object->shadow_count++;
1907 				vm_object_reference_locked(new_backing_object);
1908 				VM_OBJECT_WUNLOCK(new_backing_object);
1909 				object->backing_object_offset +=
1910 					backing_object->backing_object_offset;
1911 			}
1912 
1913 			/*
1914 			 * Drop the reference count on backing_object. Since
1915 			 * its ref_count was at least 2, it will not vanish.
1916 			 */
1917 			backing_object->ref_count--;
1918 			VM_OBJECT_WUNLOCK(backing_object);
1919 			object_bypasses++;
1920 		}
1921 
1922 		/*
1923 		 * Try again with this object's new backing object.
1924 		 */
1925 	}
1926 }
1927 
1928 /*
1929  *	vm_object_page_remove:
1930  *
1931  *	For the given object, either frees or invalidates each of the
1932  *	specified pages.  In general, a page is freed.  However, if a page is
1933  *	wired for any reason other than the existence of a managed, wired
1934  *	mapping, then it may be invalidated but not removed from the object.
1935  *	Pages are specified by the given range ["start", "end") and the option
1936  *	OBJPR_CLEANONLY.  As a special case, if "end" is zero, then the range
1937  *	extends from "start" to the end of the object.  If the option
1938  *	OBJPR_CLEANONLY is specified, then only the non-dirty pages within the
1939  *	specified range are affected.  If the option OBJPR_NOTMAPPED is
1940  *	specified, then the pages within the specified range must have no
1941  *	mappings.  Otherwise, if this option is not specified, any mappings to
1942  *	the specified pages are removed before the pages are freed or
1943  *	invalidated.
1944  *
1945  *	In general, this operation should only be performed on objects that
1946  *	contain managed pages.  There are, however, two exceptions.  First, it
1947  *	is performed on the kernel and kmem objects by vm_map_entry_delete().
1948  *	Second, it is used by msync(..., MS_INVALIDATE) to invalidate device-
1949  *	backed pages.  In both of these cases, the option OBJPR_CLEANONLY must
1950  *	not be specified and the option OBJPR_NOTMAPPED must be specified.
1951  *
1952  *	The object must be locked.
1953  */
1954 void
1955 vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
1956     int options)
1957 {
1958 	vm_page_t p, next;
1959 	struct mtx *mtx;
1960 	struct pglist pgl;
1961 
1962 	VM_OBJECT_ASSERT_WLOCKED(object);
1963 	KASSERT((object->flags & OBJ_UNMANAGED) == 0 ||
1964 	    (options & (OBJPR_CLEANONLY | OBJPR_NOTMAPPED)) == OBJPR_NOTMAPPED,
1965 	    ("vm_object_page_remove: illegal options for object %p", object));
1966 	if (object->resident_page_count == 0)
1967 		return;
1968 	vm_object_pip_add(object, 1);
1969 	TAILQ_INIT(&pgl);
1970 again:
1971 	p = vm_page_find_least(object, start);
1972 	mtx = NULL;
1973 
1974 	/*
1975 	 * Here, the variable "p" is either (1) the page with the least pindex
1976 	 * greater than or equal to the parameter "start" or (2) NULL.
1977 	 */
1978 	for (; p != NULL && (p->pindex < end || end == 0); p = next) {
1979 		next = TAILQ_NEXT(p, listq);
1980 
1981 		/*
1982 		 * If the page is wired for any reason besides the existence
1983 		 * of managed, wired mappings, then it cannot be freed.  For
1984 		 * example, fictitious pages, which represent device memory,
1985 		 * are inherently wired and cannot be freed.  They can,
1986 		 * however, be invalidated if the option OBJPR_CLEANONLY is
1987 		 * not specified.
1988 		 */
1989 		vm_page_change_lock(p, &mtx);
1990 		if (vm_page_xbusied(p)) {
1991 			VM_OBJECT_WUNLOCK(object);
1992 			vm_page_busy_sleep(p, "vmopax", true);
1993 			VM_OBJECT_WLOCK(object);
1994 			goto again;
1995 		}
1996 		if (p->wire_count != 0) {
1997 			if ((options & OBJPR_NOTMAPPED) == 0 &&
1998 			    object->ref_count != 0)
1999 				pmap_remove_all(p);
2000 			if ((options & OBJPR_CLEANONLY) == 0) {
2001 				p->valid = 0;
2002 				vm_page_undirty(p);
2003 			}
2004 			continue;
2005 		}
2006 		if (vm_page_busied(p)) {
2007 			VM_OBJECT_WUNLOCK(object);
2008 			vm_page_busy_sleep(p, "vmopar", false);
2009 			VM_OBJECT_WLOCK(object);
2010 			goto again;
2011 		}
2012 		KASSERT((p->flags & PG_FICTITIOUS) == 0,
2013 		    ("vm_object_page_remove: page %p is fictitious", p));
2014 		if ((options & OBJPR_CLEANONLY) != 0 && p->valid != 0) {
2015 			if ((options & OBJPR_NOTMAPPED) == 0 &&
2016 			    object->ref_count != 0)
2017 				pmap_remove_write(p);
2018 			if (p->dirty != 0)
2019 				continue;
2020 		}
2021 		if ((options & OBJPR_NOTMAPPED) == 0 && object->ref_count != 0)
2022 			pmap_remove_all(p);
2023 		p->flags &= ~PG_ZERO;
2024 		if (vm_page_free_prep(p, false))
2025 			TAILQ_INSERT_TAIL(&pgl, p, listq);
2026 	}
2027 	if (mtx != NULL)
2028 		mtx_unlock(mtx);
2029 	vm_page_free_phys_pglist(&pgl);
2030 	vm_object_pip_wakeup(object);
2031 }
2032 
2033 /*
2034  *	vm_object_page_noreuse:
2035  *
2036  *	For the given object, attempt to move the specified pages to
2037  *	the head of the inactive queue.  This bypasses regular LRU
2038  *	operation and allows the pages to be reused quickly under memory
2039  *	pressure.  If a page is wired for any reason, then it will not
2040  *	be queued.  Pages are specified by the range ["start", "end").
2041  *	As a special case, if "end" is zero, then the range extends from
2042  *	"start" to the end of the object.
2043  *
2044  *	This operation should only be performed on objects that
2045  *	contain non-fictitious, managed pages.
2046  *
2047  *	The object must be locked.
2048  */
2049 void
2050 vm_object_page_noreuse(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
2051 {
2052 	struct mtx *mtx;
2053 	vm_page_t p, next;
2054 
2055 	VM_OBJECT_ASSERT_LOCKED(object);
2056 	KASSERT((object->flags & (OBJ_FICTITIOUS | OBJ_UNMANAGED)) == 0,
2057 	    ("vm_object_page_noreuse: illegal object %p", object));
2058 	if (object->resident_page_count == 0)
2059 		return;
2060 	p = vm_page_find_least(object, start);
2061 
2062 	/*
2063 	 * Here, the variable "p" is either (1) the page with the least pindex
2064 	 * greater than or equal to the parameter "start" or (2) NULL.
2065 	 */
2066 	mtx = NULL;
2067 	for (; p != NULL && (p->pindex < end || end == 0); p = next) {
2068 		next = TAILQ_NEXT(p, listq);
2069 		vm_page_change_lock(p, &mtx);
2070 		vm_page_deactivate_noreuse(p);
2071 	}
2072 	if (mtx != NULL)
2073 		mtx_unlock(mtx);
2074 }
2075 
2076 /*
2077  *	Populate the specified range of the object with valid pages.  Returns
2078  *	TRUE if the range is successfully populated and FALSE otherwise.
2079  *
2080  *	Note: This function should be optimized to pass a larger array of
2081  *	pages to vm_pager_get_pages() before it is applied to a non-
2082  *	OBJT_DEVICE object.
2083  *
2084  *	The object must be locked.
2085  */
2086 boolean_t
2087 vm_object_populate(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
2088 {
2089 	vm_page_t m;
2090 	vm_pindex_t pindex;
2091 	int rv;
2092 
2093 	VM_OBJECT_ASSERT_WLOCKED(object);
2094 	for (pindex = start; pindex < end; pindex++) {
2095 		m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL);
2096 		if (m->valid != VM_PAGE_BITS_ALL) {
2097 			rv = vm_pager_get_pages(object, &m, 1, NULL, NULL);
2098 			if (rv != VM_PAGER_OK) {
2099 				vm_page_lock(m);
2100 				vm_page_free(m);
2101 				vm_page_unlock(m);
2102 				break;
2103 			}
2104 		}
2105 		/*
2106 		 * Keep "m" busy because a subsequent iteration may unlock
2107 		 * the object.
2108 		 */
2109 	}
2110 	if (pindex > start) {
2111 		m = vm_page_lookup(object, start);
2112 		while (m != NULL && m->pindex < pindex) {
2113 			vm_page_xunbusy(m);
2114 			m = TAILQ_NEXT(m, listq);
2115 		}
2116 	}
2117 	return (pindex == end);
2118 }
2119 
2120 /*
2121  *	Routine:	vm_object_coalesce
2122  *	Function:	Coalesces two objects backing up adjoining
2123  *			regions of memory into a single object.
2124  *
2125  *	returns TRUE if objects were combined.
2126  *
2127  *	NOTE:	Only works at the moment if the second object is NULL -
2128  *		if it's not, which object do we lock first?
2129  *
2130  *	Parameters:
2131  *		prev_object	First object to coalesce
2132  *		prev_offset	Offset into prev_object
2133  *		prev_size	Size of reference to prev_object
2134  *		next_size	Size of reference to the second object
2135  *		reserved	Indicator that extension region has
2136  *				swap accounted for
2137  *
2138  *	Conditions:
2139  *	The object must *not* be locked.
2140  */
2141 boolean_t
2142 vm_object_coalesce(vm_object_t prev_object, vm_ooffset_t prev_offset,
2143     vm_size_t prev_size, vm_size_t next_size, boolean_t reserved)
2144 {
2145 	vm_pindex_t next_pindex;
2146 
2147 	if (prev_object == NULL)
2148 		return (TRUE);
2149 	VM_OBJECT_WLOCK(prev_object);
2150 	if ((prev_object->type != OBJT_DEFAULT &&
2151 	    prev_object->type != OBJT_SWAP) ||
2152 	    (prev_object->flags & OBJ_TMPFS_NODE) != 0) {
2153 		VM_OBJECT_WUNLOCK(prev_object);
2154 		return (FALSE);
2155 	}
2156 
2157 	/*
2158 	 * Try to collapse the object first
2159 	 */
2160 	vm_object_collapse(prev_object);
2161 
2162 	/*
2163 	 * Can't coalesce if: . more than one reference . paged out . shadows
2164 	 * another object . has a copy elsewhere (any of which mean that the
2165 	 * pages not mapped to prev_entry may be in use anyway)
2166 	 */
2167 	if (prev_object->backing_object != NULL) {
2168 		VM_OBJECT_WUNLOCK(prev_object);
2169 		return (FALSE);
2170 	}
2171 
2172 	prev_size >>= PAGE_SHIFT;
2173 	next_size >>= PAGE_SHIFT;
2174 	next_pindex = OFF_TO_IDX(prev_offset) + prev_size;
2175 
2176 	if ((prev_object->ref_count > 1) &&
2177 	    (prev_object->size != next_pindex)) {
2178 		VM_OBJECT_WUNLOCK(prev_object);
2179 		return (FALSE);
2180 	}
2181 
2182 	/*
2183 	 * Account for the charge.
2184 	 */
2185 	if (prev_object->cred != NULL) {
2186 
2187 		/*
2188 		 * If prev_object was charged, then this mapping,
2189 		 * although not charged now, may become writable
2190 		 * later. Non-NULL cred in the object would prevent
2191 		 * swap reservation during enabling of the write
2192 		 * access, so reserve swap now. Failed reservation
2193 		 * cause allocation of the separate object for the map
2194 		 * entry, and swap reservation for this entry is
2195 		 * managed in appropriate time.
2196 		 */
2197 		if (!reserved && !swap_reserve_by_cred(ptoa(next_size),
2198 		    prev_object->cred)) {
2199 			VM_OBJECT_WUNLOCK(prev_object);
2200 			return (FALSE);
2201 		}
2202 		prev_object->charge += ptoa(next_size);
2203 	}
2204 
2205 	/*
2206 	 * Remove any pages that may still be in the object from a previous
2207 	 * deallocation.
2208 	 */
2209 	if (next_pindex < prev_object->size) {
2210 		vm_object_page_remove(prev_object, next_pindex, next_pindex +
2211 		    next_size, 0);
2212 		if (prev_object->type == OBJT_SWAP)
2213 			swap_pager_freespace(prev_object,
2214 					     next_pindex, next_size);
2215 #if 0
2216 		if (prev_object->cred != NULL) {
2217 			KASSERT(prev_object->charge >=
2218 			    ptoa(prev_object->size - next_pindex),
2219 			    ("object %p overcharged 1 %jx %jx", prev_object,
2220 				(uintmax_t)next_pindex, (uintmax_t)next_size));
2221 			prev_object->charge -= ptoa(prev_object->size -
2222 			    next_pindex);
2223 		}
2224 #endif
2225 	}
2226 
2227 	/*
2228 	 * Extend the object if necessary.
2229 	 */
2230 	if (next_pindex + next_size > prev_object->size)
2231 		prev_object->size = next_pindex + next_size;
2232 
2233 	VM_OBJECT_WUNLOCK(prev_object);
2234 	return (TRUE);
2235 }
2236 
2237 void
2238 vm_object_set_writeable_dirty(vm_object_t object)
2239 {
2240 
2241 	VM_OBJECT_ASSERT_WLOCKED(object);
2242 	if (object->type != OBJT_VNODE) {
2243 		if ((object->flags & OBJ_TMPFS_NODE) != 0) {
2244 			KASSERT(object->type == OBJT_SWAP, ("non-swap tmpfs"));
2245 			vm_object_set_flag(object, OBJ_TMPFS_DIRTY);
2246 		}
2247 		return;
2248 	}
2249 	object->generation++;
2250 	if ((object->flags & OBJ_MIGHTBEDIRTY) != 0)
2251 		return;
2252 	vm_object_set_flag(object, OBJ_MIGHTBEDIRTY);
2253 }
2254 
2255 /*
2256  *	vm_object_unwire:
2257  *
2258  *	For each page offset within the specified range of the given object,
2259  *	find the highest-level page in the shadow chain and unwire it.  A page
2260  *	must exist at every page offset, and the highest-level page must be
2261  *	wired.
2262  */
2263 void
2264 vm_object_unwire(vm_object_t object, vm_ooffset_t offset, vm_size_t length,
2265     uint8_t queue)
2266 {
2267 	vm_object_t tobject;
2268 	vm_page_t m, tm;
2269 	vm_pindex_t end_pindex, pindex, tpindex;
2270 	int depth, locked_depth;
2271 
2272 	KASSERT((offset & PAGE_MASK) == 0,
2273 	    ("vm_object_unwire: offset is not page aligned"));
2274 	KASSERT((length & PAGE_MASK) == 0,
2275 	    ("vm_object_unwire: length is not a multiple of PAGE_SIZE"));
2276 	/* The wired count of a fictitious page never changes. */
2277 	if ((object->flags & OBJ_FICTITIOUS) != 0)
2278 		return;
2279 	pindex = OFF_TO_IDX(offset);
2280 	end_pindex = pindex + atop(length);
2281 	locked_depth = 1;
2282 	VM_OBJECT_RLOCK(object);
2283 	m = vm_page_find_least(object, pindex);
2284 	while (pindex < end_pindex) {
2285 		if (m == NULL || pindex < m->pindex) {
2286 			/*
2287 			 * The first object in the shadow chain doesn't
2288 			 * contain a page at the current index.  Therefore,
2289 			 * the page must exist in a backing object.
2290 			 */
2291 			tobject = object;
2292 			tpindex = pindex;
2293 			depth = 0;
2294 			do {
2295 				tpindex +=
2296 				    OFF_TO_IDX(tobject->backing_object_offset);
2297 				tobject = tobject->backing_object;
2298 				KASSERT(tobject != NULL,
2299 				    ("vm_object_unwire: missing page"));
2300 				if ((tobject->flags & OBJ_FICTITIOUS) != 0)
2301 					goto next_page;
2302 				depth++;
2303 				if (depth == locked_depth) {
2304 					locked_depth++;
2305 					VM_OBJECT_RLOCK(tobject);
2306 				}
2307 			} while ((tm = vm_page_lookup(tobject, tpindex)) ==
2308 			    NULL);
2309 		} else {
2310 			tm = m;
2311 			m = TAILQ_NEXT(m, listq);
2312 		}
2313 		vm_page_lock(tm);
2314 		vm_page_unwire(tm, queue);
2315 		vm_page_unlock(tm);
2316 next_page:
2317 		pindex++;
2318 	}
2319 	/* Release the accumulated object locks. */
2320 	for (depth = 0; depth < locked_depth; depth++) {
2321 		tobject = object->backing_object;
2322 		VM_OBJECT_RUNLOCK(object);
2323 		object = tobject;
2324 	}
2325 }
2326 
2327 struct vnode *
2328 vm_object_vnode(vm_object_t object)
2329 {
2330 
2331 	VM_OBJECT_ASSERT_LOCKED(object);
2332 	if (object->type == OBJT_VNODE)
2333 		return (object->handle);
2334 	if (object->type == OBJT_SWAP && (object->flags & OBJ_TMPFS) != 0)
2335 		return (object->un_pager.swp.swp_tmpfs);
2336 	return (NULL);
2337 }
2338 
2339 static int
2340 sysctl_vm_object_list(SYSCTL_HANDLER_ARGS)
2341 {
2342 	struct kinfo_vmobject *kvo;
2343 	char *fullpath, *freepath;
2344 	struct vnode *vp;
2345 	struct vattr va;
2346 	vm_object_t obj;
2347 	vm_page_t m;
2348 	int count, error;
2349 
2350 	if (req->oldptr == NULL) {
2351 		/*
2352 		 * If an old buffer has not been provided, generate an
2353 		 * estimate of the space needed for a subsequent call.
2354 		 */
2355 		mtx_lock(&vm_object_list_mtx);
2356 		count = 0;
2357 		TAILQ_FOREACH(obj, &vm_object_list, object_list) {
2358 			if (obj->type == OBJT_DEAD)
2359 				continue;
2360 			count++;
2361 		}
2362 		mtx_unlock(&vm_object_list_mtx);
2363 		return (SYSCTL_OUT(req, NULL, sizeof(struct kinfo_vmobject) *
2364 		    count * 11 / 10));
2365 	}
2366 
2367 	kvo = malloc(sizeof(*kvo), M_TEMP, M_WAITOK);
2368 	error = 0;
2369 
2370 	/*
2371 	 * VM objects are type stable and are never removed from the
2372 	 * list once added.  This allows us to safely read obj->object_list
2373 	 * after reacquiring the VM object lock.
2374 	 */
2375 	mtx_lock(&vm_object_list_mtx);
2376 	TAILQ_FOREACH(obj, &vm_object_list, object_list) {
2377 		if (obj->type == OBJT_DEAD)
2378 			continue;
2379 		VM_OBJECT_RLOCK(obj);
2380 		if (obj->type == OBJT_DEAD) {
2381 			VM_OBJECT_RUNLOCK(obj);
2382 			continue;
2383 		}
2384 		mtx_unlock(&vm_object_list_mtx);
2385 		kvo->kvo_size = ptoa(obj->size);
2386 		kvo->kvo_resident = obj->resident_page_count;
2387 		kvo->kvo_ref_count = obj->ref_count;
2388 		kvo->kvo_shadow_count = obj->shadow_count;
2389 		kvo->kvo_memattr = obj->memattr;
2390 		kvo->kvo_active = 0;
2391 		kvo->kvo_inactive = 0;
2392 		TAILQ_FOREACH(m, &obj->memq, listq) {
2393 			/*
2394 			 * A page may belong to the object but be
2395 			 * dequeued and set to PQ_NONE while the
2396 			 * object lock is not held.  This makes the
2397 			 * reads of m->queue below racy, and we do not
2398 			 * count pages set to PQ_NONE.  However, this
2399 			 * sysctl is only meant to give an
2400 			 * approximation of the system anyway.
2401 			 */
2402 			if (vm_page_active(m))
2403 				kvo->kvo_active++;
2404 			else if (vm_page_inactive(m))
2405 				kvo->kvo_inactive++;
2406 		}
2407 
2408 		kvo->kvo_vn_fileid = 0;
2409 		kvo->kvo_vn_fsid = 0;
2410 		kvo->kvo_vn_fsid_freebsd11 = 0;
2411 		freepath = NULL;
2412 		fullpath = "";
2413 		vp = NULL;
2414 		switch (obj->type) {
2415 		case OBJT_DEFAULT:
2416 			kvo->kvo_type = KVME_TYPE_DEFAULT;
2417 			break;
2418 		case OBJT_VNODE:
2419 			kvo->kvo_type = KVME_TYPE_VNODE;
2420 			vp = obj->handle;
2421 			vref(vp);
2422 			break;
2423 		case OBJT_SWAP:
2424 			kvo->kvo_type = KVME_TYPE_SWAP;
2425 			break;
2426 		case OBJT_DEVICE:
2427 			kvo->kvo_type = KVME_TYPE_DEVICE;
2428 			break;
2429 		case OBJT_PHYS:
2430 			kvo->kvo_type = KVME_TYPE_PHYS;
2431 			break;
2432 		case OBJT_DEAD:
2433 			kvo->kvo_type = KVME_TYPE_DEAD;
2434 			break;
2435 		case OBJT_SG:
2436 			kvo->kvo_type = KVME_TYPE_SG;
2437 			break;
2438 		case OBJT_MGTDEVICE:
2439 			kvo->kvo_type = KVME_TYPE_MGTDEVICE;
2440 			break;
2441 		default:
2442 			kvo->kvo_type = KVME_TYPE_UNKNOWN;
2443 			break;
2444 		}
2445 		VM_OBJECT_RUNLOCK(obj);
2446 		if (vp != NULL) {
2447 			vn_fullpath(curthread, vp, &fullpath, &freepath);
2448 			vn_lock(vp, LK_SHARED | LK_RETRY);
2449 			if (VOP_GETATTR(vp, &va, curthread->td_ucred) == 0) {
2450 				kvo->kvo_vn_fileid = va.va_fileid;
2451 				kvo->kvo_vn_fsid = va.va_fsid;
2452 				kvo->kvo_vn_fsid_freebsd11 = va.va_fsid;
2453 								/* truncate */
2454 			}
2455 			vput(vp);
2456 		}
2457 
2458 		strlcpy(kvo->kvo_path, fullpath, sizeof(kvo->kvo_path));
2459 		if (freepath != NULL)
2460 			free(freepath, M_TEMP);
2461 
2462 		/* Pack record size down */
2463 		kvo->kvo_structsize = offsetof(struct kinfo_vmobject, kvo_path)
2464 		    + strlen(kvo->kvo_path) + 1;
2465 		kvo->kvo_structsize = roundup(kvo->kvo_structsize,
2466 		    sizeof(uint64_t));
2467 		error = SYSCTL_OUT(req, kvo, kvo->kvo_structsize);
2468 		mtx_lock(&vm_object_list_mtx);
2469 		if (error)
2470 			break;
2471 	}
2472 	mtx_unlock(&vm_object_list_mtx);
2473 	free(kvo, M_TEMP);
2474 	return (error);
2475 }
2476 SYSCTL_PROC(_vm, OID_AUTO, objects, CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_SKIP |
2477     CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_object_list, "S,kinfo_vmobject",
2478     "List of VM objects");
2479 
2480 #include "opt_ddb.h"
2481 #ifdef DDB
2482 #include <sys/kernel.h>
2483 
2484 #include <sys/cons.h>
2485 
2486 #include <ddb/ddb.h>
2487 
2488 static int
2489 _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry)
2490 {
2491 	vm_map_t tmpm;
2492 	vm_map_entry_t tmpe;
2493 	vm_object_t obj;
2494 	int entcount;
2495 
2496 	if (map == 0)
2497 		return 0;
2498 
2499 	if (entry == 0) {
2500 		tmpe = map->header.next;
2501 		entcount = map->nentries;
2502 		while (entcount-- && (tmpe != &map->header)) {
2503 			if (_vm_object_in_map(map, object, tmpe)) {
2504 				return 1;
2505 			}
2506 			tmpe = tmpe->next;
2507 		}
2508 	} else if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
2509 		tmpm = entry->object.sub_map;
2510 		tmpe = tmpm->header.next;
2511 		entcount = tmpm->nentries;
2512 		while (entcount-- && tmpe != &tmpm->header) {
2513 			if (_vm_object_in_map(tmpm, object, tmpe)) {
2514 				return 1;
2515 			}
2516 			tmpe = tmpe->next;
2517 		}
2518 	} else if ((obj = entry->object.vm_object) != NULL) {
2519 		for (; obj; obj = obj->backing_object)
2520 			if (obj == object) {
2521 				return 1;
2522 			}
2523 	}
2524 	return 0;
2525 }
2526 
2527 static int
2528 vm_object_in_map(vm_object_t object)
2529 {
2530 	struct proc *p;
2531 
2532 	/* sx_slock(&allproc_lock); */
2533 	FOREACH_PROC_IN_SYSTEM(p) {
2534 		if (!p->p_vmspace /* || (p->p_flag & (P_SYSTEM|P_WEXIT)) */)
2535 			continue;
2536 		if (_vm_object_in_map(&p->p_vmspace->vm_map, object, 0)) {
2537 			/* sx_sunlock(&allproc_lock); */
2538 			return 1;
2539 		}
2540 	}
2541 	/* sx_sunlock(&allproc_lock); */
2542 	if (_vm_object_in_map(kernel_map, object, 0))
2543 		return 1;
2544 	return 0;
2545 }
2546 
2547 DB_SHOW_COMMAND(vmochk, vm_object_check)
2548 {
2549 	vm_object_t object;
2550 
2551 	/*
2552 	 * make sure that internal objs are in a map somewhere
2553 	 * and none have zero ref counts.
2554 	 */
2555 	TAILQ_FOREACH(object, &vm_object_list, object_list) {
2556 		if (object->handle == NULL &&
2557 		    (object->type == OBJT_DEFAULT || object->type == OBJT_SWAP)) {
2558 			if (object->ref_count == 0) {
2559 				db_printf("vmochk: internal obj has zero ref count: %ld\n",
2560 					(long)object->size);
2561 			}
2562 			if (!vm_object_in_map(object)) {
2563 				db_printf(
2564 			"vmochk: internal obj is not in a map: "
2565 			"ref: %d, size: %lu: 0x%lx, backing_object: %p\n",
2566 				    object->ref_count, (u_long)object->size,
2567 				    (u_long)object->size,
2568 				    (void *)object->backing_object);
2569 			}
2570 		}
2571 	}
2572 }
2573 
2574 /*
2575  *	vm_object_print:	[ debug ]
2576  */
2577 DB_SHOW_COMMAND(object, vm_object_print_static)
2578 {
2579 	/* XXX convert args. */
2580 	vm_object_t object = (vm_object_t)addr;
2581 	boolean_t full = have_addr;
2582 
2583 	vm_page_t p;
2584 
2585 	/* XXX count is an (unused) arg.  Avoid shadowing it. */
2586 #define	count	was_count
2587 
2588 	int count;
2589 
2590 	if (object == NULL)
2591 		return;
2592 
2593 	db_iprintf(
2594 	    "Object %p: type=%d, size=0x%jx, res=%d, ref=%d, flags=0x%x ruid %d charge %jx\n",
2595 	    object, (int)object->type, (uintmax_t)object->size,
2596 	    object->resident_page_count, object->ref_count, object->flags,
2597 	    object->cred ? object->cred->cr_ruid : -1, (uintmax_t)object->charge);
2598 	db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%jx\n",
2599 	    object->shadow_count,
2600 	    object->backing_object ? object->backing_object->ref_count : 0,
2601 	    object->backing_object, (uintmax_t)object->backing_object_offset);
2602 
2603 	if (!full)
2604 		return;
2605 
2606 	db_indent += 2;
2607 	count = 0;
2608 	TAILQ_FOREACH(p, &object->memq, listq) {
2609 		if (count == 0)
2610 			db_iprintf("memory:=");
2611 		else if (count == 6) {
2612 			db_printf("\n");
2613 			db_iprintf(" ...");
2614 			count = 0;
2615 		} else
2616 			db_printf(",");
2617 		count++;
2618 
2619 		db_printf("(off=0x%jx,page=0x%jx)",
2620 		    (uintmax_t)p->pindex, (uintmax_t)VM_PAGE_TO_PHYS(p));
2621 	}
2622 	if (count != 0)
2623 		db_printf("\n");
2624 	db_indent -= 2;
2625 }
2626 
2627 /* XXX. */
2628 #undef count
2629 
2630 /* XXX need this non-static entry for calling from vm_map_print. */
2631 void
2632 vm_object_print(
2633         /* db_expr_t */ long addr,
2634 	boolean_t have_addr,
2635 	/* db_expr_t */ long count,
2636 	char *modif)
2637 {
2638 	vm_object_print_static(addr, have_addr, count, modif);
2639 }
2640 
2641 DB_SHOW_COMMAND(vmopag, vm_object_print_pages)
2642 {
2643 	vm_object_t object;
2644 	vm_pindex_t fidx;
2645 	vm_paddr_t pa;
2646 	vm_page_t m, prev_m;
2647 	int rcount, nl, c;
2648 
2649 	nl = 0;
2650 	TAILQ_FOREACH(object, &vm_object_list, object_list) {
2651 		db_printf("new object: %p\n", (void *)object);
2652 		if (nl > 18) {
2653 			c = cngetc();
2654 			if (c != ' ')
2655 				return;
2656 			nl = 0;
2657 		}
2658 		nl++;
2659 		rcount = 0;
2660 		fidx = 0;
2661 		pa = -1;
2662 		TAILQ_FOREACH(m, &object->memq, listq) {
2663 			if (m->pindex > 128)
2664 				break;
2665 			if ((prev_m = TAILQ_PREV(m, pglist, listq)) != NULL &&
2666 			    prev_m->pindex + 1 != m->pindex) {
2667 				if (rcount) {
2668 					db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2669 						(long)fidx, rcount, (long)pa);
2670 					if (nl > 18) {
2671 						c = cngetc();
2672 						if (c != ' ')
2673 							return;
2674 						nl = 0;
2675 					}
2676 					nl++;
2677 					rcount = 0;
2678 				}
2679 			}
2680 			if (rcount &&
2681 				(VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) {
2682 				++rcount;
2683 				continue;
2684 			}
2685 			if (rcount) {
2686 				db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2687 					(long)fidx, rcount, (long)pa);
2688 				if (nl > 18) {
2689 					c = cngetc();
2690 					if (c != ' ')
2691 						return;
2692 					nl = 0;
2693 				}
2694 				nl++;
2695 			}
2696 			fidx = m->pindex;
2697 			pa = VM_PAGE_TO_PHYS(m);
2698 			rcount = 1;
2699 		}
2700 		if (rcount) {
2701 			db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2702 				(long)fidx, rcount, (long)pa);
2703 			if (nl > 18) {
2704 				c = cngetc();
2705 				if (c != ' ')
2706 					return;
2707 				nl = 0;
2708 			}
2709 			nl++;
2710 		}
2711 	}
2712 }
2713 #endif /* DDB */
2714