xref: /freebsd/sys/vm/vm_object.c (revision e72055b7feba695a760d45f01f0f8268b1cb4a74)
1 /*-
2  * Copyright (c) 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * This code is derived from software contributed to Berkeley by
6  * The Mach Operating System project at Carnegie-Mellon University.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  * 4. Neither the name of the University nor the names of its contributors
17  *    may be used to endorse or promote products derived from this software
18  *    without specific prior written permission.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30  * SUCH DAMAGE.
31  *
32  *	from: @(#)vm_object.c	8.5 (Berkeley) 3/22/94
33  *
34  *
35  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
36  * All rights reserved.
37  *
38  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
39  *
40  * Permission to use, copy, modify and distribute this software and
41  * its documentation is hereby granted, provided that both the copyright
42  * notice and this permission notice appear in all copies of the
43  * software, derivative works or modified versions, and any portions
44  * thereof, and that both notices appear in supporting documentation.
45  *
46  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
47  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
48  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
49  *
50  * Carnegie Mellon requests users of this software to return to
51  *
52  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
53  *  School of Computer Science
54  *  Carnegie Mellon University
55  *  Pittsburgh PA 15213-3890
56  *
57  * any improvements or extensions that they make and grant Carnegie the
58  * rights to redistribute these changes.
59  */
60 
61 /*
62  *	Virtual memory object module.
63  */
64 
65 #include <sys/cdefs.h>
66 __FBSDID("$FreeBSD$");
67 
68 #include "opt_vm.h"
69 
70 #include <sys/param.h>
71 #include <sys/systm.h>
72 #include <sys/lock.h>
73 #include <sys/mman.h>
74 #include <sys/mount.h>
75 #include <sys/kernel.h>
76 #include <sys/sysctl.h>
77 #include <sys/mutex.h>
78 #include <sys/proc.h>		/* for curproc, pageproc */
79 #include <sys/socket.h>
80 #include <sys/resourcevar.h>
81 #include <sys/rwlock.h>
82 #include <sys/vnode.h>
83 #include <sys/vmmeter.h>
84 #include <sys/sx.h>
85 
86 #include <vm/vm.h>
87 #include <vm/vm_param.h>
88 #include <vm/pmap.h>
89 #include <vm/vm_map.h>
90 #include <vm/vm_object.h>
91 #include <vm/vm_page.h>
92 #include <vm/vm_pageout.h>
93 #include <vm/vm_pager.h>
94 #include <vm/swap_pager.h>
95 #include <vm/vm_kern.h>
96 #include <vm/vm_extern.h>
97 #include <vm/vm_radix.h>
98 #include <vm/vm_reserv.h>
99 #include <vm/uma.h>
100 
101 static int old_msync;
102 SYSCTL_INT(_vm, OID_AUTO, old_msync, CTLFLAG_RW, &old_msync, 0,
103     "Use old (insecure) msync behavior");
104 
105 static int	vm_object_page_collect_flush(vm_object_t object, vm_page_t p,
106 		    int pagerflags, int flags, boolean_t *clearobjflags,
107 		    boolean_t *eio);
108 static boolean_t vm_object_page_remove_write(vm_page_t p, int flags,
109 		    boolean_t *clearobjflags);
110 static void	vm_object_qcollapse(vm_object_t object);
111 static void	vm_object_vndeallocate(vm_object_t object);
112 
113 /*
114  *	Virtual memory objects maintain the actual data
115  *	associated with allocated virtual memory.  A given
116  *	page of memory exists within exactly one object.
117  *
118  *	An object is only deallocated when all "references"
119  *	are given up.  Only one "reference" to a given
120  *	region of an object should be writeable.
121  *
122  *	Associated with each object is a list of all resident
123  *	memory pages belonging to that object; this list is
124  *	maintained by the "vm_page" module, and locked by the object's
125  *	lock.
126  *
127  *	Each object also records a "pager" routine which is
128  *	used to retrieve (and store) pages to the proper backing
129  *	storage.  In addition, objects may be backed by other
130  *	objects from which they were virtual-copied.
131  *
132  *	The only items within the object structure which are
133  *	modified after time of creation are:
134  *		reference count		locked by object's lock
135  *		pager routine		locked by object's lock
136  *
137  */
138 
139 struct object_q vm_object_list;
140 struct mtx vm_object_list_mtx;	/* lock for object list and count */
141 
142 struct vm_object kernel_object_store;
143 struct vm_object kmem_object_store;
144 
145 static SYSCTL_NODE(_vm_stats, OID_AUTO, object, CTLFLAG_RD, 0,
146     "VM object stats");
147 
148 static long object_collapses;
149 SYSCTL_LONG(_vm_stats_object, OID_AUTO, collapses, CTLFLAG_RD,
150     &object_collapses, 0, "VM object collapses");
151 
152 static long object_bypasses;
153 SYSCTL_LONG(_vm_stats_object, OID_AUTO, bypasses, CTLFLAG_RD,
154     &object_bypasses, 0, "VM object bypasses");
155 
156 static uma_zone_t obj_zone;
157 
158 static int vm_object_zinit(void *mem, int size, int flags);
159 
160 #ifdef INVARIANTS
161 static void vm_object_zdtor(void *mem, int size, void *arg);
162 
163 static void
164 vm_object_zdtor(void *mem, int size, void *arg)
165 {
166 	vm_object_t object;
167 
168 	object = (vm_object_t)mem;
169 	KASSERT(TAILQ_EMPTY(&object->memq),
170 	    ("object %p has resident pages in its memq", object));
171 	KASSERT(vm_radix_is_empty(&object->rtree),
172 	    ("object %p has resident pages in its trie", object));
173 #if VM_NRESERVLEVEL > 0
174 	KASSERT(LIST_EMPTY(&object->rvq),
175 	    ("object %p has reservations",
176 	    object));
177 #endif
178 	KASSERT(vm_object_cache_is_empty(object),
179 	    ("object %p has cached pages",
180 	    object));
181 	KASSERT(object->paging_in_progress == 0,
182 	    ("object %p paging_in_progress = %d",
183 	    object, object->paging_in_progress));
184 	KASSERT(object->resident_page_count == 0,
185 	    ("object %p resident_page_count = %d",
186 	    object, object->resident_page_count));
187 	KASSERT(object->shadow_count == 0,
188 	    ("object %p shadow_count = %d",
189 	    object, object->shadow_count));
190 }
191 #endif
192 
193 static int
194 vm_object_zinit(void *mem, int size, int flags)
195 {
196 	vm_object_t object;
197 
198 	object = (vm_object_t)mem;
199 	bzero(&object->lock, sizeof(object->lock));
200 	rw_init_flags(&object->lock, "vm object", RW_DUPOK);
201 
202 	/* These are true for any object that has been freed */
203 	object->rtree.rt_root = 0;
204 	object->rtree.rt_flags = 0;
205 	object->paging_in_progress = 0;
206 	object->resident_page_count = 0;
207 	object->shadow_count = 0;
208 	object->cache.rt_root = 0;
209 	object->cache.rt_flags = 0;
210 	return (0);
211 }
212 
213 static void
214 _vm_object_allocate(objtype_t type, vm_pindex_t size, vm_object_t object)
215 {
216 
217 	TAILQ_INIT(&object->memq);
218 	LIST_INIT(&object->shadow_head);
219 
220 	object->type = type;
221 	switch (type) {
222 	case OBJT_DEAD:
223 		panic("_vm_object_allocate: can't create OBJT_DEAD");
224 	case OBJT_DEFAULT:
225 	case OBJT_SWAP:
226 		object->flags = OBJ_ONEMAPPING;
227 		break;
228 	case OBJT_DEVICE:
229 	case OBJT_SG:
230 		object->flags = OBJ_FICTITIOUS | OBJ_UNMANAGED;
231 		break;
232 	case OBJT_MGTDEVICE:
233 		object->flags = OBJ_FICTITIOUS;
234 		break;
235 	case OBJT_PHYS:
236 		object->flags = OBJ_UNMANAGED;
237 		break;
238 	case OBJT_VNODE:
239 		object->flags = 0;
240 		break;
241 	default:
242 		panic("_vm_object_allocate: type %d is undefined", type);
243 	}
244 	object->size = size;
245 	object->generation = 1;
246 	object->ref_count = 1;
247 	object->memattr = VM_MEMATTR_DEFAULT;
248 	object->cred = NULL;
249 	object->charge = 0;
250 	object->handle = NULL;
251 	object->backing_object = NULL;
252 	object->backing_object_offset = (vm_ooffset_t) 0;
253 #if VM_NRESERVLEVEL > 0
254 	LIST_INIT(&object->rvq);
255 #endif
256 
257 	mtx_lock(&vm_object_list_mtx);
258 	TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);
259 	mtx_unlock(&vm_object_list_mtx);
260 }
261 
262 /*
263  *	vm_object_init:
264  *
265  *	Initialize the VM objects module.
266  */
267 void
268 vm_object_init(void)
269 {
270 	TAILQ_INIT(&vm_object_list);
271 	mtx_init(&vm_object_list_mtx, "vm object_list", NULL, MTX_DEF);
272 
273 	rw_init(&kernel_object->lock, "kernel vm object");
274 	_vm_object_allocate(OBJT_PHYS, OFF_TO_IDX(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS),
275 	    kernel_object);
276 #if VM_NRESERVLEVEL > 0
277 	kernel_object->flags |= OBJ_COLORED;
278 	kernel_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS);
279 #endif
280 
281 	rw_init(&kmem_object->lock, "kmem vm object");
282 	_vm_object_allocate(OBJT_PHYS, OFF_TO_IDX(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS),
283 	    kmem_object);
284 #if VM_NRESERVLEVEL > 0
285 	kmem_object->flags |= OBJ_COLORED;
286 	kmem_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS);
287 #endif
288 
289 	/*
290 	 * The lock portion of struct vm_object must be type stable due
291 	 * to vm_pageout_fallback_object_lock locking a vm object
292 	 * without holding any references to it.
293 	 */
294 	obj_zone = uma_zcreate("VM OBJECT", sizeof (struct vm_object), NULL,
295 #ifdef INVARIANTS
296 	    vm_object_zdtor,
297 #else
298 	    NULL,
299 #endif
300 	    vm_object_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
301 
302 	vm_radix_init();
303 }
304 
305 void
306 vm_object_clear_flag(vm_object_t object, u_short bits)
307 {
308 
309 	VM_OBJECT_ASSERT_WLOCKED(object);
310 	object->flags &= ~bits;
311 }
312 
313 /*
314  *	Sets the default memory attribute for the specified object.  Pages
315  *	that are allocated to this object are by default assigned this memory
316  *	attribute.
317  *
318  *	Presently, this function must be called before any pages are allocated
319  *	to the object.  In the future, this requirement may be relaxed for
320  *	"default" and "swap" objects.
321  */
322 int
323 vm_object_set_memattr(vm_object_t object, vm_memattr_t memattr)
324 {
325 
326 	VM_OBJECT_ASSERT_WLOCKED(object);
327 	switch (object->type) {
328 	case OBJT_DEFAULT:
329 	case OBJT_DEVICE:
330 	case OBJT_MGTDEVICE:
331 	case OBJT_PHYS:
332 	case OBJT_SG:
333 	case OBJT_SWAP:
334 	case OBJT_VNODE:
335 		if (!TAILQ_EMPTY(&object->memq))
336 			return (KERN_FAILURE);
337 		break;
338 	case OBJT_DEAD:
339 		return (KERN_INVALID_ARGUMENT);
340 	default:
341 		panic("vm_object_set_memattr: object %p is of undefined type",
342 		    object);
343 	}
344 	object->memattr = memattr;
345 	return (KERN_SUCCESS);
346 }
347 
348 void
349 vm_object_pip_add(vm_object_t object, short i)
350 {
351 
352 	VM_OBJECT_ASSERT_WLOCKED(object);
353 	object->paging_in_progress += i;
354 }
355 
356 void
357 vm_object_pip_subtract(vm_object_t object, short i)
358 {
359 
360 	VM_OBJECT_ASSERT_WLOCKED(object);
361 	object->paging_in_progress -= i;
362 }
363 
364 void
365 vm_object_pip_wakeup(vm_object_t object)
366 {
367 
368 	VM_OBJECT_ASSERT_WLOCKED(object);
369 	object->paging_in_progress--;
370 	if ((object->flags & OBJ_PIPWNT) && object->paging_in_progress == 0) {
371 		vm_object_clear_flag(object, OBJ_PIPWNT);
372 		wakeup(object);
373 	}
374 }
375 
376 void
377 vm_object_pip_wakeupn(vm_object_t object, short i)
378 {
379 
380 	VM_OBJECT_ASSERT_WLOCKED(object);
381 	if (i)
382 		object->paging_in_progress -= i;
383 	if ((object->flags & OBJ_PIPWNT) && object->paging_in_progress == 0) {
384 		vm_object_clear_flag(object, OBJ_PIPWNT);
385 		wakeup(object);
386 	}
387 }
388 
389 void
390 vm_object_pip_wait(vm_object_t object, char *waitid)
391 {
392 
393 	VM_OBJECT_ASSERT_WLOCKED(object);
394 	while (object->paging_in_progress) {
395 		object->flags |= OBJ_PIPWNT;
396 		VM_OBJECT_SLEEP(object, object, PVM, waitid, 0);
397 	}
398 }
399 
400 /*
401  *	vm_object_allocate:
402  *
403  *	Returns a new object with the given size.
404  */
405 vm_object_t
406 vm_object_allocate(objtype_t type, vm_pindex_t size)
407 {
408 	vm_object_t object;
409 
410 	object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);
411 	_vm_object_allocate(type, size, object);
412 	return (object);
413 }
414 
415 
416 /*
417  *	vm_object_reference:
418  *
419  *	Gets another reference to the given object.  Note: OBJ_DEAD
420  *	objects can be referenced during final cleaning.
421  */
422 void
423 vm_object_reference(vm_object_t object)
424 {
425 	if (object == NULL)
426 		return;
427 	VM_OBJECT_WLOCK(object);
428 	vm_object_reference_locked(object);
429 	VM_OBJECT_WUNLOCK(object);
430 }
431 
432 /*
433  *	vm_object_reference_locked:
434  *
435  *	Gets another reference to the given object.
436  *
437  *	The object must be locked.
438  */
439 void
440 vm_object_reference_locked(vm_object_t object)
441 {
442 	struct vnode *vp;
443 
444 	VM_OBJECT_ASSERT_WLOCKED(object);
445 	object->ref_count++;
446 	if (object->type == OBJT_VNODE) {
447 		vp = object->handle;
448 		vref(vp);
449 	}
450 }
451 
452 /*
453  * Handle deallocating an object of type OBJT_VNODE.
454  */
455 static void
456 vm_object_vndeallocate(vm_object_t object)
457 {
458 	struct vnode *vp = (struct vnode *) object->handle;
459 
460 	VM_OBJECT_ASSERT_WLOCKED(object);
461 	KASSERT(object->type == OBJT_VNODE,
462 	    ("vm_object_vndeallocate: not a vnode object"));
463 	KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp"));
464 #ifdef INVARIANTS
465 	if (object->ref_count == 0) {
466 		vprint("vm_object_vndeallocate", vp);
467 		panic("vm_object_vndeallocate: bad object reference count");
468 	}
469 #endif
470 
471 	if (object->ref_count > 1) {
472 		object->ref_count--;
473 		VM_OBJECT_WUNLOCK(object);
474 		/* vrele may need the vnode lock. */
475 		vrele(vp);
476 	} else {
477 		vhold(vp);
478 		VM_OBJECT_WUNLOCK(object);
479 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
480 		vdrop(vp);
481 		VM_OBJECT_WLOCK(object);
482 		object->ref_count--;
483 		if (object->type == OBJT_DEAD) {
484 			VM_OBJECT_WUNLOCK(object);
485 			VOP_UNLOCK(vp, 0);
486 		} else {
487 			if (object->ref_count == 0)
488 				VOP_UNSET_TEXT(vp);
489 			VM_OBJECT_WUNLOCK(object);
490 			vput(vp);
491 		}
492 	}
493 }
494 
495 /*
496  *	vm_object_deallocate:
497  *
498  *	Release a reference to the specified object,
499  *	gained either through a vm_object_allocate
500  *	or a vm_object_reference call.  When all references
501  *	are gone, storage associated with this object
502  *	may be relinquished.
503  *
504  *	No object may be locked.
505  */
506 void
507 vm_object_deallocate(vm_object_t object)
508 {
509 	vm_object_t temp;
510 	struct vnode *vp;
511 
512 	while (object != NULL) {
513 		VM_OBJECT_WLOCK(object);
514 		if (object->type == OBJT_VNODE) {
515 			vm_object_vndeallocate(object);
516 			return;
517 		}
518 
519 		KASSERT(object->ref_count != 0,
520 			("vm_object_deallocate: object deallocated too many times: %d", object->type));
521 
522 		/*
523 		 * If the reference count goes to 0 we start calling
524 		 * vm_object_terminate() on the object chain.
525 		 * A ref count of 1 may be a special case depending on the
526 		 * shadow count being 0 or 1.
527 		 */
528 		object->ref_count--;
529 		if (object->ref_count > 1) {
530 			VM_OBJECT_WUNLOCK(object);
531 			return;
532 		} else if (object->ref_count == 1) {
533 			if (object->type == OBJT_SWAP &&
534 			    (object->flags & OBJ_TMPFS) != 0) {
535 				vp = object->un_pager.swp.swp_tmpfs;
536 				vhold(vp);
537 				VM_OBJECT_WUNLOCK(object);
538 				vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
539 				VM_OBJECT_WLOCK(object);
540 				if (object->type == OBJT_DEAD ||
541 				    object->ref_count != 1) {
542 					VM_OBJECT_WUNLOCK(object);
543 					VOP_UNLOCK(vp, 0);
544 					vdrop(vp);
545 					return;
546 				}
547 				if ((object->flags & OBJ_TMPFS) != 0)
548 					VOP_UNSET_TEXT(vp);
549 				VOP_UNLOCK(vp, 0);
550 				vdrop(vp);
551 			}
552 			if (object->shadow_count == 0 &&
553 			    object->handle == NULL &&
554 			    (object->type == OBJT_DEFAULT ||
555 			    (object->type == OBJT_SWAP &&
556 			    (object->flags & OBJ_TMPFS_NODE) == 0))) {
557 				vm_object_set_flag(object, OBJ_ONEMAPPING);
558 			} else if ((object->shadow_count == 1) &&
559 			    (object->handle == NULL) &&
560 			    (object->type == OBJT_DEFAULT ||
561 			     object->type == OBJT_SWAP)) {
562 				vm_object_t robject;
563 
564 				robject = LIST_FIRST(&object->shadow_head);
565 				KASSERT(robject != NULL,
566 				    ("vm_object_deallocate: ref_count: %d, shadow_count: %d",
567 					 object->ref_count,
568 					 object->shadow_count));
569 				KASSERT((robject->flags & OBJ_TMPFS_NODE) == 0,
570 				    ("shadowed tmpfs v_object %p", object));
571 				if (!VM_OBJECT_TRYWLOCK(robject)) {
572 					/*
573 					 * Avoid a potential deadlock.
574 					 */
575 					object->ref_count++;
576 					VM_OBJECT_WUNLOCK(object);
577 					/*
578 					 * More likely than not the thread
579 					 * holding robject's lock has lower
580 					 * priority than the current thread.
581 					 * Let the lower priority thread run.
582 					 */
583 					pause("vmo_de", 1);
584 					continue;
585 				}
586 				/*
587 				 * Collapse object into its shadow unless its
588 				 * shadow is dead.  In that case, object will
589 				 * be deallocated by the thread that is
590 				 * deallocating its shadow.
591 				 */
592 				if ((robject->flags & OBJ_DEAD) == 0 &&
593 				    (robject->handle == NULL) &&
594 				    (robject->type == OBJT_DEFAULT ||
595 				     robject->type == OBJT_SWAP)) {
596 
597 					robject->ref_count++;
598 retry:
599 					if (robject->paging_in_progress) {
600 						VM_OBJECT_WUNLOCK(object);
601 						vm_object_pip_wait(robject,
602 						    "objde1");
603 						temp = robject->backing_object;
604 						if (object == temp) {
605 							VM_OBJECT_WLOCK(object);
606 							goto retry;
607 						}
608 					} else if (object->paging_in_progress) {
609 						VM_OBJECT_WUNLOCK(robject);
610 						object->flags |= OBJ_PIPWNT;
611 						VM_OBJECT_SLEEP(object, object,
612 						    PDROP | PVM, "objde2", 0);
613 						VM_OBJECT_WLOCK(robject);
614 						temp = robject->backing_object;
615 						if (object == temp) {
616 							VM_OBJECT_WLOCK(object);
617 							goto retry;
618 						}
619 					} else
620 						VM_OBJECT_WUNLOCK(object);
621 
622 					if (robject->ref_count == 1) {
623 						robject->ref_count--;
624 						object = robject;
625 						goto doterm;
626 					}
627 					object = robject;
628 					vm_object_collapse(object);
629 					VM_OBJECT_WUNLOCK(object);
630 					continue;
631 				}
632 				VM_OBJECT_WUNLOCK(robject);
633 			}
634 			VM_OBJECT_WUNLOCK(object);
635 			return;
636 		}
637 doterm:
638 		temp = object->backing_object;
639 		if (temp != NULL) {
640 			KASSERT((object->flags & OBJ_TMPFS_NODE) == 0,
641 			    ("shadowed tmpfs v_object 2 %p", object));
642 			VM_OBJECT_WLOCK(temp);
643 			LIST_REMOVE(object, shadow_list);
644 			temp->shadow_count--;
645 			VM_OBJECT_WUNLOCK(temp);
646 			object->backing_object = NULL;
647 		}
648 		/*
649 		 * Don't double-terminate, we could be in a termination
650 		 * recursion due to the terminate having to sync data
651 		 * to disk.
652 		 */
653 		if ((object->flags & OBJ_DEAD) == 0)
654 			vm_object_terminate(object);
655 		else
656 			VM_OBJECT_WUNLOCK(object);
657 		object = temp;
658 	}
659 }
660 
661 /*
662  *	vm_object_destroy removes the object from the global object list
663  *      and frees the space for the object.
664  */
665 void
666 vm_object_destroy(vm_object_t object)
667 {
668 
669 	/*
670 	 * Remove the object from the global object list.
671 	 */
672 	mtx_lock(&vm_object_list_mtx);
673 	TAILQ_REMOVE(&vm_object_list, object, object_list);
674 	mtx_unlock(&vm_object_list_mtx);
675 
676 	/*
677 	 * Release the allocation charge.
678 	 */
679 	if (object->cred != NULL) {
680 		KASSERT(object->type == OBJT_DEFAULT ||
681 		    object->type == OBJT_SWAP,
682 		    ("%s: non-swap obj %p has cred", __func__, object));
683 		swap_release_by_cred(object->charge, object->cred);
684 		object->charge = 0;
685 		crfree(object->cred);
686 		object->cred = NULL;
687 	}
688 
689 	/*
690 	 * Free the space for the object.
691 	 */
692 	uma_zfree(obj_zone, object);
693 }
694 
695 /*
696  *	vm_object_terminate actually destroys the specified object, freeing
697  *	up all previously used resources.
698  *
699  *	The object must be locked.
700  *	This routine may block.
701  */
702 void
703 vm_object_terminate(vm_object_t object)
704 {
705 	vm_page_t p, p_next;
706 
707 	VM_OBJECT_ASSERT_WLOCKED(object);
708 
709 	/*
710 	 * Make sure no one uses us.
711 	 */
712 	vm_object_set_flag(object, OBJ_DEAD);
713 
714 	/*
715 	 * wait for the pageout daemon to be done with the object
716 	 */
717 	vm_object_pip_wait(object, "objtrm");
718 
719 	KASSERT(!object->paging_in_progress,
720 		("vm_object_terminate: pageout in progress"));
721 
722 	/*
723 	 * Clean and free the pages, as appropriate. All references to the
724 	 * object are gone, so we don't need to lock it.
725 	 */
726 	if (object->type == OBJT_VNODE) {
727 		struct vnode *vp = (struct vnode *)object->handle;
728 
729 		/*
730 		 * Clean pages and flush buffers.
731 		 */
732 		vm_object_page_clean(object, 0, 0, OBJPC_SYNC);
733 		VM_OBJECT_WUNLOCK(object);
734 
735 		vinvalbuf(vp, V_SAVE, 0, 0);
736 
737 		VM_OBJECT_WLOCK(object);
738 	}
739 
740 	KASSERT(object->ref_count == 0,
741 		("vm_object_terminate: object with references, ref_count=%d",
742 		object->ref_count));
743 
744 	/*
745 	 * Free any remaining pageable pages.  This also removes them from the
746 	 * paging queues.  However, don't free wired pages, just remove them
747 	 * from the object.  Rather than incrementally removing each page from
748 	 * the object, the page and object are reset to any empty state.
749 	 */
750 	TAILQ_FOREACH_SAFE(p, &object->memq, listq, p_next) {
751 		vm_page_assert_unbusied(p);
752 		vm_page_lock(p);
753 		/*
754 		 * Optimize the page's removal from the object by resetting
755 		 * its "object" field.  Specifically, if the page is not
756 		 * wired, then the effect of this assignment is that
757 		 * vm_page_free()'s call to vm_page_remove() will return
758 		 * immediately without modifying the page or the object.
759 		 */
760 		p->object = NULL;
761 		if (p->wire_count == 0) {
762 			vm_page_free(p);
763 			PCPU_INC(cnt.v_pfree);
764 		}
765 		vm_page_unlock(p);
766 	}
767 	/*
768 	 * If the object contained any pages, then reset it to an empty state.
769 	 * None of the object's fields, including "resident_page_count", were
770 	 * modified by the preceding loop.
771 	 */
772 	if (object->resident_page_count != 0) {
773 		vm_radix_reclaim_allnodes(&object->rtree);
774 		TAILQ_INIT(&object->memq);
775 		object->resident_page_count = 0;
776 		if (object->type == OBJT_VNODE)
777 			vdrop(object->handle);
778 	}
779 
780 #if VM_NRESERVLEVEL > 0
781 	if (__predict_false(!LIST_EMPTY(&object->rvq)))
782 		vm_reserv_break_all(object);
783 #endif
784 	if (__predict_false(!vm_object_cache_is_empty(object)))
785 		vm_page_cache_free(object, 0, 0);
786 
787 	/*
788 	 * Let the pager know object is dead.
789 	 */
790 	vm_pager_deallocate(object);
791 	VM_OBJECT_WUNLOCK(object);
792 
793 	vm_object_destroy(object);
794 }
795 
796 /*
797  * Make the page read-only so that we can clear the object flags.  However, if
798  * this is a nosync mmap then the object is likely to stay dirty so do not
799  * mess with the page and do not clear the object flags.  Returns TRUE if the
800  * page should be flushed, and FALSE otherwise.
801  */
802 static boolean_t
803 vm_object_page_remove_write(vm_page_t p, int flags, boolean_t *clearobjflags)
804 {
805 
806 	/*
807 	 * If we have been asked to skip nosync pages and this is a
808 	 * nosync page, skip it.  Note that the object flags were not
809 	 * cleared in this case so we do not have to set them.
810 	 */
811 	if ((flags & OBJPC_NOSYNC) != 0 && (p->oflags & VPO_NOSYNC) != 0) {
812 		*clearobjflags = FALSE;
813 		return (FALSE);
814 	} else {
815 		pmap_remove_write(p);
816 		return (p->dirty != 0);
817 	}
818 }
819 
820 /*
821  *	vm_object_page_clean
822  *
823  *	Clean all dirty pages in the specified range of object.  Leaves page
824  * 	on whatever queue it is currently on.   If NOSYNC is set then do not
825  *	write out pages with VPO_NOSYNC set (originally comes from MAP_NOSYNC),
826  *	leaving the object dirty.
827  *
828  *	When stuffing pages asynchronously, allow clustering.  XXX we need a
829  *	synchronous clustering mode implementation.
830  *
831  *	Odd semantics: if start == end, we clean everything.
832  *
833  *	The object must be locked.
834  *
835  *	Returns FALSE if some page from the range was not written, as
836  *	reported by the pager, and TRUE otherwise.
837  */
838 boolean_t
839 vm_object_page_clean(vm_object_t object, vm_ooffset_t start, vm_ooffset_t end,
840     int flags)
841 {
842 	vm_page_t np, p;
843 	vm_pindex_t pi, tend, tstart;
844 	int curgeneration, n, pagerflags;
845 	boolean_t clearobjflags, eio, res;
846 
847 	VM_OBJECT_ASSERT_WLOCKED(object);
848 
849 	/*
850 	 * The OBJ_MIGHTBEDIRTY flag is only set for OBJT_VNODE
851 	 * objects.  The check below prevents the function from
852 	 * operating on non-vnode objects.
853 	 */
854 	if ((object->flags & OBJ_MIGHTBEDIRTY) == 0 ||
855 	    object->resident_page_count == 0)
856 		return (TRUE);
857 
858 	pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) != 0 ?
859 	    VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK;
860 	pagerflags |= (flags & OBJPC_INVAL) != 0 ? VM_PAGER_PUT_INVAL : 0;
861 
862 	tstart = OFF_TO_IDX(start);
863 	tend = (end == 0) ? object->size : OFF_TO_IDX(end + PAGE_MASK);
864 	clearobjflags = tstart == 0 && tend >= object->size;
865 	res = TRUE;
866 
867 rescan:
868 	curgeneration = object->generation;
869 
870 	for (p = vm_page_find_least(object, tstart); p != NULL; p = np) {
871 		pi = p->pindex;
872 		if (pi >= tend)
873 			break;
874 		np = TAILQ_NEXT(p, listq);
875 		if (p->valid == 0)
876 			continue;
877 		if (vm_page_sleep_if_busy(p, "vpcwai")) {
878 			if (object->generation != curgeneration) {
879 				if ((flags & OBJPC_SYNC) != 0)
880 					goto rescan;
881 				else
882 					clearobjflags = FALSE;
883 			}
884 			np = vm_page_find_least(object, pi);
885 			continue;
886 		}
887 		if (!vm_object_page_remove_write(p, flags, &clearobjflags))
888 			continue;
889 
890 		n = vm_object_page_collect_flush(object, p, pagerflags,
891 		    flags, &clearobjflags, &eio);
892 		if (eio) {
893 			res = FALSE;
894 			clearobjflags = FALSE;
895 		}
896 		if (object->generation != curgeneration) {
897 			if ((flags & OBJPC_SYNC) != 0)
898 				goto rescan;
899 			else
900 				clearobjflags = FALSE;
901 		}
902 
903 		/*
904 		 * If the VOP_PUTPAGES() did a truncated write, so
905 		 * that even the first page of the run is not fully
906 		 * written, vm_pageout_flush() returns 0 as the run
907 		 * length.  Since the condition that caused truncated
908 		 * write may be permanent, e.g. exhausted free space,
909 		 * accepting n == 0 would cause an infinite loop.
910 		 *
911 		 * Forwarding the iterator leaves the unwritten page
912 		 * behind, but there is not much we can do there if
913 		 * filesystem refuses to write it.
914 		 */
915 		if (n == 0) {
916 			n = 1;
917 			clearobjflags = FALSE;
918 		}
919 		np = vm_page_find_least(object, pi + n);
920 	}
921 #if 0
922 	VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC) ? MNT_WAIT : 0);
923 #endif
924 
925 	if (clearobjflags)
926 		vm_object_clear_flag(object, OBJ_MIGHTBEDIRTY);
927 	return (res);
928 }
929 
930 static int
931 vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags,
932     int flags, boolean_t *clearobjflags, boolean_t *eio)
933 {
934 	vm_page_t ma[vm_pageout_page_count], p_first, tp;
935 	int count, i, mreq, runlen;
936 
937 	vm_page_lock_assert(p, MA_NOTOWNED);
938 	VM_OBJECT_ASSERT_WLOCKED(object);
939 
940 	count = 1;
941 	mreq = 0;
942 
943 	for (tp = p; count < vm_pageout_page_count; count++) {
944 		tp = vm_page_next(tp);
945 		if (tp == NULL || vm_page_busied(tp))
946 			break;
947 		if (!vm_object_page_remove_write(tp, flags, clearobjflags))
948 			break;
949 	}
950 
951 	for (p_first = p; count < vm_pageout_page_count; count++) {
952 		tp = vm_page_prev(p_first);
953 		if (tp == NULL || vm_page_busied(tp))
954 			break;
955 		if (!vm_object_page_remove_write(tp, flags, clearobjflags))
956 			break;
957 		p_first = tp;
958 		mreq++;
959 	}
960 
961 	for (tp = p_first, i = 0; i < count; tp = TAILQ_NEXT(tp, listq), i++)
962 		ma[i] = tp;
963 
964 	vm_pageout_flush(ma, count, pagerflags, mreq, &runlen, eio);
965 	return (runlen);
966 }
967 
968 /*
969  * Note that there is absolutely no sense in writing out
970  * anonymous objects, so we track down the vnode object
971  * to write out.
972  * We invalidate (remove) all pages from the address space
973  * for semantic correctness.
974  *
975  * If the backing object is a device object with unmanaged pages, then any
976  * mappings to the specified range of pages must be removed before this
977  * function is called.
978  *
979  * Note: certain anonymous maps, such as MAP_NOSYNC maps,
980  * may start out with a NULL object.
981  */
982 boolean_t
983 vm_object_sync(vm_object_t object, vm_ooffset_t offset, vm_size_t size,
984     boolean_t syncio, boolean_t invalidate)
985 {
986 	vm_object_t backing_object;
987 	struct vnode *vp;
988 	struct mount *mp;
989 	int error, flags, fsync_after;
990 	boolean_t res;
991 
992 	if (object == NULL)
993 		return (TRUE);
994 	res = TRUE;
995 	error = 0;
996 	VM_OBJECT_WLOCK(object);
997 	while ((backing_object = object->backing_object) != NULL) {
998 		VM_OBJECT_WLOCK(backing_object);
999 		offset += object->backing_object_offset;
1000 		VM_OBJECT_WUNLOCK(object);
1001 		object = backing_object;
1002 		if (object->size < OFF_TO_IDX(offset + size))
1003 			size = IDX_TO_OFF(object->size) - offset;
1004 	}
1005 	/*
1006 	 * Flush pages if writing is allowed, invalidate them
1007 	 * if invalidation requested.  Pages undergoing I/O
1008 	 * will be ignored by vm_object_page_remove().
1009 	 *
1010 	 * We cannot lock the vnode and then wait for paging
1011 	 * to complete without deadlocking against vm_fault.
1012 	 * Instead we simply call vm_object_page_remove() and
1013 	 * allow it to block internally on a page-by-page
1014 	 * basis when it encounters pages undergoing async
1015 	 * I/O.
1016 	 */
1017 	if (object->type == OBJT_VNODE &&
1018 	    (object->flags & OBJ_MIGHTBEDIRTY) != 0) {
1019 		vp = object->handle;
1020 		VM_OBJECT_WUNLOCK(object);
1021 		(void) vn_start_write(vp, &mp, V_WAIT);
1022 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1023 		if (syncio && !invalidate && offset == 0 &&
1024 		    OFF_TO_IDX(size) == object->size) {
1025 			/*
1026 			 * If syncing the whole mapping of the file,
1027 			 * it is faster to schedule all the writes in
1028 			 * async mode, also allowing the clustering,
1029 			 * and then wait for i/o to complete.
1030 			 */
1031 			flags = 0;
1032 			fsync_after = TRUE;
1033 		} else {
1034 			flags = (syncio || invalidate) ? OBJPC_SYNC : 0;
1035 			flags |= invalidate ? (OBJPC_SYNC | OBJPC_INVAL) : 0;
1036 			fsync_after = FALSE;
1037 		}
1038 		VM_OBJECT_WLOCK(object);
1039 		res = vm_object_page_clean(object, offset, offset + size,
1040 		    flags);
1041 		VM_OBJECT_WUNLOCK(object);
1042 		if (fsync_after)
1043 			error = VOP_FSYNC(vp, MNT_WAIT, curthread);
1044 		VOP_UNLOCK(vp, 0);
1045 		vn_finished_write(mp);
1046 		if (error != 0)
1047 			res = FALSE;
1048 		VM_OBJECT_WLOCK(object);
1049 	}
1050 	if ((object->type == OBJT_VNODE ||
1051 	     object->type == OBJT_DEVICE) && invalidate) {
1052 		if (object->type == OBJT_DEVICE)
1053 			/*
1054 			 * The option OBJPR_NOTMAPPED must be passed here
1055 			 * because vm_object_page_remove() cannot remove
1056 			 * unmanaged mappings.
1057 			 */
1058 			flags = OBJPR_NOTMAPPED;
1059 		else if (old_msync)
1060 			flags = OBJPR_NOTWIRED;
1061 		else
1062 			flags = OBJPR_CLEANONLY | OBJPR_NOTWIRED;
1063 		vm_object_page_remove(object, OFF_TO_IDX(offset),
1064 		    OFF_TO_IDX(offset + size + PAGE_MASK), flags);
1065 	}
1066 	VM_OBJECT_WUNLOCK(object);
1067 	return (res);
1068 }
1069 
1070 /*
1071  *	vm_object_madvise:
1072  *
1073  *	Implements the madvise function at the object/page level.
1074  *
1075  *	MADV_WILLNEED	(any object)
1076  *
1077  *	    Activate the specified pages if they are resident.
1078  *
1079  *	MADV_DONTNEED	(any object)
1080  *
1081  *	    Deactivate the specified pages if they are resident.
1082  *
1083  *	MADV_FREE	(OBJT_DEFAULT/OBJT_SWAP objects,
1084  *			 OBJ_ONEMAPPING only)
1085  *
1086  *	    Deactivate and clean the specified pages if they are
1087  *	    resident.  This permits the process to reuse the pages
1088  *	    without faulting or the kernel to reclaim the pages
1089  *	    without I/O.
1090  */
1091 void
1092 vm_object_madvise(vm_object_t object, vm_pindex_t pindex, vm_pindex_t end,
1093     int advise)
1094 {
1095 	vm_pindex_t tpindex;
1096 	vm_object_t backing_object, tobject;
1097 	vm_page_t m;
1098 
1099 	if (object == NULL)
1100 		return;
1101 	VM_OBJECT_WLOCK(object);
1102 	/*
1103 	 * Locate and adjust resident pages
1104 	 */
1105 	for (; pindex < end; pindex += 1) {
1106 relookup:
1107 		tobject = object;
1108 		tpindex = pindex;
1109 shadowlookup:
1110 		/*
1111 		 * MADV_FREE only operates on OBJT_DEFAULT or OBJT_SWAP pages
1112 		 * and those pages must be OBJ_ONEMAPPING.
1113 		 */
1114 		if (advise == MADV_FREE) {
1115 			if ((tobject->type != OBJT_DEFAULT &&
1116 			     tobject->type != OBJT_SWAP) ||
1117 			    (tobject->flags & OBJ_ONEMAPPING) == 0) {
1118 				goto unlock_tobject;
1119 			}
1120 		} else if ((tobject->flags & OBJ_UNMANAGED) != 0)
1121 			goto unlock_tobject;
1122 		m = vm_page_lookup(tobject, tpindex);
1123 		if (m == NULL && advise == MADV_WILLNEED) {
1124 			/*
1125 			 * If the page is cached, reactivate it.
1126 			 */
1127 			m = vm_page_alloc(tobject, tpindex, VM_ALLOC_IFCACHED |
1128 			    VM_ALLOC_NOBUSY);
1129 		}
1130 		if (m == NULL) {
1131 			/*
1132 			 * There may be swap even if there is no backing page
1133 			 */
1134 			if (advise == MADV_FREE && tobject->type == OBJT_SWAP)
1135 				swap_pager_freespace(tobject, tpindex, 1);
1136 			/*
1137 			 * next object
1138 			 */
1139 			backing_object = tobject->backing_object;
1140 			if (backing_object == NULL)
1141 				goto unlock_tobject;
1142 			VM_OBJECT_WLOCK(backing_object);
1143 			tpindex += OFF_TO_IDX(tobject->backing_object_offset);
1144 			if (tobject != object)
1145 				VM_OBJECT_WUNLOCK(tobject);
1146 			tobject = backing_object;
1147 			goto shadowlookup;
1148 		} else if (m->valid != VM_PAGE_BITS_ALL)
1149 			goto unlock_tobject;
1150 		/*
1151 		 * If the page is not in a normal state, skip it.
1152 		 */
1153 		vm_page_lock(m);
1154 		if (m->hold_count != 0 || m->wire_count != 0) {
1155 			vm_page_unlock(m);
1156 			goto unlock_tobject;
1157 		}
1158 		KASSERT((m->flags & PG_FICTITIOUS) == 0,
1159 		    ("vm_object_madvise: page %p is fictitious", m));
1160 		KASSERT((m->oflags & VPO_UNMANAGED) == 0,
1161 		    ("vm_object_madvise: page %p is not managed", m));
1162 		if (vm_page_busied(m)) {
1163 			if (advise == MADV_WILLNEED) {
1164 				/*
1165 				 * Reference the page before unlocking and
1166 				 * sleeping so that the page daemon is less
1167 				 * likely to reclaim it.
1168 				 */
1169 				vm_page_aflag_set(m, PGA_REFERENCED);
1170 			}
1171 			if (object != tobject)
1172 				VM_OBJECT_WUNLOCK(object);
1173 			VM_OBJECT_WUNLOCK(tobject);
1174 			vm_page_busy_sleep(m, "madvpo");
1175 			VM_OBJECT_WLOCK(object);
1176   			goto relookup;
1177 		}
1178 		if (advise == MADV_WILLNEED) {
1179 			vm_page_activate(m);
1180 		} else {
1181 			vm_page_advise(m, advise);
1182 		}
1183 		vm_page_unlock(m);
1184 		if (advise == MADV_FREE && tobject->type == OBJT_SWAP)
1185 			swap_pager_freespace(tobject, tpindex, 1);
1186 unlock_tobject:
1187 		if (tobject != object)
1188 			VM_OBJECT_WUNLOCK(tobject);
1189 	}
1190 	VM_OBJECT_WUNLOCK(object);
1191 }
1192 
1193 /*
1194  *	vm_object_shadow:
1195  *
1196  *	Create a new object which is backed by the
1197  *	specified existing object range.  The source
1198  *	object reference is deallocated.
1199  *
1200  *	The new object and offset into that object
1201  *	are returned in the source parameters.
1202  */
1203 void
1204 vm_object_shadow(
1205 	vm_object_t *object,	/* IN/OUT */
1206 	vm_ooffset_t *offset,	/* IN/OUT */
1207 	vm_size_t length)
1208 {
1209 	vm_object_t source;
1210 	vm_object_t result;
1211 
1212 	source = *object;
1213 
1214 	/*
1215 	 * Don't create the new object if the old object isn't shared.
1216 	 */
1217 	if (source != NULL) {
1218 		VM_OBJECT_WLOCK(source);
1219 		if (source->ref_count == 1 &&
1220 		    source->handle == NULL &&
1221 		    (source->type == OBJT_DEFAULT ||
1222 		     source->type == OBJT_SWAP)) {
1223 			VM_OBJECT_WUNLOCK(source);
1224 			return;
1225 		}
1226 		VM_OBJECT_WUNLOCK(source);
1227 	}
1228 
1229 	/*
1230 	 * Allocate a new object with the given length.
1231 	 */
1232 	result = vm_object_allocate(OBJT_DEFAULT, atop(length));
1233 
1234 	/*
1235 	 * The new object shadows the source object, adding a reference to it.
1236 	 * Our caller changes his reference to point to the new object,
1237 	 * removing a reference to the source object.  Net result: no change
1238 	 * of reference count.
1239 	 *
1240 	 * Try to optimize the result object's page color when shadowing
1241 	 * in order to maintain page coloring consistency in the combined
1242 	 * shadowed object.
1243 	 */
1244 	result->backing_object = source;
1245 	/*
1246 	 * Store the offset into the source object, and fix up the offset into
1247 	 * the new object.
1248 	 */
1249 	result->backing_object_offset = *offset;
1250 	if (source != NULL) {
1251 		VM_OBJECT_WLOCK(source);
1252 		LIST_INSERT_HEAD(&source->shadow_head, result, shadow_list);
1253 		source->shadow_count++;
1254 #if VM_NRESERVLEVEL > 0
1255 		result->flags |= source->flags & OBJ_COLORED;
1256 		result->pg_color = (source->pg_color + OFF_TO_IDX(*offset)) &
1257 		    ((1 << (VM_NFREEORDER - 1)) - 1);
1258 #endif
1259 		VM_OBJECT_WUNLOCK(source);
1260 	}
1261 
1262 
1263 	/*
1264 	 * Return the new things
1265 	 */
1266 	*offset = 0;
1267 	*object = result;
1268 }
1269 
1270 /*
1271  *	vm_object_split:
1272  *
1273  * Split the pages in a map entry into a new object.  This affords
1274  * easier removal of unused pages, and keeps object inheritance from
1275  * being a negative impact on memory usage.
1276  */
1277 void
1278 vm_object_split(vm_map_entry_t entry)
1279 {
1280 	vm_page_t m, m_next;
1281 	vm_object_t orig_object, new_object, source;
1282 	vm_pindex_t idx, offidxstart;
1283 	vm_size_t size;
1284 
1285 	orig_object = entry->object.vm_object;
1286 	if (orig_object->type != OBJT_DEFAULT && orig_object->type != OBJT_SWAP)
1287 		return;
1288 	if (orig_object->ref_count <= 1)
1289 		return;
1290 	VM_OBJECT_WUNLOCK(orig_object);
1291 
1292 	offidxstart = OFF_TO_IDX(entry->offset);
1293 	size = atop(entry->end - entry->start);
1294 
1295 	/*
1296 	 * If swap_pager_copy() is later called, it will convert new_object
1297 	 * into a swap object.
1298 	 */
1299 	new_object = vm_object_allocate(OBJT_DEFAULT, size);
1300 
1301 	/*
1302 	 * At this point, the new object is still private, so the order in
1303 	 * which the original and new objects are locked does not matter.
1304 	 */
1305 	VM_OBJECT_WLOCK(new_object);
1306 	VM_OBJECT_WLOCK(orig_object);
1307 	source = orig_object->backing_object;
1308 	if (source != NULL) {
1309 		VM_OBJECT_WLOCK(source);
1310 		if ((source->flags & OBJ_DEAD) != 0) {
1311 			VM_OBJECT_WUNLOCK(source);
1312 			VM_OBJECT_WUNLOCK(orig_object);
1313 			VM_OBJECT_WUNLOCK(new_object);
1314 			vm_object_deallocate(new_object);
1315 			VM_OBJECT_WLOCK(orig_object);
1316 			return;
1317 		}
1318 		LIST_INSERT_HEAD(&source->shadow_head,
1319 				  new_object, shadow_list);
1320 		source->shadow_count++;
1321 		vm_object_reference_locked(source);	/* for new_object */
1322 		vm_object_clear_flag(source, OBJ_ONEMAPPING);
1323 		VM_OBJECT_WUNLOCK(source);
1324 		new_object->backing_object_offset =
1325 			orig_object->backing_object_offset + entry->offset;
1326 		new_object->backing_object = source;
1327 	}
1328 	if (orig_object->cred != NULL) {
1329 		new_object->cred = orig_object->cred;
1330 		crhold(orig_object->cred);
1331 		new_object->charge = ptoa(size);
1332 		KASSERT(orig_object->charge >= ptoa(size),
1333 		    ("orig_object->charge < 0"));
1334 		orig_object->charge -= ptoa(size);
1335 	}
1336 retry:
1337 	m = vm_page_find_least(orig_object, offidxstart);
1338 	for (; m != NULL && (idx = m->pindex - offidxstart) < size;
1339 	    m = m_next) {
1340 		m_next = TAILQ_NEXT(m, listq);
1341 
1342 		/*
1343 		 * We must wait for pending I/O to complete before we can
1344 		 * rename the page.
1345 		 *
1346 		 * We do not have to VM_PROT_NONE the page as mappings should
1347 		 * not be changed by this operation.
1348 		 */
1349 		if (vm_page_busied(m)) {
1350 			VM_OBJECT_WUNLOCK(new_object);
1351 			vm_page_lock(m);
1352 			VM_OBJECT_WUNLOCK(orig_object);
1353 			vm_page_busy_sleep(m, "spltwt");
1354 			VM_OBJECT_WLOCK(orig_object);
1355 			VM_OBJECT_WLOCK(new_object);
1356 			goto retry;
1357 		}
1358 
1359 		/* vm_page_rename() will handle dirty and cache. */
1360 		if (vm_page_rename(m, new_object, idx)) {
1361 			VM_OBJECT_WUNLOCK(new_object);
1362 			VM_OBJECT_WUNLOCK(orig_object);
1363 			VM_WAIT;
1364 			VM_OBJECT_WLOCK(orig_object);
1365 			VM_OBJECT_WLOCK(new_object);
1366 			goto retry;
1367 		}
1368 #if VM_NRESERVLEVEL > 0
1369 		/*
1370 		 * If some of the reservation's allocated pages remain with
1371 		 * the original object, then transferring the reservation to
1372 		 * the new object is neither particularly beneficial nor
1373 		 * particularly harmful as compared to leaving the reservation
1374 		 * with the original object.  If, however, all of the
1375 		 * reservation's allocated pages are transferred to the new
1376 		 * object, then transferring the reservation is typically
1377 		 * beneficial.  Determining which of these two cases applies
1378 		 * would be more costly than unconditionally renaming the
1379 		 * reservation.
1380 		 */
1381 		vm_reserv_rename(m, new_object, orig_object, offidxstart);
1382 #endif
1383 		if (orig_object->type == OBJT_SWAP)
1384 			vm_page_xbusy(m);
1385 	}
1386 	if (orig_object->type == OBJT_SWAP) {
1387 		/*
1388 		 * swap_pager_copy() can sleep, in which case the orig_object's
1389 		 * and new_object's locks are released and reacquired.
1390 		 */
1391 		swap_pager_copy(orig_object, new_object, offidxstart, 0);
1392 		TAILQ_FOREACH(m, &new_object->memq, listq)
1393 			vm_page_xunbusy(m);
1394 
1395 		/*
1396 		 * Transfer any cached pages from orig_object to new_object.
1397 		 * If swap_pager_copy() found swapped out pages within the
1398 		 * specified range of orig_object, then it changed
1399 		 * new_object's type to OBJT_SWAP when it transferred those
1400 		 * pages to new_object.  Otherwise, new_object's type
1401 		 * should still be OBJT_DEFAULT and orig_object should not
1402 		 * contain any cached pages within the specified range.
1403 		 */
1404 		if (__predict_false(!vm_object_cache_is_empty(orig_object)))
1405 			vm_page_cache_transfer(orig_object, offidxstart,
1406 			    new_object);
1407 	}
1408 	VM_OBJECT_WUNLOCK(orig_object);
1409 	VM_OBJECT_WUNLOCK(new_object);
1410 	entry->object.vm_object = new_object;
1411 	entry->offset = 0LL;
1412 	vm_object_deallocate(orig_object);
1413 	VM_OBJECT_WLOCK(new_object);
1414 }
1415 
1416 #define	OBSC_TEST_ALL_SHADOWED	0x0001
1417 #define	OBSC_COLLAPSE_NOWAIT	0x0002
1418 #define	OBSC_COLLAPSE_WAIT	0x0004
1419 
1420 static int
1421 vm_object_backing_scan(vm_object_t object, int op)
1422 {
1423 	int r = 1;
1424 	vm_page_t p;
1425 	vm_object_t backing_object;
1426 	vm_pindex_t backing_offset_index;
1427 
1428 	VM_OBJECT_ASSERT_WLOCKED(object);
1429 	VM_OBJECT_ASSERT_WLOCKED(object->backing_object);
1430 
1431 	backing_object = object->backing_object;
1432 	backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
1433 
1434 	/*
1435 	 * Initial conditions
1436 	 */
1437 	if (op & OBSC_TEST_ALL_SHADOWED) {
1438 		/*
1439 		 * We do not want to have to test for the existence of cache
1440 		 * or swap pages in the backing object.  XXX but with the
1441 		 * new swapper this would be pretty easy to do.
1442 		 *
1443 		 * XXX what about anonymous MAP_SHARED memory that hasn't
1444 		 * been ZFOD faulted yet?  If we do not test for this, the
1445 		 * shadow test may succeed! XXX
1446 		 */
1447 		if (backing_object->type != OBJT_DEFAULT) {
1448 			return (0);
1449 		}
1450 	}
1451 	if (op & OBSC_COLLAPSE_WAIT) {
1452 		vm_object_set_flag(backing_object, OBJ_DEAD);
1453 	}
1454 
1455 	/*
1456 	 * Our scan
1457 	 */
1458 	p = TAILQ_FIRST(&backing_object->memq);
1459 	while (p) {
1460 		vm_page_t next = TAILQ_NEXT(p, listq);
1461 		vm_pindex_t new_pindex = p->pindex - backing_offset_index;
1462 
1463 		if (op & OBSC_TEST_ALL_SHADOWED) {
1464 			vm_page_t pp;
1465 
1466 			/*
1467 			 * Ignore pages outside the parent object's range
1468 			 * and outside the parent object's mapping of the
1469 			 * backing object.
1470 			 *
1471 			 * note that we do not busy the backing object's
1472 			 * page.
1473 			 */
1474 			if (
1475 			    p->pindex < backing_offset_index ||
1476 			    new_pindex >= object->size
1477 			) {
1478 				p = next;
1479 				continue;
1480 			}
1481 
1482 			/*
1483 			 * See if the parent has the page or if the parent's
1484 			 * object pager has the page.  If the parent has the
1485 			 * page but the page is not valid, the parent's
1486 			 * object pager must have the page.
1487 			 *
1488 			 * If this fails, the parent does not completely shadow
1489 			 * the object and we might as well give up now.
1490 			 */
1491 
1492 			pp = vm_page_lookup(object, new_pindex);
1493 			if (
1494 			    (pp == NULL || pp->valid == 0) &&
1495 			    !vm_pager_has_page(object, new_pindex, NULL, NULL)
1496 			) {
1497 				r = 0;
1498 				break;
1499 			}
1500 		}
1501 
1502 		/*
1503 		 * Check for busy page
1504 		 */
1505 		if (op & (OBSC_COLLAPSE_WAIT | OBSC_COLLAPSE_NOWAIT)) {
1506 			vm_page_t pp;
1507 
1508 			if (op & OBSC_COLLAPSE_NOWAIT) {
1509 				if (!p->valid || vm_page_busied(p)) {
1510 					p = next;
1511 					continue;
1512 				}
1513 			} else if (op & OBSC_COLLAPSE_WAIT) {
1514 				if (vm_page_busied(p)) {
1515 					VM_OBJECT_WUNLOCK(object);
1516 					vm_page_lock(p);
1517 					VM_OBJECT_WUNLOCK(backing_object);
1518 					vm_page_busy_sleep(p, "vmocol");
1519 					VM_OBJECT_WLOCK(object);
1520 					VM_OBJECT_WLOCK(backing_object);
1521 					/*
1522 					 * If we slept, anything could have
1523 					 * happened.  Since the object is
1524 					 * marked dead, the backing offset
1525 					 * should not have changed so we
1526 					 * just restart our scan.
1527 					 */
1528 					p = TAILQ_FIRST(&backing_object->memq);
1529 					continue;
1530 				}
1531 			}
1532 
1533 			KASSERT(
1534 			    p->object == backing_object,
1535 			    ("vm_object_backing_scan: object mismatch")
1536 			);
1537 
1538 			if (
1539 			    p->pindex < backing_offset_index ||
1540 			    new_pindex >= object->size
1541 			) {
1542 				if (backing_object->type == OBJT_SWAP)
1543 					swap_pager_freespace(backing_object,
1544 					    p->pindex, 1);
1545 
1546 				/*
1547 				 * Page is out of the parent object's range, we
1548 				 * can simply destroy it.
1549 				 */
1550 				vm_page_lock(p);
1551 				KASSERT(!pmap_page_is_mapped(p),
1552 				    ("freeing mapped page %p", p));
1553 				if (p->wire_count == 0)
1554 					vm_page_free(p);
1555 				else
1556 					vm_page_remove(p);
1557 				vm_page_unlock(p);
1558 				p = next;
1559 				continue;
1560 			}
1561 
1562 			pp = vm_page_lookup(object, new_pindex);
1563 			if (
1564 			    (op & OBSC_COLLAPSE_NOWAIT) != 0 &&
1565 			    (pp != NULL && pp->valid == 0)
1566 			) {
1567 				if (backing_object->type == OBJT_SWAP)
1568 					swap_pager_freespace(backing_object,
1569 					    p->pindex, 1);
1570 
1571 				/*
1572 				 * The page in the parent is not (yet) valid.
1573 				 * We don't know anything about the state of
1574 				 * the original page.  It might be mapped,
1575 				 * so we must avoid the next if here.
1576 				 *
1577 				 * This is due to a race in vm_fault() where
1578 				 * we must unbusy the original (backing_obj)
1579 				 * page before we can (re)lock the parent.
1580 				 * Hence we can get here.
1581 				 */
1582 				p = next;
1583 				continue;
1584 			}
1585 			if (
1586 			    pp != NULL ||
1587 			    vm_pager_has_page(object, new_pindex, NULL, NULL)
1588 			) {
1589 				if (backing_object->type == OBJT_SWAP)
1590 					swap_pager_freespace(backing_object,
1591 					    p->pindex, 1);
1592 
1593 				/*
1594 				 * page already exists in parent OR swap exists
1595 				 * for this location in the parent.  Destroy
1596 				 * the original page from the backing object.
1597 				 *
1598 				 * Leave the parent's page alone
1599 				 */
1600 				vm_page_lock(p);
1601 				KASSERT(!pmap_page_is_mapped(p),
1602 				    ("freeing mapped page %p", p));
1603 				if (p->wire_count == 0)
1604 					vm_page_free(p);
1605 				else
1606 					vm_page_remove(p);
1607 				vm_page_unlock(p);
1608 				p = next;
1609 				continue;
1610 			}
1611 
1612 			/*
1613 			 * Page does not exist in parent, rename the
1614 			 * page from the backing object to the main object.
1615 			 *
1616 			 * If the page was mapped to a process, it can remain
1617 			 * mapped through the rename.
1618 			 * vm_page_rename() will handle dirty and cache.
1619 			 */
1620 			if (vm_page_rename(p, object, new_pindex)) {
1621 				if (op & OBSC_COLLAPSE_NOWAIT) {
1622 					p = next;
1623 					continue;
1624 				}
1625 				VM_OBJECT_WLOCK(backing_object);
1626 				VM_OBJECT_WUNLOCK(object);
1627 				VM_WAIT;
1628 				VM_OBJECT_WLOCK(object);
1629 				VM_OBJECT_WLOCK(backing_object);
1630 				p = TAILQ_FIRST(&backing_object->memq);
1631 				continue;
1632 			}
1633 
1634 			/* Use the old pindex to free the right page. */
1635 			if (backing_object->type == OBJT_SWAP)
1636 				swap_pager_freespace(backing_object,
1637 				    new_pindex + backing_offset_index, 1);
1638 
1639 #if VM_NRESERVLEVEL > 0
1640 			/*
1641 			 * Rename the reservation.
1642 			 */
1643 			vm_reserv_rename(p, object, backing_object,
1644 			    backing_offset_index);
1645 #endif
1646 		}
1647 		p = next;
1648 	}
1649 	return (r);
1650 }
1651 
1652 
1653 /*
1654  * this version of collapse allows the operation to occur earlier and
1655  * when paging_in_progress is true for an object...  This is not a complete
1656  * operation, but should plug 99.9% of the rest of the leaks.
1657  */
1658 static void
1659 vm_object_qcollapse(vm_object_t object)
1660 {
1661 	vm_object_t backing_object = object->backing_object;
1662 
1663 	VM_OBJECT_ASSERT_WLOCKED(object);
1664 	VM_OBJECT_ASSERT_WLOCKED(backing_object);
1665 
1666 	if (backing_object->ref_count != 1)
1667 		return;
1668 
1669 	vm_object_backing_scan(object, OBSC_COLLAPSE_NOWAIT);
1670 }
1671 
1672 /*
1673  *	vm_object_collapse:
1674  *
1675  *	Collapse an object with the object backing it.
1676  *	Pages in the backing object are moved into the
1677  *	parent, and the backing object is deallocated.
1678  */
1679 void
1680 vm_object_collapse(vm_object_t object)
1681 {
1682 	VM_OBJECT_ASSERT_WLOCKED(object);
1683 
1684 	while (TRUE) {
1685 		vm_object_t backing_object;
1686 
1687 		/*
1688 		 * Verify that the conditions are right for collapse:
1689 		 *
1690 		 * The object exists and the backing object exists.
1691 		 */
1692 		if ((backing_object = object->backing_object) == NULL)
1693 			break;
1694 
1695 		/*
1696 		 * we check the backing object first, because it is most likely
1697 		 * not collapsable.
1698 		 */
1699 		VM_OBJECT_WLOCK(backing_object);
1700 		if (backing_object->handle != NULL ||
1701 		    (backing_object->type != OBJT_DEFAULT &&
1702 		     backing_object->type != OBJT_SWAP) ||
1703 		    (backing_object->flags & OBJ_DEAD) ||
1704 		    object->handle != NULL ||
1705 		    (object->type != OBJT_DEFAULT &&
1706 		     object->type != OBJT_SWAP) ||
1707 		    (object->flags & OBJ_DEAD)) {
1708 			VM_OBJECT_WUNLOCK(backing_object);
1709 			break;
1710 		}
1711 
1712 		if (
1713 		    object->paging_in_progress != 0 ||
1714 		    backing_object->paging_in_progress != 0
1715 		) {
1716 			vm_object_qcollapse(object);
1717 			VM_OBJECT_WUNLOCK(backing_object);
1718 			break;
1719 		}
1720 		/*
1721 		 * We know that we can either collapse the backing object (if
1722 		 * the parent is the only reference to it) or (perhaps) have
1723 		 * the parent bypass the object if the parent happens to shadow
1724 		 * all the resident pages in the entire backing object.
1725 		 *
1726 		 * This is ignoring pager-backed pages such as swap pages.
1727 		 * vm_object_backing_scan fails the shadowing test in this
1728 		 * case.
1729 		 */
1730 		if (backing_object->ref_count == 1) {
1731 			/*
1732 			 * If there is exactly one reference to the backing
1733 			 * object, we can collapse it into the parent.
1734 			 */
1735 			vm_object_backing_scan(object, OBSC_COLLAPSE_WAIT);
1736 
1737 #if VM_NRESERVLEVEL > 0
1738 			/*
1739 			 * Break any reservations from backing_object.
1740 			 */
1741 			if (__predict_false(!LIST_EMPTY(&backing_object->rvq)))
1742 				vm_reserv_break_all(backing_object);
1743 #endif
1744 
1745 			/*
1746 			 * Move the pager from backing_object to object.
1747 			 */
1748 			if (backing_object->type == OBJT_SWAP) {
1749 				/*
1750 				 * swap_pager_copy() can sleep, in which case
1751 				 * the backing_object's and object's locks are
1752 				 * released and reacquired.
1753 				 * Since swap_pager_copy() is being asked to
1754 				 * destroy the source, it will change the
1755 				 * backing_object's type to OBJT_DEFAULT.
1756 				 */
1757 				swap_pager_copy(
1758 				    backing_object,
1759 				    object,
1760 				    OFF_TO_IDX(object->backing_object_offset), TRUE);
1761 
1762 				/*
1763 				 * Free any cached pages from backing_object.
1764 				 */
1765 				if (__predict_false(
1766 				    !vm_object_cache_is_empty(backing_object)))
1767 					vm_page_cache_free(backing_object, 0, 0);
1768 			}
1769 			/*
1770 			 * Object now shadows whatever backing_object did.
1771 			 * Note that the reference to
1772 			 * backing_object->backing_object moves from within
1773 			 * backing_object to within object.
1774 			 */
1775 			LIST_REMOVE(object, shadow_list);
1776 			backing_object->shadow_count--;
1777 			if (backing_object->backing_object) {
1778 				VM_OBJECT_WLOCK(backing_object->backing_object);
1779 				LIST_REMOVE(backing_object, shadow_list);
1780 				LIST_INSERT_HEAD(
1781 				    &backing_object->backing_object->shadow_head,
1782 				    object, shadow_list);
1783 				/*
1784 				 * The shadow_count has not changed.
1785 				 */
1786 				VM_OBJECT_WUNLOCK(backing_object->backing_object);
1787 			}
1788 			object->backing_object = backing_object->backing_object;
1789 			object->backing_object_offset +=
1790 			    backing_object->backing_object_offset;
1791 
1792 			/*
1793 			 * Discard backing_object.
1794 			 *
1795 			 * Since the backing object has no pages, no pager left,
1796 			 * and no object references within it, all that is
1797 			 * necessary is to dispose of it.
1798 			 */
1799 			KASSERT(backing_object->ref_count == 1, (
1800 "backing_object %p was somehow re-referenced during collapse!",
1801 			    backing_object));
1802 			VM_OBJECT_WUNLOCK(backing_object);
1803 			vm_object_destroy(backing_object);
1804 
1805 			object_collapses++;
1806 		} else {
1807 			vm_object_t new_backing_object;
1808 
1809 			/*
1810 			 * If we do not entirely shadow the backing object,
1811 			 * there is nothing we can do so we give up.
1812 			 */
1813 			if (object->resident_page_count != object->size &&
1814 			    vm_object_backing_scan(object,
1815 			    OBSC_TEST_ALL_SHADOWED) == 0) {
1816 				VM_OBJECT_WUNLOCK(backing_object);
1817 				break;
1818 			}
1819 
1820 			/*
1821 			 * Make the parent shadow the next object in the
1822 			 * chain.  Deallocating backing_object will not remove
1823 			 * it, since its reference count is at least 2.
1824 			 */
1825 			LIST_REMOVE(object, shadow_list);
1826 			backing_object->shadow_count--;
1827 
1828 			new_backing_object = backing_object->backing_object;
1829 			if ((object->backing_object = new_backing_object) != NULL) {
1830 				VM_OBJECT_WLOCK(new_backing_object);
1831 				LIST_INSERT_HEAD(
1832 				    &new_backing_object->shadow_head,
1833 				    object,
1834 				    shadow_list
1835 				);
1836 				new_backing_object->shadow_count++;
1837 				vm_object_reference_locked(new_backing_object);
1838 				VM_OBJECT_WUNLOCK(new_backing_object);
1839 				object->backing_object_offset +=
1840 					backing_object->backing_object_offset;
1841 			}
1842 
1843 			/*
1844 			 * Drop the reference count on backing_object. Since
1845 			 * its ref_count was at least 2, it will not vanish.
1846 			 */
1847 			backing_object->ref_count--;
1848 			VM_OBJECT_WUNLOCK(backing_object);
1849 			object_bypasses++;
1850 		}
1851 
1852 		/*
1853 		 * Try again with this object's new backing object.
1854 		 */
1855 	}
1856 }
1857 
1858 /*
1859  *	vm_object_page_remove:
1860  *
1861  *	For the given object, either frees or invalidates each of the
1862  *	specified pages.  In general, a page is freed.  However, if a page is
1863  *	wired for any reason other than the existence of a managed, wired
1864  *	mapping, then it may be invalidated but not removed from the object.
1865  *	Pages are specified by the given range ["start", "end") and the option
1866  *	OBJPR_CLEANONLY.  As a special case, if "end" is zero, then the range
1867  *	extends from "start" to the end of the object.  If the option
1868  *	OBJPR_CLEANONLY is specified, then only the non-dirty pages within the
1869  *	specified range are affected.  If the option OBJPR_NOTMAPPED is
1870  *	specified, then the pages within the specified range must have no
1871  *	mappings.  Otherwise, if this option is not specified, any mappings to
1872  *	the specified pages are removed before the pages are freed or
1873  *	invalidated.
1874  *
1875  *	In general, this operation should only be performed on objects that
1876  *	contain managed pages.  There are, however, two exceptions.  First, it
1877  *	is performed on the kernel and kmem objects by vm_map_entry_delete().
1878  *	Second, it is used by msync(..., MS_INVALIDATE) to invalidate device-
1879  *	backed pages.  In both of these cases, the option OBJPR_CLEANONLY must
1880  *	not be specified and the option OBJPR_NOTMAPPED must be specified.
1881  *
1882  *	The object must be locked.
1883  */
1884 void
1885 vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
1886     int options)
1887 {
1888 	vm_page_t p, next;
1889 	int wirings;
1890 
1891 	VM_OBJECT_ASSERT_WLOCKED(object);
1892 	KASSERT((object->flags & OBJ_UNMANAGED) == 0 ||
1893 	    (options & (OBJPR_CLEANONLY | OBJPR_NOTMAPPED)) == OBJPR_NOTMAPPED,
1894 	    ("vm_object_page_remove: illegal options for object %p", object));
1895 	if (object->resident_page_count == 0)
1896 		goto skipmemq;
1897 	vm_object_pip_add(object, 1);
1898 again:
1899 	p = vm_page_find_least(object, start);
1900 
1901 	/*
1902 	 * Here, the variable "p" is either (1) the page with the least pindex
1903 	 * greater than or equal to the parameter "start" or (2) NULL.
1904 	 */
1905 	for (; p != NULL && (p->pindex < end || end == 0); p = next) {
1906 		next = TAILQ_NEXT(p, listq);
1907 
1908 		/*
1909 		 * If the page is wired for any reason besides the existence
1910 		 * of managed, wired mappings, then it cannot be freed.  For
1911 		 * example, fictitious pages, which represent device memory,
1912 		 * are inherently wired and cannot be freed.  They can,
1913 		 * however, be invalidated if the option OBJPR_CLEANONLY is
1914 		 * not specified.
1915 		 */
1916 		vm_page_lock(p);
1917 		if (vm_page_xbusied(p)) {
1918 			VM_OBJECT_WUNLOCK(object);
1919 			vm_page_busy_sleep(p, "vmopax");
1920 			VM_OBJECT_WLOCK(object);
1921 			goto again;
1922 		}
1923 		if ((wirings = p->wire_count) != 0 &&
1924 		    (wirings = pmap_page_wired_mappings(p)) != p->wire_count) {
1925 			if ((options & (OBJPR_NOTWIRED | OBJPR_NOTMAPPED)) ==
1926 			    0) {
1927 				pmap_remove_all(p);
1928 				/* Account for removal of wired mappings. */
1929 				if (wirings != 0)
1930 					p->wire_count -= wirings;
1931 			}
1932 			if ((options & OBJPR_CLEANONLY) == 0) {
1933 				p->valid = 0;
1934 				vm_page_undirty(p);
1935 			}
1936 			goto next;
1937 		}
1938 		if (vm_page_busied(p)) {
1939 			VM_OBJECT_WUNLOCK(object);
1940 			vm_page_busy_sleep(p, "vmopar");
1941 			VM_OBJECT_WLOCK(object);
1942 			goto again;
1943 		}
1944 		KASSERT((p->flags & PG_FICTITIOUS) == 0,
1945 		    ("vm_object_page_remove: page %p is fictitious", p));
1946 		if ((options & OBJPR_CLEANONLY) != 0 && p->valid != 0) {
1947 			if ((options & OBJPR_NOTMAPPED) == 0)
1948 				pmap_remove_write(p);
1949 			if (p->dirty)
1950 				goto next;
1951 		}
1952 		if ((options & OBJPR_NOTMAPPED) == 0) {
1953 			if ((options & OBJPR_NOTWIRED) != 0 && wirings != 0)
1954 				goto next;
1955 			pmap_remove_all(p);
1956 			/* Account for removal of wired mappings. */
1957 			if (wirings != 0) {
1958 				KASSERT(p->wire_count == wirings,
1959 				    ("inconsistent wire count %d %d %p",
1960 				    p->wire_count, wirings, p));
1961 				p->wire_count = 0;
1962 				atomic_subtract_int(&vm_cnt.v_wire_count, 1);
1963 			}
1964 		}
1965 		vm_page_free(p);
1966 next:
1967 		vm_page_unlock(p);
1968 	}
1969 	vm_object_pip_wakeup(object);
1970 skipmemq:
1971 	if (__predict_false(!vm_object_cache_is_empty(object)))
1972 		vm_page_cache_free(object, start, end);
1973 }
1974 
1975 /*
1976  *	vm_object_page_cache:
1977  *
1978  *	For the given object, attempt to move the specified clean
1979  *	pages to the cache queue.  If a page is wired for any reason,
1980  *	then it will not be changed.  Pages are specified by the given
1981  *	range ["start", "end").  As a special case, if "end" is zero,
1982  *	then the range extends from "start" to the end of the object.
1983  *	Any mappings to the specified pages are removed before the
1984  *	pages are moved to the cache queue.
1985  *
1986  *	This operation should only be performed on objects that
1987  *	contain non-fictitious, managed pages.
1988  *
1989  *	The object must be locked.
1990  */
1991 void
1992 vm_object_page_cache(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
1993 {
1994 	struct mtx *mtx, *new_mtx;
1995 	vm_page_t p, next;
1996 
1997 	VM_OBJECT_ASSERT_WLOCKED(object);
1998 	KASSERT((object->flags & (OBJ_FICTITIOUS | OBJ_UNMANAGED)) == 0,
1999 	    ("vm_object_page_cache: illegal object %p", object));
2000 	if (object->resident_page_count == 0)
2001 		return;
2002 	p = vm_page_find_least(object, start);
2003 
2004 	/*
2005 	 * Here, the variable "p" is either (1) the page with the least pindex
2006 	 * greater than or equal to the parameter "start" or (2) NULL.
2007 	 */
2008 	mtx = NULL;
2009 	for (; p != NULL && (p->pindex < end || end == 0); p = next) {
2010 		next = TAILQ_NEXT(p, listq);
2011 
2012 		/*
2013 		 * Avoid releasing and reacquiring the same page lock.
2014 		 */
2015 		new_mtx = vm_page_lockptr(p);
2016 		if (mtx != new_mtx) {
2017 			if (mtx != NULL)
2018 				mtx_unlock(mtx);
2019 			mtx = new_mtx;
2020 			mtx_lock(mtx);
2021 		}
2022 		vm_page_try_to_cache(p);
2023 	}
2024 	if (mtx != NULL)
2025 		mtx_unlock(mtx);
2026 }
2027 
2028 /*
2029  *	Populate the specified range of the object with valid pages.  Returns
2030  *	TRUE if the range is successfully populated and FALSE otherwise.
2031  *
2032  *	Note: This function should be optimized to pass a larger array of
2033  *	pages to vm_pager_get_pages() before it is applied to a non-
2034  *	OBJT_DEVICE object.
2035  *
2036  *	The object must be locked.
2037  */
2038 boolean_t
2039 vm_object_populate(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
2040 {
2041 	vm_page_t m, ma[1];
2042 	vm_pindex_t pindex;
2043 	int rv;
2044 
2045 	VM_OBJECT_ASSERT_WLOCKED(object);
2046 	for (pindex = start; pindex < end; pindex++) {
2047 		m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL);
2048 		if (m->valid != VM_PAGE_BITS_ALL) {
2049 			ma[0] = m;
2050 			rv = vm_pager_get_pages(object, ma, 1, 0);
2051 			m = vm_page_lookup(object, pindex);
2052 			if (m == NULL)
2053 				break;
2054 			if (rv != VM_PAGER_OK) {
2055 				vm_page_lock(m);
2056 				vm_page_free(m);
2057 				vm_page_unlock(m);
2058 				break;
2059 			}
2060 		}
2061 		/*
2062 		 * Keep "m" busy because a subsequent iteration may unlock
2063 		 * the object.
2064 		 */
2065 	}
2066 	if (pindex > start) {
2067 		m = vm_page_lookup(object, start);
2068 		while (m != NULL && m->pindex < pindex) {
2069 			vm_page_xunbusy(m);
2070 			m = TAILQ_NEXT(m, listq);
2071 		}
2072 	}
2073 	return (pindex == end);
2074 }
2075 
2076 /*
2077  *	Routine:	vm_object_coalesce
2078  *	Function:	Coalesces two objects backing up adjoining
2079  *			regions of memory into a single object.
2080  *
2081  *	returns TRUE if objects were combined.
2082  *
2083  *	NOTE:	Only works at the moment if the second object is NULL -
2084  *		if it's not, which object do we lock first?
2085  *
2086  *	Parameters:
2087  *		prev_object	First object to coalesce
2088  *		prev_offset	Offset into prev_object
2089  *		prev_size	Size of reference to prev_object
2090  *		next_size	Size of reference to the second object
2091  *		reserved	Indicator that extension region has
2092  *				swap accounted for
2093  *
2094  *	Conditions:
2095  *	The object must *not* be locked.
2096  */
2097 boolean_t
2098 vm_object_coalesce(vm_object_t prev_object, vm_ooffset_t prev_offset,
2099     vm_size_t prev_size, vm_size_t next_size, boolean_t reserved)
2100 {
2101 	vm_pindex_t next_pindex;
2102 
2103 	if (prev_object == NULL)
2104 		return (TRUE);
2105 	VM_OBJECT_WLOCK(prev_object);
2106 	if ((prev_object->type != OBJT_DEFAULT &&
2107 	    prev_object->type != OBJT_SWAP) ||
2108 	    (prev_object->flags & OBJ_TMPFS_NODE) != 0) {
2109 		VM_OBJECT_WUNLOCK(prev_object);
2110 		return (FALSE);
2111 	}
2112 
2113 	/*
2114 	 * Try to collapse the object first
2115 	 */
2116 	vm_object_collapse(prev_object);
2117 
2118 	/*
2119 	 * Can't coalesce if: . more than one reference . paged out . shadows
2120 	 * another object . has a copy elsewhere (any of which mean that the
2121 	 * pages not mapped to prev_entry may be in use anyway)
2122 	 */
2123 	if (prev_object->backing_object != NULL) {
2124 		VM_OBJECT_WUNLOCK(prev_object);
2125 		return (FALSE);
2126 	}
2127 
2128 	prev_size >>= PAGE_SHIFT;
2129 	next_size >>= PAGE_SHIFT;
2130 	next_pindex = OFF_TO_IDX(prev_offset) + prev_size;
2131 
2132 	if ((prev_object->ref_count > 1) &&
2133 	    (prev_object->size != next_pindex)) {
2134 		VM_OBJECT_WUNLOCK(prev_object);
2135 		return (FALSE);
2136 	}
2137 
2138 	/*
2139 	 * Account for the charge.
2140 	 */
2141 	if (prev_object->cred != NULL) {
2142 
2143 		/*
2144 		 * If prev_object was charged, then this mapping,
2145 		 * althought not charged now, may become writable
2146 		 * later. Non-NULL cred in the object would prevent
2147 		 * swap reservation during enabling of the write
2148 		 * access, so reserve swap now. Failed reservation
2149 		 * cause allocation of the separate object for the map
2150 		 * entry, and swap reservation for this entry is
2151 		 * managed in appropriate time.
2152 		 */
2153 		if (!reserved && !swap_reserve_by_cred(ptoa(next_size),
2154 		    prev_object->cred)) {
2155 			return (FALSE);
2156 		}
2157 		prev_object->charge += ptoa(next_size);
2158 	}
2159 
2160 	/*
2161 	 * Remove any pages that may still be in the object from a previous
2162 	 * deallocation.
2163 	 */
2164 	if (next_pindex < prev_object->size) {
2165 		vm_object_page_remove(prev_object, next_pindex, next_pindex +
2166 		    next_size, 0);
2167 		if (prev_object->type == OBJT_SWAP)
2168 			swap_pager_freespace(prev_object,
2169 					     next_pindex, next_size);
2170 #if 0
2171 		if (prev_object->cred != NULL) {
2172 			KASSERT(prev_object->charge >=
2173 			    ptoa(prev_object->size - next_pindex),
2174 			    ("object %p overcharged 1 %jx %jx", prev_object,
2175 				(uintmax_t)next_pindex, (uintmax_t)next_size));
2176 			prev_object->charge -= ptoa(prev_object->size -
2177 			    next_pindex);
2178 		}
2179 #endif
2180 	}
2181 
2182 	/*
2183 	 * Extend the object if necessary.
2184 	 */
2185 	if (next_pindex + next_size > prev_object->size)
2186 		prev_object->size = next_pindex + next_size;
2187 
2188 	VM_OBJECT_WUNLOCK(prev_object);
2189 	return (TRUE);
2190 }
2191 
2192 void
2193 vm_object_set_writeable_dirty(vm_object_t object)
2194 {
2195 
2196 	VM_OBJECT_ASSERT_WLOCKED(object);
2197 	if (object->type != OBJT_VNODE)
2198 		return;
2199 	object->generation++;
2200 	if ((object->flags & OBJ_MIGHTBEDIRTY) != 0)
2201 		return;
2202 	vm_object_set_flag(object, OBJ_MIGHTBEDIRTY);
2203 }
2204 
2205 /*
2206  *	vm_object_unwire:
2207  *
2208  *	For each page offset within the specified range of the given object,
2209  *	find the highest-level page in the shadow chain and unwire it.  A page
2210  *	must exist at every page offset, and the highest-level page must be
2211  *	wired.
2212  */
2213 void
2214 vm_object_unwire(vm_object_t object, vm_ooffset_t offset, vm_size_t length,
2215     uint8_t queue)
2216 {
2217 	vm_object_t tobject;
2218 	vm_page_t m, tm;
2219 	vm_pindex_t end_pindex, pindex, tpindex;
2220 	int depth, locked_depth;
2221 
2222 	KASSERT((offset & PAGE_MASK) == 0,
2223 	    ("vm_object_unwire: offset is not page aligned"));
2224 	KASSERT((length & PAGE_MASK) == 0,
2225 	    ("vm_object_unwire: length is not a multiple of PAGE_SIZE"));
2226 	/* The wired count of a fictitious page never changes. */
2227 	if ((object->flags & OBJ_FICTITIOUS) != 0)
2228 		return;
2229 	pindex = OFF_TO_IDX(offset);
2230 	end_pindex = pindex + atop(length);
2231 	locked_depth = 1;
2232 	VM_OBJECT_RLOCK(object);
2233 	m = vm_page_find_least(object, pindex);
2234 	while (pindex < end_pindex) {
2235 		if (m == NULL || pindex < m->pindex) {
2236 			/*
2237 			 * The first object in the shadow chain doesn't
2238 			 * contain a page at the current index.  Therefore,
2239 			 * the page must exist in a backing object.
2240 			 */
2241 			tobject = object;
2242 			tpindex = pindex;
2243 			depth = 0;
2244 			do {
2245 				tpindex +=
2246 				    OFF_TO_IDX(tobject->backing_object_offset);
2247 				tobject = tobject->backing_object;
2248 				KASSERT(tobject != NULL,
2249 				    ("vm_object_unwire: missing page"));
2250 				if ((tobject->flags & OBJ_FICTITIOUS) != 0)
2251 					goto next_page;
2252 				depth++;
2253 				if (depth == locked_depth) {
2254 					locked_depth++;
2255 					VM_OBJECT_RLOCK(tobject);
2256 				}
2257 			} while ((tm = vm_page_lookup(tobject, tpindex)) ==
2258 			    NULL);
2259 		} else {
2260 			tm = m;
2261 			m = TAILQ_NEXT(m, listq);
2262 		}
2263 		vm_page_lock(tm);
2264 		vm_page_unwire(tm, queue);
2265 		vm_page_unlock(tm);
2266 next_page:
2267 		pindex++;
2268 	}
2269 	/* Release the accumulated object locks. */
2270 	for (depth = 0; depth < locked_depth; depth++) {
2271 		tobject = object->backing_object;
2272 		VM_OBJECT_RUNLOCK(object);
2273 		object = tobject;
2274 	}
2275 }
2276 
2277 #include "opt_ddb.h"
2278 #ifdef DDB
2279 #include <sys/kernel.h>
2280 
2281 #include <sys/cons.h>
2282 
2283 #include <ddb/ddb.h>
2284 
2285 static int
2286 _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry)
2287 {
2288 	vm_map_t tmpm;
2289 	vm_map_entry_t tmpe;
2290 	vm_object_t obj;
2291 	int entcount;
2292 
2293 	if (map == 0)
2294 		return 0;
2295 
2296 	if (entry == 0) {
2297 		tmpe = map->header.next;
2298 		entcount = map->nentries;
2299 		while (entcount-- && (tmpe != &map->header)) {
2300 			if (_vm_object_in_map(map, object, tmpe)) {
2301 				return 1;
2302 			}
2303 			tmpe = tmpe->next;
2304 		}
2305 	} else if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
2306 		tmpm = entry->object.sub_map;
2307 		tmpe = tmpm->header.next;
2308 		entcount = tmpm->nentries;
2309 		while (entcount-- && tmpe != &tmpm->header) {
2310 			if (_vm_object_in_map(tmpm, object, tmpe)) {
2311 				return 1;
2312 			}
2313 			tmpe = tmpe->next;
2314 		}
2315 	} else if ((obj = entry->object.vm_object) != NULL) {
2316 		for (; obj; obj = obj->backing_object)
2317 			if (obj == object) {
2318 				return 1;
2319 			}
2320 	}
2321 	return 0;
2322 }
2323 
2324 static int
2325 vm_object_in_map(vm_object_t object)
2326 {
2327 	struct proc *p;
2328 
2329 	/* sx_slock(&allproc_lock); */
2330 	FOREACH_PROC_IN_SYSTEM(p) {
2331 		if (!p->p_vmspace /* || (p->p_flag & (P_SYSTEM|P_WEXIT)) */)
2332 			continue;
2333 		if (_vm_object_in_map(&p->p_vmspace->vm_map, object, 0)) {
2334 			/* sx_sunlock(&allproc_lock); */
2335 			return 1;
2336 		}
2337 	}
2338 	/* sx_sunlock(&allproc_lock); */
2339 	if (_vm_object_in_map(kernel_map, object, 0))
2340 		return 1;
2341 	return 0;
2342 }
2343 
2344 DB_SHOW_COMMAND(vmochk, vm_object_check)
2345 {
2346 	vm_object_t object;
2347 
2348 	/*
2349 	 * make sure that internal objs are in a map somewhere
2350 	 * and none have zero ref counts.
2351 	 */
2352 	TAILQ_FOREACH(object, &vm_object_list, object_list) {
2353 		if (object->handle == NULL &&
2354 		    (object->type == OBJT_DEFAULT || object->type == OBJT_SWAP)) {
2355 			if (object->ref_count == 0) {
2356 				db_printf("vmochk: internal obj has zero ref count: %ld\n",
2357 					(long)object->size);
2358 			}
2359 			if (!vm_object_in_map(object)) {
2360 				db_printf(
2361 			"vmochk: internal obj is not in a map: "
2362 			"ref: %d, size: %lu: 0x%lx, backing_object: %p\n",
2363 				    object->ref_count, (u_long)object->size,
2364 				    (u_long)object->size,
2365 				    (void *)object->backing_object);
2366 			}
2367 		}
2368 	}
2369 }
2370 
2371 /*
2372  *	vm_object_print:	[ debug ]
2373  */
2374 DB_SHOW_COMMAND(object, vm_object_print_static)
2375 {
2376 	/* XXX convert args. */
2377 	vm_object_t object = (vm_object_t)addr;
2378 	boolean_t full = have_addr;
2379 
2380 	vm_page_t p;
2381 
2382 	/* XXX count is an (unused) arg.  Avoid shadowing it. */
2383 #define	count	was_count
2384 
2385 	int count;
2386 
2387 	if (object == NULL)
2388 		return;
2389 
2390 	db_iprintf(
2391 	    "Object %p: type=%d, size=0x%jx, res=%d, ref=%d, flags=0x%x ruid %d charge %jx\n",
2392 	    object, (int)object->type, (uintmax_t)object->size,
2393 	    object->resident_page_count, object->ref_count, object->flags,
2394 	    object->cred ? object->cred->cr_ruid : -1, (uintmax_t)object->charge);
2395 	db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%jx\n",
2396 	    object->shadow_count,
2397 	    object->backing_object ? object->backing_object->ref_count : 0,
2398 	    object->backing_object, (uintmax_t)object->backing_object_offset);
2399 
2400 	if (!full)
2401 		return;
2402 
2403 	db_indent += 2;
2404 	count = 0;
2405 	TAILQ_FOREACH(p, &object->memq, listq) {
2406 		if (count == 0)
2407 			db_iprintf("memory:=");
2408 		else if (count == 6) {
2409 			db_printf("\n");
2410 			db_iprintf(" ...");
2411 			count = 0;
2412 		} else
2413 			db_printf(",");
2414 		count++;
2415 
2416 		db_printf("(off=0x%jx,page=0x%jx)",
2417 		    (uintmax_t)p->pindex, (uintmax_t)VM_PAGE_TO_PHYS(p));
2418 	}
2419 	if (count != 0)
2420 		db_printf("\n");
2421 	db_indent -= 2;
2422 }
2423 
2424 /* XXX. */
2425 #undef count
2426 
2427 /* XXX need this non-static entry for calling from vm_map_print. */
2428 void
2429 vm_object_print(
2430         /* db_expr_t */ long addr,
2431 	boolean_t have_addr,
2432 	/* db_expr_t */ long count,
2433 	char *modif)
2434 {
2435 	vm_object_print_static(addr, have_addr, count, modif);
2436 }
2437 
2438 DB_SHOW_COMMAND(vmopag, vm_object_print_pages)
2439 {
2440 	vm_object_t object;
2441 	vm_pindex_t fidx;
2442 	vm_paddr_t pa;
2443 	vm_page_t m, prev_m;
2444 	int rcount, nl, c;
2445 
2446 	nl = 0;
2447 	TAILQ_FOREACH(object, &vm_object_list, object_list) {
2448 		db_printf("new object: %p\n", (void *)object);
2449 		if (nl > 18) {
2450 			c = cngetc();
2451 			if (c != ' ')
2452 				return;
2453 			nl = 0;
2454 		}
2455 		nl++;
2456 		rcount = 0;
2457 		fidx = 0;
2458 		pa = -1;
2459 		TAILQ_FOREACH(m, &object->memq, listq) {
2460 			if (m->pindex > 128)
2461 				break;
2462 			if ((prev_m = TAILQ_PREV(m, pglist, listq)) != NULL &&
2463 			    prev_m->pindex + 1 != m->pindex) {
2464 				if (rcount) {
2465 					db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2466 						(long)fidx, rcount, (long)pa);
2467 					if (nl > 18) {
2468 						c = cngetc();
2469 						if (c != ' ')
2470 							return;
2471 						nl = 0;
2472 					}
2473 					nl++;
2474 					rcount = 0;
2475 				}
2476 			}
2477 			if (rcount &&
2478 				(VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) {
2479 				++rcount;
2480 				continue;
2481 			}
2482 			if (rcount) {
2483 				db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2484 					(long)fidx, rcount, (long)pa);
2485 				if (nl > 18) {
2486 					c = cngetc();
2487 					if (c != ' ')
2488 						return;
2489 					nl = 0;
2490 				}
2491 				nl++;
2492 			}
2493 			fidx = m->pindex;
2494 			pa = VM_PAGE_TO_PHYS(m);
2495 			rcount = 1;
2496 		}
2497 		if (rcount) {
2498 			db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2499 				(long)fidx, rcount, (long)pa);
2500 			if (nl > 18) {
2501 				c = cngetc();
2502 				if (c != ' ')
2503 					return;
2504 				nl = 0;
2505 			}
2506 			nl++;
2507 		}
2508 	}
2509 }
2510 #endif /* DDB */
2511