xref: /freebsd/sys/vm/vm_object.c (revision 2ad1a3f7295aa9393833907df43945f5278b3eef)
1df8bae1dSRodney W. Grimes /*
2df8bae1dSRodney W. Grimes  * Copyright (c) 1991, 1993
3df8bae1dSRodney W. Grimes  *	The Regents of the University of California.  All rights reserved.
4df8bae1dSRodney W. Grimes  *
5df8bae1dSRodney W. Grimes  * This code is derived from software contributed to Berkeley by
6df8bae1dSRodney W. Grimes  * The Mach Operating System project at Carnegie-Mellon University.
7df8bae1dSRodney W. Grimes  *
8df8bae1dSRodney W. Grimes  * Redistribution and use in source and binary forms, with or without
9df8bae1dSRodney W. Grimes  * modification, are permitted provided that the following conditions
10df8bae1dSRodney W. Grimes  * are met:
11df8bae1dSRodney W. Grimes  * 1. Redistributions of source code must retain the above copyright
12df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer.
13df8bae1dSRodney W. Grimes  * 2. Redistributions in binary form must reproduce the above copyright
14df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer in the
15df8bae1dSRodney W. Grimes  *    documentation and/or other materials provided with the distribution.
16df8bae1dSRodney W. Grimes  * 3. All advertising materials mentioning features or use of this software
17df8bae1dSRodney W. Grimes  *    must display the following acknowledgement:
18df8bae1dSRodney W. Grimes  *	This product includes software developed by the University of
19df8bae1dSRodney W. Grimes  *	California, Berkeley and its contributors.
20df8bae1dSRodney W. Grimes  * 4. Neither the name of the University nor the names of its contributors
21df8bae1dSRodney W. Grimes  *    may be used to endorse or promote products derived from this software
22df8bae1dSRodney W. Grimes  *    without specific prior written permission.
23df8bae1dSRodney W. Grimes  *
24df8bae1dSRodney W. Grimes  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25df8bae1dSRodney W. Grimes  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26df8bae1dSRodney W. Grimes  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27df8bae1dSRodney W. Grimes  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28df8bae1dSRodney W. Grimes  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29df8bae1dSRodney W. Grimes  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30df8bae1dSRodney W. Grimes  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31df8bae1dSRodney W. Grimes  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32df8bae1dSRodney W. Grimes  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33df8bae1dSRodney W. Grimes  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34df8bae1dSRodney W. Grimes  * SUCH DAMAGE.
35df8bae1dSRodney W. Grimes  *
363c4dd356SDavid Greenman  *	from: @(#)vm_object.c	8.5 (Berkeley) 3/22/94
37df8bae1dSRodney W. Grimes  *
38df8bae1dSRodney W. Grimes  *
39df8bae1dSRodney W. Grimes  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
40df8bae1dSRodney W. Grimes  * All rights reserved.
41df8bae1dSRodney W. Grimes  *
42df8bae1dSRodney W. Grimes  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
43df8bae1dSRodney W. Grimes  *
44df8bae1dSRodney W. Grimes  * Permission to use, copy, modify and distribute this software and
45df8bae1dSRodney W. Grimes  * its documentation is hereby granted, provided that both the copyright
46df8bae1dSRodney W. Grimes  * notice and this permission notice appear in all copies of the
47df8bae1dSRodney W. Grimes  * software, derivative works or modified versions, and any portions
48df8bae1dSRodney W. Grimes  * thereof, and that both notices appear in supporting documentation.
49df8bae1dSRodney W. Grimes  *
50df8bae1dSRodney W. Grimes  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
51df8bae1dSRodney W. Grimes  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
52df8bae1dSRodney W. Grimes  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
53df8bae1dSRodney W. Grimes  *
54df8bae1dSRodney W. Grimes  * Carnegie Mellon requests users of this software to return to
55df8bae1dSRodney W. Grimes  *
56df8bae1dSRodney W. Grimes  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
57df8bae1dSRodney W. Grimes  *  School of Computer Science
58df8bae1dSRodney W. Grimes  *  Carnegie Mellon University
59df8bae1dSRodney W. Grimes  *  Pittsburgh PA 15213-3890
60df8bae1dSRodney W. Grimes  *
61df8bae1dSRodney W. Grimes  * any improvements or extensions that they make and grant Carnegie the
62df8bae1dSRodney W. Grimes  * rights to redistribute these changes.
633c4dd356SDavid Greenman  *
642ad1a3f7SMatthew Dillon  * $Id: vm_object.c,v 1.147 1999/02/08 05:15:54 dillon Exp $
65df8bae1dSRodney W. Grimes  */
66df8bae1dSRodney W. Grimes 
67df8bae1dSRodney W. Grimes /*
68df8bae1dSRodney W. Grimes  *	Virtual memory object module.
69df8bae1dSRodney W. Grimes  */
70df8bae1dSRodney W. Grimes 
71df8bae1dSRodney W. Grimes #include <sys/param.h>
72df8bae1dSRodney W. Grimes #include <sys/systm.h>
73f23b4c91SGarrett Wollman #include <sys/proc.h>		/* for curproc, pageproc */
740d94caffSDavid Greenman #include <sys/vnode.h>
75efeaf95aSDavid Greenman #include <sys/vmmeter.h>
76867a482dSJohn Dyson #include <sys/mman.h>
77cf2819ccSJohn Dyson #include <sys/mount.h>
78df8bae1dSRodney W. Grimes 
79df8bae1dSRodney W. Grimes #include <vm/vm.h>
80efeaf95aSDavid Greenman #include <vm/vm_param.h>
81efeaf95aSDavid Greenman #include <vm/vm_prot.h>
82efeaf95aSDavid Greenman #include <vm/pmap.h>
83efeaf95aSDavid Greenman #include <vm/vm_map.h>
84efeaf95aSDavid Greenman #include <vm/vm_object.h>
85df8bae1dSRodney W. Grimes #include <vm/vm_page.h>
8626f9a767SRodney W. Grimes #include <vm/vm_pageout.h>
870d94caffSDavid Greenman #include <vm/vm_pager.h>
8805f0fdd2SPoul-Henning Kamp #include <vm/swap_pager.h>
89a1f6d91cSDavid Greenman #include <vm/vm_kern.h>
90efeaf95aSDavid Greenman #include <vm/vm_extern.h>
9199448ed1SJohn Dyson #include <vm/vm_zone.h>
9226f9a767SRodney W. Grimes 
93cac597e4SBruce Evans static void	vm_object_qcollapse __P((vm_object_t object));
94f6b04d2bSDavid Greenman 
95df8bae1dSRodney W. Grimes /*
96df8bae1dSRodney W. Grimes  *	Virtual memory objects maintain the actual data
97df8bae1dSRodney W. Grimes  *	associated with allocated virtual memory.  A given
98df8bae1dSRodney W. Grimes  *	page of memory exists within exactly one object.
99df8bae1dSRodney W. Grimes  *
100df8bae1dSRodney W. Grimes  *	An object is only deallocated when all "references"
101df8bae1dSRodney W. Grimes  *	are given up.  Only one "reference" to a given
102df8bae1dSRodney W. Grimes  *	region of an object should be writeable.
103df8bae1dSRodney W. Grimes  *
104df8bae1dSRodney W. Grimes  *	Associated with each object is a list of all resident
105df8bae1dSRodney W. Grimes  *	memory pages belonging to that object; this list is
106df8bae1dSRodney W. Grimes  *	maintained by the "vm_page" module, and locked by the object's
107df8bae1dSRodney W. Grimes  *	lock.
108df8bae1dSRodney W. Grimes  *
109df8bae1dSRodney W. Grimes  *	Each object also records a "pager" routine which is
110df8bae1dSRodney W. Grimes  *	used to retrieve (and store) pages to the proper backing
111df8bae1dSRodney W. Grimes  *	storage.  In addition, objects may be backed by other
112df8bae1dSRodney W. Grimes  *	objects from which they were virtual-copied.
113df8bae1dSRodney W. Grimes  *
114df8bae1dSRodney W. Grimes  *	The only items within the object structure which are
115df8bae1dSRodney W. Grimes  *	modified after time of creation are:
116df8bae1dSRodney W. Grimes  *		reference count		locked by object's lock
117df8bae1dSRodney W. Grimes  *		pager routine		locked by object's lock
118df8bae1dSRodney W. Grimes  *
119df8bae1dSRodney W. Grimes  */
120df8bae1dSRodney W. Grimes 
12128f8db14SBruce Evans struct object_q vm_object_list;
122289bdf33SBruce Evans #ifndef NULL_SIMPLELOCKS
123303b270bSEivind Eklund static struct simplelock vm_object_list_lock;
124289bdf33SBruce Evans #endif
1254de628deSBruce Evans static long vm_object_count;		/* count of all objects */
12628f8db14SBruce Evans vm_object_t kernel_object;
12728f8db14SBruce Evans vm_object_t kmem_object;
128f708ef1bSPoul-Henning Kamp static struct vm_object kernel_object_store;
129f708ef1bSPoul-Henning Kamp static struct vm_object kmem_object_store;
130aef922f5SJohn Dyson extern int vm_pageout_page_count;
131df8bae1dSRodney W. Grimes 
132f708ef1bSPoul-Henning Kamp static long object_collapses;
133f708ef1bSPoul-Henning Kamp static long object_bypasses;
1345070c7f8SJohn Dyson static int next_index;
13599448ed1SJohn Dyson static vm_zone_t obj_zone;
13699448ed1SJohn Dyson static struct vm_zone obj_zone_store;
1371c7c3c6aSMatthew Dillon static int object_hash_rand;
13899448ed1SJohn Dyson #define VM_OBJECTS_INIT 256
139303b270bSEivind Eklund static struct vm_object vm_objects_init[VM_OBJECTS_INIT];
140df8bae1dSRodney W. Grimes 
1413075778bSJohn Dyson void
14224a1cce3SDavid Greenman _vm_object_allocate(type, size, object)
14324a1cce3SDavid Greenman 	objtype_t type;
144df8bae1dSRodney W. Grimes 	vm_size_t size;
145d031cff1SMatthew Dillon 	vm_object_t object;
146df8bae1dSRodney W. Grimes {
14799448ed1SJohn Dyson 	int incr;
148df8bae1dSRodney W. Grimes 	TAILQ_INIT(&object->memq);
14924a1cce3SDavid Greenman 	TAILQ_INIT(&object->shadow_head);
150a1f6d91cSDavid Greenman 
15124a1cce3SDavid Greenman 	object->type = type;
152df8bae1dSRodney W. Grimes 	object->size = size;
153a1f6d91cSDavid Greenman 	object->ref_count = 1;
15424a1cce3SDavid Greenman 	object->flags = 0;
155c0877f10SJohn Dyson 	if ((object->type == OBJT_DEFAULT) || (object->type == OBJT_SWAP))
156069e9bc1SDoug Rabson 		vm_object_set_flag(object, OBJ_ONEMAPPING);
157867a482dSJohn Dyson 	object->behavior = OBJ_NORMAL;
158df8bae1dSRodney W. Grimes 	object->paging_in_progress = 0;
159a1f6d91cSDavid Greenman 	object->resident_page_count = 0;
160eaf13dd7SJohn Dyson 	object->cache_count = 0;
161eaf13dd7SJohn Dyson 	object->wire_count = 0;
162de5f6a77SJohn Dyson 	object->shadow_count = 0;
1635070c7f8SJohn Dyson 	object->pg_color = next_index;
16499448ed1SJohn Dyson 	if ( size > (PQ_L2_SIZE / 3 + PQ_PRIME1))
16599448ed1SJohn Dyson 		incr = PQ_L2_SIZE / 3 + PQ_PRIME1;
16699448ed1SJohn Dyson 	else
16799448ed1SJohn Dyson 		incr = size;
16899448ed1SJohn Dyson 	next_index = (next_index + incr) & PQ_L2_MASK;
16924a1cce3SDavid Greenman 	object->handle = NULL;
17024a1cce3SDavid Greenman 	object->backing_object = NULL;
171a316d390SJohn Dyson 	object->backing_object_offset = (vm_ooffset_t) 0;
1721c7c3c6aSMatthew Dillon 	/*
1731c7c3c6aSMatthew Dillon 	 * Try to generate a number that will spread objects out in the
1741c7c3c6aSMatthew Dillon 	 * hash table.  We 'wipe' new objects across the hash in 128 page
1751c7c3c6aSMatthew Dillon 	 * increments plus 1 more to offset it a little more by the time
1761c7c3c6aSMatthew Dillon 	 * it wraps around.
1771c7c3c6aSMatthew Dillon 	 */
1781c7c3c6aSMatthew Dillon 	object->hash_rand = object_hash_rand - 129;
179a1f6d91cSDavid Greenman 
180a1f6d91cSDavid Greenman 	object->last_read = 0;
1812d8acc0fSJohn Dyson 	object->generation++;
182df8bae1dSRodney W. Grimes 
183df8bae1dSRodney W. Grimes 	TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);
184df8bae1dSRodney W. Grimes 	vm_object_count++;
1851c7c3c6aSMatthew Dillon 	object_hash_rand = object->hash_rand;
186df8bae1dSRodney W. Grimes }
187df8bae1dSRodney W. Grimes 
188df8bae1dSRodney W. Grimes /*
18926f9a767SRodney W. Grimes  *	vm_object_init:
19026f9a767SRodney W. Grimes  *
19126f9a767SRodney W. Grimes  *	Initialize the VM objects module.
19226f9a767SRodney W. Grimes  */
19326f9a767SRodney W. Grimes void
194a316d390SJohn Dyson vm_object_init()
19526f9a767SRodney W. Grimes {
19626f9a767SRodney W. Grimes 	TAILQ_INIT(&vm_object_list);
197996c772fSJohn Dyson 	simple_lock_init(&vm_object_list_lock);
19826f9a767SRodney W. Grimes 	vm_object_count = 0;
1990217125fSDavid Greenman 
20026f9a767SRodney W. Grimes 	kernel_object = &kernel_object_store;
201a316d390SJohn Dyson 	_vm_object_allocate(OBJT_DEFAULT, OFF_TO_IDX(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS),
20226f9a767SRodney W. Grimes 	    kernel_object);
20326f9a767SRodney W. Grimes 
20426f9a767SRodney W. Grimes 	kmem_object = &kmem_object_store;
205a316d390SJohn Dyson 	_vm_object_allocate(OBJT_DEFAULT, OFF_TO_IDX(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS),
20626f9a767SRodney W. Grimes 	    kmem_object);
20799448ed1SJohn Dyson 
20899448ed1SJohn Dyson 	obj_zone = &obj_zone_store;
20999448ed1SJohn Dyson 	zbootinit(obj_zone, "VM OBJECT", sizeof (struct vm_object),
21099448ed1SJohn Dyson 		vm_objects_init, VM_OBJECTS_INIT);
21199448ed1SJohn Dyson }
21299448ed1SJohn Dyson 
21399448ed1SJohn Dyson void
21499448ed1SJohn Dyson vm_object_init2() {
2150a80f406SJohn Dyson 	zinitna(obj_zone, NULL, NULL, 0, 0, 0, 1);
21626f9a767SRodney W. Grimes }
21726f9a767SRodney W. Grimes 
21826f9a767SRodney W. Grimes /*
21926f9a767SRodney W. Grimes  *	vm_object_allocate:
22026f9a767SRodney W. Grimes  *
22126f9a767SRodney W. Grimes  *	Returns a new object with the given size.
22226f9a767SRodney W. Grimes  */
22326f9a767SRodney W. Grimes 
22426f9a767SRodney W. Grimes vm_object_t
22524a1cce3SDavid Greenman vm_object_allocate(type, size)
22624a1cce3SDavid Greenman 	objtype_t type;
22726f9a767SRodney W. Grimes 	vm_size_t size;
22826f9a767SRodney W. Grimes {
229d031cff1SMatthew Dillon 	vm_object_t result;
230d031cff1SMatthew Dillon 
23199448ed1SJohn Dyson 	result = (vm_object_t) zalloc(obj_zone);
23267bf6868SJohn Dyson 
23324a1cce3SDavid Greenman 	_vm_object_allocate(type, size, result);
23426f9a767SRodney W. Grimes 
23526f9a767SRodney W. Grimes 	return (result);
23626f9a767SRodney W. Grimes }
23726f9a767SRodney W. Grimes 
23826f9a767SRodney W. Grimes 
23926f9a767SRodney W. Grimes /*
240df8bae1dSRodney W. Grimes  *	vm_object_reference:
241df8bae1dSRodney W. Grimes  *
242df8bae1dSRodney W. Grimes  *	Gets another reference to the given object.
243df8bae1dSRodney W. Grimes  */
2446476c0d2SJohn Dyson void
24526f9a767SRodney W. Grimes vm_object_reference(object)
246d031cff1SMatthew Dillon 	vm_object_t object;
247df8bae1dSRodney W. Grimes {
248df8bae1dSRodney W. Grimes 	if (object == NULL)
249df8bae1dSRodney W. Grimes 		return;
25095e5e988SJohn Dyson 
2515526d2d9SEivind Eklund 	KASSERT(!(object->flags & OBJ_DEAD),
2525526d2d9SEivind Eklund 	    ("vm_object_reference: attempting to reference dead obj"));
25395e5e988SJohn Dyson 
254df8bae1dSRodney W. Grimes 	object->ref_count++;
25547221757SJohn Dyson 	if (object->type == OBJT_VNODE) {
25647221757SJohn Dyson 		while (vget((struct vnode *) object->handle, LK_RETRY|LK_NOOBJ, curproc)) {
25795461b45SJohn Dyson #if !defined(MAX_PERF)
25847221757SJohn Dyson 			printf("vm_object_reference: delay in getting object\n");
25995461b45SJohn Dyson #endif
26047221757SJohn Dyson 		}
26147221757SJohn Dyson 	}
26295e5e988SJohn Dyson }
26395e5e988SJohn Dyson 
264bf27292bSJohn Dyson void
26595e5e988SJohn Dyson vm_object_vndeallocate(object)
26695e5e988SJohn Dyson 	vm_object_t object;
26795e5e988SJohn Dyson {
26895e5e988SJohn Dyson 	struct vnode *vp = (struct vnode *) object->handle;
269219cbf59SEivind Eklund 
2705526d2d9SEivind Eklund 	KASSERT(object->type == OBJT_VNODE,
2715526d2d9SEivind Eklund 	    ("vm_object_vndeallocate: not a vnode object"));
272219cbf59SEivind Eklund 	KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp"));
273219cbf59SEivind Eklund #ifdef INVARIANTS
27495e5e988SJohn Dyson 	if (object->ref_count == 0) {
27595e5e988SJohn Dyson 		vprint("vm_object_vndeallocate", vp);
27695e5e988SJohn Dyson 		panic("vm_object_vndeallocate: bad object reference count");
27795e5e988SJohn Dyson 	}
27895e5e988SJohn Dyson #endif
27995e5e988SJohn Dyson 
28095e5e988SJohn Dyson 	object->ref_count--;
28147221757SJohn Dyson 	if (object->ref_count == 0) {
282bf27292bSJohn Dyson 		vp->v_flag &= ~VTEXT;
283069e9bc1SDoug Rabson 		vm_object_clear_flag(object, OBJ_OPT);
2842be70f79SJohn Dyson 	}
28547221757SJohn Dyson 	vrele(vp);
286df8bae1dSRodney W. Grimes }
287df8bae1dSRodney W. Grimes 
288df8bae1dSRodney W. Grimes /*
289df8bae1dSRodney W. Grimes  *	vm_object_deallocate:
290df8bae1dSRodney W. Grimes  *
291df8bae1dSRodney W. Grimes  *	Release a reference to the specified object,
292df8bae1dSRodney W. Grimes  *	gained either through a vm_object_allocate
293df8bae1dSRodney W. Grimes  *	or a vm_object_reference call.  When all references
294df8bae1dSRodney W. Grimes  *	are gone, storage associated with this object
295df8bae1dSRodney W. Grimes  *	may be relinquished.
296df8bae1dSRodney W. Grimes  *
297df8bae1dSRodney W. Grimes  *	No object may be locked.
298df8bae1dSRodney W. Grimes  */
29926f9a767SRodney W. Grimes void
30026f9a767SRodney W. Grimes vm_object_deallocate(object)
30126f9a767SRodney W. Grimes 	vm_object_t object;
302df8bae1dSRodney W. Grimes {
303df8bae1dSRodney W. Grimes 	vm_object_t temp;
304df8bae1dSRodney W. Grimes 
305df8bae1dSRodney W. Grimes 	while (object != NULL) {
306df8bae1dSRodney W. Grimes 
30795e5e988SJohn Dyson 		if (object->type == OBJT_VNODE) {
30895e5e988SJohn Dyson 			vm_object_vndeallocate(object);
30995e5e988SJohn Dyson 			return;
31095e5e988SJohn Dyson 		}
31195e5e988SJohn Dyson 
3122be70f79SJohn Dyson 		if (object->ref_count == 0) {
31347221757SJohn Dyson 			panic("vm_object_deallocate: object deallocated too many times: %d", object->type);
3142be70f79SJohn Dyson 		} else if (object->ref_count > 2) {
3152be70f79SJohn Dyson 			object->ref_count--;
3162be70f79SJohn Dyson 			return;
3172be70f79SJohn Dyson 		}
3182be70f79SJohn Dyson 
3192be70f79SJohn Dyson 		/*
3202be70f79SJohn Dyson 		 * Here on ref_count of one or two, which are special cases for
3212be70f79SJohn Dyson 		 * objects.
3222be70f79SJohn Dyson 		 */
323c0877f10SJohn Dyson 		if ((object->ref_count == 2) && (object->shadow_count == 0)) {
324069e9bc1SDoug Rabson 			vm_object_set_flag(object, OBJ_ONEMAPPING);
325c0877f10SJohn Dyson 			object->ref_count--;
326c0877f10SJohn Dyson 			return;
327c0877f10SJohn Dyson 		} else if ((object->ref_count == 2) && (object->shadow_count == 1)) {
328be6d5bfaSDavid Greenman 			object->ref_count--;
3292be70f79SJohn Dyson 			if ((object->handle == NULL) &&
33024a1cce3SDavid Greenman 			    (object->type == OBJT_DEFAULT ||
33124a1cce3SDavid Greenman 			     object->type == OBJT_SWAP)) {
332a1f6d91cSDavid Greenman 				vm_object_t robject;
33395e5e988SJohn Dyson 
334b18bfc3dSJohn Dyson 				robject = TAILQ_FIRST(&object->shadow_head);
3355526d2d9SEivind Eklund 				KASSERT(robject != NULL,
336219cbf59SEivind Eklund 				    ("vm_object_deallocate: ref_count: %d, shadow_count: %d",
3375526d2d9SEivind Eklund 					 object->ref_count,
3385526d2d9SEivind Eklund 					 object->shadow_count));
33995e5e988SJohn Dyson 				if ((robject->handle == NULL) &&
34024a1cce3SDavid Greenman 				    (robject->type == OBJT_DEFAULT ||
34124a1cce3SDavid Greenman 				     robject->type == OBJT_SWAP)) {
342a1f6d91cSDavid Greenman 
34395e5e988SJohn Dyson 					robject->ref_count++;
34495e5e988SJohn Dyson 
3451c7c3c6aSMatthew Dillon 					while (
3461c7c3c6aSMatthew Dillon 						robject->paging_in_progress ||
3471c7c3c6aSMatthew Dillon 						object->paging_in_progress
3481c7c3c6aSMatthew Dillon 					) {
34966095752SJohn Dyson 						vm_object_pip_sleep(robject, "objde1");
35066095752SJohn Dyson 						vm_object_pip_sleep(object, "objde2");
351a1f6d91cSDavid Greenman 					}
352a1f6d91cSDavid Greenman 
35395e5e988SJohn Dyson 					if (robject->ref_count == 1) {
35495e5e988SJohn Dyson 						robject->ref_count--;
355ba8da839SDavid Greenman 						object = robject;
35695e5e988SJohn Dyson 						goto doterm;
35795e5e988SJohn Dyson 					}
35895e5e988SJohn Dyson 
35995e5e988SJohn Dyson 					object = robject;
36095e5e988SJohn Dyson 					vm_object_collapse(object);
361ba8da839SDavid Greenman 					continue;
362a1f6d91cSDavid Greenman 				}
36395e5e988SJohn Dyson 			}
36495e5e988SJohn Dyson 
365a1f6d91cSDavid Greenman 			return;
366df8bae1dSRodney W. Grimes 
36795e5e988SJohn Dyson 		} else {
3681efb74fbSJohn Dyson 			object->ref_count--;
36995e5e988SJohn Dyson 			if (object->ref_count != 0)
37095e5e988SJohn Dyson 				return;
37195e5e988SJohn Dyson 		}
37295e5e988SJohn Dyson 
37395e5e988SJohn Dyson doterm:
3741efb74fbSJohn Dyson 
37524a1cce3SDavid Greenman 		temp = object->backing_object;
376de5f6a77SJohn Dyson 		if (temp) {
37724a1cce3SDavid Greenman 			TAILQ_REMOVE(&temp->shadow_head, object, shadow_list);
37895e5e988SJohn Dyson 			temp->shadow_count--;
3792d8acc0fSJohn Dyson 			if (temp->ref_count == 0)
380069e9bc1SDoug Rabson 				vm_object_clear_flag(temp, OBJ_OPT);
381eaf13dd7SJohn Dyson 			temp->generation++;
38295461b45SJohn Dyson 			object->backing_object = NULL;
383de5f6a77SJohn Dyson 		}
384df8bae1dSRodney W. Grimes 		vm_object_terminate(object);
385df8bae1dSRodney W. Grimes 		/* unlocks and deallocates object */
386df8bae1dSRodney W. Grimes 		object = temp;
387df8bae1dSRodney W. Grimes 	}
388df8bae1dSRodney W. Grimes }
389df8bae1dSRodney W. Grimes 
390df8bae1dSRodney W. Grimes /*
391df8bae1dSRodney W. Grimes  *	vm_object_terminate actually destroys the specified object, freeing
392df8bae1dSRodney W. Grimes  *	up all previously used resources.
393df8bae1dSRodney W. Grimes  *
394df8bae1dSRodney W. Grimes  *	The object must be locked.
3951c7c3c6aSMatthew Dillon  *	This routine may block.
396df8bae1dSRodney W. Grimes  */
39795e5e988SJohn Dyson void
39826f9a767SRodney W. Grimes vm_object_terminate(object)
399d031cff1SMatthew Dillon 	vm_object_t object;
400df8bae1dSRodney W. Grimes {
401d031cff1SMatthew Dillon 	vm_page_t p;
402e4b7635dSDavid Greenman 	int s;
403df8bae1dSRodney W. Grimes 
40495e5e988SJohn Dyson 	/*
40595e5e988SJohn Dyson 	 * Make sure no one uses us.
40695e5e988SJohn Dyson 	 */
407069e9bc1SDoug Rabson 	vm_object_set_flag(object, OBJ_DEAD);
4083c631446SJohn Dyson 
409df8bae1dSRodney W. Grimes 	/*
410f6b04d2bSDavid Greenman 	 * wait for the pageout daemon to be done with the object
411df8bae1dSRodney W. Grimes 	 */
41266095752SJohn Dyson 	vm_object_pip_wait(object, "objtrm");
413df8bae1dSRodney W. Grimes 
4145526d2d9SEivind Eklund 	KASSERT(!object->paging_in_progress,
4155526d2d9SEivind Eklund 		("vm_object_terminate: pageout in progress"));
41626f9a767SRodney W. Grimes 
41726f9a767SRodney W. Grimes 	/*
4180d94caffSDavid Greenman 	 * Clean and free the pages, as appropriate. All references to the
4190d94caffSDavid Greenman 	 * object are gone, so we don't need to lock it.
42026f9a767SRodney W. Grimes 	 */
42124a1cce3SDavid Greenman 	if (object->type == OBJT_VNODE) {
42295e5e988SJohn Dyson 		struct vnode *vp;
42395e5e988SJohn Dyson 
42495e5e988SJohn Dyson 		/*
42595e5e988SJohn Dyson 		 * Freeze optimized copies.
42695e5e988SJohn Dyson 		 */
42795e5e988SJohn Dyson 		vm_freeze_copyopts(object, 0, object->size);
42895e5e988SJohn Dyson 
42995e5e988SJohn Dyson 		/*
43095e5e988SJohn Dyson 		 * Clean pages and flush buffers.
43195e5e988SJohn Dyson 		 */
4328f9110f6SJohn Dyson 		vm_object_page_clean(object, 0, 0, OBJPC_SYNC);
43395e5e988SJohn Dyson 
43495e5e988SJohn Dyson 		vp = (struct vnode *) object->handle;
435f6b04d2bSDavid Greenman 		vinvalbuf(vp, V_SAVE, NOCRED, NULL, 0, 0);
436bef608bdSJohn Dyson 	}
437bef608bdSJohn Dyson 
4380b10ba98SDavid Greenman 	if (object->ref_count != 0)
4390b10ba98SDavid Greenman 		panic("vm_object_terminate: object with references, ref_count=%d", object->ref_count);
440996c772fSJohn Dyson 
4410d94caffSDavid Greenman 	/*
442356863ebSDavid Greenman 	 * Now free any remaining pages. For internal objects, this also
443356863ebSDavid Greenman 	 * removes them from paging queues. Don't free wired pages, just
444356863ebSDavid Greenman 	 * remove them from the object.
445df8bae1dSRodney W. Grimes 	 */
446e4b7635dSDavid Greenman 	s = splvm();
447b18bfc3dSJohn Dyson 	while ((p = TAILQ_FIRST(&object->memq)) != NULL) {
44895461b45SJohn Dyson #if !defined(MAX_PERF)
449a2f4a846SJohn Dyson 		if (p->busy || (p->flags & PG_BUSY))
4501c7c3c6aSMatthew Dillon 			panic("vm_object_terminate: freeing busy page %p\n", p);
45195461b45SJohn Dyson #endif
4520b10ba98SDavid Greenman 		if (p->wire_count == 0) {
453e69763a3SDoug Rabson 			vm_page_busy(p);
454df8bae1dSRodney W. Grimes 			vm_page_free(p);
455df8bae1dSRodney W. Grimes 			cnt.v_pfree++;
4560b10ba98SDavid Greenman 		} else {
457356863ebSDavid Greenman 			vm_page_busy(p);
4580b10ba98SDavid Greenman 			vm_page_remove(p);
4590b10ba98SDavid Greenman 		}
460df8bae1dSRodney W. Grimes 	}
461e4b7635dSDavid Greenman 	splx(s);
462bef608bdSJohn Dyson 
4632d8acc0fSJohn Dyson 	/*
4649fcfb650SDavid Greenman 	 * Let the pager know object is dead.
4659fcfb650SDavid Greenman 	 */
4669fcfb650SDavid Greenman 	vm_pager_deallocate(object);
4679fcfb650SDavid Greenman 
4689fcfb650SDavid Greenman 	/*
4690b10ba98SDavid Greenman 	 * Remove the object from the global object list.
4702d8acc0fSJohn Dyson 	 */
471996c772fSJohn Dyson 	simple_lock(&vm_object_list_lock);
472df8bae1dSRodney W. Grimes 	TAILQ_REMOVE(&vm_object_list, object, object_list);
473996c772fSJohn Dyson 	simple_unlock(&vm_object_list_lock);
4740b10ba98SDavid Greenman 
4750b10ba98SDavid Greenman 	wakeup(object);
4760b10ba98SDavid Greenman 
477df8bae1dSRodney W. Grimes 	/*
478df8bae1dSRodney W. Grimes 	 * Free the space for the object.
479df8bae1dSRodney W. Grimes 	 */
48099448ed1SJohn Dyson 	zfree(obj_zone, object);
48147221757SJohn Dyson }
482df8bae1dSRodney W. Grimes 
483df8bae1dSRodney W. Grimes /*
484df8bae1dSRodney W. Grimes  *	vm_object_page_clean
485df8bae1dSRodney W. Grimes  *
486df8bae1dSRodney W. Grimes  *	Clean all dirty pages in the specified range of object.
48726f9a767SRodney W. Grimes  *	Leaves page on whatever queue it is currently on.
48826f9a767SRodney W. Grimes  *
48926f9a767SRodney W. Grimes  *	Odd semantics: if start == end, we clean everything.
49026f9a767SRodney W. Grimes  *
49126f9a767SRodney W. Grimes  *	The object must be locked.
49226f9a767SRodney W. Grimes  */
493f6b04d2bSDavid Greenman 
494f6b04d2bSDavid Greenman void
4958f9110f6SJohn Dyson vm_object_page_clean(object, start, end, flags)
496f6b04d2bSDavid Greenman 	vm_object_t object;
497a316d390SJohn Dyson 	vm_pindex_t start;
498a316d390SJohn Dyson 	vm_pindex_t end;
4998f9110f6SJohn Dyson 	int flags;
500f6b04d2bSDavid Greenman {
501d031cff1SMatthew Dillon 	vm_page_t p, np, tp;
502d031cff1SMatthew Dillon 	vm_offset_t tstart, tend;
503bd7e5f99SJohn Dyson 	vm_pindex_t pi;
504aef922f5SJohn Dyson 	int s;
50524a1cce3SDavid Greenman 	struct vnode *vp;
506aef922f5SJohn Dyson 	int runlen;
507bd7e5f99SJohn Dyson 	int maxf;
508bd7e5f99SJohn Dyson 	int chkb;
509bd7e5f99SJohn Dyson 	int maxb;
510bd7e5f99SJohn Dyson 	int i;
5118f9110f6SJohn Dyson 	int pagerflags;
512bd7e5f99SJohn Dyson 	vm_page_t maf[vm_pageout_page_count];
513bd7e5f99SJohn Dyson 	vm_page_t mab[vm_pageout_page_count];
514aef922f5SJohn Dyson 	vm_page_t ma[vm_pageout_page_count];
5152d8acc0fSJohn Dyson 	int curgeneration;
516f6b04d2bSDavid Greenman 
517aef922f5SJohn Dyson 	if (object->type != OBJT_VNODE ||
518aef922f5SJohn Dyson 		(object->flags & OBJ_MIGHTBEDIRTY) == 0)
519f6b04d2bSDavid Greenman 		return;
520f6b04d2bSDavid Greenman 
5218f9110f6SJohn Dyson 	pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) ? VM_PAGER_PUT_SYNC : 0;
5228f9110f6SJohn Dyson 	pagerflags |= (flags & OBJPC_INVAL) ? VM_PAGER_PUT_INVAL : 0;
5238f9110f6SJohn Dyson 
52424a1cce3SDavid Greenman 	vp = object->handle;
52524a1cce3SDavid Greenman 
526069e9bc1SDoug Rabson 	vm_object_set_flag(object, OBJ_CLEANING);
52724a1cce3SDavid Greenman 
528f6b04d2bSDavid Greenman 	tstart = start;
529f6b04d2bSDavid Greenman 	if (end == 0) {
530f6b04d2bSDavid Greenman 		tend = object->size;
531f6b04d2bSDavid Greenman 	} else {
532f6b04d2bSDavid Greenman 		tend = end;
533f6b04d2bSDavid Greenman 	}
534eaf13dd7SJohn Dyson 
535eaf13dd7SJohn Dyson 	for(p = TAILQ_FIRST(&object->memq); p; p = TAILQ_NEXT(p, listq)) {
536e69763a3SDoug Rabson 		vm_page_flag_set(p, PG_CLEANCHK);
537eaf13dd7SJohn Dyson 		vm_page_protect(p, VM_PROT_READ);
538eaf13dd7SJohn Dyson 	}
539eaf13dd7SJohn Dyson 
540a316d390SJohn Dyson 	if ((tstart == 0) && (tend == object->size)) {
541069e9bc1SDoug Rabson 		vm_object_clear_flag(object, OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY);
542ec4f9fb0SDavid Greenman 	}
543f6b04d2bSDavid Greenman 
544bd7e5f99SJohn Dyson rescan:
5452d8acc0fSJohn Dyson 	curgeneration = object->generation;
5462d8acc0fSJohn Dyson 
547b18bfc3dSJohn Dyson 	for(p = TAILQ_FIRST(&object->memq); p; p = np) {
548b18bfc3dSJohn Dyson 		np = TAILQ_NEXT(p, listq);
549bd7e5f99SJohn Dyson 
550bd7e5f99SJohn Dyson 		pi = p->pindex;
551bd7e5f99SJohn Dyson 		if (((p->flags & PG_CLEANCHK) == 0) ||
552bd7e5f99SJohn Dyson 			(pi < tstart) || (pi >= tend) ||
5535070c7f8SJohn Dyson 			(p->valid == 0) ||
5545070c7f8SJohn Dyson 			((p->queue - p->pc) == PQ_CACHE)) {
555e69763a3SDoug Rabson 			vm_page_flag_clear(p, PG_CLEANCHK);
556aef922f5SJohn Dyson 			continue;
557f6b04d2bSDavid Greenman 		}
558f6b04d2bSDavid Greenman 
559bd7e5f99SJohn Dyson 		vm_page_test_dirty(p);
560bd7e5f99SJohn Dyson 		if ((p->dirty & p->valid) == 0) {
561e69763a3SDoug Rabson 			vm_page_flag_clear(p, PG_CLEANCHK);
562bd7e5f99SJohn Dyson 			continue;
563bd7e5f99SJohn Dyson 		}
564ec4f9fb0SDavid Greenman 
565b18bfc3dSJohn Dyson 		s = splvm();
5661c7c3c6aSMatthew Dillon 		while (vm_page_sleep_busy(p, TRUE, "vpcwai")) {
5672d8acc0fSJohn Dyson 			if (object->generation != curgeneration) {
568f6b04d2bSDavid Greenman 				splx(s);
569bd7e5f99SJohn Dyson 				goto rescan;
570f6b04d2bSDavid Greenman 			}
5712d8acc0fSJohn Dyson 		}
572f6b04d2bSDavid Greenman 
573bd7e5f99SJohn Dyson 		maxf = 0;
574bd7e5f99SJohn Dyson 		for(i=1;i<vm_pageout_page_count;i++) {
5758aef1712SMatthew Dillon 			if ((tp = vm_page_lookup(object, pi + i)) != NULL) {
576bd7e5f99SJohn Dyson 				if ((tp->flags & PG_BUSY) ||
577ffc82b0aSJohn Dyson 					(tp->flags & PG_CLEANCHK) == 0 ||
578ffc82b0aSJohn Dyson 					(tp->busy != 0))
579bd7e5f99SJohn Dyson 					break;
5805070c7f8SJohn Dyson 				if((tp->queue - tp->pc) == PQ_CACHE) {
581e69763a3SDoug Rabson 					vm_page_flag_clear(tp, PG_CLEANCHK);
5823077a9c2SJohn Dyson 					break;
5833077a9c2SJohn Dyson 				}
584bd7e5f99SJohn Dyson 				vm_page_test_dirty(tp);
585bd7e5f99SJohn Dyson 				if ((tp->dirty & tp->valid) == 0) {
586e69763a3SDoug Rabson 					vm_page_flag_clear(tp, PG_CLEANCHK);
587bd7e5f99SJohn Dyson 					break;
588bd7e5f99SJohn Dyson 				}
589bd7e5f99SJohn Dyson 				maf[ i - 1 ] = tp;
590bd7e5f99SJohn Dyson 				maxf++;
591bd7e5f99SJohn Dyson 				continue;
592bd7e5f99SJohn Dyson 			}
593bd7e5f99SJohn Dyson 			break;
594bd7e5f99SJohn Dyson 		}
595aef922f5SJohn Dyson 
596bd7e5f99SJohn Dyson 		maxb = 0;
597bd7e5f99SJohn Dyson 		chkb = vm_pageout_page_count -  maxf;
598bd7e5f99SJohn Dyson 		if (chkb) {
599bd7e5f99SJohn Dyson 			for(i = 1; i < chkb;i++) {
6008aef1712SMatthew Dillon 				if ((tp = vm_page_lookup(object, pi - i)) != NULL) {
601bd7e5f99SJohn Dyson 					if ((tp->flags & PG_BUSY) ||
602ffc82b0aSJohn Dyson 						(tp->flags & PG_CLEANCHK) == 0 ||
603ffc82b0aSJohn Dyson 						(tp->busy != 0))
604bd7e5f99SJohn Dyson 						break;
6055070c7f8SJohn Dyson 					if((tp->queue - tp->pc) == PQ_CACHE) {
606e69763a3SDoug Rabson 						vm_page_flag_clear(tp, PG_CLEANCHK);
6073077a9c2SJohn Dyson 						break;
6083077a9c2SJohn Dyson 					}
609bd7e5f99SJohn Dyson 					vm_page_test_dirty(tp);
610bd7e5f99SJohn Dyson 					if ((tp->dirty & tp->valid) == 0) {
611e69763a3SDoug Rabson 						vm_page_flag_clear(tp, PG_CLEANCHK);
612bd7e5f99SJohn Dyson 						break;
613bd7e5f99SJohn Dyson 					}
614bd7e5f99SJohn Dyson 					mab[ i - 1 ] = tp;
615bd7e5f99SJohn Dyson 					maxb++;
616bd7e5f99SJohn Dyson 					continue;
617bd7e5f99SJohn Dyson 				}
618bd7e5f99SJohn Dyson 				break;
619bd7e5f99SJohn Dyson 			}
620bd7e5f99SJohn Dyson 		}
621bd7e5f99SJohn Dyson 
622bd7e5f99SJohn Dyson 		for(i=0;i<maxb;i++) {
623bd7e5f99SJohn Dyson 			int index = (maxb - i) - 1;
624bd7e5f99SJohn Dyson 			ma[index] = mab[i];
625e69763a3SDoug Rabson 			vm_page_flag_clear(ma[index], PG_CLEANCHK);
626bd7e5f99SJohn Dyson 		}
627e69763a3SDoug Rabson 		vm_page_flag_clear(p, PG_CLEANCHK);
628bd7e5f99SJohn Dyson 		ma[maxb] = p;
629bd7e5f99SJohn Dyson 		for(i=0;i<maxf;i++) {
630bd7e5f99SJohn Dyson 			int index = (maxb + i) + 1;
631bd7e5f99SJohn Dyson 			ma[index] = maf[i];
632e69763a3SDoug Rabson 			vm_page_flag_clear(ma[index], PG_CLEANCHK);
633f6b04d2bSDavid Greenman 		}
634bd7e5f99SJohn Dyson 		runlen = maxb + maxf + 1;
635cf2819ccSJohn Dyson 
636f35329acSJohn Dyson 		splx(s);
6378f9110f6SJohn Dyson 		vm_pageout_flush(ma, runlen, pagerflags);
638cf2819ccSJohn Dyson 		for (i = 0; i<runlen; i++) {
639cf2819ccSJohn Dyson 			if (ma[i]->valid & ma[i]->dirty) {
640cf2819ccSJohn Dyson 				vm_page_protect(ma[i], VM_PROT_READ);
641e69763a3SDoug Rabson 				vm_page_flag_set(ma[i], PG_CLEANCHK);
642cf2819ccSJohn Dyson 			}
643cf2819ccSJohn Dyson 		}
6442d8acc0fSJohn Dyson 		if (object->generation != curgeneration)
645bd7e5f99SJohn Dyson 			goto rescan;
646f6b04d2bSDavid Greenman 	}
647aef922f5SJohn Dyson 
648cf2819ccSJohn Dyson 	VOP_FSYNC(vp, NULL, (pagerflags & VM_PAGER_PUT_SYNC)?MNT_WAIT:0, curproc);
649aef922f5SJohn Dyson 
650069e9bc1SDoug Rabson 	vm_object_clear_flag(object, OBJ_CLEANING);
651f5cf85d4SDavid Greenman 	return;
65226f9a767SRodney W. Grimes }
653df8bae1dSRodney W. Grimes 
654f708ef1bSPoul-Henning Kamp #ifdef not_used
655f708ef1bSPoul-Henning Kamp /* XXX I cannot tell if this should be an exported symbol */
656df8bae1dSRodney W. Grimes /*
657df8bae1dSRodney W. Grimes  *	vm_object_deactivate_pages
658df8bae1dSRodney W. Grimes  *
659df8bae1dSRodney W. Grimes  *	Deactivate all pages in the specified object.  (Keep its pages
660df8bae1dSRodney W. Grimes  *	in memory even though it is no longer referenced.)
661df8bae1dSRodney W. Grimes  *
662df8bae1dSRodney W. Grimes  *	The object must be locked.
663df8bae1dSRodney W. Grimes  */
664f708ef1bSPoul-Henning Kamp static void
665df8bae1dSRodney W. Grimes vm_object_deactivate_pages(object)
666d031cff1SMatthew Dillon 	vm_object_t object;
667df8bae1dSRodney W. Grimes {
668d031cff1SMatthew Dillon 	vm_page_t p, next;
669df8bae1dSRodney W. Grimes 
670b18bfc3dSJohn Dyson 	for (p = TAILQ_FIRST(&object->memq); p != NULL; p = next) {
671b18bfc3dSJohn Dyson 		next = TAILQ_NEXT(p, listq);
672df8bae1dSRodney W. Grimes 		vm_page_deactivate(p);
673df8bae1dSRodney W. Grimes 	}
674df8bae1dSRodney W. Grimes }
675f708ef1bSPoul-Henning Kamp #endif
676df8bae1dSRodney W. Grimes 
677df8bae1dSRodney W. Grimes /*
6788e3ad7c9SMatthew Dillon  * Same as vm_object_pmap_copy, except range checking really
6791efb74fbSJohn Dyson  * works, and is meant for small sections of an object.
6808e3ad7c9SMatthew Dillon  *
6818e3ad7c9SMatthew Dillon  * This code protects resident pages by making them read-only
6828e3ad7c9SMatthew Dillon  * and is typically called on a fork or split when a page
6838e3ad7c9SMatthew Dillon  * is converted to copy-on-write.
6848e3ad7c9SMatthew Dillon  *
6858e3ad7c9SMatthew Dillon  * NOTE: If the page is already at VM_PROT_NONE, calling
6868e3ad7c9SMatthew Dillon  * vm_page_protect will have no effect.
6871efb74fbSJohn Dyson  */
6888e3ad7c9SMatthew Dillon 
6891efb74fbSJohn Dyson void
6901efb74fbSJohn Dyson vm_object_pmap_copy_1(object, start, end)
691d031cff1SMatthew Dillon 	vm_object_t object;
692d031cff1SMatthew Dillon 	vm_pindex_t start;
693d031cff1SMatthew Dillon 	vm_pindex_t end;
6941efb74fbSJohn Dyson {
6951efb74fbSJohn Dyson 	vm_pindex_t idx;
696d031cff1SMatthew Dillon 	vm_page_t p;
6971efb74fbSJohn Dyson 
6981efb74fbSJohn Dyson 	if (object == NULL || (object->flags & OBJ_WRITEABLE) == 0)
6991efb74fbSJohn Dyson 		return;
7001efb74fbSJohn Dyson 
7011efb74fbSJohn Dyson 	for (idx = start; idx < end; idx++) {
7021efb74fbSJohn Dyson 		p = vm_page_lookup(object, idx);
7031efb74fbSJohn Dyson 		if (p == NULL)
7041efb74fbSJohn Dyson 			continue;
7051efb74fbSJohn Dyson 		vm_page_protect(p, VM_PROT_READ);
7061efb74fbSJohn Dyson 	}
7071efb74fbSJohn Dyson }
7081efb74fbSJohn Dyson 
7091efb74fbSJohn Dyson /*
710df8bae1dSRodney W. Grimes  *	vm_object_pmap_remove:
711df8bae1dSRodney W. Grimes  *
712df8bae1dSRodney W. Grimes  *	Removes all physical pages in the specified
713df8bae1dSRodney W. Grimes  *	object range from all physical maps.
714df8bae1dSRodney W. Grimes  *
715df8bae1dSRodney W. Grimes  *	The object must *not* be locked.
716df8bae1dSRodney W. Grimes  */
71726f9a767SRodney W. Grimes void
71826f9a767SRodney W. Grimes vm_object_pmap_remove(object, start, end)
719d031cff1SMatthew Dillon 	vm_object_t object;
720d031cff1SMatthew Dillon 	vm_pindex_t start;
721d031cff1SMatthew Dillon 	vm_pindex_t end;
722df8bae1dSRodney W. Grimes {
723d031cff1SMatthew Dillon 	vm_page_t p;
724d031cff1SMatthew Dillon 
725df8bae1dSRodney W. Grimes 	if (object == NULL)
726df8bae1dSRodney W. Grimes 		return;
727b5b40fa6SJohn Dyson 	for (p = TAILQ_FIRST(&object->memq);
728b5b40fa6SJohn Dyson 		p != NULL;
729b5b40fa6SJohn Dyson 		p = TAILQ_NEXT(p, listq)) {
730bd7e5f99SJohn Dyson 		if (p->pindex >= start && p->pindex < end)
731f919ebdeSDavid Greenman 			vm_page_protect(p, VM_PROT_NONE);
73226f9a767SRodney W. Grimes 	}
7336e20a165SJohn Dyson 	if ((start == 0) && (object->size == end))
734069e9bc1SDoug Rabson 		vm_object_clear_flag(object, OBJ_WRITEABLE);
73526f9a767SRodney W. Grimes }
736df8bae1dSRodney W. Grimes 
737df8bae1dSRodney W. Grimes /*
738867a482dSJohn Dyson  *	vm_object_madvise:
739867a482dSJohn Dyson  *
740867a482dSJohn Dyson  *	Implements the madvise function at the object/page level.
7411c7c3c6aSMatthew Dillon  *
7421c7c3c6aSMatthew Dillon  *	Currently, madvise() functions are limited to the default and
7431c7c3c6aSMatthew Dillon  *	swap object types only, and also limited to only the unshared portions
7441c7c3c6aSMatthew Dillon  *	of a process's address space.  MADV_FREE, certainly, could never be
7451c7c3c6aSMatthew Dillon  *	run on anything else.  The others are more flexible and the code could
7461c7c3c6aSMatthew Dillon  *	be adjusted in the future to handle expanded cases for them.
747867a482dSJohn Dyson  */
748867a482dSJohn Dyson void
749867a482dSJohn Dyson vm_object_madvise(object, pindex, count, advise)
750867a482dSJohn Dyson 	vm_object_t object;
751867a482dSJohn Dyson 	vm_pindex_t pindex;
752867a482dSJohn Dyson 	int count;
753867a482dSJohn Dyson 	int advise;
754867a482dSJohn Dyson {
7556e20a165SJohn Dyson 	vm_pindex_t end, tpindex;
7566e20a165SJohn Dyson 	vm_object_t tobject;
757867a482dSJohn Dyson 	vm_page_t m;
758867a482dSJohn Dyson 
759867a482dSJohn Dyson 	if (object == NULL)
760867a482dSJohn Dyson 		return;
761867a482dSJohn Dyson 
762867a482dSJohn Dyson 	end = pindex + count;
763867a482dSJohn Dyson 
7641c7c3c6aSMatthew Dillon 	/*
7651c7c3c6aSMatthew Dillon 	 * MADV_FREE special case - free any swap backing store (as well
7661c7c3c6aSMatthew Dillon 	 * as resident pages later on).
7671c7c3c6aSMatthew Dillon 	 */
7686e20a165SJohn Dyson 
7691c7c3c6aSMatthew Dillon 	if (advise == MADV_FREE) {
7701c7c3c6aSMatthew Dillon 		tobject = object;
7711c7c3c6aSMatthew Dillon 		tpindex = pindex;
7721c7c3c6aSMatthew Dillon 
7731c7c3c6aSMatthew Dillon 		while (
7741c7c3c6aSMatthew Dillon 		    (tobject->type == OBJT_DEFAULT ||
7751c7c3c6aSMatthew Dillon 		     tobject->type == OBJT_SWAP) &&
7761c7c3c6aSMatthew Dillon 		    (tobject->flags & OBJ_ONEMAPPING)
7771c7c3c6aSMatthew Dillon 		) {
7781c7c3c6aSMatthew Dillon 			if (tobject->type == OBJT_SWAP) {
7791c7c3c6aSMatthew Dillon 				swap_pager_freespace(tobject, tpindex, count);
7801c7c3c6aSMatthew Dillon 			}
7811c7c3c6aSMatthew Dillon 			if ((tobject = tobject->backing_object) == NULL)
7821c7c3c6aSMatthew Dillon 				break;
7831c7c3c6aSMatthew Dillon 			tpindex += OFF_TO_IDX(tobject->backing_object_offset);
7841c7c3c6aSMatthew Dillon 		}
7851c7c3c6aSMatthew Dillon 	}
7861c7c3c6aSMatthew Dillon 
7871c7c3c6aSMatthew Dillon 	/*
7881c7c3c6aSMatthew Dillon 	 * Locate and adjust resident pages
7891c7c3c6aSMatthew Dillon 	 */
7901c7c3c6aSMatthew Dillon 
7911c7c3c6aSMatthew Dillon 	for (; pindex < end; pindex += 1) {
7926e20a165SJohn Dyson relookup:
7936e20a165SJohn Dyson 		tobject = object;
7946e20a165SJohn Dyson 		tpindex = pindex;
7956e20a165SJohn Dyson shadowlookup:
7961c7c3c6aSMatthew Dillon 
7971c7c3c6aSMatthew Dillon 		if (tobject->type != OBJT_DEFAULT &&
7981c7c3c6aSMatthew Dillon 		    tobject->type != OBJT_SWAP
7991c7c3c6aSMatthew Dillon 		) {
8006e20a165SJohn Dyson 			continue;
8016e20a165SJohn Dyson 		}
8026e20a165SJohn Dyson 
8031c7c3c6aSMatthew Dillon 		if ((tobject->flags & OBJ_ONEMAPPING) == 0)
8041c7c3c6aSMatthew Dillon 			continue;
8051c7c3c6aSMatthew Dillon 
8061c7c3c6aSMatthew Dillon 		m = vm_page_lookup(tobject, tpindex);
8071c7c3c6aSMatthew Dillon 
8081c7c3c6aSMatthew Dillon 		if (m == NULL) {
8096e20a165SJohn Dyson 			tobject = tobject->backing_object;
8101c7c3c6aSMatthew Dillon 			if (tobject == NULL)
8111c7c3c6aSMatthew Dillon 				continue;
8126e20a165SJohn Dyson 			tpindex += OFF_TO_IDX(tobject->backing_object_offset);
8136e20a165SJohn Dyson 			goto shadowlookup;
8146e20a165SJohn Dyson 		}
815867a482dSJohn Dyson 
816867a482dSJohn Dyson 		/*
817867a482dSJohn Dyson 		 * If the page is busy or not in a normal active state,
818867a482dSJohn Dyson 		 * we skip it.  Things can break if we mess with pages
819867a482dSJohn Dyson 		 * in any of the below states.
820867a482dSJohn Dyson 		 */
8211c7c3c6aSMatthew Dillon 		if (
8221c7c3c6aSMatthew Dillon 		    m->hold_count ||
8231c7c3c6aSMatthew Dillon 		    m->wire_count ||
8241c7c3c6aSMatthew Dillon 		    m->valid != VM_PAGE_BITS_ALL
8251c7c3c6aSMatthew Dillon 		) {
826867a482dSJohn Dyson 			continue;
8276e20a165SJohn Dyson 		}
8286e20a165SJohn Dyson 
8291c7c3c6aSMatthew Dillon  		if (vm_page_sleep_busy(m, TRUE, "madvpo"))
8306e20a165SJohn Dyson   			goto relookup;
831867a482dSJohn Dyson 
832867a482dSJohn Dyson 		if (advise == MADV_WILLNEED) {
833867a482dSJohn Dyson 			vm_page_activate(m);
8346e20a165SJohn Dyson 		} else if (advise == MADV_DONTNEED) {
835867a482dSJohn Dyson 			vm_page_deactivate(m);
8360a47b48bSJohn Dyson 		} else if (advise == MADV_FREE) {
8371c7c3c6aSMatthew Dillon 			/*
8381c7c3c6aSMatthew Dillon 			 * If MADV_FREE_FORCE_FREE is defined, we attempt to
8391c7c3c6aSMatthew Dillon 			 * immediately free the page.  Otherwise we just
8401c7c3c6aSMatthew Dillon 			 * destroy any swap backing store, mark it clean,
8411c7c3c6aSMatthew Dillon 			 * and stuff it into the cache.
8421c7c3c6aSMatthew Dillon 			 */
8436e20a165SJohn Dyson 			pmap_clear_modify(VM_PAGE_TO_PHYS(m));
8446e20a165SJohn Dyson 			m->dirty = 0;
8451c7c3c6aSMatthew Dillon 
8461c7c3c6aSMatthew Dillon #ifdef MADV_FREE_FORCE_FREE
8471c7c3c6aSMatthew Dillon 			if (tobject->resident_page_count > 1) {
8481c7c3c6aSMatthew Dillon 				vm_page_busy(m);
8491c7c3c6aSMatthew Dillon 				vm_page_protect(m, VM_PROT_NONE);
8501c7c3c6aSMatthew Dillon 				vm_page_free(m);
8511c7c3c6aSMatthew Dillon 			} else
8521c7c3c6aSMatthew Dillon #endif
8531c7c3c6aSMatthew Dillon 			{
8541c7c3c6aSMatthew Dillon 				vm_page_cache(m);
8551c7c3c6aSMatthew Dillon 			}
856867a482dSJohn Dyson 		}
857867a482dSJohn Dyson 	}
858867a482dSJohn Dyson }
859867a482dSJohn Dyson 
860867a482dSJohn Dyson /*
861df8bae1dSRodney W. Grimes  *	vm_object_shadow:
862df8bae1dSRodney W. Grimes  *
863df8bae1dSRodney W. Grimes  *	Create a new object which is backed by the
864df8bae1dSRodney W. Grimes  *	specified existing object range.  The source
865df8bae1dSRodney W. Grimes  *	object reference is deallocated.
866df8bae1dSRodney W. Grimes  *
867df8bae1dSRodney W. Grimes  *	The new object and offset into that object
868df8bae1dSRodney W. Grimes  *	are returned in the source parameters.
869df8bae1dSRodney W. Grimes  */
870df8bae1dSRodney W. Grimes 
87126f9a767SRodney W. Grimes void
87226f9a767SRodney W. Grimes vm_object_shadow(object, offset, length)
873df8bae1dSRodney W. Grimes 	vm_object_t *object;	/* IN/OUT */
874a316d390SJohn Dyson 	vm_ooffset_t *offset;	/* IN/OUT */
875df8bae1dSRodney W. Grimes 	vm_size_t length;
876df8bae1dSRodney W. Grimes {
877d031cff1SMatthew Dillon 	vm_object_t source;
878d031cff1SMatthew Dillon 	vm_object_t result;
879df8bae1dSRodney W. Grimes 
880df8bae1dSRodney W. Grimes 	source = *object;
881df8bae1dSRodney W. Grimes 
882df8bae1dSRodney W. Grimes 	/*
883df8bae1dSRodney W. Grimes 	 * Allocate a new object with the given length
884df8bae1dSRodney W. Grimes 	 */
885df8bae1dSRodney W. Grimes 
88624a1cce3SDavid Greenman 	if ((result = vm_object_allocate(OBJT_DEFAULT, length)) == NULL)
887df8bae1dSRodney W. Grimes 		panic("vm_object_shadow: no object for shadowing");
888df8bae1dSRodney W. Grimes 
889df8bae1dSRodney W. Grimes 	/*
8900d94caffSDavid Greenman 	 * The new object shadows the source object, adding a reference to it.
8910d94caffSDavid Greenman 	 * Our caller changes his reference to point to the new object,
8920d94caffSDavid Greenman 	 * removing a reference to the source object.  Net result: no change
8930d94caffSDavid Greenman 	 * of reference count.
8949b09fe24SMatthew Dillon 	 *
8959b09fe24SMatthew Dillon 	 * Try to optimize the result object's page color when shadowing
8969b09fe24SMatthew Dillon 	 * in order to maintain page coloring consistancy in the combined
8979b09fe24SMatthew Dillon 	 * shadowed object.
898df8bae1dSRodney W. Grimes 	 */
89924a1cce3SDavid Greenman 	result->backing_object = source;
900de5f6a77SJohn Dyson 	if (source) {
901de5f6a77SJohn Dyson 		TAILQ_INSERT_TAIL(&source->shadow_head, result, shadow_list);
902069e9bc1SDoug Rabson 		vm_object_clear_flag(source, OBJ_ONEMAPPING);
903eaf13dd7SJohn Dyson 		source->shadow_count++;
904eaf13dd7SJohn Dyson 		source->generation++;
9059b09fe24SMatthew Dillon 		result->pg_color = (source->pg_color + OFF_TO_IDX(*offset)) & PQ_L2_MASK;
906de5f6a77SJohn Dyson 	}
907df8bae1dSRodney W. Grimes 
908df8bae1dSRodney W. Grimes 	/*
9090d94caffSDavid Greenman 	 * Store the offset into the source object, and fix up the offset into
9100d94caffSDavid Greenman 	 * the new object.
911df8bae1dSRodney W. Grimes 	 */
912df8bae1dSRodney W. Grimes 
91324a1cce3SDavid Greenman 	result->backing_object_offset = *offset;
914df8bae1dSRodney W. Grimes 
915df8bae1dSRodney W. Grimes 	/*
916df8bae1dSRodney W. Grimes 	 * Return the new things
917df8bae1dSRodney W. Grimes 	 */
918df8bae1dSRodney W. Grimes 
919df8bae1dSRodney W. Grimes 	*offset = 0;
920df8bae1dSRodney W. Grimes 	*object = result;
921df8bae1dSRodney W. Grimes }
922df8bae1dSRodney W. Grimes 
9232ad1a3f7SMatthew Dillon #define	OBSC_TEST_ALL_SHADOWED	0x0001
9242ad1a3f7SMatthew Dillon #define	OBSC_COLLAPSE_NOWAIT	0x0002
9252ad1a3f7SMatthew Dillon #define	OBSC_COLLAPSE_WAIT	0x0004
9262ad1a3f7SMatthew Dillon 
9272ad1a3f7SMatthew Dillon static __inline int
9282ad1a3f7SMatthew Dillon vm_object_backing_scan(vm_object_t object, int op)
9292ad1a3f7SMatthew Dillon {
9302ad1a3f7SMatthew Dillon 	int s;
9312ad1a3f7SMatthew Dillon 	int r = 1;
9322ad1a3f7SMatthew Dillon 	vm_page_t p;
9332ad1a3f7SMatthew Dillon 	vm_object_t backing_object;
9342ad1a3f7SMatthew Dillon 	vm_pindex_t backing_offset_index;
9352ad1a3f7SMatthew Dillon 
9362ad1a3f7SMatthew Dillon 	s = splvm();
9372ad1a3f7SMatthew Dillon 
9382ad1a3f7SMatthew Dillon 	backing_object = object->backing_object;
9392ad1a3f7SMatthew Dillon 	backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
9402ad1a3f7SMatthew Dillon 
9412ad1a3f7SMatthew Dillon 	/*
9422ad1a3f7SMatthew Dillon 	 * Initial conditions
9432ad1a3f7SMatthew Dillon 	 */
9442ad1a3f7SMatthew Dillon 
9452ad1a3f7SMatthew Dillon 	if (op & OBSC_TEST_ALL_SHADOWED) {
9462ad1a3f7SMatthew Dillon 		/*
9472ad1a3f7SMatthew Dillon 		 * We do not want to have to test for the existance of
9482ad1a3f7SMatthew Dillon 		 * swap pages in the backing object.  XXX but with the
9492ad1a3f7SMatthew Dillon 		 * new swapper this would be pretty easy to do.
9502ad1a3f7SMatthew Dillon 		 *
9512ad1a3f7SMatthew Dillon 		 * XXX what about anonymous MAP_SHARED memory that hasn't
9522ad1a3f7SMatthew Dillon 		 * been ZFOD faulted yet?  If we do not test for this, the
9532ad1a3f7SMatthew Dillon 		 * shadow test may succeed! XXX
9542ad1a3f7SMatthew Dillon 		 */
9552ad1a3f7SMatthew Dillon 		if (backing_object->type != OBJT_DEFAULT) {
9562ad1a3f7SMatthew Dillon 			splx(s);
9572ad1a3f7SMatthew Dillon 			return(0);
9582ad1a3f7SMatthew Dillon 		}
9592ad1a3f7SMatthew Dillon 	}
9602ad1a3f7SMatthew Dillon 	if (op & OBSC_COLLAPSE_WAIT) {
9612ad1a3f7SMatthew Dillon 		vm_object_set_flag(backing_object, OBJ_DEAD);
9622ad1a3f7SMatthew Dillon 	}
9632ad1a3f7SMatthew Dillon 
9642ad1a3f7SMatthew Dillon 	/*
9652ad1a3f7SMatthew Dillon 	 * Our scan
9662ad1a3f7SMatthew Dillon 	 */
9672ad1a3f7SMatthew Dillon 
9682ad1a3f7SMatthew Dillon 	p = TAILQ_FIRST(&backing_object->memq);
9692ad1a3f7SMatthew Dillon 	while (p) {
9702ad1a3f7SMatthew Dillon 		vm_page_t next = TAILQ_NEXT(p, listq);
9712ad1a3f7SMatthew Dillon 		vm_pindex_t new_pindex = p->pindex - backing_offset_index;
9722ad1a3f7SMatthew Dillon 
9732ad1a3f7SMatthew Dillon 		if (op & OBSC_TEST_ALL_SHADOWED) {
9742ad1a3f7SMatthew Dillon 			vm_page_t pp;
9752ad1a3f7SMatthew Dillon 
9762ad1a3f7SMatthew Dillon 			/*
9772ad1a3f7SMatthew Dillon 			 * Ignore pages outside the parent object's range
9782ad1a3f7SMatthew Dillon 			 * and outside the parent object's mapping of the
9792ad1a3f7SMatthew Dillon 			 * backing object.
9802ad1a3f7SMatthew Dillon 			 *
9812ad1a3f7SMatthew Dillon 			 * note that we do not busy the backing object's
9822ad1a3f7SMatthew Dillon 			 * page.
9832ad1a3f7SMatthew Dillon 			 */
9842ad1a3f7SMatthew Dillon 
9852ad1a3f7SMatthew Dillon 			if (
9862ad1a3f7SMatthew Dillon 			    p->pindex < backing_offset_index ||
9872ad1a3f7SMatthew Dillon 			    new_pindex >= object->size
9882ad1a3f7SMatthew Dillon 			) {
9892ad1a3f7SMatthew Dillon 				p = next;
9902ad1a3f7SMatthew Dillon 				continue;
9912ad1a3f7SMatthew Dillon 			}
9922ad1a3f7SMatthew Dillon 
9932ad1a3f7SMatthew Dillon 			/*
9942ad1a3f7SMatthew Dillon 			 * See if the parent has the page or if the parent's
9952ad1a3f7SMatthew Dillon 			 * object pager has the page.  If the parent has the
9962ad1a3f7SMatthew Dillon 			 * page but the page is not valid, the parent's
9972ad1a3f7SMatthew Dillon 			 * object pager must have the page.
9982ad1a3f7SMatthew Dillon 			 *
9992ad1a3f7SMatthew Dillon 			 * If this fails, the parent does not completely shadow
10002ad1a3f7SMatthew Dillon 			 * the object and we might as well give up now.
10012ad1a3f7SMatthew Dillon 			 */
10022ad1a3f7SMatthew Dillon 
10032ad1a3f7SMatthew Dillon 			pp = vm_page_lookup(object, new_pindex);
10042ad1a3f7SMatthew Dillon 			if (
10052ad1a3f7SMatthew Dillon 			    (pp == NULL || pp->valid == 0) &&
10062ad1a3f7SMatthew Dillon 			    !vm_pager_has_page(object, new_pindex, NULL, NULL)
10072ad1a3f7SMatthew Dillon 			) {
10082ad1a3f7SMatthew Dillon 				r = 0;
10092ad1a3f7SMatthew Dillon 				break;
10102ad1a3f7SMatthew Dillon 			}
10112ad1a3f7SMatthew Dillon 		}
10122ad1a3f7SMatthew Dillon 
10132ad1a3f7SMatthew Dillon 		/*
10142ad1a3f7SMatthew Dillon 		 * Check for busy page
10152ad1a3f7SMatthew Dillon 		 */
10162ad1a3f7SMatthew Dillon 
10172ad1a3f7SMatthew Dillon 		if (op & (OBSC_COLLAPSE_WAIT | OBSC_COLLAPSE_NOWAIT)) {
10182ad1a3f7SMatthew Dillon 			vm_page_t pp;
10192ad1a3f7SMatthew Dillon 
10202ad1a3f7SMatthew Dillon 			if (op & OBSC_COLLAPSE_NOWAIT) {
10212ad1a3f7SMatthew Dillon 				if (
10222ad1a3f7SMatthew Dillon 				    (p->flags & PG_BUSY) ||
10232ad1a3f7SMatthew Dillon 				    !p->valid ||
10242ad1a3f7SMatthew Dillon 				    p->hold_count ||
10252ad1a3f7SMatthew Dillon 				    p->wire_count ||
10262ad1a3f7SMatthew Dillon 				    p->busy
10272ad1a3f7SMatthew Dillon 				) {
10282ad1a3f7SMatthew Dillon 					p = next;
10292ad1a3f7SMatthew Dillon 					continue;
10302ad1a3f7SMatthew Dillon 				}
10312ad1a3f7SMatthew Dillon 			} else if (op & OBSC_COLLAPSE_WAIT) {
10322ad1a3f7SMatthew Dillon 				if (vm_page_sleep_busy(p, TRUE, "vmocol")) {
10332ad1a3f7SMatthew Dillon 					/*
10342ad1a3f7SMatthew Dillon 					 * If we slept, anything could have
10352ad1a3f7SMatthew Dillon 					 * happened.  Since the object is
10362ad1a3f7SMatthew Dillon 					 * marked dead, the backing offset
10372ad1a3f7SMatthew Dillon 					 * should not have changed so we
10382ad1a3f7SMatthew Dillon 					 * just restart our scan.
10392ad1a3f7SMatthew Dillon 					 */
10402ad1a3f7SMatthew Dillon 					p = TAILQ_FIRST(&backing_object->memq);
10412ad1a3f7SMatthew Dillon 					continue;
10422ad1a3f7SMatthew Dillon 				}
10432ad1a3f7SMatthew Dillon 			}
10442ad1a3f7SMatthew Dillon 
10452ad1a3f7SMatthew Dillon 			/*
10462ad1a3f7SMatthew Dillon 			 * Busy the page
10472ad1a3f7SMatthew Dillon 			 */
10482ad1a3f7SMatthew Dillon 			vm_page_busy(p);
10492ad1a3f7SMatthew Dillon 
10502ad1a3f7SMatthew Dillon 			KASSERT(
10512ad1a3f7SMatthew Dillon 			    p->object == backing_object,
10522ad1a3f7SMatthew Dillon 			    ("vm_object_qcollapse(): object mismatch")
10532ad1a3f7SMatthew Dillon 			);
10542ad1a3f7SMatthew Dillon 
10552ad1a3f7SMatthew Dillon 			/*
10562ad1a3f7SMatthew Dillon 			 * Destroy any associated swap
10572ad1a3f7SMatthew Dillon 			 */
10582ad1a3f7SMatthew Dillon 			if (backing_object->type == OBJT_SWAP) {
10592ad1a3f7SMatthew Dillon 				swap_pager_freespace(
10602ad1a3f7SMatthew Dillon 				    backing_object,
10612ad1a3f7SMatthew Dillon 				    p->pindex,
10622ad1a3f7SMatthew Dillon 				    1
10632ad1a3f7SMatthew Dillon 				);
10642ad1a3f7SMatthew Dillon 			}
10652ad1a3f7SMatthew Dillon 
10662ad1a3f7SMatthew Dillon 			if (
10672ad1a3f7SMatthew Dillon 			    p->pindex < backing_offset_index ||
10682ad1a3f7SMatthew Dillon 			    new_pindex >= object->size
10692ad1a3f7SMatthew Dillon 			) {
10702ad1a3f7SMatthew Dillon 				/*
10712ad1a3f7SMatthew Dillon 				 * Page is out of the parent object's range, we
10722ad1a3f7SMatthew Dillon 				 * can simply destroy it.
10732ad1a3f7SMatthew Dillon 				 */
10742ad1a3f7SMatthew Dillon 				vm_page_protect(p, VM_PROT_NONE);
10752ad1a3f7SMatthew Dillon 				vm_page_free(p);
10762ad1a3f7SMatthew Dillon 				p = next;
10772ad1a3f7SMatthew Dillon 				continue;
10782ad1a3f7SMatthew Dillon 			}
10792ad1a3f7SMatthew Dillon 
10802ad1a3f7SMatthew Dillon 			pp = vm_page_lookup(object, new_pindex);
10812ad1a3f7SMatthew Dillon 			if (
10822ad1a3f7SMatthew Dillon 			    pp != NULL ||
10832ad1a3f7SMatthew Dillon 			    vm_pager_has_page(object, new_pindex, NULL, NULL)
10842ad1a3f7SMatthew Dillon 			) {
10852ad1a3f7SMatthew Dillon 				/*
10862ad1a3f7SMatthew Dillon 				 * page already exists in parent OR swap exists
10872ad1a3f7SMatthew Dillon 				 * for this location in the parent.  Destroy
10882ad1a3f7SMatthew Dillon 				 * the original page from the backing object.
10892ad1a3f7SMatthew Dillon 				 *
10902ad1a3f7SMatthew Dillon 				 * Leave the parent's page alone
10912ad1a3f7SMatthew Dillon 				 */
10922ad1a3f7SMatthew Dillon 				vm_page_protect(p, VM_PROT_NONE);
10932ad1a3f7SMatthew Dillon 				vm_page_free(p);
10942ad1a3f7SMatthew Dillon 				p = next;
10952ad1a3f7SMatthew Dillon 				continue;
10962ad1a3f7SMatthew Dillon 			}
10972ad1a3f7SMatthew Dillon 
10982ad1a3f7SMatthew Dillon 			/*
10992ad1a3f7SMatthew Dillon 			 * Page does not exist in parent, rename the
11002ad1a3f7SMatthew Dillon 			 * page from the backing object to the main object.
11012ad1a3f7SMatthew Dillon 			 */
11022ad1a3f7SMatthew Dillon 			if ((p->queue - p->pc) == PQ_CACHE)
11032ad1a3f7SMatthew Dillon 				vm_page_deactivate(p);
11042ad1a3f7SMatthew Dillon 			else
11052ad1a3f7SMatthew Dillon 				vm_page_protect(p, VM_PROT_NONE);
11062ad1a3f7SMatthew Dillon 
11072ad1a3f7SMatthew Dillon 			vm_page_rename(p, object, new_pindex);
11082ad1a3f7SMatthew Dillon 			/* page automatically made dirty by rename */
11092ad1a3f7SMatthew Dillon 		}
11102ad1a3f7SMatthew Dillon 		p = next;
11112ad1a3f7SMatthew Dillon 	}
11122ad1a3f7SMatthew Dillon 	splx(s);
11132ad1a3f7SMatthew Dillon 	return(r);
11142ad1a3f7SMatthew Dillon }
11152ad1a3f7SMatthew Dillon 
1116df8bae1dSRodney W. Grimes 
1117df8bae1dSRodney W. Grimes /*
11182fe6e4d7SDavid Greenman  * this version of collapse allows the operation to occur earlier and
11192fe6e4d7SDavid Greenman  * when paging_in_progress is true for an object...  This is not a complete
11202fe6e4d7SDavid Greenman  * operation, but should plug 99.9% of the rest of the leaks.
11212fe6e4d7SDavid Greenman  */
11222fe6e4d7SDavid Greenman static void
11232fe6e4d7SDavid Greenman vm_object_qcollapse(object)
1124d031cff1SMatthew Dillon 	vm_object_t object;
11252fe6e4d7SDavid Greenman {
11262ad1a3f7SMatthew Dillon 	vm_object_t backing_object = object->backing_object;
11272fe6e4d7SDavid Greenman 
11282fe6e4d7SDavid Greenman 	if (backing_object->ref_count != 1)
11292fe6e4d7SDavid Greenman 		return;
11302fe6e4d7SDavid Greenman 
1131010cf3b9SDavid Greenman 	backing_object->ref_count += 2;
1132010cf3b9SDavid Greenman 
11332ad1a3f7SMatthew Dillon 	vm_object_backing_scan(object, OBSC_COLLAPSE_NOWAIT);
11341c7c3c6aSMatthew Dillon 
1135010cf3b9SDavid Greenman 	backing_object->ref_count -= 2;
11362fe6e4d7SDavid Greenman }
11372fe6e4d7SDavid Greenman 
1138df8bae1dSRodney W. Grimes /*
1139df8bae1dSRodney W. Grimes  *	vm_object_collapse:
1140df8bae1dSRodney W. Grimes  *
1141df8bae1dSRodney W. Grimes  *	Collapse an object with the object backing it.
1142df8bae1dSRodney W. Grimes  *	Pages in the backing object are moved into the
1143df8bae1dSRodney W. Grimes  *	parent, and the backing object is deallocated.
1144df8bae1dSRodney W. Grimes  */
114526f9a767SRodney W. Grimes void
114626f9a767SRodney W. Grimes vm_object_collapse(object)
114724a1cce3SDavid Greenman 	vm_object_t object;
1148df8bae1dSRodney W. Grimes {
1149df8bae1dSRodney W. Grimes 	while (TRUE) {
11502ad1a3f7SMatthew Dillon 		vm_object_t backing_object;
11512ad1a3f7SMatthew Dillon 
1152df8bae1dSRodney W. Grimes 		/*
1153df8bae1dSRodney W. Grimes 		 * Verify that the conditions are right for collapse:
1154df8bae1dSRodney W. Grimes 		 *
11552ad1a3f7SMatthew Dillon 		 * The object exists and the backing object exists.
1156df8bae1dSRodney W. Grimes 		 */
11572fe6e4d7SDavid Greenman 		if (object == NULL)
11582ad1a3f7SMatthew Dillon 			break;
1159df8bae1dSRodney W. Grimes 
116024a1cce3SDavid Greenman 		if ((backing_object = object->backing_object) == NULL)
11612ad1a3f7SMatthew Dillon 			break;
1162df8bae1dSRodney W. Grimes 
1163f919ebdeSDavid Greenman 		/*
1164f919ebdeSDavid Greenman 		 * we check the backing object first, because it is most likely
116524a1cce3SDavid Greenman 		 * not collapsable.
1166f919ebdeSDavid Greenman 		 */
116724a1cce3SDavid Greenman 		if (backing_object->handle != NULL ||
116824a1cce3SDavid Greenman 		    (backing_object->type != OBJT_DEFAULT &&
116924a1cce3SDavid Greenman 		     backing_object->type != OBJT_SWAP) ||
1170f919ebdeSDavid Greenman 		    (backing_object->flags & OBJ_DEAD) ||
117124a1cce3SDavid Greenman 		    object->handle != NULL ||
117224a1cce3SDavid Greenman 		    (object->type != OBJT_DEFAULT &&
117324a1cce3SDavid Greenman 		     object->type != OBJT_SWAP) ||
117424a1cce3SDavid Greenman 		    (object->flags & OBJ_DEAD)) {
11752ad1a3f7SMatthew Dillon 			break;
117624a1cce3SDavid Greenman 		}
11779b4814bbSDavid Greenman 
11782ad1a3f7SMatthew Dillon 		if (
11792ad1a3f7SMatthew Dillon 		    object->paging_in_progress != 0 ||
11802ad1a3f7SMatthew Dillon 		    backing_object->paging_in_progress != 0
11812ad1a3f7SMatthew Dillon 		) {
1182b9921222SDavid Greenman 			vm_object_qcollapse(object);
11832ad1a3f7SMatthew Dillon 			break;
1184df8bae1dSRodney W. Grimes 		}
1185f919ebdeSDavid Greenman 
118626f9a767SRodney W. Grimes 		/*
11870d94caffSDavid Greenman 		 * We know that we can either collapse the backing object (if
11882ad1a3f7SMatthew Dillon 		 * the parent is the only reference to it) or (perhaps) have
11892ad1a3f7SMatthew Dillon 		 * the parent bypass the object if the parent happens to shadow
11902ad1a3f7SMatthew Dillon 		 * all the resident pages in the entire backing object.
11912ad1a3f7SMatthew Dillon 		 *
11922ad1a3f7SMatthew Dillon 		 * This is ignoring pager-backed pages such as swap pages.
11932ad1a3f7SMatthew Dillon 		 * vm_object_backing_scan fails the shadowing test in this
11942ad1a3f7SMatthew Dillon 		 * case.
1195df8bae1dSRodney W. Grimes 		 */
1196df8bae1dSRodney W. Grimes 
1197df8bae1dSRodney W. Grimes 		if (backing_object->ref_count == 1) {
1198df8bae1dSRodney W. Grimes 			/*
11992ad1a3f7SMatthew Dillon 			 * If there is exactly one reference to the backing
12002ad1a3f7SMatthew Dillon 			 * object, we can collapse it into the parent.
1201df8bae1dSRodney W. Grimes 			 */
1202df8bae1dSRodney W. Grimes 
12032ad1a3f7SMatthew Dillon 			vm_object_backing_scan(object, OBSC_COLLAPSE_WAIT);
1204df8bae1dSRodney W. Grimes 
1205df8bae1dSRodney W. Grimes 			/*
1206df8bae1dSRodney W. Grimes 			 * Move the pager from backing_object to object.
1207df8bae1dSRodney W. Grimes 			 */
1208df8bae1dSRodney W. Grimes 
120924a1cce3SDavid Greenman 			if (backing_object->type == OBJT_SWAP) {
1210d474eaaaSDoug Rabson 				vm_object_pip_add(backing_object, 1);
121124a1cce3SDavid Greenman 
121224a1cce3SDavid Greenman 				/*
12131c7c3c6aSMatthew Dillon 				 * scrap the paging_offset junk and do a
12141c7c3c6aSMatthew Dillon 				 * discrete copy.  This also removes major
12151c7c3c6aSMatthew Dillon 				 * assumptions about how the swap-pager
12161c7c3c6aSMatthew Dillon 				 * works from where it doesn't belong.  The
12171c7c3c6aSMatthew Dillon 				 * new swapper is able to optimize the
12181c7c3c6aSMatthew Dillon 				 * destroy-source case.
121924a1cce3SDavid Greenman 				 */
12201c7c3c6aSMatthew Dillon 
12211c7c3c6aSMatthew Dillon 				vm_object_pip_add(object, 1);
12221c7c3c6aSMatthew Dillon 				swap_pager_copy(
12231c7c3c6aSMatthew Dillon 				    backing_object,
12241c7c3c6aSMatthew Dillon 				    object,
12251c7c3c6aSMatthew Dillon 				    OFF_TO_IDX(object->backing_object_offset), TRUE);
1226f919ebdeSDavid Greenman 				vm_object_pip_wakeup(object);
1227c0503609SDavid Greenman 
1228f919ebdeSDavid Greenman 				vm_object_pip_wakeup(backing_object);
1229c0503609SDavid Greenman 			}
1230df8bae1dSRodney W. Grimes 			/*
1231df8bae1dSRodney W. Grimes 			 * Object now shadows whatever backing_object did.
12322ad1a3f7SMatthew Dillon 			 * Note that the reference to
12332ad1a3f7SMatthew Dillon 			 * backing_object->backing_object moves from within
12342ad1a3f7SMatthew Dillon 			 * backing_object to within object.
1235df8bae1dSRodney W. Grimes 			 */
1236df8bae1dSRodney W. Grimes 
12372ad1a3f7SMatthew Dillon 			TAILQ_REMOVE(
12382ad1a3f7SMatthew Dillon 			    &object->backing_object->shadow_head,
12392ad1a3f7SMatthew Dillon 			    object,
12402ad1a3f7SMatthew Dillon 			    shadow_list
12412ad1a3f7SMatthew Dillon 			);
1242eaf13dd7SJohn Dyson 			object->backing_object->shadow_count--;
1243eaf13dd7SJohn Dyson 			object->backing_object->generation++;
1244de5f6a77SJohn Dyson 			if (backing_object->backing_object) {
12452ad1a3f7SMatthew Dillon 				TAILQ_REMOVE(
12462ad1a3f7SMatthew Dillon 				    &backing_object->backing_object->shadow_head,
12472ad1a3f7SMatthew Dillon 				    backing_object,
12482ad1a3f7SMatthew Dillon 				    shadow_list
12492ad1a3f7SMatthew Dillon 				);
1250eaf13dd7SJohn Dyson 				backing_object->backing_object->shadow_count--;
1251eaf13dd7SJohn Dyson 				backing_object->backing_object->generation++;
1252de5f6a77SJohn Dyson 			}
125324a1cce3SDavid Greenman 			object->backing_object = backing_object->backing_object;
1254de5f6a77SJohn Dyson 			if (object->backing_object) {
12552ad1a3f7SMatthew Dillon 				TAILQ_INSERT_TAIL(
12562ad1a3f7SMatthew Dillon 				    &object->backing_object->shadow_head,
12572ad1a3f7SMatthew Dillon 				    object,
12582ad1a3f7SMatthew Dillon 				    shadow_list
12592ad1a3f7SMatthew Dillon 				);
1260eaf13dd7SJohn Dyson 				object->backing_object->shadow_count++;
1261eaf13dd7SJohn Dyson 				object->backing_object->generation++;
1262de5f6a77SJohn Dyson 			}
12632fe6e4d7SDavid Greenman 
12642ad1a3f7SMatthew Dillon 			object->backing_object_offset +=
12652ad1a3f7SMatthew Dillon 			    backing_object->backing_object_offset;
12662ad1a3f7SMatthew Dillon 
1267df8bae1dSRodney W. Grimes 			/*
1268df8bae1dSRodney W. Grimes 			 * Discard backing_object.
1269df8bae1dSRodney W. Grimes 			 *
12700d94caffSDavid Greenman 			 * Since the backing object has no pages, no pager left,
12710d94caffSDavid Greenman 			 * and no object references within it, all that is
12720d94caffSDavid Greenman 			 * necessary is to dispose of it.
1273df8bae1dSRodney W. Grimes 			 */
1274df8bae1dSRodney W. Grimes 
12752ad1a3f7SMatthew Dillon 			TAILQ_REMOVE(
12762ad1a3f7SMatthew Dillon 			    &vm_object_list,
12772ad1a3f7SMatthew Dillon 			    backing_object,
12782ad1a3f7SMatthew Dillon 			    object_list
12792ad1a3f7SMatthew Dillon 			);
1280df8bae1dSRodney W. Grimes 			vm_object_count--;
1281df8bae1dSRodney W. Grimes 
128299448ed1SJohn Dyson 			zfree(obj_zone, backing_object);
1283df8bae1dSRodney W. Grimes 
1284df8bae1dSRodney W. Grimes 			object_collapses++;
12850d94caffSDavid Greenman 		} else {
128695e5e988SJohn Dyson 			vm_object_t new_backing_object;
1287df8bae1dSRodney W. Grimes 
1288df8bae1dSRodney W. Grimes 			/*
12892ad1a3f7SMatthew Dillon 			 * If we do not entirely shadow the backing object,
12902ad1a3f7SMatthew Dillon 			 * there is nothing we can do so we give up.
1291df8bae1dSRodney W. Grimes 			 */
1292df8bae1dSRodney W. Grimes 
12932ad1a3f7SMatthew Dillon 			if (vm_object_backing_scan(object, OBSC_TEST_ALL_SHADOWED) == 0) {
12942ad1a3f7SMatthew Dillon 				break;
129524a1cce3SDavid Greenman 			}
1296df8bae1dSRodney W. Grimes 
1297df8bae1dSRodney W. Grimes 			/*
12980d94caffSDavid Greenman 			 * Make the parent shadow the next object in the
12990d94caffSDavid Greenman 			 * chain.  Deallocating backing_object will not remove
13000d94caffSDavid Greenman 			 * it, since its reference count is at least 2.
1301df8bae1dSRodney W. Grimes 			 */
1302df8bae1dSRodney W. Grimes 
13032ad1a3f7SMatthew Dillon 			TAILQ_REMOVE(
13042ad1a3f7SMatthew Dillon 			    &backing_object->shadow_head,
13052ad1a3f7SMatthew Dillon 			    object,
13062ad1a3f7SMatthew Dillon 			    shadow_list
13072ad1a3f7SMatthew Dillon 			);
1308eaf13dd7SJohn Dyson 			backing_object->shadow_count--;
1309eaf13dd7SJohn Dyson 			backing_object->generation++;
131095e5e988SJohn Dyson 
131195e5e988SJohn Dyson 			new_backing_object = backing_object->backing_object;
13128aef1712SMatthew Dillon 			if ((object->backing_object = new_backing_object) != NULL) {
131395e5e988SJohn Dyson 				vm_object_reference(new_backing_object);
13142ad1a3f7SMatthew Dillon 				TAILQ_INSERT_TAIL(
13152ad1a3f7SMatthew Dillon 				    &new_backing_object->shadow_head,
13162ad1a3f7SMatthew Dillon 				    object,
13172ad1a3f7SMatthew Dillon 				    shadow_list
13182ad1a3f7SMatthew Dillon 				);
1319eaf13dd7SJohn Dyson 				new_backing_object->shadow_count++;
1320eaf13dd7SJohn Dyson 				new_backing_object->generation++;
132195e5e988SJohn Dyson 				object->backing_object_offset +=
132295e5e988SJohn Dyson 					backing_object->backing_object_offset;
1323de5f6a77SJohn Dyson 			}
1324df8bae1dSRodney W. Grimes 
1325df8bae1dSRodney W. Grimes 			/*
13260d94caffSDavid Greenman 			 * Drop the reference count on backing_object. Since
13270d94caffSDavid Greenman 			 * its ref_count was at least 2, it will not vanish;
1328eaf13dd7SJohn Dyson 			 * so we don't need to call vm_object_deallocate, but
1329eaf13dd7SJohn Dyson 			 * we do anyway.
1330df8bae1dSRodney W. Grimes 			 */
133195e5e988SJohn Dyson 			vm_object_deallocate(backing_object);
1332df8bae1dSRodney W. Grimes 			object_bypasses++;
1333df8bae1dSRodney W. Grimes 		}
1334df8bae1dSRodney W. Grimes 
1335df8bae1dSRodney W. Grimes 		/*
1336df8bae1dSRodney W. Grimes 		 * Try again with this object's new backing object.
1337df8bae1dSRodney W. Grimes 		 */
1338df8bae1dSRodney W. Grimes 	}
1339df8bae1dSRodney W. Grimes }
1340df8bae1dSRodney W. Grimes 
1341df8bae1dSRodney W. Grimes /*
1342df8bae1dSRodney W. Grimes  *	vm_object_page_remove: [internal]
1343df8bae1dSRodney W. Grimes  *
1344df8bae1dSRodney W. Grimes  *	Removes all physical pages in the specified
1345df8bae1dSRodney W. Grimes  *	object range from the object's list of pages.
1346df8bae1dSRodney W. Grimes  *
1347df8bae1dSRodney W. Grimes  *	The object must be locked.
1348df8bae1dSRodney W. Grimes  */
134926f9a767SRodney W. Grimes void
13507c1f6cedSDavid Greenman vm_object_page_remove(object, start, end, clean_only)
1351d031cff1SMatthew Dillon 	vm_object_t object;
1352d031cff1SMatthew Dillon 	vm_pindex_t start;
1353d031cff1SMatthew Dillon 	vm_pindex_t end;
13547c1f6cedSDavid Greenman 	boolean_t clean_only;
1355df8bae1dSRodney W. Grimes {
1356d031cff1SMatthew Dillon 	vm_page_t p, next;
1357a316d390SJohn Dyson 	unsigned int size;
1358f5ef029eSPoul-Henning Kamp 	int all;
1359df8bae1dSRodney W. Grimes 
1360df8bae1dSRodney W. Grimes 	if (object == NULL)
1361df8bae1dSRodney W. Grimes 		return;
1362df8bae1dSRodney W. Grimes 
136395e5e988SJohn Dyson 	all = ((end == 0) && (start == 0));
136495e5e988SJohn Dyson 
1365d474eaaaSDoug Rabson 	vm_object_pip_add(object, 1);
136626f9a767SRodney W. Grimes again:
136726f9a767SRodney W. Grimes 	size = end - start;
136895e5e988SJohn Dyson 	if (all || size > 4 || size >= object->size / 4) {
1369b18bfc3dSJohn Dyson 		for (p = TAILQ_FIRST(&object->memq); p != NULL; p = next) {
1370b18bfc3dSJohn Dyson 			next = TAILQ_NEXT(p, listq);
137195e5e988SJohn Dyson 			if (all || ((start <= p->pindex) && (p->pindex < end))) {
1372bd7e5f99SJohn Dyson 				if (p->wire_count != 0) {
1373bd7e5f99SJohn Dyson 					vm_page_protect(p, VM_PROT_NONE);
1374ce65e68cSDavid Greenman 					if (!clean_only)
1375bd7e5f99SJohn Dyson 						p->valid = 0;
13760d94caffSDavid Greenman 					continue;
13770d94caffSDavid Greenman 				}
13780891ef4cSJohn Dyson 
1379b18bfc3dSJohn Dyson 				/*
1380b18bfc3dSJohn Dyson 				 * The busy flags are only cleared at
1381b18bfc3dSJohn Dyson 				 * interrupt -- minimize the spl transitions
1382b18bfc3dSJohn Dyson 				 */
1383ffc82b0aSJohn Dyson 
13841c7c3c6aSMatthew Dillon  				if (vm_page_sleep_busy(p, TRUE, "vmopar"))
138526f9a767SRodney W. Grimes  					goto again;
13860891ef4cSJohn Dyson 
13878f9110f6SJohn Dyson 				if (clean_only && p->valid) {
13887c1f6cedSDavid Greenman 					vm_page_test_dirty(p);
13897c1f6cedSDavid Greenman 					if (p->valid & p->dirty)
13907c1f6cedSDavid Greenman 						continue;
13917c1f6cedSDavid Greenman 				}
1392eaf13dd7SJohn Dyson 
1393e69763a3SDoug Rabson 				vm_page_busy(p);
1394f919ebdeSDavid Greenman 				vm_page_protect(p, VM_PROT_NONE);
1395df8bae1dSRodney W. Grimes 				vm_page_free(p);
139626f9a767SRodney W. Grimes 			}
139726f9a767SRodney W. Grimes 		}
139826f9a767SRodney W. Grimes 	} else {
139926f9a767SRodney W. Grimes 		while (size > 0) {
1400bd7e5f99SJohn Dyson 			if ((p = vm_page_lookup(object, start)) != 0) {
1401eaf13dd7SJohn Dyson 
1402bd7e5f99SJohn Dyson 				if (p->wire_count != 0) {
1403bd7e5f99SJohn Dyson 					vm_page_protect(p, VM_PROT_NONE);
1404ce65e68cSDavid Greenman 					if (!clean_only)
1405ce65e68cSDavid Greenman 						p->valid = 0;
1406bd7e5f99SJohn Dyson 					start += 1;
1407bd7e5f99SJohn Dyson 					size -= 1;
1408bd7e5f99SJohn Dyson 					continue;
14090d94caffSDavid Greenman 				}
1410eaf13dd7SJohn Dyson 
1411b18bfc3dSJohn Dyson 				/*
1412b18bfc3dSJohn Dyson 				 * The busy flags are only cleared at
1413b18bfc3dSJohn Dyson 				 * interrupt -- minimize the spl transitions
1414b18bfc3dSJohn Dyson 				 */
14151c7c3c6aSMatthew Dillon  				if (vm_page_sleep_busy(p, TRUE, "vmopar"))
141626f9a767SRodney W. Grimes 					goto again;
1417eaf13dd7SJohn Dyson 
14188f9110f6SJohn Dyson 				if (clean_only && p->valid) {
14197c1f6cedSDavid Greenman 					vm_page_test_dirty(p);
1420bd7e5f99SJohn Dyson 					if (p->valid & p->dirty) {
1421bd7e5f99SJohn Dyson 						start += 1;
1422bd7e5f99SJohn Dyson 						size -= 1;
14237c1f6cedSDavid Greenman 						continue;
14247c1f6cedSDavid Greenman 					}
1425bd7e5f99SJohn Dyson 				}
1426eaf13dd7SJohn Dyson 
1427e69763a3SDoug Rabson 				vm_page_busy(p);
1428f919ebdeSDavid Greenman 				vm_page_protect(p, VM_PROT_NONE);
142926f9a767SRodney W. Grimes 				vm_page_free(p);
143026f9a767SRodney W. Grimes 			}
1431a316d390SJohn Dyson 			start += 1;
1432a316d390SJohn Dyson 			size -= 1;
1433df8bae1dSRodney W. Grimes 		}
1434df8bae1dSRodney W. Grimes 	}
1435f919ebdeSDavid Greenman 	vm_object_pip_wakeup(object);
1436c0503609SDavid Greenman }
1437df8bae1dSRodney W. Grimes 
1438df8bae1dSRodney W. Grimes /*
1439df8bae1dSRodney W. Grimes  *	Routine:	vm_object_coalesce
1440df8bae1dSRodney W. Grimes  *	Function:	Coalesces two objects backing up adjoining
1441df8bae1dSRodney W. Grimes  *			regions of memory into a single object.
1442df8bae1dSRodney W. Grimes  *
1443df8bae1dSRodney W. Grimes  *	returns TRUE if objects were combined.
1444df8bae1dSRodney W. Grimes  *
1445df8bae1dSRodney W. Grimes  *	NOTE:	Only works at the moment if the second object is NULL -
1446df8bae1dSRodney W. Grimes  *		if it's not, which object do we lock first?
1447df8bae1dSRodney W. Grimes  *
1448df8bae1dSRodney W. Grimes  *	Parameters:
1449df8bae1dSRodney W. Grimes  *		prev_object	First object to coalesce
1450df8bae1dSRodney W. Grimes  *		prev_offset	Offset into prev_object
1451df8bae1dSRodney W. Grimes  *		next_object	Second object into coalesce
1452df8bae1dSRodney W. Grimes  *		next_offset	Offset into next_object
1453df8bae1dSRodney W. Grimes  *
1454df8bae1dSRodney W. Grimes  *		prev_size	Size of reference to prev_object
1455df8bae1dSRodney W. Grimes  *		next_size	Size of reference to next_object
1456df8bae1dSRodney W. Grimes  *
1457df8bae1dSRodney W. Grimes  *	Conditions:
1458df8bae1dSRodney W. Grimes  *	The object must *not* be locked.
1459df8bae1dSRodney W. Grimes  */
14600d94caffSDavid Greenman boolean_t
1461a316d390SJohn Dyson vm_object_coalesce(prev_object, prev_pindex, prev_size, next_size)
1462d031cff1SMatthew Dillon 	vm_object_t prev_object;
1463a316d390SJohn Dyson 	vm_pindex_t prev_pindex;
1464df8bae1dSRodney W. Grimes 	vm_size_t prev_size, next_size;
1465df8bae1dSRodney W. Grimes {
1466df8bae1dSRodney W. Grimes 	vm_size_t newsize;
1467df8bae1dSRodney W. Grimes 
1468df8bae1dSRodney W. Grimes 	if (prev_object == NULL) {
1469df8bae1dSRodney W. Grimes 		return (TRUE);
1470df8bae1dSRodney W. Grimes 	}
1471df8bae1dSRodney W. Grimes 
14724112823fSMatthew Dillon 	if (prev_object->type != OBJT_DEFAULT &&
14734112823fSMatthew Dillon 	    prev_object->type != OBJT_SWAP) {
147430dcfc09SJohn Dyson 		return (FALSE);
147530dcfc09SJohn Dyson 	}
147630dcfc09SJohn Dyson 
1477df8bae1dSRodney W. Grimes 	/*
1478df8bae1dSRodney W. Grimes 	 * Try to collapse the object first
1479df8bae1dSRodney W. Grimes 	 */
1480df8bae1dSRodney W. Grimes 	vm_object_collapse(prev_object);
1481df8bae1dSRodney W. Grimes 
1482df8bae1dSRodney W. Grimes 	/*
14830d94caffSDavid Greenman 	 * Can't coalesce if: . more than one reference . paged out . shadows
14840d94caffSDavid Greenman 	 * another object . has a copy elsewhere (any of which mean that the
14850d94caffSDavid Greenman 	 * pages not mapped to prev_entry may be in use anyway)
1486df8bae1dSRodney W. Grimes 	 */
1487df8bae1dSRodney W. Grimes 
14888cc7e047SJohn Dyson 	if (prev_object->backing_object != NULL) {
1489df8bae1dSRodney W. Grimes 		return (FALSE);
1490df8bae1dSRodney W. Grimes 	}
1491a316d390SJohn Dyson 
1492a316d390SJohn Dyson 	prev_size >>= PAGE_SHIFT;
1493a316d390SJohn Dyson 	next_size >>= PAGE_SHIFT;
14948cc7e047SJohn Dyson 
14958cc7e047SJohn Dyson 	if ((prev_object->ref_count > 1) &&
14968cc7e047SJohn Dyson 	    (prev_object->size != prev_pindex + prev_size)) {
14978cc7e047SJohn Dyson 		return (FALSE);
14988cc7e047SJohn Dyson 	}
14998cc7e047SJohn Dyson 
1500df8bae1dSRodney W. Grimes 	/*
15010d94caffSDavid Greenman 	 * Remove any pages that may still be in the object from a previous
15020d94caffSDavid Greenman 	 * deallocation.
1503df8bae1dSRodney W. Grimes 	 */
1504df8bae1dSRodney W. Grimes 
1505df8bae1dSRodney W. Grimes 	vm_object_page_remove(prev_object,
1506a316d390SJohn Dyson 	    prev_pindex + prev_size,
1507a316d390SJohn Dyson 	    prev_pindex + prev_size + next_size, FALSE);
1508df8bae1dSRodney W. Grimes 
1509df8bae1dSRodney W. Grimes 	/*
1510df8bae1dSRodney W. Grimes 	 * Extend the object if necessary.
1511df8bae1dSRodney W. Grimes 	 */
1512a316d390SJohn Dyson 	newsize = prev_pindex + prev_size + next_size;
1513df8bae1dSRodney W. Grimes 	if (newsize > prev_object->size)
1514df8bae1dSRodney W. Grimes 		prev_object->size = newsize;
1515df8bae1dSRodney W. Grimes 
1516df8bae1dSRodney W. Grimes 	return (TRUE);
1517df8bae1dSRodney W. Grimes }
1518df8bae1dSRodney W. Grimes 
1519c7c34a24SBruce Evans #include "opt_ddb.h"
1520c3cb3e12SDavid Greenman #ifdef DDB
1521c7c34a24SBruce Evans #include <sys/kernel.h>
1522c7c34a24SBruce Evans 
1523c7c34a24SBruce Evans #include <machine/cons.h>
1524c7c34a24SBruce Evans 
1525c7c34a24SBruce Evans #include <ddb/ddb.h>
1526c7c34a24SBruce Evans 
1527c7c34a24SBruce Evans static int	_vm_object_in_map __P((vm_map_t map, vm_object_t object,
1528c7c34a24SBruce Evans 				       vm_map_entry_t entry));
1529c7c34a24SBruce Evans static int	vm_object_in_map __P((vm_object_t object));
1530c3cb3e12SDavid Greenman 
1531cac597e4SBruce Evans static int
1532a1f6d91cSDavid Greenman _vm_object_in_map(map, object, entry)
1533a1f6d91cSDavid Greenman 	vm_map_t map;
1534a1f6d91cSDavid Greenman 	vm_object_t object;
1535a1f6d91cSDavid Greenman 	vm_map_entry_t entry;
1536a1f6d91cSDavid Greenman {
1537a1f6d91cSDavid Greenman 	vm_map_t tmpm;
1538a1f6d91cSDavid Greenman 	vm_map_entry_t tmpe;
1539a1f6d91cSDavid Greenman 	vm_object_t obj;
1540a1f6d91cSDavid Greenman 	int entcount;
1541a1f6d91cSDavid Greenman 
1542a1f6d91cSDavid Greenman 	if (map == 0)
1543a1f6d91cSDavid Greenman 		return 0;
1544a1f6d91cSDavid Greenman 
1545a1f6d91cSDavid Greenman 	if (entry == 0) {
1546a1f6d91cSDavid Greenman 		tmpe = map->header.next;
1547a1f6d91cSDavid Greenman 		entcount = map->nentries;
1548a1f6d91cSDavid Greenman 		while (entcount-- && (tmpe != &map->header)) {
1549a1f6d91cSDavid Greenman 			if( _vm_object_in_map(map, object, tmpe)) {
1550a1f6d91cSDavid Greenman 				return 1;
1551a1f6d91cSDavid Greenman 			}
1552a1f6d91cSDavid Greenman 			tmpe = tmpe->next;
1553a1f6d91cSDavid Greenman 		}
15549fdfe602SMatthew Dillon 	} else if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
15559fdfe602SMatthew Dillon 		tmpm = entry->object.sub_map;
1556a1f6d91cSDavid Greenman 		tmpe = tmpm->header.next;
1557a1f6d91cSDavid Greenman 		entcount = tmpm->nentries;
1558a1f6d91cSDavid Greenman 		while (entcount-- && tmpe != &tmpm->header) {
1559a1f6d91cSDavid Greenman 			if( _vm_object_in_map(tmpm, object, tmpe)) {
1560a1f6d91cSDavid Greenman 				return 1;
1561a1f6d91cSDavid Greenman 			}
1562a1f6d91cSDavid Greenman 			tmpe = tmpe->next;
1563a1f6d91cSDavid Greenman 		}
15648aef1712SMatthew Dillon 	} else if ((obj = entry->object.vm_object) != NULL) {
156524a1cce3SDavid Greenman 		for(; obj; obj=obj->backing_object)
1566a1f6d91cSDavid Greenman 			if( obj == object) {
1567a1f6d91cSDavid Greenman 				return 1;
1568a1f6d91cSDavid Greenman 			}
1569a1f6d91cSDavid Greenman 	}
1570a1f6d91cSDavid Greenman 	return 0;
1571a1f6d91cSDavid Greenman }
1572a1f6d91cSDavid Greenman 
1573cac597e4SBruce Evans static int
1574a1f6d91cSDavid Greenman vm_object_in_map( object)
1575a1f6d91cSDavid Greenman 	vm_object_t object;
1576a1f6d91cSDavid Greenman {
1577a1f6d91cSDavid Greenman 	struct proc *p;
15781b67ec6dSJeffrey Hsu 	for (p = allproc.lh_first; p != 0; p = p->p_list.le_next) {
1579a1f6d91cSDavid Greenman 		if( !p->p_vmspace /* || (p->p_flag & (P_SYSTEM|P_WEXIT)) */)
1580a1f6d91cSDavid Greenman 			continue;
1581a1f6d91cSDavid Greenman 		if( _vm_object_in_map(&p->p_vmspace->vm_map, object, 0))
1582a1f6d91cSDavid Greenman 			return 1;
1583a1f6d91cSDavid Greenman 	}
1584a1f6d91cSDavid Greenman 	if( _vm_object_in_map( kernel_map, object, 0))
1585a1f6d91cSDavid Greenman 		return 1;
1586a1f6d91cSDavid Greenman 	if( _vm_object_in_map( kmem_map, object, 0))
1587a1f6d91cSDavid Greenman 		return 1;
1588a1f6d91cSDavid Greenman 	if( _vm_object_in_map( pager_map, object, 0))
1589a1f6d91cSDavid Greenman 		return 1;
1590a1f6d91cSDavid Greenman 	if( _vm_object_in_map( buffer_map, object, 0))
1591a1f6d91cSDavid Greenman 		return 1;
1592a1f6d91cSDavid Greenman 	if( _vm_object_in_map( io_map, object, 0))
1593a1f6d91cSDavid Greenman 		return 1;
1594a1f6d91cSDavid Greenman 	if( _vm_object_in_map( phys_map, object, 0))
1595a1f6d91cSDavid Greenman 		return 1;
1596a1f6d91cSDavid Greenman 	if( _vm_object_in_map( mb_map, object, 0))
1597a1f6d91cSDavid Greenman 		return 1;
1598a1f6d91cSDavid Greenman 	if( _vm_object_in_map( u_map, object, 0))
1599a1f6d91cSDavid Greenman 		return 1;
1600a1f6d91cSDavid Greenman 	return 0;
1601a1f6d91cSDavid Greenman }
1602a1f6d91cSDavid Greenman 
1603c7c34a24SBruce Evans DB_SHOW_COMMAND(vmochk, vm_object_check)
1604f708ef1bSPoul-Henning Kamp {
1605a1f6d91cSDavid Greenman 	vm_object_t object;
1606a1f6d91cSDavid Greenman 
1607a1f6d91cSDavid Greenman 	/*
1608a1f6d91cSDavid Greenman 	 * make sure that internal objs are in a map somewhere
1609a1f6d91cSDavid Greenman 	 * and none have zero ref counts.
1610a1f6d91cSDavid Greenman 	 */
1611b18bfc3dSJohn Dyson 	for (object = TAILQ_FIRST(&vm_object_list);
1612a1f6d91cSDavid Greenman 			object != NULL;
1613b18bfc3dSJohn Dyson 			object = TAILQ_NEXT(object, object_list)) {
161424a1cce3SDavid Greenman 		if (object->handle == NULL &&
161524a1cce3SDavid Greenman 		    (object->type == OBJT_DEFAULT || object->type == OBJT_SWAP)) {
1616a1f6d91cSDavid Greenman 			if (object->ref_count == 0) {
1617c7c34a24SBruce Evans 				db_printf("vmochk: internal obj has zero ref count: %d\n",
1618a1f6d91cSDavid Greenman 					object->size);
1619a1f6d91cSDavid Greenman 			}
1620a1f6d91cSDavid Greenman 			if (!vm_object_in_map(object)) {
1621fc62ef1fSBruce Evans 				db_printf(
1622fc62ef1fSBruce Evans 			"vmochk: internal obj is not in a map: "
1623fc62ef1fSBruce Evans 			"ref: %d, size: %lu: 0x%lx, backing_object: %p\n",
1624fc62ef1fSBruce Evans 				    object->ref_count, (u_long)object->size,
1625fc62ef1fSBruce Evans 				    (u_long)object->size,
1626fc62ef1fSBruce Evans 				    (void *)object->backing_object);
1627a1f6d91cSDavid Greenman 			}
1628a1f6d91cSDavid Greenman 		}
1629a1f6d91cSDavid Greenman 	}
1630a1f6d91cSDavid Greenman }
1631a1f6d91cSDavid Greenman 
163226f9a767SRodney W. Grimes /*
1633df8bae1dSRodney W. Grimes  *	vm_object_print:	[ debug ]
1634df8bae1dSRodney W. Grimes  */
1635c7c34a24SBruce Evans DB_SHOW_COMMAND(object, vm_object_print_static)
1636df8bae1dSRodney W. Grimes {
1637c7c34a24SBruce Evans 	/* XXX convert args. */
1638c7c34a24SBruce Evans 	vm_object_t object = (vm_object_t)addr;
1639c7c34a24SBruce Evans 	boolean_t full = have_addr;
1640c7c34a24SBruce Evans 
1641d031cff1SMatthew Dillon 	vm_page_t p;
1642df8bae1dSRodney W. Grimes 
1643c7c34a24SBruce Evans 	/* XXX count is an (unused) arg.  Avoid shadowing it. */
1644c7c34a24SBruce Evans #define	count	was_count
1645c7c34a24SBruce Evans 
1646d031cff1SMatthew Dillon 	int count;
1647df8bae1dSRodney W. Grimes 
1648df8bae1dSRodney W. Grimes 	if (object == NULL)
1649df8bae1dSRodney W. Grimes 		return;
1650df8bae1dSRodney W. Grimes 
1651eb95adefSBruce Evans 	db_iprintf(
1652eb95adefSBruce Evans 	    "Object %p: type=%d, size=0x%lx, res=%d, ref=%d, flags=0x%x\n",
1653eb95adefSBruce Evans 	    object, (int)object->type, (u_long)object->size,
1654eb95adefSBruce Evans 	    object->resident_page_count, object->ref_count, object->flags);
1655eb95adefSBruce Evans 	/*
16561c7c3c6aSMatthew Dillon 	 * XXX no %qd in kernel.  Truncate object->backing_object_offset.
1657eb95adefSBruce Evans 	 */
16581c7c3c6aSMatthew Dillon 	db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%lx\n",
16591c7c3c6aSMatthew Dillon 	    object->shadow_count,
1660eb95adefSBruce Evans 	    object->backing_object ? object->backing_object->ref_count : 0,
1661eb95adefSBruce Evans 	    object->backing_object, (long)object->backing_object_offset);
1662df8bae1dSRodney W. Grimes 
1663df8bae1dSRodney W. Grimes 	if (!full)
1664df8bae1dSRodney W. Grimes 		return;
1665df8bae1dSRodney W. Grimes 
1666c7c34a24SBruce Evans 	db_indent += 2;
1667df8bae1dSRodney W. Grimes 	count = 0;
1668b18bfc3dSJohn Dyson 	for (p = TAILQ_FIRST(&object->memq); p != NULL; p = TAILQ_NEXT(p, listq)) {
1669df8bae1dSRodney W. Grimes 		if (count == 0)
1670c7c34a24SBruce Evans 			db_iprintf("memory:=");
1671df8bae1dSRodney W. Grimes 		else if (count == 6) {
1672c7c34a24SBruce Evans 			db_printf("\n");
1673c7c34a24SBruce Evans 			db_iprintf(" ...");
1674df8bae1dSRodney W. Grimes 			count = 0;
1675df8bae1dSRodney W. Grimes 		} else
1676c7c34a24SBruce Evans 			db_printf(",");
1677df8bae1dSRodney W. Grimes 		count++;
1678df8bae1dSRodney W. Grimes 
1679c7c34a24SBruce Evans 		db_printf("(off=0x%lx,page=0x%lx)",
1680a316d390SJohn Dyson 		    (u_long) p->pindex, (u_long) VM_PAGE_TO_PHYS(p));
1681df8bae1dSRodney W. Grimes 	}
1682df8bae1dSRodney W. Grimes 	if (count != 0)
1683c7c34a24SBruce Evans 		db_printf("\n");
1684c7c34a24SBruce Evans 	db_indent -= 2;
1685df8bae1dSRodney W. Grimes }
16865070c7f8SJohn Dyson 
1687c7c34a24SBruce Evans /* XXX. */
1688c7c34a24SBruce Evans #undef count
1689c7c34a24SBruce Evans 
1690c7c34a24SBruce Evans /* XXX need this non-static entry for calling from vm_map_print. */
16915070c7f8SJohn Dyson void
1692c7c34a24SBruce Evans vm_object_print(addr, have_addr, count, modif)
1693ecbb00a2SDoug Rabson         /* db_expr_t */ long addr;
1694c7c34a24SBruce Evans 	boolean_t have_addr;
1695ecbb00a2SDoug Rabson 	/* db_expr_t */ long count;
1696c7c34a24SBruce Evans 	char *modif;
1697c7c34a24SBruce Evans {
1698c7c34a24SBruce Evans 	vm_object_print_static(addr, have_addr, count, modif);
1699c7c34a24SBruce Evans }
1700c7c34a24SBruce Evans 
1701c7c34a24SBruce Evans DB_SHOW_COMMAND(vmopag, vm_object_print_pages)
17025070c7f8SJohn Dyson {
17035070c7f8SJohn Dyson 	vm_object_t object;
17045070c7f8SJohn Dyson 	int nl = 0;
17055070c7f8SJohn Dyson 	int c;
17065070c7f8SJohn Dyson 	for (object = TAILQ_FIRST(&vm_object_list);
17075070c7f8SJohn Dyson 			object != NULL;
17085070c7f8SJohn Dyson 			object = TAILQ_NEXT(object, object_list)) {
17095070c7f8SJohn Dyson 		vm_pindex_t idx, fidx;
17105070c7f8SJohn Dyson 		vm_pindex_t osize;
17115070c7f8SJohn Dyson 		vm_offset_t pa = -1, padiff;
17125070c7f8SJohn Dyson 		int rcount;
17135070c7f8SJohn Dyson 		vm_page_t m;
17145070c7f8SJohn Dyson 
1715fc62ef1fSBruce Evans 		db_printf("new object: %p\n", (void *)object);
17165070c7f8SJohn Dyson 		if ( nl > 18) {
17175070c7f8SJohn Dyson 			c = cngetc();
17185070c7f8SJohn Dyson 			if (c != ' ')
17195070c7f8SJohn Dyson 				return;
17205070c7f8SJohn Dyson 			nl = 0;
17215070c7f8SJohn Dyson 		}
17225070c7f8SJohn Dyson 		nl++;
17235070c7f8SJohn Dyson 		rcount = 0;
17245070c7f8SJohn Dyson 		fidx = 0;
17255070c7f8SJohn Dyson 		osize = object->size;
17265070c7f8SJohn Dyson 		if (osize > 128)
17275070c7f8SJohn Dyson 			osize = 128;
17285070c7f8SJohn Dyson 		for(idx=0;idx<osize;idx++) {
17295070c7f8SJohn Dyson 			m = vm_page_lookup(object, idx);
17305070c7f8SJohn Dyson 			if (m == NULL) {
17315070c7f8SJohn Dyson 				if (rcount) {
17325070c7f8SJohn Dyson 					db_printf(" index(%d)run(%d)pa(0x%x)\n",
17335070c7f8SJohn Dyson 						fidx, rcount, pa);
17345070c7f8SJohn Dyson 					if ( nl > 18) {
17355070c7f8SJohn Dyson 						c = cngetc();
17365070c7f8SJohn Dyson 						if (c != ' ')
17375070c7f8SJohn Dyson 							return;
17385070c7f8SJohn Dyson 						nl = 0;
17395070c7f8SJohn Dyson 					}
17405070c7f8SJohn Dyson 					nl++;
17415070c7f8SJohn Dyson 					rcount = 0;
17425070c7f8SJohn Dyson 				}
17435070c7f8SJohn Dyson 				continue;
17445070c7f8SJohn Dyson 			}
17455070c7f8SJohn Dyson 
17465070c7f8SJohn Dyson 
17475070c7f8SJohn Dyson 			if (rcount &&
17485070c7f8SJohn Dyson 				(VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) {
17495070c7f8SJohn Dyson 				++rcount;
17505070c7f8SJohn Dyson 				continue;
17515070c7f8SJohn Dyson 			}
17525070c7f8SJohn Dyson 			if (rcount) {
17535070c7f8SJohn Dyson 				padiff = pa + rcount * PAGE_SIZE - VM_PAGE_TO_PHYS(m);
17545070c7f8SJohn Dyson 				padiff >>= PAGE_SHIFT;
17555070c7f8SJohn Dyson 				padiff &= PQ_L2_MASK;
17565070c7f8SJohn Dyson 				if (padiff == 0) {
17575070c7f8SJohn Dyson 					pa = VM_PAGE_TO_PHYS(m) - rcount * PAGE_SIZE;
17585070c7f8SJohn Dyson 					++rcount;
17595070c7f8SJohn Dyson 					continue;
17605070c7f8SJohn Dyson 				}
17615070c7f8SJohn Dyson 				db_printf(" index(%d)run(%d)pa(0x%x)", fidx, rcount, pa);
17625070c7f8SJohn Dyson 				db_printf("pd(%d)\n", padiff);
17635070c7f8SJohn Dyson 				if ( nl > 18) {
17645070c7f8SJohn Dyson 					c = cngetc();
17655070c7f8SJohn Dyson 					if (c != ' ')
17665070c7f8SJohn Dyson 						return;
17675070c7f8SJohn Dyson 					nl = 0;
17685070c7f8SJohn Dyson 				}
17695070c7f8SJohn Dyson 				nl++;
17705070c7f8SJohn Dyson 			}
17715070c7f8SJohn Dyson 			fidx = idx;
17725070c7f8SJohn Dyson 			pa = VM_PAGE_TO_PHYS(m);
17735070c7f8SJohn Dyson 			rcount = 1;
17745070c7f8SJohn Dyson 		}
17755070c7f8SJohn Dyson 		if (rcount) {
17765070c7f8SJohn Dyson 			db_printf(" index(%d)run(%d)pa(0x%x)\n", fidx, rcount, pa);
17775070c7f8SJohn Dyson 			if ( nl > 18) {
17785070c7f8SJohn Dyson 				c = cngetc();
17795070c7f8SJohn Dyson 				if (c != ' ')
17805070c7f8SJohn Dyson 					return;
17815070c7f8SJohn Dyson 				nl = 0;
17825070c7f8SJohn Dyson 			}
17835070c7f8SJohn Dyson 			nl++;
17845070c7f8SJohn Dyson 		}
17855070c7f8SJohn Dyson 	}
17865070c7f8SJohn Dyson }
1787c3cb3e12SDavid Greenman #endif /* DDB */
1788