1df8bae1dSRodney W. Grimes /* 2df8bae1dSRodney W. Grimes * Copyright (c) 1991, 1993 3df8bae1dSRodney W. Grimes * The Regents of the University of California. All rights reserved. 4df8bae1dSRodney W. Grimes * 5df8bae1dSRodney W. Grimes * This code is derived from software contributed to Berkeley by 6df8bae1dSRodney W. Grimes * The Mach Operating System project at Carnegie-Mellon University. 7df8bae1dSRodney W. Grimes * 8df8bae1dSRodney W. Grimes * Redistribution and use in source and binary forms, with or without 9df8bae1dSRodney W. Grimes * modification, are permitted provided that the following conditions 10df8bae1dSRodney W. Grimes * are met: 11df8bae1dSRodney W. Grimes * 1. Redistributions of source code must retain the above copyright 12df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer. 13df8bae1dSRodney W. Grimes * 2. Redistributions in binary form must reproduce the above copyright 14df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer in the 15df8bae1dSRodney W. Grimes * documentation and/or other materials provided with the distribution. 16df8bae1dSRodney W. Grimes * 3. All advertising materials mentioning features or use of this software 17df8bae1dSRodney W. Grimes * must display the following acknowledgement: 18df8bae1dSRodney W. Grimes * This product includes software developed by the University of 19df8bae1dSRodney W. Grimes * California, Berkeley and its contributors. 20df8bae1dSRodney W. Grimes * 4. Neither the name of the University nor the names of its contributors 21df8bae1dSRodney W. Grimes * may be used to endorse or promote products derived from this software 22df8bae1dSRodney W. Grimes * without specific prior written permission. 23df8bae1dSRodney W. Grimes * 24df8bae1dSRodney W. Grimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25df8bae1dSRodney W. Grimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26df8bae1dSRodney W. Grimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27df8bae1dSRodney W. Grimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28df8bae1dSRodney W. Grimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29df8bae1dSRodney W. Grimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30df8bae1dSRodney W. Grimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31df8bae1dSRodney W. Grimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32df8bae1dSRodney W. Grimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33df8bae1dSRodney W. Grimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34df8bae1dSRodney W. Grimes * SUCH DAMAGE. 35df8bae1dSRodney W. Grimes * 363c4dd356SDavid Greenman * from: @(#)vm_object.c 8.5 (Berkeley) 3/22/94 37df8bae1dSRodney W. Grimes * 38df8bae1dSRodney W. Grimes * 39df8bae1dSRodney W. Grimes * Copyright (c) 1987, 1990 Carnegie-Mellon University. 40df8bae1dSRodney W. Grimes * All rights reserved. 41df8bae1dSRodney W. Grimes * 42df8bae1dSRodney W. Grimes * Authors: Avadis Tevanian, Jr., Michael Wayne Young 43df8bae1dSRodney W. Grimes * 44df8bae1dSRodney W. Grimes * Permission to use, copy, modify and distribute this software and 45df8bae1dSRodney W. Grimes * its documentation is hereby granted, provided that both the copyright 46df8bae1dSRodney W. Grimes * notice and this permission notice appear in all copies of the 47df8bae1dSRodney W. Grimes * software, derivative works or modified versions, and any portions 48df8bae1dSRodney W. Grimes * thereof, and that both notices appear in supporting documentation. 49df8bae1dSRodney W. Grimes * 50df8bae1dSRodney W. Grimes * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 51df8bae1dSRodney W. Grimes * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 52df8bae1dSRodney W. Grimes * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 53df8bae1dSRodney W. Grimes * 54df8bae1dSRodney W. Grimes * Carnegie Mellon requests users of this software to return to 55df8bae1dSRodney W. Grimes * 56df8bae1dSRodney W. Grimes * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 57df8bae1dSRodney W. Grimes * School of Computer Science 58df8bae1dSRodney W. Grimes * Carnegie Mellon University 59df8bae1dSRodney W. Grimes * Pittsburgh PA 15213-3890 60df8bae1dSRodney W. Grimes * 61df8bae1dSRodney W. Grimes * any improvements or extensions that they make and grant Carnegie the 62df8bae1dSRodney W. Grimes * rights to redistribute these changes. 633c4dd356SDavid Greenman * 64f6b04d2bSDavid Greenman * $Id: vm_object.c,v 1.40 1995/03/25 08:42:14 davidg Exp $ 65df8bae1dSRodney W. Grimes */ 66df8bae1dSRodney W. Grimes 67df8bae1dSRodney W. Grimes /* 68df8bae1dSRodney W. Grimes * Virtual memory object module. 69df8bae1dSRodney W. Grimes */ 70df8bae1dSRodney W. Grimes 71df8bae1dSRodney W. Grimes #include <sys/param.h> 72df8bae1dSRodney W. Grimes #include <sys/systm.h> 73f23b4c91SGarrett Wollman #include <sys/kernel.h> 74f23b4c91SGarrett Wollman #include <sys/proc.h> /* for curproc, pageproc */ 75df8bae1dSRodney W. Grimes #include <sys/malloc.h> 760d94caffSDavid Greenman #include <sys/vnode.h> 770d94caffSDavid Greenman #include <sys/mount.h> 78df8bae1dSRodney W. Grimes 79df8bae1dSRodney W. Grimes #include <vm/vm.h> 80df8bae1dSRodney W. Grimes #include <vm/vm_page.h> 8126f9a767SRodney W. Grimes #include <vm/vm_pageout.h> 820d94caffSDavid Greenman #include <vm/vm_pager.h> 8305f0fdd2SPoul-Henning Kamp #include <vm/swap_pager.h> 840d94caffSDavid Greenman #include <vm/vnode_pager.h> 85a1f6d91cSDavid Greenman #include <vm/vm_kern.h> 8626f9a767SRodney W. Grimes 8726f9a767SRodney W. Grimes static void _vm_object_allocate(vm_size_t, vm_object_t); 88df8bae1dSRodney W. Grimes 89f6b04d2bSDavid Greenman 90df8bae1dSRodney W. Grimes /* 91df8bae1dSRodney W. Grimes * Virtual memory objects maintain the actual data 92df8bae1dSRodney W. Grimes * associated with allocated virtual memory. A given 93df8bae1dSRodney W. Grimes * page of memory exists within exactly one object. 94df8bae1dSRodney W. Grimes * 95df8bae1dSRodney W. Grimes * An object is only deallocated when all "references" 96df8bae1dSRodney W. Grimes * are given up. Only one "reference" to a given 97df8bae1dSRodney W. Grimes * region of an object should be writeable. 98df8bae1dSRodney W. Grimes * 99df8bae1dSRodney W. Grimes * Associated with each object is a list of all resident 100df8bae1dSRodney W. Grimes * memory pages belonging to that object; this list is 101df8bae1dSRodney W. Grimes * maintained by the "vm_page" module, and locked by the object's 102df8bae1dSRodney W. Grimes * lock. 103df8bae1dSRodney W. Grimes * 104df8bae1dSRodney W. Grimes * Each object also records a "pager" routine which is 105df8bae1dSRodney W. Grimes * used to retrieve (and store) pages to the proper backing 106df8bae1dSRodney W. Grimes * storage. In addition, objects may be backed by other 107df8bae1dSRodney W. Grimes * objects from which they were virtual-copied. 108df8bae1dSRodney W. Grimes * 109df8bae1dSRodney W. Grimes * The only items within the object structure which are 110df8bae1dSRodney W. Grimes * modified after time of creation are: 111df8bae1dSRodney W. Grimes * reference count locked by object's lock 112df8bae1dSRodney W. Grimes * pager routine locked by object's lock 113df8bae1dSRodney W. Grimes * 114df8bae1dSRodney W. Grimes */ 115df8bae1dSRodney W. Grimes 11626f9a767SRodney W. Grimes 117df8bae1dSRodney W. Grimes struct vm_object kernel_object_store; 118df8bae1dSRodney W. Grimes struct vm_object kmem_object_store; 119df8bae1dSRodney W. Grimes 1200d94caffSDavid Greenman int vm_object_cache_max; 1210d94caffSDavid Greenman 1220d94caffSDavid Greenman #define VM_OBJECT_HASH_COUNT 509 123df8bae1dSRodney W. Grimes 124df8bae1dSRodney W. Grimes struct vm_object_hash_head vm_object_hashtable[VM_OBJECT_HASH_COUNT]; 125df8bae1dSRodney W. Grimes 126df8bae1dSRodney W. Grimes long object_collapses = 0; 127df8bae1dSRodney W. Grimes long object_bypasses = 0; 128df8bae1dSRodney W. Grimes 129df8bae1dSRodney W. Grimes static void 130df8bae1dSRodney W. Grimes _vm_object_allocate(size, object) 131df8bae1dSRodney W. Grimes vm_size_t size; 132df8bae1dSRodney W. Grimes register vm_object_t object; 133df8bae1dSRodney W. Grimes { 134df8bae1dSRodney W. Grimes TAILQ_INIT(&object->memq); 1352fe6e4d7SDavid Greenman TAILQ_INIT(&object->reverse_shadow_head); 136a1f6d91cSDavid Greenman 137df8bae1dSRodney W. Grimes object->size = size; 138a1f6d91cSDavid Greenman object->ref_count = 1; 139a1f6d91cSDavid Greenman vm_object_lock_init(object); 140c4ed5a07SDavid Greenman object->flags = OBJ_INTERNAL; /* pager will reset */ 141df8bae1dSRodney W. Grimes object->paging_in_progress = 0; 142a1f6d91cSDavid Greenman object->resident_page_count = 0; 143df8bae1dSRodney W. Grimes 144df8bae1dSRodney W. Grimes object->pager = NULL; 145df8bae1dSRodney W. Grimes object->paging_offset = 0; 146df8bae1dSRodney W. Grimes object->shadow = NULL; 147df8bae1dSRodney W. Grimes object->shadow_offset = (vm_offset_t) 0; 148a1f6d91cSDavid Greenman object->copy = NULL; 149a1f6d91cSDavid Greenman 150a1f6d91cSDavid Greenman object->last_read = 0; 151df8bae1dSRodney W. Grimes 152df8bae1dSRodney W. Grimes simple_lock(&vm_object_list_lock); 153df8bae1dSRodney W. Grimes TAILQ_INSERT_TAIL(&vm_object_list, object, object_list); 154df8bae1dSRodney W. Grimes vm_object_count++; 155df8bae1dSRodney W. Grimes simple_unlock(&vm_object_list_lock); 156df8bae1dSRodney W. Grimes } 157df8bae1dSRodney W. Grimes 158df8bae1dSRodney W. Grimes /* 15926f9a767SRodney W. Grimes * vm_object_init: 16026f9a767SRodney W. Grimes * 16126f9a767SRodney W. Grimes * Initialize the VM objects module. 16226f9a767SRodney W. Grimes */ 16326f9a767SRodney W. Grimes void 16426f9a767SRodney W. Grimes vm_object_init(vm_offset_t nothing) 16526f9a767SRodney W. Grimes { 16626f9a767SRodney W. Grimes register int i; 16726f9a767SRodney W. Grimes 16826f9a767SRodney W. Grimes TAILQ_INIT(&vm_object_cached_list); 16926f9a767SRodney W. Grimes TAILQ_INIT(&vm_object_list); 17026f9a767SRodney W. Grimes vm_object_count = 0; 17126f9a767SRodney W. Grimes simple_lock_init(&vm_cache_lock); 17226f9a767SRodney W. Grimes simple_lock_init(&vm_object_list_lock); 1730217125fSDavid Greenman 1740217125fSDavid Greenman vm_object_cache_max = 84; 1750217125fSDavid Greenman if (cnt.v_page_count > 1000) 1760217125fSDavid Greenman vm_object_cache_max += (cnt.v_page_count - 1000) / 4; 17726f9a767SRodney W. Grimes 17826f9a767SRodney W. Grimes for (i = 0; i < VM_OBJECT_HASH_COUNT; i++) 17926f9a767SRodney W. Grimes TAILQ_INIT(&vm_object_hashtable[i]); 18026f9a767SRodney W. Grimes 18126f9a767SRodney W. Grimes kernel_object = &kernel_object_store; 18226f9a767SRodney W. Grimes _vm_object_allocate(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS, 18326f9a767SRodney W. Grimes kernel_object); 18426f9a767SRodney W. Grimes 18526f9a767SRodney W. Grimes kmem_object = &kmem_object_store; 18626f9a767SRodney W. Grimes _vm_object_allocate(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS, 18726f9a767SRodney W. Grimes kmem_object); 18826f9a767SRodney W. Grimes } 18926f9a767SRodney W. Grimes 19026f9a767SRodney W. Grimes /* 19126f9a767SRodney W. Grimes * vm_object_allocate: 19226f9a767SRodney W. Grimes * 19326f9a767SRodney W. Grimes * Returns a new object with the given size. 19426f9a767SRodney W. Grimes */ 19526f9a767SRodney W. Grimes 19626f9a767SRodney W. Grimes vm_object_t 19726f9a767SRodney W. Grimes vm_object_allocate(size) 19826f9a767SRodney W. Grimes vm_size_t size; 19926f9a767SRodney W. Grimes { 20026f9a767SRodney W. Grimes register vm_object_t result; 20126f9a767SRodney W. Grimes 20226f9a767SRodney W. Grimes result = (vm_object_t) 20326f9a767SRodney W. Grimes malloc((u_long) sizeof *result, M_VMOBJ, M_WAITOK); 20426f9a767SRodney W. Grimes 20526f9a767SRodney W. Grimes 20626f9a767SRodney W. Grimes _vm_object_allocate(size, result); 20726f9a767SRodney W. Grimes 20826f9a767SRodney W. Grimes return (result); 20926f9a767SRodney W. Grimes } 21026f9a767SRodney W. Grimes 21126f9a767SRodney W. Grimes 21226f9a767SRodney W. Grimes /* 213df8bae1dSRodney W. Grimes * vm_object_reference: 214df8bae1dSRodney W. Grimes * 215df8bae1dSRodney W. Grimes * Gets another reference to the given object. 216df8bae1dSRodney W. Grimes */ 21726f9a767SRodney W. Grimes inline void 21826f9a767SRodney W. Grimes vm_object_reference(object) 219df8bae1dSRodney W. Grimes register vm_object_t object; 220df8bae1dSRodney W. Grimes { 221df8bae1dSRodney W. Grimes if (object == NULL) 222df8bae1dSRodney W. Grimes return; 223df8bae1dSRodney W. Grimes 224df8bae1dSRodney W. Grimes vm_object_lock(object); 225df8bae1dSRodney W. Grimes object->ref_count++; 226df8bae1dSRodney W. Grimes vm_object_unlock(object); 227df8bae1dSRodney W. Grimes } 228df8bae1dSRodney W. Grimes 229df8bae1dSRodney W. Grimes /* 230df8bae1dSRodney W. Grimes * vm_object_deallocate: 231df8bae1dSRodney W. Grimes * 232df8bae1dSRodney W. Grimes * Release a reference to the specified object, 233df8bae1dSRodney W. Grimes * gained either through a vm_object_allocate 234df8bae1dSRodney W. Grimes * or a vm_object_reference call. When all references 235df8bae1dSRodney W. Grimes * are gone, storage associated with this object 236df8bae1dSRodney W. Grimes * may be relinquished. 237df8bae1dSRodney W. Grimes * 238df8bae1dSRodney W. Grimes * No object may be locked. 239df8bae1dSRodney W. Grimes */ 24026f9a767SRodney W. Grimes void 24126f9a767SRodney W. Grimes vm_object_deallocate(object) 24226f9a767SRodney W. Grimes vm_object_t object; 243df8bae1dSRodney W. Grimes { 244df8bae1dSRodney W. Grimes vm_object_t temp; 245f5cf85d4SDavid Greenman vm_pager_t pager; 246df8bae1dSRodney W. Grimes 247df8bae1dSRodney W. Grimes while (object != NULL) { 248df8bae1dSRodney W. Grimes 249ba8da839SDavid Greenman if (object->ref_count == 0) 250ba8da839SDavid Greenman panic("vm_object_deallocate: object deallocated too many times"); 251ba8da839SDavid Greenman 252df8bae1dSRodney W. Grimes /* 2530d94caffSDavid Greenman * The cache holds a reference (uncounted) to the object; we 2540d94caffSDavid Greenman * must lock it before removing the object. 255df8bae1dSRodney W. Grimes */ 256df8bae1dSRodney W. Grimes 257df8bae1dSRodney W. Grimes vm_object_cache_lock(); 258df8bae1dSRodney W. Grimes 259df8bae1dSRodney W. Grimes /* 260df8bae1dSRodney W. Grimes * Lose the reference 261df8bae1dSRodney W. Grimes */ 262df8bae1dSRodney W. Grimes vm_object_lock(object); 263be6d5bfaSDavid Greenman 264be6d5bfaSDavid Greenman object->ref_count--; 265be6d5bfaSDavid Greenman 266be6d5bfaSDavid Greenman if (object->ref_count != 0) { 267be6d5bfaSDavid Greenman if ((object->ref_count == 1) && 268be6d5bfaSDavid Greenman (object->flags & OBJ_INTERNAL)) { 269a1f6d91cSDavid Greenman vm_object_t robject; 270a1f6d91cSDavid Greenman robject = object->reverse_shadow_head.tqh_first; 271be6d5bfaSDavid Greenman if ((robject != NULL) && 272be6d5bfaSDavid Greenman (robject->flags & OBJ_INTERNAL)) { 273a1f6d91cSDavid Greenman int s; 274a1f6d91cSDavid Greenman robject->ref_count += 2; 2759b4814bbSDavid Greenman object->ref_count += 2; 276a1f6d91cSDavid Greenman 277a1f6d91cSDavid Greenman do { 278a1f6d91cSDavid Greenman s = splhigh(); 279a1f6d91cSDavid Greenman while (robject->paging_in_progress) { 280c0503609SDavid Greenman robject->flags |= OBJ_PIPWNT; 281a1f6d91cSDavid Greenman tsleep(robject, PVM, "objde1", 0); 2820d94caffSDavid Greenman } 283a1f6d91cSDavid Greenman 284a1f6d91cSDavid Greenman while (object->paging_in_progress) { 285c0503609SDavid Greenman object->flags |= OBJ_PIPWNT; 286a1f6d91cSDavid Greenman tsleep(object, PVM, "objde2", 0); 287a1f6d91cSDavid Greenman } 288a1f6d91cSDavid Greenman splx(s); 289a1f6d91cSDavid Greenman 290a1f6d91cSDavid Greenman } while( object->paging_in_progress || robject->paging_in_progress); 291a1f6d91cSDavid Greenman 2929b4814bbSDavid Greenman object->ref_count -= 2; 293a1f6d91cSDavid Greenman robject->ref_count -= 2; 294a1f6d91cSDavid Greenman if( robject->ref_count == 0) { 295a1f6d91cSDavid Greenman vm_object_unlock(object); 296a1f6d91cSDavid Greenman vm_object_cache_unlock(); 297a1f6d91cSDavid Greenman robject->ref_count += 1; 298ba8da839SDavid Greenman object = robject; 299ba8da839SDavid Greenman continue; 300a1f6d91cSDavid Greenman } 301a1f6d91cSDavid Greenman vm_object_cache_unlock(); 302a1f6d91cSDavid Greenman vm_object_unlock(object); 303a1f6d91cSDavid Greenman vm_object_lock(robject); 304a1f6d91cSDavid Greenman vm_object_collapse(robject); 305a1f6d91cSDavid Greenman return; 306010cf3b9SDavid Greenman } 307010cf3b9SDavid Greenman } 30826f9a767SRodney W. Grimes vm_object_unlock(object); 309df8bae1dSRodney W. Grimes /* 3100d94caffSDavid Greenman * If there are still references, then we are done. 311df8bae1dSRodney W. Grimes */ 312df8bae1dSRodney W. Grimes vm_object_cache_unlock(); 313df8bae1dSRodney W. Grimes return; 314df8bae1dSRodney W. Grimes } 315df8bae1dSRodney W. Grimes 316f5cf85d4SDavid Greenman pager = object->pager; 317f5cf85d4SDavid Greenman 318f5cf85d4SDavid Greenman if (pager && pager->pg_type == PG_VNODE) { 319f5cf85d4SDavid Greenman vn_pager_t vnp = (vn_pager_t) pager->pg_data; 320f5cf85d4SDavid Greenman 321f5cf85d4SDavid Greenman vnp->vnp_vp->v_flag &= ~VTEXT; 322f5cf85d4SDavid Greenman } 323f5cf85d4SDavid Greenman 324f919ebdeSDavid Greenman /* 325f919ebdeSDavid Greenman * See if this object can persist and has some resident 326f919ebdeSDavid Greenman * pages. If so, enter it in the cache. 327f919ebdeSDavid Greenman */ 328f5cf85d4SDavid Greenman if (object->flags & OBJ_CANPERSIST) { 329f5cf85d4SDavid Greenman if (object->resident_page_count != 0) { 330df8bae1dSRodney W. Grimes TAILQ_INSERT_TAIL(&vm_object_cached_list, object, 331df8bae1dSRodney W. Grimes cached_list); 332df8bae1dSRodney W. Grimes vm_object_cached++; 333df8bae1dSRodney W. Grimes vm_object_cache_unlock(); 334df8bae1dSRodney W. Grimes 335df8bae1dSRodney W. Grimes vm_object_unlock(object); 336df8bae1dSRodney W. Grimes 337df8bae1dSRodney W. Grimes vm_object_cache_trim(); 338df8bae1dSRodney W. Grimes return; 339f5cf85d4SDavid Greenman } else { 340f5cf85d4SDavid Greenman object->flags &= ~OBJ_CANPERSIST; 341f5cf85d4SDavid Greenman } 342df8bae1dSRodney W. Grimes } 343f919ebdeSDavid Greenman 344df8bae1dSRodney W. Grimes /* 345df8bae1dSRodney W. Grimes * Make sure no one can look us up now. 346df8bae1dSRodney W. Grimes */ 3470d94caffSDavid Greenman object->flags |= OBJ_DEAD; 348d7a0fc93SDavid Greenman if ((object->flags & OBJ_INTERNAL) == 0) 349f5cf85d4SDavid Greenman vm_object_remove(pager); 350df8bae1dSRodney W. Grimes vm_object_cache_unlock(); 351df8bae1dSRodney W. Grimes 352df8bae1dSRodney W. Grimes temp = object->shadow; 3532fe6e4d7SDavid Greenman if (temp) 3542fe6e4d7SDavid Greenman TAILQ_REMOVE(&temp->reverse_shadow_head, object, reverse_shadow_list); 355df8bae1dSRodney W. Grimes vm_object_terminate(object); 356df8bae1dSRodney W. Grimes /* unlocks and deallocates object */ 357df8bae1dSRodney W. Grimes object = temp; 358df8bae1dSRodney W. Grimes } 359df8bae1dSRodney W. Grimes } 360df8bae1dSRodney W. Grimes 361df8bae1dSRodney W. Grimes /* 362df8bae1dSRodney W. Grimes * vm_object_terminate actually destroys the specified object, freeing 363df8bae1dSRodney W. Grimes * up all previously used resources. 364df8bae1dSRodney W. Grimes * 365df8bae1dSRodney W. Grimes * The object must be locked. 366df8bae1dSRodney W. Grimes */ 36726f9a767SRodney W. Grimes void 36826f9a767SRodney W. Grimes vm_object_terminate(object) 369df8bae1dSRodney W. Grimes register vm_object_t object; 370df8bae1dSRodney W. Grimes { 3710d94caffSDavid Greenman register vm_page_t p, next; 372df8bae1dSRodney W. Grimes vm_object_t shadow_object; 37326f9a767SRodney W. Grimes int s; 374df8bae1dSRodney W. Grimes 375df8bae1dSRodney W. Grimes /* 3760d94caffSDavid Greenman * Detach the object from its shadow if we are the shadow's copy. 377df8bae1dSRodney W. Grimes */ 378df8bae1dSRodney W. Grimes if ((shadow_object = object->shadow) != NULL) { 379df8bae1dSRodney W. Grimes vm_object_lock(shadow_object); 380df8bae1dSRodney W. Grimes if (shadow_object->copy == object) 381df8bae1dSRodney W. Grimes shadow_object->copy = NULL; 382df8bae1dSRodney W. Grimes vm_object_unlock(shadow_object); 383df8bae1dSRodney W. Grimes } 384df8bae1dSRodney W. Grimes 385df8bae1dSRodney W. Grimes /* 386f6b04d2bSDavid Greenman * wait for the pageout daemon to be done with the object 387df8bae1dSRodney W. Grimes */ 3880d94caffSDavid Greenman s = splhigh(); 389df8bae1dSRodney W. Grimes while (object->paging_in_progress) { 3900d94caffSDavid Greenman vm_object_unlock(object); 391c0503609SDavid Greenman object->flags |= OBJ_PIPWNT; 3920d94caffSDavid Greenman tsleep((caddr_t) object, PVM, "objtrm", 0); 393df8bae1dSRodney W. Grimes vm_object_lock(object); 394df8bae1dSRodney W. Grimes } 3950d94caffSDavid Greenman splx(s); 396df8bae1dSRodney W. Grimes 39726f9a767SRodney W. Grimes if (object->paging_in_progress != 0) 39826f9a767SRodney W. Grimes panic("vm_object_deallocate: pageout in progress"); 39926f9a767SRodney W. Grimes 40026f9a767SRodney W. Grimes /* 4010d94caffSDavid Greenman * Clean and free the pages, as appropriate. All references to the 4020d94caffSDavid Greenman * object are gone, so we don't need to lock it. 40326f9a767SRodney W. Grimes */ 404f6b04d2bSDavid Greenman if (object->pager && (object->pager->pg_type == PG_VNODE)) { 405f6b04d2bSDavid Greenman vn_pager_t vnp = object->pager->pg_data; 406f6b04d2bSDavid Greenman struct vnode *vp; 407f6b04d2bSDavid Greenman 408f6b04d2bSDavid Greenman vp = vnp->vnp_vp; 4090426122fSDavid Greenman VOP_LOCK(vp); 410f6b04d2bSDavid Greenman (void) _vm_object_page_clean(object, 0, 0, TRUE); 411f6b04d2bSDavid Greenman vinvalbuf(vp, V_SAVE, NOCRED, NULL, 0, 0); 4120426122fSDavid Greenman VOP_UNLOCK(vp); 413df8bae1dSRodney W. Grimes } 41483edfd47SDavid Greenman 4150d94caffSDavid Greenman /* 4160d94caffSDavid Greenman * Now free the pages. For internal objects, this also removes them 4170d94caffSDavid Greenman * from paging queues. 418df8bae1dSRodney W. Grimes */ 419df8bae1dSRodney W. Grimes while ((p = object->memq.tqh_first) != NULL) { 420df8bae1dSRodney W. Grimes VM_PAGE_CHECK(p); 421df8bae1dSRodney W. Grimes vm_page_lock_queues(); 422be6d5bfaSDavid Greenman if (p->flags & PG_BUSY) 423be6d5bfaSDavid Greenman printf("vm_object_terminate: freeing busy page\n"); 4240d94caffSDavid Greenman PAGE_WAKEUP(p); 425df8bae1dSRodney W. Grimes vm_page_free(p); 426df8bae1dSRodney W. Grimes cnt.v_pfree++; 427df8bae1dSRodney W. Grimes vm_page_unlock_queues(); 428df8bae1dSRodney W. Grimes } 4290d94caffSDavid Greenman vm_object_unlock(object); 430df8bae1dSRodney W. Grimes 431df8bae1dSRodney W. Grimes /* 432df8bae1dSRodney W. Grimes * Let the pager know object is dead. 433df8bae1dSRodney W. Grimes */ 434df8bae1dSRodney W. Grimes if (object->pager != NULL) 435df8bae1dSRodney W. Grimes vm_pager_deallocate(object->pager); 436df8bae1dSRodney W. Grimes 437df8bae1dSRodney W. Grimes simple_lock(&vm_object_list_lock); 438df8bae1dSRodney W. Grimes TAILQ_REMOVE(&vm_object_list, object, object_list); 439df8bae1dSRodney W. Grimes vm_object_count--; 440df8bae1dSRodney W. Grimes simple_unlock(&vm_object_list_lock); 441df8bae1dSRodney W. Grimes 4420426122fSDavid Greenman wakeup(object); 4430426122fSDavid Greenman 444df8bae1dSRodney W. Grimes /* 445df8bae1dSRodney W. Grimes * Free the space for the object. 446df8bae1dSRodney W. Grimes */ 447df8bae1dSRodney W. Grimes free((caddr_t) object, M_VMOBJ); 448df8bae1dSRodney W. Grimes } 449df8bae1dSRodney W. Grimes 450df8bae1dSRodney W. Grimes /* 451df8bae1dSRodney W. Grimes * vm_object_page_clean 452df8bae1dSRodney W. Grimes * 453df8bae1dSRodney W. Grimes * Clean all dirty pages in the specified range of object. 45426f9a767SRodney W. Grimes * Leaves page on whatever queue it is currently on. 45526f9a767SRodney W. Grimes * 45626f9a767SRodney W. Grimes * Odd semantics: if start == end, we clean everything. 45726f9a767SRodney W. Grimes * 45826f9a767SRodney W. Grimes * The object must be locked. 45926f9a767SRodney W. Grimes */ 460f6b04d2bSDavid Greenman 461f6b04d2bSDavid Greenman void 462f6b04d2bSDavid Greenman _vm_object_page_clean(object, start, end, syncio) 463f6b04d2bSDavid Greenman vm_object_t object; 464f6b04d2bSDavid Greenman vm_offset_t start; 465f6b04d2bSDavid Greenman vm_offset_t end; 466f6b04d2bSDavid Greenman boolean_t syncio; 467f6b04d2bSDavid Greenman { 468f6b04d2bSDavid Greenman register vm_page_t p; 469f6b04d2bSDavid Greenman register vm_offset_t tstart, tend; 470f6b04d2bSDavid Greenman int pass; 471f6b04d2bSDavid Greenman int pgcount, s; 472f6b04d2bSDavid Greenman int allclean; 473f6b04d2bSDavid Greenman 474f6b04d2bSDavid Greenman if (object->pager == NULL || (object->flags & OBJ_WRITEABLE) == 0) 475f6b04d2bSDavid Greenman return; 476f6b04d2bSDavid Greenman 477f6b04d2bSDavid Greenman if (start != end) { 478f6b04d2bSDavid Greenman start = trunc_page(start); 479f6b04d2bSDavid Greenman end = round_page(end); 480f6b04d2bSDavid Greenman } 481f6b04d2bSDavid Greenman 482f6b04d2bSDavid Greenman object->flags &= ~OBJ_WRITEABLE; 483f6b04d2bSDavid Greenman 484f6b04d2bSDavid Greenman pass = 0; 485f6b04d2bSDavid Greenman startover: 486f6b04d2bSDavid Greenman tstart = start; 487f6b04d2bSDavid Greenman if (end == 0) { 488f6b04d2bSDavid Greenman tend = object->size; 489f6b04d2bSDavid Greenman } else { 490f6b04d2bSDavid Greenman tend = end; 491f6b04d2bSDavid Greenman } 492f6b04d2bSDavid Greenman /* 493f6b04d2bSDavid Greenman * Wait until potential collapse operation is complete 494f6b04d2bSDavid Greenman */ 495f6b04d2bSDavid Greenman if (object->flags & OBJ_INTERNAL) { 496f6b04d2bSDavid Greenman s = splhigh(); 497f6b04d2bSDavid Greenman while (object->paging_in_progress) { 498f6b04d2bSDavid Greenman object->flags |= OBJ_PIPWNT; 499f6b04d2bSDavid Greenman tsleep(object, PVM, "objpcw", 0); 500f6b04d2bSDavid Greenman } 501f6b04d2bSDavid Greenman splx(s); 502f6b04d2bSDavid Greenman } 503f6b04d2bSDavid Greenman 504f6b04d2bSDavid Greenman pgcount = object->resident_page_count; 505f6b04d2bSDavid Greenman 506f6b04d2bSDavid Greenman if (pass == 0 && 507f6b04d2bSDavid Greenman (pgcount < 128 || pgcount > (object->size / (8 * PAGE_SIZE)))) { 508f6b04d2bSDavid Greenman allclean = 1; 509f6b04d2bSDavid Greenman for(; pgcount && (tstart < tend); tstart += PAGE_SIZE) { 510f6b04d2bSDavid Greenman p = vm_page_lookup(object, tstart); 511f6b04d2bSDavid Greenman if (!p) 512f6b04d2bSDavid Greenman continue; 513f6b04d2bSDavid Greenman --pgcount; 514f6b04d2bSDavid Greenman s = splhigh(); 515f6b04d2bSDavid Greenman TAILQ_REMOVE(&object->memq, p, listq); 516f6b04d2bSDavid Greenman TAILQ_INSERT_TAIL(&object->memq, p, listq); 517f6b04d2bSDavid Greenman splx(s); 518f6b04d2bSDavid Greenman if ((p->flags & (PG_BUSY|PG_CACHE)) || p->busy || p->valid == 0 || 519f6b04d2bSDavid Greenman p->bmapped) { 520f6b04d2bSDavid Greenman continue; 521f6b04d2bSDavid Greenman } 522f6b04d2bSDavid Greenman vm_page_test_dirty(p); 523f6b04d2bSDavid Greenman if ((p->valid & p->dirty) != 0) { 524f6b04d2bSDavid Greenman vm_offset_t tincr; 525f6b04d2bSDavid Greenman tincr = vm_pageout_clean(p, VM_PAGEOUT_FORCE); 526f6b04d2bSDavid Greenman pgcount -= (tincr - 1); 527f6b04d2bSDavid Greenman tincr *= PAGE_SIZE; 528f6b04d2bSDavid Greenman tstart += tincr - PAGE_SIZE; 529f6b04d2bSDavid Greenman allclean = 0; 530f6b04d2bSDavid Greenman } 531f6b04d2bSDavid Greenman } 532f6b04d2bSDavid Greenman if (!allclean) { 533f6b04d2bSDavid Greenman pass = 1; 534f6b04d2bSDavid Greenman goto startover; 535f6b04d2bSDavid Greenman } 536f6b04d2bSDavid Greenman object->flags &= ~OBJ_WRITEABLE; 537f6b04d2bSDavid Greenman return; 538f6b04d2bSDavid Greenman } 539f6b04d2bSDavid Greenman 540f6b04d2bSDavid Greenman allclean = 1; 541f6b04d2bSDavid Greenman while ((p = object->memq.tqh_first) != NULL && pgcount > 0) { 542f6b04d2bSDavid Greenman 543f6b04d2bSDavid Greenman if (p->flags & PG_CACHE) { 544f6b04d2bSDavid Greenman goto donext; 545f6b04d2bSDavid Greenman } 546f6b04d2bSDavid Greenman 547f6b04d2bSDavid Greenman if (p->offset >= tstart && p->offset < tend) { 548f6b04d2bSDavid Greenman if (p->valid == 0 || p->bmapped) { 549f6b04d2bSDavid Greenman goto donext; 550f6b04d2bSDavid Greenman } 551f6b04d2bSDavid Greenman 552f6b04d2bSDavid Greenman s = splhigh(); 553f6b04d2bSDavid Greenman if ((p->flags & PG_BUSY) || p->busy) { 554f6b04d2bSDavid Greenman allclean = 0; 555f6b04d2bSDavid Greenman if (pass > 0) { 556f6b04d2bSDavid Greenman p->flags |= PG_WANTED; 557f6b04d2bSDavid Greenman tsleep(p, PVM, "objpcn", 0); 558f6b04d2bSDavid Greenman splx(s); 559f6b04d2bSDavid Greenman continue; 560f6b04d2bSDavid Greenman } else { 561f6b04d2bSDavid Greenman splx(s); 562f6b04d2bSDavid Greenman goto donext; 563f6b04d2bSDavid Greenman } 564f6b04d2bSDavid Greenman } 565f6b04d2bSDavid Greenman 566f6b04d2bSDavid Greenman TAILQ_REMOVE(&object->memq, p, listq); 567f6b04d2bSDavid Greenman TAILQ_INSERT_TAIL(&object->memq, p, listq); 568f6b04d2bSDavid Greenman splx(s); 569f6b04d2bSDavid Greenman 570f6b04d2bSDavid Greenman pgcount--; 571f6b04d2bSDavid Greenman vm_page_test_dirty(p); 572f6b04d2bSDavid Greenman if ((p->valid & p->dirty) != 0) { 573f6b04d2bSDavid Greenman vm_pageout_clean(p, VM_PAGEOUT_FORCE); 574f6b04d2bSDavid Greenman allclean = 0; 575f6b04d2bSDavid Greenman } 576f6b04d2bSDavid Greenman continue; 577f6b04d2bSDavid Greenman } 578f6b04d2bSDavid Greenman donext: 579f6b04d2bSDavid Greenman TAILQ_REMOVE(&object->memq, p, listq); 580f6b04d2bSDavid Greenman TAILQ_INSERT_TAIL(&object->memq, p, listq); 581f6b04d2bSDavid Greenman pgcount--; 582f6b04d2bSDavid Greenman } 583f6b04d2bSDavid Greenman if ((!allclean && (pass == 0)) || (object->flags & OBJ_WRITEABLE)) { 584f6b04d2bSDavid Greenman pass = 1; 585f6b04d2bSDavid Greenman object->flags &= ~OBJ_WRITEABLE; 586f6b04d2bSDavid Greenman goto startover; 587f6b04d2bSDavid Greenman } 588f6b04d2bSDavid Greenman return; 589f6b04d2bSDavid Greenman } 590f6b04d2bSDavid Greenman 591f6b04d2bSDavid Greenman 592f5cf85d4SDavid Greenman void 593f5cf85d4SDavid Greenman vm_object_page_clean(object, start, end, syncio) 59426f9a767SRodney W. Grimes register vm_object_t object; 59526f9a767SRodney W. Grimes register vm_offset_t start; 59626f9a767SRodney W. Grimes register vm_offset_t end; 59726f9a767SRodney W. Grimes boolean_t syncio; 59826f9a767SRodney W. Grimes { 599f6b04d2bSDavid Greenman if (object->pager && (object->flags & OBJ_WRITEABLE) && 600f6b04d2bSDavid Greenman (object->pager->pg_type == PG_VNODE)) { 601f6b04d2bSDavid Greenman vn_pager_t vnp = (vn_pager_t) object->pager->pg_data; 602f6b04d2bSDavid Greenman struct vnode *vp; 60326f9a767SRodney W. Grimes 604f6b04d2bSDavid Greenman vp = vnp->vnp_vp; 605f6b04d2bSDavid Greenman vget(vp, 1); 606f6b04d2bSDavid Greenman _vm_object_page_clean(object, start, end, syncio); 607f6b04d2bSDavid Greenman vput(vp); 608f6b04d2bSDavid Greenman } else { 609f6b04d2bSDavid Greenman _vm_object_page_clean(object, start, end, syncio); 61026f9a767SRodney W. Grimes } 61126f9a767SRodney W. Grimes } 612f6b04d2bSDavid Greenman 613f6b04d2bSDavid Greenman void 614f6b04d2bSDavid Greenman vm_object_cache_clean() 615f6b04d2bSDavid Greenman { 616f6b04d2bSDavid Greenman vm_object_t object; 617f6b04d2bSDavid Greenman vm_object_cache_lock(); 618f6b04d2bSDavid Greenman while(1) { 619f6b04d2bSDavid Greenman object = vm_object_cached_list.tqh_first; 620f6b04d2bSDavid Greenman while( object) { 621f6b04d2bSDavid Greenman if( (object->flags & OBJ_WRITEABLE) && 622f6b04d2bSDavid Greenman object->pager && 623f6b04d2bSDavid Greenman object->pager->pg_type == PG_VNODE) { 624f6b04d2bSDavid Greenman vm_object_page_clean(object, 0, 0, 0); 625f6b04d2bSDavid Greenman goto loop; 626f6b04d2bSDavid Greenman } 627f6b04d2bSDavid Greenman object = object->cached_list.tqe_next; 62826f9a767SRodney W. Grimes } 629f5cf85d4SDavid Greenman return; 630f6b04d2bSDavid Greenman loop: 631f6b04d2bSDavid Greenman } 63226f9a767SRodney W. Grimes } 633df8bae1dSRodney W. Grimes 634df8bae1dSRodney W. Grimes /* 635df8bae1dSRodney W. Grimes * vm_object_deactivate_pages 636df8bae1dSRodney W. Grimes * 637df8bae1dSRodney W. Grimes * Deactivate all pages in the specified object. (Keep its pages 638df8bae1dSRodney W. Grimes * in memory even though it is no longer referenced.) 639df8bae1dSRodney W. Grimes * 640df8bae1dSRodney W. Grimes * The object must be locked. 641df8bae1dSRodney W. Grimes */ 642df8bae1dSRodney W. Grimes void 643df8bae1dSRodney W. Grimes vm_object_deactivate_pages(object) 644df8bae1dSRodney W. Grimes register vm_object_t object; 645df8bae1dSRodney W. Grimes { 646df8bae1dSRodney W. Grimes register vm_page_t p, next; 647df8bae1dSRodney W. Grimes 648df8bae1dSRodney W. Grimes for (p = object->memq.tqh_first; p != NULL; p = next) { 649df8bae1dSRodney W. Grimes next = p->listq.tqe_next; 650df8bae1dSRodney W. Grimes vm_page_lock_queues(); 651df8bae1dSRodney W. Grimes vm_page_deactivate(p); 652df8bae1dSRodney W. Grimes vm_page_unlock_queues(); 653df8bae1dSRodney W. Grimes } 654df8bae1dSRodney W. Grimes } 655df8bae1dSRodney W. Grimes 656df8bae1dSRodney W. Grimes /* 657df8bae1dSRodney W. Grimes * Trim the object cache to size. 658df8bae1dSRodney W. Grimes */ 659df8bae1dSRodney W. Grimes void 660df8bae1dSRodney W. Grimes vm_object_cache_trim() 661df8bae1dSRodney W. Grimes { 662df8bae1dSRodney W. Grimes register vm_object_t object; 663df8bae1dSRodney W. Grimes 664df8bae1dSRodney W. Grimes vm_object_cache_lock(); 6650d94caffSDavid Greenman while (vm_object_cached > vm_object_cache_max) { 666df8bae1dSRodney W. Grimes object = vm_object_cached_list.tqh_first; 667df8bae1dSRodney W. Grimes vm_object_cache_unlock(); 668df8bae1dSRodney W. Grimes 669df8bae1dSRodney W. Grimes if (object != vm_object_lookup(object->pager)) 6708e58bf68SDavid Greenman panic("vm_object_cache_trim: I'm sooo confused."); 671df8bae1dSRodney W. Grimes 672df8bae1dSRodney W. Grimes pager_cache(object, FALSE); 673df8bae1dSRodney W. Grimes 674df8bae1dSRodney W. Grimes vm_object_cache_lock(); 675df8bae1dSRodney W. Grimes } 676df8bae1dSRodney W. Grimes vm_object_cache_unlock(); 677df8bae1dSRodney W. Grimes } 678df8bae1dSRodney W. Grimes 67926f9a767SRodney W. Grimes 680df8bae1dSRodney W. Grimes /* 681df8bae1dSRodney W. Grimes * vm_object_pmap_copy: 682df8bae1dSRodney W. Grimes * 683df8bae1dSRodney W. Grimes * Makes all physical pages in the specified 684df8bae1dSRodney W. Grimes * object range copy-on-write. No writeable 685df8bae1dSRodney W. Grimes * references to these pages should remain. 686df8bae1dSRodney W. Grimes * 687df8bae1dSRodney W. Grimes * The object must *not* be locked. 688df8bae1dSRodney W. Grimes */ 6890d94caffSDavid Greenman void 6900d94caffSDavid Greenman vm_object_pmap_copy(object, start, end) 691df8bae1dSRodney W. Grimes register vm_object_t object; 692df8bae1dSRodney W. Grimes register vm_offset_t start; 693df8bae1dSRodney W. Grimes register vm_offset_t end; 694df8bae1dSRodney W. Grimes { 695df8bae1dSRodney W. Grimes register vm_page_t p; 696df8bae1dSRodney W. Grimes 697df8bae1dSRodney W. Grimes if (object == NULL) 698df8bae1dSRodney W. Grimes return; 699df8bae1dSRodney W. Grimes 700df8bae1dSRodney W. Grimes vm_object_lock(object); 701df8bae1dSRodney W. Grimes for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 702df8bae1dSRodney W. Grimes if ((start <= p->offset) && (p->offset < end)) { 703f919ebdeSDavid Greenman vm_page_protect(p, VM_PROT_READ); 704df8bae1dSRodney W. Grimes p->flags |= PG_COPYONWRITE; 705df8bae1dSRodney W. Grimes } 706df8bae1dSRodney W. Grimes } 707df8bae1dSRodney W. Grimes vm_object_unlock(object); 708df8bae1dSRodney W. Grimes } 709df8bae1dSRodney W. Grimes 710df8bae1dSRodney W. Grimes /* 711df8bae1dSRodney W. Grimes * vm_object_pmap_remove: 712df8bae1dSRodney W. Grimes * 713df8bae1dSRodney W. Grimes * Removes all physical pages in the specified 714df8bae1dSRodney W. Grimes * object range from all physical maps. 715df8bae1dSRodney W. Grimes * 716df8bae1dSRodney W. Grimes * The object must *not* be locked. 717df8bae1dSRodney W. Grimes */ 71826f9a767SRodney W. Grimes void 71926f9a767SRodney W. Grimes vm_object_pmap_remove(object, start, end) 720df8bae1dSRodney W. Grimes register vm_object_t object; 721df8bae1dSRodney W. Grimes register vm_offset_t start; 722df8bae1dSRodney W. Grimes register vm_offset_t end; 723df8bae1dSRodney W. Grimes { 724df8bae1dSRodney W. Grimes register vm_page_t p; 725a481f200SDavid Greenman int s; 726df8bae1dSRodney W. Grimes 727df8bae1dSRodney W. Grimes if (object == NULL) 728df8bae1dSRodney W. Grimes return; 7292fe6e4d7SDavid Greenman ++object->paging_in_progress; 730df8bae1dSRodney W. Grimes 731df8bae1dSRodney W. Grimes vm_object_lock(object); 73226f9a767SRodney W. Grimes again: 73326f9a767SRodney W. Grimes for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 73426f9a767SRodney W. Grimes if ((start <= p->offset) && (p->offset < end)) { 735a481f200SDavid Greenman s = splhigh(); 7360d94caffSDavid Greenman if ((p->flags & PG_BUSY) || p->busy) { 73726f9a767SRodney W. Grimes p->flags |= PG_WANTED; 73826f9a767SRodney W. Grimes tsleep((caddr_t) p, PVM, "vmopmr", 0); 739a481f200SDavid Greenman splx(s); 74026f9a767SRodney W. Grimes goto again; 74126f9a767SRodney W. Grimes } 742a481f200SDavid Greenman splx(s); 743f919ebdeSDavid Greenman vm_page_protect(p, VM_PROT_NONE); 74426f9a767SRodney W. Grimes } 74526f9a767SRodney W. Grimes } 746df8bae1dSRodney W. Grimes vm_object_unlock(object); 747f919ebdeSDavid Greenman vm_object_pip_wakeup(object); 748c0503609SDavid Greenman } 749df8bae1dSRodney W. Grimes 750df8bae1dSRodney W. Grimes /* 751df8bae1dSRodney W. Grimes * vm_object_copy: 752df8bae1dSRodney W. Grimes * 753df8bae1dSRodney W. Grimes * Create a new object which is a copy of an existing 754df8bae1dSRodney W. Grimes * object, and mark all of the pages in the existing 755df8bae1dSRodney W. Grimes * object 'copy-on-write'. The new object has one reference. 756df8bae1dSRodney W. Grimes * Returns the new object. 757df8bae1dSRodney W. Grimes * 758df8bae1dSRodney W. Grimes * May defer the copy until later if the object is not backed 759df8bae1dSRodney W. Grimes * up by a non-default pager. 760df8bae1dSRodney W. Grimes */ 7610d94caffSDavid Greenman void 7620d94caffSDavid Greenman vm_object_copy(src_object, src_offset, size, 763df8bae1dSRodney W. Grimes dst_object, dst_offset, src_needs_copy) 764df8bae1dSRodney W. Grimes register vm_object_t src_object; 765df8bae1dSRodney W. Grimes vm_offset_t src_offset; 766df8bae1dSRodney W. Grimes vm_size_t size; 767df8bae1dSRodney W. Grimes vm_object_t *dst_object;/* OUT */ 768df8bae1dSRodney W. Grimes vm_offset_t *dst_offset;/* OUT */ 769df8bae1dSRodney W. Grimes boolean_t *src_needs_copy; /* OUT */ 770df8bae1dSRodney W. Grimes { 771df8bae1dSRodney W. Grimes register vm_object_t new_copy; 772df8bae1dSRodney W. Grimes register vm_object_t old_copy; 773df8bae1dSRodney W. Grimes vm_offset_t new_start, new_end; 774df8bae1dSRodney W. Grimes 775df8bae1dSRodney W. Grimes register vm_page_t p; 776df8bae1dSRodney W. Grimes 777df8bae1dSRodney W. Grimes if (src_object == NULL) { 778df8bae1dSRodney W. Grimes /* 779df8bae1dSRodney W. Grimes * Nothing to copy 780df8bae1dSRodney W. Grimes */ 781df8bae1dSRodney W. Grimes *dst_object = NULL; 782df8bae1dSRodney W. Grimes *dst_offset = 0; 783df8bae1dSRodney W. Grimes *src_needs_copy = FALSE; 784df8bae1dSRodney W. Grimes return; 785df8bae1dSRodney W. Grimes } 786df8bae1dSRodney W. Grimes /* 7870d94caffSDavid Greenman * If the object's pager is null_pager or the default pager, we don't 7880d94caffSDavid Greenman * have to make a copy of it. Instead, we set the needs copy flag and 789df8bae1dSRodney W. Grimes * make a shadow later. 790df8bae1dSRodney W. Grimes */ 791df8bae1dSRodney W. Grimes 792df8bae1dSRodney W. Grimes vm_object_lock(src_object); 79326f9a767SRodney W. Grimes 79426f9a767SRodney W. Grimes /* 79526f9a767SRodney W. Grimes * Try to collapse the object before copying it. 79626f9a767SRodney W. Grimes */ 79726f9a767SRodney W. Grimes 79826f9a767SRodney W. Grimes vm_object_collapse(src_object); 79926f9a767SRodney W. Grimes 800df8bae1dSRodney W. Grimes if (src_object->pager == NULL || 801df8bae1dSRodney W. Grimes (src_object->flags & OBJ_INTERNAL)) { 802df8bae1dSRodney W. Grimes 803df8bae1dSRodney W. Grimes /* 804df8bae1dSRodney W. Grimes * Make another reference to the object 805df8bae1dSRodney W. Grimes */ 806df8bae1dSRodney W. Grimes src_object->ref_count++; 807df8bae1dSRodney W. Grimes 808df8bae1dSRodney W. Grimes /* 809df8bae1dSRodney W. Grimes * Mark all of the pages copy-on-write. 810df8bae1dSRodney W. Grimes */ 811df8bae1dSRodney W. Grimes for (p = src_object->memq.tqh_first; p; p = p->listq.tqe_next) 812df8bae1dSRodney W. Grimes if (src_offset <= p->offset && 813df8bae1dSRodney W. Grimes p->offset < src_offset + size) 814df8bae1dSRodney W. Grimes p->flags |= PG_COPYONWRITE; 815df8bae1dSRodney W. Grimes vm_object_unlock(src_object); 816df8bae1dSRodney W. Grimes 817df8bae1dSRodney W. Grimes *dst_object = src_object; 818df8bae1dSRodney W. Grimes *dst_offset = src_offset; 819df8bae1dSRodney W. Grimes 820df8bae1dSRodney W. Grimes /* 821df8bae1dSRodney W. Grimes * Must make a shadow when write is desired 822df8bae1dSRodney W. Grimes */ 823df8bae1dSRodney W. Grimes *src_needs_copy = TRUE; 824df8bae1dSRodney W. Grimes return; 825df8bae1dSRodney W. Grimes } 826df8bae1dSRodney W. Grimes /* 8270d94caffSDavid Greenman * If the object has a pager, the pager wants to see all of the 8280d94caffSDavid Greenman * changes. We need a copy-object for the changed pages. 829df8bae1dSRodney W. Grimes * 8300d94caffSDavid Greenman * If there is a copy-object, and it is empty, no changes have been made 8310d94caffSDavid Greenman * to the object since the copy-object was made. We can use the same 8320d94caffSDavid Greenman * copy- object. 833df8bae1dSRodney W. Grimes */ 834df8bae1dSRodney W. Grimes 835df8bae1dSRodney W. Grimes Retry1: 836df8bae1dSRodney W. Grimes old_copy = src_object->copy; 837df8bae1dSRodney W. Grimes if (old_copy != NULL) { 838df8bae1dSRodney W. Grimes /* 839df8bae1dSRodney W. Grimes * Try to get the locks (out of order) 840df8bae1dSRodney W. Grimes */ 841df8bae1dSRodney W. Grimes if (!vm_object_lock_try(old_copy)) { 842df8bae1dSRodney W. Grimes vm_object_unlock(src_object); 843df8bae1dSRodney W. Grimes 844df8bae1dSRodney W. Grimes /* should spin a bit here... */ 8450d94caffSDavid Greenman tsleep((caddr_t) old_copy, PVM, "cpylck", 1); 846df8bae1dSRodney W. Grimes vm_object_lock(src_object); 847df8bae1dSRodney W. Grimes goto Retry1; 848df8bae1dSRodney W. Grimes } 849df8bae1dSRodney W. Grimes if (old_copy->resident_page_count == 0 && 850df8bae1dSRodney W. Grimes old_copy->pager == NULL) { 851df8bae1dSRodney W. Grimes /* 8520d94caffSDavid Greenman * Return another reference to the existing 8530d94caffSDavid Greenman * copy-object. 854df8bae1dSRodney W. Grimes */ 855df8bae1dSRodney W. Grimes old_copy->ref_count++; 856df8bae1dSRodney W. Grimes vm_object_unlock(old_copy); 857df8bae1dSRodney W. Grimes vm_object_unlock(src_object); 858df8bae1dSRodney W. Grimes *dst_object = old_copy; 859df8bae1dSRodney W. Grimes *dst_offset = src_offset; 860df8bae1dSRodney W. Grimes *src_needs_copy = FALSE; 861df8bae1dSRodney W. Grimes return; 862df8bae1dSRodney W. Grimes } 863df8bae1dSRodney W. Grimes vm_object_unlock(old_copy); 864df8bae1dSRodney W. Grimes } 865df8bae1dSRodney W. Grimes vm_object_unlock(src_object); 866df8bae1dSRodney W. Grimes 867df8bae1dSRodney W. Grimes /* 8680d94caffSDavid Greenman * If the object has a pager, the pager wants to see all of the 8690d94caffSDavid Greenman * changes. We must make a copy-object and put the changed pages 8700d94caffSDavid Greenman * there. 871df8bae1dSRodney W. Grimes * 8720d94caffSDavid Greenman * The copy-object is always made large enough to completely shadow the 8730d94caffSDavid Greenman * original object, since it may have several users who want to shadow 874df8bae1dSRodney W. Grimes * the original object at different points. 875df8bae1dSRodney W. Grimes */ 876df8bae1dSRodney W. Grimes 877df8bae1dSRodney W. Grimes new_copy = vm_object_allocate(src_object->size); 878df8bae1dSRodney W. Grimes 879df8bae1dSRodney W. Grimes Retry2: 880df8bae1dSRodney W. Grimes vm_object_lock(src_object); 881df8bae1dSRodney W. Grimes /* 882df8bae1dSRodney W. Grimes * Copy object may have changed while we were unlocked 883df8bae1dSRodney W. Grimes */ 884df8bae1dSRodney W. Grimes old_copy = src_object->copy; 885df8bae1dSRodney W. Grimes if (old_copy != NULL) { 886df8bae1dSRodney W. Grimes /* 887df8bae1dSRodney W. Grimes * Try to get the locks (out of order) 888df8bae1dSRodney W. Grimes */ 889df8bae1dSRodney W. Grimes if (!vm_object_lock_try(old_copy)) { 890df8bae1dSRodney W. Grimes vm_object_unlock(src_object); 8910d94caffSDavid Greenman tsleep((caddr_t) old_copy, PVM, "cpylck", 1); 892df8bae1dSRodney W. Grimes goto Retry2; 893df8bae1dSRodney W. Grimes } 894df8bae1dSRodney W. Grimes /* 895df8bae1dSRodney W. Grimes * Consistency check 896df8bae1dSRodney W. Grimes */ 897df8bae1dSRodney W. Grimes if (old_copy->shadow != src_object || 898df8bae1dSRodney W. Grimes old_copy->shadow_offset != (vm_offset_t) 0) 899df8bae1dSRodney W. Grimes panic("vm_object_copy: copy/shadow inconsistency"); 900df8bae1dSRodney W. Grimes 901df8bae1dSRodney W. Grimes /* 9020d94caffSDavid Greenman * Make the old copy-object shadow the new one. It will 9030d94caffSDavid Greenman * receive no more pages from the original object. 904df8bae1dSRodney W. Grimes */ 905df8bae1dSRodney W. Grimes 906df8bae1dSRodney W. Grimes src_object->ref_count--; /* remove ref. from old_copy */ 9072fe6e4d7SDavid Greenman if (old_copy->shadow) 9082fe6e4d7SDavid Greenman TAILQ_REMOVE(&old_copy->shadow->reverse_shadow_head, old_copy, reverse_shadow_list); 909df8bae1dSRodney W. Grimes old_copy->shadow = new_copy; 9102fe6e4d7SDavid Greenman TAILQ_INSERT_TAIL(&old_copy->shadow->reverse_shadow_head, old_copy, reverse_shadow_list); 9110d94caffSDavid Greenman new_copy->ref_count++; /* locking not needed - we have the 9120d94caffSDavid Greenman * only pointer */ 913df8bae1dSRodney W. Grimes vm_object_unlock(old_copy); /* done with old_copy */ 914df8bae1dSRodney W. Grimes } 915df8bae1dSRodney W. Grimes new_start = (vm_offset_t) 0; /* always shadow original at 0 */ 916df8bae1dSRodney W. Grimes new_end = (vm_offset_t) new_copy->size; /* for the whole object */ 917df8bae1dSRodney W. Grimes 918df8bae1dSRodney W. Grimes /* 919df8bae1dSRodney W. Grimes * Point the new copy at the existing object. 920df8bae1dSRodney W. Grimes */ 921df8bae1dSRodney W. Grimes 922df8bae1dSRodney W. Grimes new_copy->shadow = src_object; 9232fe6e4d7SDavid Greenman TAILQ_INSERT_TAIL(&new_copy->shadow->reverse_shadow_head, new_copy, reverse_shadow_list); 924df8bae1dSRodney W. Grimes new_copy->shadow_offset = new_start; 925df8bae1dSRodney W. Grimes src_object->ref_count++; 926df8bae1dSRodney W. Grimes src_object->copy = new_copy; 927df8bae1dSRodney W. Grimes 928df8bae1dSRodney W. Grimes /* 9290d94caffSDavid Greenman * Mark all the affected pages of the existing object copy-on-write. 930df8bae1dSRodney W. Grimes */ 931df8bae1dSRodney W. Grimes for (p = src_object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) 932df8bae1dSRodney W. Grimes if ((new_start <= p->offset) && (p->offset < new_end)) 933df8bae1dSRodney W. Grimes p->flags |= PG_COPYONWRITE; 934df8bae1dSRodney W. Grimes 935df8bae1dSRodney W. Grimes vm_object_unlock(src_object); 936df8bae1dSRodney W. Grimes 937df8bae1dSRodney W. Grimes *dst_object = new_copy; 938df8bae1dSRodney W. Grimes *dst_offset = src_offset - new_start; 939df8bae1dSRodney W. Grimes *src_needs_copy = FALSE; 940df8bae1dSRodney W. Grimes } 941df8bae1dSRodney W. Grimes 942df8bae1dSRodney W. Grimes /* 943df8bae1dSRodney W. Grimes * vm_object_shadow: 944df8bae1dSRodney W. Grimes * 945df8bae1dSRodney W. Grimes * Create a new object which is backed by the 946df8bae1dSRodney W. Grimes * specified existing object range. The source 947df8bae1dSRodney W. Grimes * object reference is deallocated. 948df8bae1dSRodney W. Grimes * 949df8bae1dSRodney W. Grimes * The new object and offset into that object 950df8bae1dSRodney W. Grimes * are returned in the source parameters. 951df8bae1dSRodney W. Grimes */ 952df8bae1dSRodney W. Grimes 95326f9a767SRodney W. Grimes void 95426f9a767SRodney W. Grimes vm_object_shadow(object, offset, length) 955df8bae1dSRodney W. Grimes vm_object_t *object; /* IN/OUT */ 956df8bae1dSRodney W. Grimes vm_offset_t *offset; /* IN/OUT */ 957df8bae1dSRodney W. Grimes vm_size_t length; 958df8bae1dSRodney W. Grimes { 959df8bae1dSRodney W. Grimes register vm_object_t source; 960df8bae1dSRodney W. Grimes register vm_object_t result; 961df8bae1dSRodney W. Grimes 962df8bae1dSRodney W. Grimes source = *object; 963df8bae1dSRodney W. Grimes 964df8bae1dSRodney W. Grimes /* 965df8bae1dSRodney W. Grimes * Allocate a new object with the given length 966df8bae1dSRodney W. Grimes */ 967df8bae1dSRodney W. Grimes 968df8bae1dSRodney W. Grimes if ((result = vm_object_allocate(length)) == NULL) 969df8bae1dSRodney W. Grimes panic("vm_object_shadow: no object for shadowing"); 970df8bae1dSRodney W. Grimes 971df8bae1dSRodney W. Grimes /* 9720d94caffSDavid Greenman * The new object shadows the source object, adding a reference to it. 9730d94caffSDavid Greenman * Our caller changes his reference to point to the new object, 9740d94caffSDavid Greenman * removing a reference to the source object. Net result: no change 9750d94caffSDavid Greenman * of reference count. 976df8bae1dSRodney W. Grimes */ 977df8bae1dSRodney W. Grimes result->shadow = source; 9787b18a718SDavid Greenman if (source) 9792fe6e4d7SDavid Greenman TAILQ_INSERT_TAIL(&result->shadow->reverse_shadow_head, result, reverse_shadow_list); 980df8bae1dSRodney W. Grimes 981df8bae1dSRodney W. Grimes /* 9820d94caffSDavid Greenman * Store the offset into the source object, and fix up the offset into 9830d94caffSDavid Greenman * the new object. 984df8bae1dSRodney W. Grimes */ 985df8bae1dSRodney W. Grimes 986df8bae1dSRodney W. Grimes result->shadow_offset = *offset; 987df8bae1dSRodney W. Grimes 988df8bae1dSRodney W. Grimes /* 989df8bae1dSRodney W. Grimes * Return the new things 990df8bae1dSRodney W. Grimes */ 991df8bae1dSRodney W. Grimes 992df8bae1dSRodney W. Grimes *offset = 0; 993df8bae1dSRodney W. Grimes *object = result; 994df8bae1dSRodney W. Grimes } 995df8bae1dSRodney W. Grimes 996df8bae1dSRodney W. Grimes /* 997df8bae1dSRodney W. Grimes * vm_object_hash hashes the pager/id pair. 998df8bae1dSRodney W. Grimes */ 999df8bae1dSRodney W. Grimes 1000df8bae1dSRodney W. Grimes #define vm_object_hash(pager) \ 100126f9a767SRodney W. Grimes (((unsigned)pager >> 5)%VM_OBJECT_HASH_COUNT) 1002df8bae1dSRodney W. Grimes 1003df8bae1dSRodney W. Grimes /* 1004df8bae1dSRodney W. Grimes * vm_object_lookup looks in the object cache for an object with the 1005df8bae1dSRodney W. Grimes * specified pager and paging id. 1006df8bae1dSRodney W. Grimes */ 1007df8bae1dSRodney W. Grimes 10080d94caffSDavid Greenman vm_object_t 10090d94caffSDavid Greenman vm_object_lookup(pager) 1010df8bae1dSRodney W. Grimes vm_pager_t pager; 1011df8bae1dSRodney W. Grimes { 1012df8bae1dSRodney W. Grimes register vm_object_hash_entry_t entry; 1013df8bae1dSRodney W. Grimes vm_object_t object; 1014df8bae1dSRodney W. Grimes 1015df8bae1dSRodney W. Grimes vm_object_cache_lock(); 1016df8bae1dSRodney W. Grimes 1017df8bae1dSRodney W. Grimes for (entry = vm_object_hashtable[vm_object_hash(pager)].tqh_first; 1018df8bae1dSRodney W. Grimes entry != NULL; 1019df8bae1dSRodney W. Grimes entry = entry->hash_links.tqe_next) { 1020df8bae1dSRodney W. Grimes object = entry->object; 1021df8bae1dSRodney W. Grimes if (object->pager == pager) { 1022df8bae1dSRodney W. Grimes vm_object_lock(object); 1023df8bae1dSRodney W. Grimes if (object->ref_count == 0) { 1024df8bae1dSRodney W. Grimes TAILQ_REMOVE(&vm_object_cached_list, object, 1025df8bae1dSRodney W. Grimes cached_list); 1026df8bae1dSRodney W. Grimes vm_object_cached--; 1027df8bae1dSRodney W. Grimes } 1028df8bae1dSRodney W. Grimes object->ref_count++; 1029df8bae1dSRodney W. Grimes vm_object_unlock(object); 1030df8bae1dSRodney W. Grimes vm_object_cache_unlock(); 1031df8bae1dSRodney W. Grimes return (object); 1032df8bae1dSRodney W. Grimes } 1033df8bae1dSRodney W. Grimes } 1034df8bae1dSRodney W. Grimes 1035df8bae1dSRodney W. Grimes vm_object_cache_unlock(); 1036df8bae1dSRodney W. Grimes return (NULL); 1037df8bae1dSRodney W. Grimes } 1038df8bae1dSRodney W. Grimes 1039df8bae1dSRodney W. Grimes /* 1040df8bae1dSRodney W. Grimes * vm_object_enter enters the specified object/pager/id into 1041df8bae1dSRodney W. Grimes * the hash table. 1042df8bae1dSRodney W. Grimes */ 1043df8bae1dSRodney W. Grimes 10440d94caffSDavid Greenman void 10450d94caffSDavid Greenman vm_object_enter(object, pager) 1046df8bae1dSRodney W. Grimes vm_object_t object; 1047df8bae1dSRodney W. Grimes vm_pager_t pager; 1048df8bae1dSRodney W. Grimes { 1049df8bae1dSRodney W. Grimes struct vm_object_hash_head *bucket; 1050df8bae1dSRodney W. Grimes register vm_object_hash_entry_t entry; 1051df8bae1dSRodney W. Grimes 1052df8bae1dSRodney W. Grimes /* 10530d94caffSDavid Greenman * We don't cache null objects, and we can't cache objects with the 10540d94caffSDavid Greenman * null pager. 1055df8bae1dSRodney W. Grimes */ 1056df8bae1dSRodney W. Grimes 1057df8bae1dSRodney W. Grimes if (object == NULL) 1058df8bae1dSRodney W. Grimes return; 1059df8bae1dSRodney W. Grimes if (pager == NULL) 1060df8bae1dSRodney W. Grimes return; 1061df8bae1dSRodney W. Grimes 1062df8bae1dSRodney W. Grimes bucket = &vm_object_hashtable[vm_object_hash(pager)]; 1063df8bae1dSRodney W. Grimes entry = (vm_object_hash_entry_t) 1064df8bae1dSRodney W. Grimes malloc((u_long) sizeof *entry, M_VMOBJHASH, M_WAITOK); 1065df8bae1dSRodney W. Grimes entry->object = object; 1066df8bae1dSRodney W. Grimes 1067df8bae1dSRodney W. Grimes vm_object_cache_lock(); 1068df8bae1dSRodney W. Grimes TAILQ_INSERT_TAIL(bucket, entry, hash_links); 1069df8bae1dSRodney W. Grimes vm_object_cache_unlock(); 1070df8bae1dSRodney W. Grimes } 1071df8bae1dSRodney W. Grimes 1072df8bae1dSRodney W. Grimes /* 1073df8bae1dSRodney W. Grimes * vm_object_remove: 1074df8bae1dSRodney W. Grimes * 1075df8bae1dSRodney W. Grimes * Remove the pager from the hash table. 1076df8bae1dSRodney W. Grimes * Note: This assumes that the object cache 1077df8bae1dSRodney W. Grimes * is locked. XXX this should be fixed 1078df8bae1dSRodney W. Grimes * by reorganizing vm_object_deallocate. 1079df8bae1dSRodney W. Grimes */ 1080df8bae1dSRodney W. Grimes void 1081df8bae1dSRodney W. Grimes vm_object_remove(pager) 1082df8bae1dSRodney W. Grimes register vm_pager_t pager; 1083df8bae1dSRodney W. Grimes { 1084df8bae1dSRodney W. Grimes struct vm_object_hash_head *bucket; 1085df8bae1dSRodney W. Grimes register vm_object_hash_entry_t entry; 1086df8bae1dSRodney W. Grimes register vm_object_t object; 1087df8bae1dSRodney W. Grimes 1088df8bae1dSRodney W. Grimes bucket = &vm_object_hashtable[vm_object_hash(pager)]; 1089df8bae1dSRodney W. Grimes 1090df8bae1dSRodney W. Grimes for (entry = bucket->tqh_first; 1091df8bae1dSRodney W. Grimes entry != NULL; 1092df8bae1dSRodney W. Grimes entry = entry->hash_links.tqe_next) { 1093df8bae1dSRodney W. Grimes object = entry->object; 1094df8bae1dSRodney W. Grimes if (object->pager == pager) { 1095df8bae1dSRodney W. Grimes TAILQ_REMOVE(bucket, entry, hash_links); 1096df8bae1dSRodney W. Grimes free((caddr_t) entry, M_VMOBJHASH); 1097df8bae1dSRodney W. Grimes break; 1098df8bae1dSRodney W. Grimes } 1099df8bae1dSRodney W. Grimes } 1100df8bae1dSRodney W. Grimes } 1101df8bae1dSRodney W. Grimes 11022fe6e4d7SDavid Greenman /* 11032fe6e4d7SDavid Greenman * this version of collapse allows the operation to occur earlier and 11042fe6e4d7SDavid Greenman * when paging_in_progress is true for an object... This is not a complete 11052fe6e4d7SDavid Greenman * operation, but should plug 99.9% of the rest of the leaks. 11062fe6e4d7SDavid Greenman */ 11072fe6e4d7SDavid Greenman static void 11082fe6e4d7SDavid Greenman vm_object_qcollapse(object) 11092fe6e4d7SDavid Greenman register vm_object_t object; 11102fe6e4d7SDavid Greenman { 11112fe6e4d7SDavid Greenman register vm_object_t backing_object; 11122fe6e4d7SDavid Greenman register vm_offset_t backing_offset, new_offset; 11132fe6e4d7SDavid Greenman register vm_page_t p, pp; 11142fe6e4d7SDavid Greenman register vm_size_t size; 11152fe6e4d7SDavid Greenman 11162fe6e4d7SDavid Greenman backing_object = object->shadow; 11172fe6e4d7SDavid Greenman if (backing_object->shadow != NULL && 11182fe6e4d7SDavid Greenman backing_object->shadow->copy == backing_object) 11192fe6e4d7SDavid Greenman return; 11202fe6e4d7SDavid Greenman if (backing_object->ref_count != 1) 11212fe6e4d7SDavid Greenman return; 11222fe6e4d7SDavid Greenman 1123010cf3b9SDavid Greenman backing_object->ref_count += 2; 1124010cf3b9SDavid Greenman 11252fe6e4d7SDavid Greenman backing_offset = object->shadow_offset; 11262fe6e4d7SDavid Greenman size = object->size; 11272fe6e4d7SDavid Greenman p = backing_object->memq.tqh_first; 11282fe6e4d7SDavid Greenman while (p) { 11292fe6e4d7SDavid Greenman vm_page_t next; 11300d94caffSDavid Greenman 11312fe6e4d7SDavid Greenman next = p->listq.tqe_next; 11320d94caffSDavid Greenman if ((p->flags & (PG_BUSY | PG_FICTITIOUS | PG_CACHE)) || 1133f6b04d2bSDavid Greenman !p->valid || p->hold_count || p->wire_count || p->busy) { 11342fe6e4d7SDavid Greenman p = next; 11352fe6e4d7SDavid Greenman continue; 11362fe6e4d7SDavid Greenman } 1137f919ebdeSDavid Greenman vm_page_protect(p, VM_PROT_NONE); 11382fe6e4d7SDavid Greenman new_offset = (p->offset - backing_offset); 11392fe6e4d7SDavid Greenman if (p->offset < backing_offset || 11402fe6e4d7SDavid Greenman new_offset >= size) { 11412fe6e4d7SDavid Greenman if (backing_object->pager) 11420d94caffSDavid Greenman swap_pager_freespace(backing_object->pager, 11430d94caffSDavid Greenman backing_object->paging_offset + p->offset, PAGE_SIZE); 11440d94caffSDavid Greenman vm_page_lock_queues(); 11452fe6e4d7SDavid Greenman vm_page_free(p); 11462fe6e4d7SDavid Greenman vm_page_unlock_queues(); 11472fe6e4d7SDavid Greenman } else { 11482fe6e4d7SDavid Greenman pp = vm_page_lookup(object, new_offset); 11492fe6e4d7SDavid Greenman if (pp != NULL || (object->pager && vm_pager_has_page(object->pager, 11502fe6e4d7SDavid Greenman object->paging_offset + new_offset))) { 11512fe6e4d7SDavid Greenman if (backing_object->pager) 11520d94caffSDavid Greenman swap_pager_freespace(backing_object->pager, 11530d94caffSDavid Greenman backing_object->paging_offset + p->offset, PAGE_SIZE); 11540d94caffSDavid Greenman vm_page_lock_queues(); 11552fe6e4d7SDavid Greenman vm_page_free(p); 11562fe6e4d7SDavid Greenman vm_page_unlock_queues(); 11572fe6e4d7SDavid Greenman } else { 1158a1f6d91cSDavid Greenman if( backing_object->pager) 1159a1f6d91cSDavid Greenman swap_pager_freespace(backing_object->pager, 1160a1f6d91cSDavid Greenman backing_object->paging_offset + p->offset, PAGE_SIZE); 11612fe6e4d7SDavid Greenman vm_page_rename(p, object, new_offset); 11629b4814bbSDavid Greenman p->dirty = VM_PAGE_BITS_ALL; 11632fe6e4d7SDavid Greenman } 11642fe6e4d7SDavid Greenman } 11652fe6e4d7SDavid Greenman p = next; 11662fe6e4d7SDavid Greenman } 1167010cf3b9SDavid Greenman backing_object->ref_count -= 2; 11682fe6e4d7SDavid Greenman } 11692fe6e4d7SDavid Greenman 1170df8bae1dSRodney W. Grimes boolean_t vm_object_collapse_allowed = TRUE; 11710d94caffSDavid Greenman 1172df8bae1dSRodney W. Grimes /* 1173df8bae1dSRodney W. Grimes * vm_object_collapse: 1174df8bae1dSRodney W. Grimes * 1175df8bae1dSRodney W. Grimes * Collapse an object with the object backing it. 1176df8bae1dSRodney W. Grimes * Pages in the backing object are moved into the 1177df8bae1dSRodney W. Grimes * parent, and the backing object is deallocated. 1178df8bae1dSRodney W. Grimes * 1179df8bae1dSRodney W. Grimes * Requires that the object be locked and the page 1180df8bae1dSRodney W. Grimes * queues be unlocked. 1181df8bae1dSRodney W. Grimes * 118226f9a767SRodney W. Grimes * This routine has significant changes by John S. Dyson 118326f9a767SRodney W. Grimes * to fix some swap memory leaks. 18 Dec 93 118426f9a767SRodney W. Grimes * 1185df8bae1dSRodney W. Grimes */ 118626f9a767SRodney W. Grimes void 118726f9a767SRodney W. Grimes vm_object_collapse(object) 1188df8bae1dSRodney W. Grimes register vm_object_t object; 1189df8bae1dSRodney W. Grimes 1190df8bae1dSRodney W. Grimes { 1191df8bae1dSRodney W. Grimes register vm_object_t backing_object; 1192df8bae1dSRodney W. Grimes register vm_offset_t backing_offset; 1193df8bae1dSRodney W. Grimes register vm_size_t size; 1194df8bae1dSRodney W. Grimes register vm_offset_t new_offset; 1195df8bae1dSRodney W. Grimes register vm_page_t p, pp; 1196df8bae1dSRodney W. Grimes 1197df8bae1dSRodney W. Grimes if (!vm_object_collapse_allowed) 1198df8bae1dSRodney W. Grimes return; 1199df8bae1dSRodney W. Grimes 1200df8bae1dSRodney W. Grimes while (TRUE) { 1201df8bae1dSRodney W. Grimes /* 1202df8bae1dSRodney W. Grimes * Verify that the conditions are right for collapse: 1203df8bae1dSRodney W. Grimes * 12040d94caffSDavid Greenman * The object exists and no pages in it are currently being paged 12050d94caffSDavid Greenman * out. 1206df8bae1dSRodney W. Grimes */ 12072fe6e4d7SDavid Greenman if (object == NULL) 1208df8bae1dSRodney W. Grimes return; 1209df8bae1dSRodney W. Grimes 1210b9921222SDavid Greenman /* 1211b9921222SDavid Greenman * Make sure there is a backing object. 1212b9921222SDavid Greenman */ 1213df8bae1dSRodney W. Grimes if ((backing_object = object->shadow) == NULL) 1214df8bae1dSRodney W. Grimes return; 1215df8bae1dSRodney W. Grimes 1216f919ebdeSDavid Greenman /* 1217f919ebdeSDavid Greenman * we check the backing object first, because it is most likely 1218f919ebdeSDavid Greenman * !OBJ_INTERNAL. 1219f919ebdeSDavid Greenman */ 1220f919ebdeSDavid Greenman if ((backing_object->flags & OBJ_INTERNAL) == 0 || 1221f919ebdeSDavid Greenman (backing_object->flags & OBJ_DEAD) || 1222f919ebdeSDavid Greenman (object->flags & OBJ_INTERNAL) == 0 || 1223f919ebdeSDavid Greenman (object->flags & OBJ_DEAD)) 12249b4814bbSDavid Greenman return; 12259b4814bbSDavid Greenman 1226f919ebdeSDavid Greenman if (object->paging_in_progress != 0 || 1227f919ebdeSDavid Greenman backing_object->paging_in_progress != 0) { 1228b9921222SDavid Greenman if (vm_object_lock_try(backing_object)) { 1229b9921222SDavid Greenman vm_object_qcollapse(object); 1230b9921222SDavid Greenman vm_object_unlock(backing_object); 1231b9921222SDavid Greenman } 1232b9921222SDavid Greenman return; 1233b9921222SDavid Greenman } 1234b9921222SDavid Greenman 1235df8bae1dSRodney W. Grimes vm_object_lock(backing_object); 1236df8bae1dSRodney W. Grimes 1237df8bae1dSRodney W. Grimes /* 12380d94caffSDavid Greenman * The backing object can't be a copy-object: the 12390d94caffSDavid Greenman * shadow_offset for the copy-object must stay as 0. 12400d94caffSDavid Greenman * Furthermore (for the 'we have all the pages' case), if we 12410d94caffSDavid Greenman * bypass backing_object and just shadow the next object in 12420d94caffSDavid Greenman * the chain, old pages from that object would then have to be 12430d94caffSDavid Greenman * copied BOTH into the (former) backing_object and into the 1244df8bae1dSRodney W. Grimes * parent object. 1245df8bae1dSRodney W. Grimes */ 1246df8bae1dSRodney W. Grimes if (backing_object->shadow != NULL && 124726f9a767SRodney W. Grimes backing_object->shadow->copy == backing_object) { 1248df8bae1dSRodney W. Grimes vm_object_unlock(backing_object); 1249df8bae1dSRodney W. Grimes return; 1250df8bae1dSRodney W. Grimes } 1251f919ebdeSDavid Greenman 125226f9a767SRodney W. Grimes /* 12530d94caffSDavid Greenman * We know that we can either collapse the backing object (if 12540d94caffSDavid Greenman * the parent is the only reference to it) or (perhaps) remove 12550d94caffSDavid Greenman * the parent's reference to it. 1256df8bae1dSRodney W. Grimes */ 1257df8bae1dSRodney W. Grimes 1258df8bae1dSRodney W. Grimes backing_offset = object->shadow_offset; 1259df8bae1dSRodney W. Grimes size = object->size; 1260df8bae1dSRodney W. Grimes 1261df8bae1dSRodney W. Grimes /* 12620d94caffSDavid Greenman * If there is exactly one reference to the backing object, we 12630d94caffSDavid Greenman * can collapse it into the parent. 1264df8bae1dSRodney W. Grimes */ 1265df8bae1dSRodney W. Grimes 1266df8bae1dSRodney W. Grimes if (backing_object->ref_count == 1) { 1267df8bae1dSRodney W. Grimes 1268a1f6d91cSDavid Greenman backing_object->flags |= OBJ_DEAD; 1269df8bae1dSRodney W. Grimes /* 1270df8bae1dSRodney W. Grimes * We can collapse the backing object. 1271df8bae1dSRodney W. Grimes * 12720d94caffSDavid Greenman * Move all in-memory pages from backing_object to the 12730d94caffSDavid Greenman * parent. Pages that have been paged out will be 12740d94caffSDavid Greenman * overwritten by any of the parent's pages that 12750d94caffSDavid Greenman * shadow them. 1276df8bae1dSRodney W. Grimes */ 1277df8bae1dSRodney W. Grimes 127805f0fdd2SPoul-Henning Kamp while ((p = backing_object->memq.tqh_first) != 0) { 127926f9a767SRodney W. Grimes 1280df8bae1dSRodney W. Grimes new_offset = (p->offset - backing_offset); 1281df8bae1dSRodney W. Grimes 1282df8bae1dSRodney W. Grimes /* 12830d94caffSDavid Greenman * If the parent has a page here, or if this 12840d94caffSDavid Greenman * page falls outside the parent, dispose of 12850d94caffSDavid Greenman * it. 1286df8bae1dSRodney W. Grimes * 1287df8bae1dSRodney W. Grimes * Otherwise, move it as planned. 1288df8bae1dSRodney W. Grimes */ 1289df8bae1dSRodney W. Grimes 1290df8bae1dSRodney W. Grimes if (p->offset < backing_offset || 1291df8bae1dSRodney W. Grimes new_offset >= size) { 1292df8bae1dSRodney W. Grimes vm_page_lock_queues(); 1293f919ebdeSDavid Greenman vm_page_protect(p, VM_PROT_NONE); 12940d94caffSDavid Greenman PAGE_WAKEUP(p); 1295df8bae1dSRodney W. Grimes vm_page_free(p); 1296df8bae1dSRodney W. Grimes vm_page_unlock_queues(); 1297df8bae1dSRodney W. Grimes } else { 1298df8bae1dSRodney W. Grimes pp = vm_page_lookup(object, new_offset); 129926f9a767SRodney W. Grimes if (pp != NULL || (object->pager && vm_pager_has_page(object->pager, 130026f9a767SRodney W. Grimes object->paging_offset + new_offset))) { 1301df8bae1dSRodney W. Grimes vm_page_lock_queues(); 1302f919ebdeSDavid Greenman vm_page_protect(p, VM_PROT_NONE); 13030d94caffSDavid Greenman PAGE_WAKEUP(p); 1304df8bae1dSRodney W. Grimes vm_page_free(p); 1305df8bae1dSRodney W. Grimes vm_page_unlock_queues(); 130626f9a767SRodney W. Grimes } else { 1307df8bae1dSRodney W. Grimes vm_page_rename(p, object, new_offset); 1308df8bae1dSRodney W. Grimes } 1309df8bae1dSRodney W. Grimes } 1310df8bae1dSRodney W. Grimes } 1311df8bae1dSRodney W. Grimes 1312df8bae1dSRodney W. Grimes /* 1313df8bae1dSRodney W. Grimes * Move the pager from backing_object to object. 1314df8bae1dSRodney W. Grimes */ 1315df8bae1dSRodney W. Grimes 1316df8bae1dSRodney W. Grimes if (backing_object->pager) { 131726f9a767SRodney W. Grimes backing_object->paging_in_progress++; 131826f9a767SRodney W. Grimes if (object->pager) { 131926f9a767SRodney W. Grimes vm_pager_t bopager; 13200d94caffSDavid Greenman 132126f9a767SRodney W. Grimes object->paging_in_progress++; 132226f9a767SRodney W. Grimes /* 132326f9a767SRodney W. Grimes * copy shadow object pages into ours 13240d94caffSDavid Greenman * and destroy unneeded pages in 13250d94caffSDavid Greenman * shadow object. 132626f9a767SRodney W. Grimes */ 132726f9a767SRodney W. Grimes bopager = backing_object->pager; 1328ba8da839SDavid Greenman backing_object->pager = NULL; 132926f9a767SRodney W. Grimes swap_pager_copy( 133026f9a767SRodney W. Grimes bopager, backing_object->paging_offset, 133126f9a767SRodney W. Grimes object->pager, object->paging_offset, 133226f9a767SRodney W. Grimes object->shadow_offset); 1333f919ebdeSDavid Greenman vm_object_pip_wakeup(object); 133426f9a767SRodney W. Grimes } else { 133526f9a767SRodney W. Grimes object->paging_in_progress++; 133626f9a767SRodney W. Grimes /* 133726f9a767SRodney W. Grimes * grab the shadow objects pager 133826f9a767SRodney W. Grimes */ 133926f9a767SRodney W. Grimes object->pager = backing_object->pager; 134026f9a767SRodney W. Grimes object->paging_offset = backing_object->paging_offset + backing_offset; 134126f9a767SRodney W. Grimes backing_object->pager = NULL; 134226f9a767SRodney W. Grimes /* 134326f9a767SRodney W. Grimes * free unnecessary blocks 134426f9a767SRodney W. Grimes */ 134526f9a767SRodney W. Grimes swap_pager_freespace(object->pager, 0, object->paging_offset); 1346f919ebdeSDavid Greenman vm_object_pip_wakeup(object); 1347c0503609SDavid Greenman } 1348c0503609SDavid Greenman 1349f919ebdeSDavid Greenman vm_object_pip_wakeup(backing_object); 1350c0503609SDavid Greenman } 1351df8bae1dSRodney W. Grimes /* 1352df8bae1dSRodney W. Grimes * Object now shadows whatever backing_object did. 1353df8bae1dSRodney W. Grimes * Note that the reference to backing_object->shadow 1354df8bae1dSRodney W. Grimes * moves from within backing_object to within object. 1355df8bae1dSRodney W. Grimes */ 1356df8bae1dSRodney W. Grimes 13570d94caffSDavid Greenman TAILQ_REMOVE(&object->shadow->reverse_shadow_head, object, 13580d94caffSDavid Greenman reverse_shadow_list); 13592fe6e4d7SDavid Greenman if (backing_object->shadow) 13600d94caffSDavid Greenman TAILQ_REMOVE(&backing_object->shadow->reverse_shadow_head, 13610d94caffSDavid Greenman backing_object, reverse_shadow_list); 1362df8bae1dSRodney W. Grimes object->shadow = backing_object->shadow; 13632fe6e4d7SDavid Greenman if (object->shadow) 13640d94caffSDavid Greenman TAILQ_INSERT_TAIL(&object->shadow->reverse_shadow_head, 13650d94caffSDavid Greenman object, reverse_shadow_list); 13662fe6e4d7SDavid Greenman 1367df8bae1dSRodney W. Grimes object->shadow_offset += backing_object->shadow_offset; 1368df8bae1dSRodney W. Grimes /* 1369df8bae1dSRodney W. Grimes * Discard backing_object. 1370df8bae1dSRodney W. Grimes * 13710d94caffSDavid Greenman * Since the backing object has no pages, no pager left, 13720d94caffSDavid Greenman * and no object references within it, all that is 13730d94caffSDavid Greenman * necessary is to dispose of it. 1374df8bae1dSRodney W. Grimes */ 1375df8bae1dSRodney W. Grimes 1376df8bae1dSRodney W. Grimes vm_object_unlock(backing_object); 1377df8bae1dSRodney W. Grimes 1378df8bae1dSRodney W. Grimes simple_lock(&vm_object_list_lock); 1379df8bae1dSRodney W. Grimes TAILQ_REMOVE(&vm_object_list, backing_object, 1380df8bae1dSRodney W. Grimes object_list); 1381df8bae1dSRodney W. Grimes vm_object_count--; 1382df8bae1dSRodney W. Grimes simple_unlock(&vm_object_list_lock); 1383df8bae1dSRodney W. Grimes 1384df8bae1dSRodney W. Grimes free((caddr_t) backing_object, M_VMOBJ); 1385df8bae1dSRodney W. Grimes 1386df8bae1dSRodney W. Grimes object_collapses++; 13870d94caffSDavid Greenman } else { 1388df8bae1dSRodney W. Grimes /* 1389df8bae1dSRodney W. Grimes * If all of the pages in the backing object are 13900d94caffSDavid Greenman * shadowed by the parent object, the parent object no 13910d94caffSDavid Greenman * longer has to shadow the backing object; it can 13920d94caffSDavid Greenman * shadow the next one in the chain. 1393df8bae1dSRodney W. Grimes * 13940d94caffSDavid Greenman * The backing object must not be paged out - we'd have 13950d94caffSDavid Greenman * to check all of the paged-out pages, as well. 1396df8bae1dSRodney W. Grimes */ 1397df8bae1dSRodney W. Grimes 1398df8bae1dSRodney W. Grimes if (backing_object->pager != NULL) { 1399df8bae1dSRodney W. Grimes vm_object_unlock(backing_object); 1400df8bae1dSRodney W. Grimes return; 1401df8bae1dSRodney W. Grimes } 1402df8bae1dSRodney W. Grimes /* 14030d94caffSDavid Greenman * Should have a check for a 'small' number of pages 14040d94caffSDavid Greenman * here. 1405df8bae1dSRodney W. Grimes */ 1406df8bae1dSRodney W. Grimes 140726f9a767SRodney W. Grimes for (p = backing_object->memq.tqh_first; p; p = p->listq.tqe_next) { 1408df8bae1dSRodney W. Grimes new_offset = (p->offset - backing_offset); 1409df8bae1dSRodney W. Grimes 1410df8bae1dSRodney W. Grimes /* 14110d94caffSDavid Greenman * If the parent has a page here, or if this 14120d94caffSDavid Greenman * page falls outside the parent, keep going. 1413df8bae1dSRodney W. Grimes * 14140d94caffSDavid Greenman * Otherwise, the backing_object must be left in 14150d94caffSDavid Greenman * the chain. 1416df8bae1dSRodney W. Grimes */ 1417df8bae1dSRodney W. Grimes 1418df8bae1dSRodney W. Grimes if (p->offset >= backing_offset && 141926f9a767SRodney W. Grimes new_offset <= size && 14200d94caffSDavid Greenman ((pp = vm_page_lookup(object, new_offset)) == NULL || 14210d94caffSDavid Greenman !pp->valid) && 142226f9a767SRodney W. Grimes (!object->pager || !vm_pager_has_page(object->pager, object->paging_offset + new_offset))) { 1423df8bae1dSRodney W. Grimes /* 14240d94caffSDavid Greenman * Page still needed. Can't go any 14250d94caffSDavid Greenman * further. 1426df8bae1dSRodney W. Grimes */ 1427df8bae1dSRodney W. Grimes vm_object_unlock(backing_object); 1428df8bae1dSRodney W. Grimes return; 1429df8bae1dSRodney W. Grimes } 1430df8bae1dSRodney W. Grimes } 1431df8bae1dSRodney W. Grimes 1432df8bae1dSRodney W. Grimes /* 14330d94caffSDavid Greenman * Make the parent shadow the next object in the 14340d94caffSDavid Greenman * chain. Deallocating backing_object will not remove 14350d94caffSDavid Greenman * it, since its reference count is at least 2. 1436df8bae1dSRodney W. Grimes */ 1437df8bae1dSRodney W. Grimes 14380d94caffSDavid Greenman TAILQ_REMOVE(&object->shadow->reverse_shadow_head, 14390d94caffSDavid Greenman object, reverse_shadow_list); 144026f9a767SRodney W. Grimes vm_object_reference(object->shadow = backing_object->shadow); 14412fe6e4d7SDavid Greenman if (object->shadow) 14420d94caffSDavid Greenman TAILQ_INSERT_TAIL(&object->shadow->reverse_shadow_head, 14430d94caffSDavid Greenman object, reverse_shadow_list); 1444df8bae1dSRodney W. Grimes object->shadow_offset += backing_object->shadow_offset; 1445df8bae1dSRodney W. Grimes 1446df8bae1dSRodney W. Grimes /* 14470d94caffSDavid Greenman * Backing object might have had a copy pointer to us. 14480d94caffSDavid Greenman * If it did, clear it. 1449df8bae1dSRodney W. Grimes */ 1450df8bae1dSRodney W. Grimes if (backing_object->copy == object) { 1451df8bae1dSRodney W. Grimes backing_object->copy = NULL; 1452df8bae1dSRodney W. Grimes } 14530d94caffSDavid Greenman /* 14540d94caffSDavid Greenman * Drop the reference count on backing_object. Since 14550d94caffSDavid Greenman * its ref_count was at least 2, it will not vanish; 14560d94caffSDavid Greenman * so we don't need to call vm_object_deallocate. 1457df8bae1dSRodney W. Grimes */ 145826f9a767SRodney W. Grimes if (backing_object->ref_count == 1) 145926f9a767SRodney W. Grimes printf("should have called obj deallocate\n"); 1460df8bae1dSRodney W. Grimes backing_object->ref_count--; 1461df8bae1dSRodney W. Grimes vm_object_unlock(backing_object); 1462df8bae1dSRodney W. Grimes 1463df8bae1dSRodney W. Grimes object_bypasses++; 1464df8bae1dSRodney W. Grimes 1465df8bae1dSRodney W. Grimes } 1466df8bae1dSRodney W. Grimes 1467df8bae1dSRodney W. Grimes /* 1468df8bae1dSRodney W. Grimes * Try again with this object's new backing object. 1469df8bae1dSRodney W. Grimes */ 1470df8bae1dSRodney W. Grimes } 1471df8bae1dSRodney W. Grimes } 1472df8bae1dSRodney W. Grimes 1473df8bae1dSRodney W. Grimes /* 1474df8bae1dSRodney W. Grimes * vm_object_page_remove: [internal] 1475df8bae1dSRodney W. Grimes * 1476df8bae1dSRodney W. Grimes * Removes all physical pages in the specified 1477df8bae1dSRodney W. Grimes * object range from the object's list of pages. 1478df8bae1dSRodney W. Grimes * 1479df8bae1dSRodney W. Grimes * The object must be locked. 1480df8bae1dSRodney W. Grimes */ 148126f9a767SRodney W. Grimes void 14827c1f6cedSDavid Greenman vm_object_page_remove(object, start, end, clean_only) 1483df8bae1dSRodney W. Grimes register vm_object_t object; 1484df8bae1dSRodney W. Grimes register vm_offset_t start; 1485df8bae1dSRodney W. Grimes register vm_offset_t end; 14867c1f6cedSDavid Greenman boolean_t clean_only; 1487df8bae1dSRodney W. Grimes { 1488df8bae1dSRodney W. Grimes register vm_page_t p, next; 148926f9a767SRodney W. Grimes vm_offset_t size; 149026f9a767SRodney W. Grimes int s; 1491df8bae1dSRodney W. Grimes 1492df8bae1dSRodney W. Grimes if (object == NULL) 1493df8bae1dSRodney W. Grimes return; 1494df8bae1dSRodney W. Grimes 14952fe6e4d7SDavid Greenman object->paging_in_progress++; 149626f9a767SRodney W. Grimes start = trunc_page(start); 149726f9a767SRodney W. Grimes end = round_page(end); 149826f9a767SRodney W. Grimes again: 149926f9a767SRodney W. Grimes size = end - start; 150026f9a767SRodney W. Grimes if (size > 4 * PAGE_SIZE || size >= object->size / 4) { 15010d94caffSDavid Greenman for (p = object->memq.tqh_first; p != NULL; p = next) { 1502df8bae1dSRodney W. Grimes next = p->listq.tqe_next; 1503df8bae1dSRodney W. Grimes if ((start <= p->offset) && (p->offset < end)) { 1504a481f200SDavid Greenman s = splhigh(); 15050d94caffSDavid Greenman if (p->bmapped) { 15060d94caffSDavid Greenman splx(s); 15070d94caffSDavid Greenman continue; 15080d94caffSDavid Greenman } 15090d94caffSDavid Greenman if ((p->flags & PG_BUSY) || p->busy) { 151026f9a767SRodney W. Grimes p->flags |= PG_WANTED; 151126f9a767SRodney W. Grimes tsleep((caddr_t) p, PVM, "vmopar", 0); 1512a481f200SDavid Greenman splx(s); 151326f9a767SRodney W. Grimes goto again; 151426f9a767SRodney W. Grimes } 1515a481f200SDavid Greenman splx(s); 15167c1f6cedSDavid Greenman if (clean_only) { 15177c1f6cedSDavid Greenman vm_page_test_dirty(p); 15187c1f6cedSDavid Greenman if (p->valid & p->dirty) 15197c1f6cedSDavid Greenman continue; 15207c1f6cedSDavid Greenman } 1521f919ebdeSDavid Greenman vm_page_protect(p, VM_PROT_NONE); 1522df8bae1dSRodney W. Grimes vm_page_lock_queues(); 15230d94caffSDavid Greenman PAGE_WAKEUP(p); 1524df8bae1dSRodney W. Grimes vm_page_free(p); 1525df8bae1dSRodney W. Grimes vm_page_unlock_queues(); 152626f9a767SRodney W. Grimes } 152726f9a767SRodney W. Grimes } 152826f9a767SRodney W. Grimes } else { 152926f9a767SRodney W. Grimes while (size > 0) { 153005f0fdd2SPoul-Henning Kamp while ((p = vm_page_lookup(object, start)) != 0) { 1531a481f200SDavid Greenman s = splhigh(); 15320d94caffSDavid Greenman if (p->bmapped) { 15330d94caffSDavid Greenman splx(s); 15340d94caffSDavid Greenman break; 15350d94caffSDavid Greenman } 15360d94caffSDavid Greenman if ((p->flags & PG_BUSY) || p->busy) { 153726f9a767SRodney W. Grimes p->flags |= PG_WANTED; 153826f9a767SRodney W. Grimes tsleep((caddr_t) p, PVM, "vmopar", 0); 1539a481f200SDavid Greenman splx(s); 154026f9a767SRodney W. Grimes goto again; 154126f9a767SRodney W. Grimes } 1542a481f200SDavid Greenman splx(s); 15437c1f6cedSDavid Greenman if (clean_only) { 15447c1f6cedSDavid Greenman vm_page_test_dirty(p); 15457c1f6cedSDavid Greenman if (p->valid & p->dirty) 15467c1f6cedSDavid Greenman continue; 15477c1f6cedSDavid Greenman } 1548f919ebdeSDavid Greenman vm_page_protect(p, VM_PROT_NONE); 154926f9a767SRodney W. Grimes vm_page_lock_queues(); 15500d94caffSDavid Greenman PAGE_WAKEUP(p); 155126f9a767SRodney W. Grimes vm_page_free(p); 155226f9a767SRodney W. Grimes vm_page_unlock_queues(); 155326f9a767SRodney W. Grimes } 155426f9a767SRodney W. Grimes start += PAGE_SIZE; 155526f9a767SRodney W. Grimes size -= PAGE_SIZE; 1556df8bae1dSRodney W. Grimes } 1557df8bae1dSRodney W. Grimes } 1558f919ebdeSDavid Greenman vm_object_pip_wakeup(object); 1559c0503609SDavid Greenman } 1560df8bae1dSRodney W. Grimes 1561df8bae1dSRodney W. Grimes /* 1562df8bae1dSRodney W. Grimes * Routine: vm_object_coalesce 1563df8bae1dSRodney W. Grimes * Function: Coalesces two objects backing up adjoining 1564df8bae1dSRodney W. Grimes * regions of memory into a single object. 1565df8bae1dSRodney W. Grimes * 1566df8bae1dSRodney W. Grimes * returns TRUE if objects were combined. 1567df8bae1dSRodney W. Grimes * 1568df8bae1dSRodney W. Grimes * NOTE: Only works at the moment if the second object is NULL - 1569df8bae1dSRodney W. Grimes * if it's not, which object do we lock first? 1570df8bae1dSRodney W. Grimes * 1571df8bae1dSRodney W. Grimes * Parameters: 1572df8bae1dSRodney W. Grimes * prev_object First object to coalesce 1573df8bae1dSRodney W. Grimes * prev_offset Offset into prev_object 1574df8bae1dSRodney W. Grimes * next_object Second object into coalesce 1575df8bae1dSRodney W. Grimes * next_offset Offset into next_object 1576df8bae1dSRodney W. Grimes * 1577df8bae1dSRodney W. Grimes * prev_size Size of reference to prev_object 1578df8bae1dSRodney W. Grimes * next_size Size of reference to next_object 1579df8bae1dSRodney W. Grimes * 1580df8bae1dSRodney W. Grimes * Conditions: 1581df8bae1dSRodney W. Grimes * The object must *not* be locked. 1582df8bae1dSRodney W. Grimes */ 15830d94caffSDavid Greenman boolean_t 15840d94caffSDavid Greenman vm_object_coalesce(prev_object, next_object, 1585df8bae1dSRodney W. Grimes prev_offset, next_offset, 1586df8bae1dSRodney W. Grimes prev_size, next_size) 1587df8bae1dSRodney W. Grimes register vm_object_t prev_object; 1588df8bae1dSRodney W. Grimes vm_object_t next_object; 1589df8bae1dSRodney W. Grimes vm_offset_t prev_offset, next_offset; 1590df8bae1dSRodney W. Grimes vm_size_t prev_size, next_size; 1591df8bae1dSRodney W. Grimes { 1592df8bae1dSRodney W. Grimes vm_size_t newsize; 1593df8bae1dSRodney W. Grimes 1594df8bae1dSRodney W. Grimes if (next_object != NULL) { 1595df8bae1dSRodney W. Grimes return (FALSE); 1596df8bae1dSRodney W. Grimes } 1597df8bae1dSRodney W. Grimes if (prev_object == NULL) { 1598df8bae1dSRodney W. Grimes return (TRUE); 1599df8bae1dSRodney W. Grimes } 1600df8bae1dSRodney W. Grimes vm_object_lock(prev_object); 1601df8bae1dSRodney W. Grimes 1602df8bae1dSRodney W. Grimes /* 1603df8bae1dSRodney W. Grimes * Try to collapse the object first 1604df8bae1dSRodney W. Grimes */ 1605df8bae1dSRodney W. Grimes vm_object_collapse(prev_object); 1606df8bae1dSRodney W. Grimes 1607df8bae1dSRodney W. Grimes /* 16080d94caffSDavid Greenman * Can't coalesce if: . more than one reference . paged out . shadows 16090d94caffSDavid Greenman * another object . has a copy elsewhere (any of which mean that the 16100d94caffSDavid Greenman * pages not mapped to prev_entry may be in use anyway) 1611df8bae1dSRodney W. Grimes */ 1612df8bae1dSRodney W. Grimes 1613df8bae1dSRodney W. Grimes if (prev_object->ref_count > 1 || 1614df8bae1dSRodney W. Grimes prev_object->pager != NULL || 1615df8bae1dSRodney W. Grimes prev_object->shadow != NULL || 1616df8bae1dSRodney W. Grimes prev_object->copy != NULL) { 1617df8bae1dSRodney W. Grimes vm_object_unlock(prev_object); 1618df8bae1dSRodney W. Grimes return (FALSE); 1619df8bae1dSRodney W. Grimes } 1620df8bae1dSRodney W. Grimes /* 16210d94caffSDavid Greenman * Remove any pages that may still be in the object from a previous 16220d94caffSDavid Greenman * deallocation. 1623df8bae1dSRodney W. Grimes */ 1624df8bae1dSRodney W. Grimes 1625df8bae1dSRodney W. Grimes vm_object_page_remove(prev_object, 1626df8bae1dSRodney W. Grimes prev_offset + prev_size, 16277c1f6cedSDavid Greenman prev_offset + prev_size + next_size, FALSE); 1628df8bae1dSRodney W. Grimes 1629df8bae1dSRodney W. Grimes /* 1630df8bae1dSRodney W. Grimes * Extend the object if necessary. 1631df8bae1dSRodney W. Grimes */ 1632df8bae1dSRodney W. Grimes newsize = prev_offset + prev_size + next_size; 1633df8bae1dSRodney W. Grimes if (newsize > prev_object->size) 1634df8bae1dSRodney W. Grimes prev_object->size = newsize; 1635df8bae1dSRodney W. Grimes 1636df8bae1dSRodney W. Grimes vm_object_unlock(prev_object); 1637df8bae1dSRodney W. Grimes return (TRUE); 1638df8bae1dSRodney W. Grimes } 1639df8bae1dSRodney W. Grimes 1640df8bae1dSRodney W. Grimes /* 164126f9a767SRodney W. Grimes * returns page after looking up in shadow chain 164226f9a767SRodney W. Grimes */ 164326f9a767SRodney W. Grimes 164426f9a767SRodney W. Grimes vm_page_t 164526f9a767SRodney W. Grimes vm_object_page_lookup(object, offset) 164626f9a767SRodney W. Grimes vm_object_t object; 164726f9a767SRodney W. Grimes vm_offset_t offset; 164826f9a767SRodney W. Grimes { 164926f9a767SRodney W. Grimes vm_page_t m; 16500d94caffSDavid Greenman 165126f9a767SRodney W. Grimes if (!(m = vm_page_lookup(object, offset))) { 165226f9a767SRodney W. Grimes if (!object->shadow) 165326f9a767SRodney W. Grimes return 0; 165426f9a767SRodney W. Grimes else 165526f9a767SRodney W. Grimes return vm_object_page_lookup(object->shadow, offset + object->shadow_offset); 165626f9a767SRodney W. Grimes } 165726f9a767SRodney W. Grimes return m; 165826f9a767SRodney W. Grimes } 165926f9a767SRodney W. Grimes 1660a1f6d91cSDavid Greenman int 1661a1f6d91cSDavid Greenman _vm_object_in_map(map, object, entry) 1662a1f6d91cSDavid Greenman vm_map_t map; 1663a1f6d91cSDavid Greenman vm_object_t object; 1664a1f6d91cSDavid Greenman vm_map_entry_t entry; 1665a1f6d91cSDavid Greenman { 1666a1f6d91cSDavid Greenman vm_map_t tmpm; 1667a1f6d91cSDavid Greenman vm_map_entry_t tmpe; 1668a1f6d91cSDavid Greenman vm_object_t obj; 1669a1f6d91cSDavid Greenman int entcount; 1670a1f6d91cSDavid Greenman 1671a1f6d91cSDavid Greenman if (map == 0) 1672a1f6d91cSDavid Greenman return 0; 1673a1f6d91cSDavid Greenman 1674a1f6d91cSDavid Greenman if (entry == 0) { 1675a1f6d91cSDavid Greenman tmpe = map->header.next; 1676a1f6d91cSDavid Greenman entcount = map->nentries; 1677a1f6d91cSDavid Greenman while (entcount-- && (tmpe != &map->header)) { 1678a1f6d91cSDavid Greenman if( _vm_object_in_map(map, object, tmpe)) { 1679a1f6d91cSDavid Greenman return 1; 1680a1f6d91cSDavid Greenman } 1681a1f6d91cSDavid Greenman tmpe = tmpe->next; 1682a1f6d91cSDavid Greenman } 1683a1f6d91cSDavid Greenman } else if (entry->is_sub_map || entry->is_a_map) { 1684a1f6d91cSDavid Greenman tmpm = entry->object.share_map; 1685a1f6d91cSDavid Greenman tmpe = tmpm->header.next; 1686a1f6d91cSDavid Greenman entcount = tmpm->nentries; 1687a1f6d91cSDavid Greenman while (entcount-- && tmpe != &tmpm->header) { 1688a1f6d91cSDavid Greenman if( _vm_object_in_map(tmpm, object, tmpe)) { 1689a1f6d91cSDavid Greenman return 1; 1690a1f6d91cSDavid Greenman } 1691a1f6d91cSDavid Greenman tmpe = tmpe->next; 1692a1f6d91cSDavid Greenman } 1693a1f6d91cSDavid Greenman } else if (obj = entry->object.vm_object) { 1694a1f6d91cSDavid Greenman for(; obj; obj=obj->shadow) 1695a1f6d91cSDavid Greenman if( obj == object) { 1696a1f6d91cSDavid Greenman return 1; 1697a1f6d91cSDavid Greenman } 1698a1f6d91cSDavid Greenman } 1699a1f6d91cSDavid Greenman return 0; 1700a1f6d91cSDavid Greenman } 1701a1f6d91cSDavid Greenman 1702a1f6d91cSDavid Greenman int 1703a1f6d91cSDavid Greenman vm_object_in_map( object) 1704a1f6d91cSDavid Greenman vm_object_t object; 1705a1f6d91cSDavid Greenman { 1706a1f6d91cSDavid Greenman struct proc *p; 1707a1f6d91cSDavid Greenman for (p = (struct proc *) allproc; p != NULL; p = p->p_next) { 1708a1f6d91cSDavid Greenman if( !p->p_vmspace /* || (p->p_flag & (P_SYSTEM|P_WEXIT)) */) 1709a1f6d91cSDavid Greenman continue; 1710a1f6d91cSDavid Greenman /* 1711a1f6d91cSDavid Greenman if (p->p_stat != SRUN && p->p_stat != SSLEEP) { 1712a1f6d91cSDavid Greenman continue; 1713a1f6d91cSDavid Greenman } 1714a1f6d91cSDavid Greenman */ 1715a1f6d91cSDavid Greenman if( _vm_object_in_map(&p->p_vmspace->vm_map, object, 0)) 1716a1f6d91cSDavid Greenman return 1; 1717a1f6d91cSDavid Greenman } 1718a1f6d91cSDavid Greenman if( _vm_object_in_map( kernel_map, object, 0)) 1719a1f6d91cSDavid Greenman return 1; 1720a1f6d91cSDavid Greenman if( _vm_object_in_map( kmem_map, object, 0)) 1721a1f6d91cSDavid Greenman return 1; 1722a1f6d91cSDavid Greenman if( _vm_object_in_map( pager_map, object, 0)) 1723a1f6d91cSDavid Greenman return 1; 1724a1f6d91cSDavid Greenman if( _vm_object_in_map( buffer_map, object, 0)) 1725a1f6d91cSDavid Greenman return 1; 1726a1f6d91cSDavid Greenman if( _vm_object_in_map( io_map, object, 0)) 1727a1f6d91cSDavid Greenman return 1; 1728a1f6d91cSDavid Greenman if( _vm_object_in_map( phys_map, object, 0)) 1729a1f6d91cSDavid Greenman return 1; 1730a1f6d91cSDavid Greenman if( _vm_object_in_map( mb_map, object, 0)) 1731a1f6d91cSDavid Greenman return 1; 1732a1f6d91cSDavid Greenman if( _vm_object_in_map( u_map, object, 0)) 1733a1f6d91cSDavid Greenman return 1; 1734a1f6d91cSDavid Greenman return 0; 1735a1f6d91cSDavid Greenman } 1736a1f6d91cSDavid Greenman 1737a1f6d91cSDavid Greenman void 1738a1f6d91cSDavid Greenman vm_object_check() { 1739a1f6d91cSDavid Greenman int i; 1740a1f6d91cSDavid Greenman int maxhash = 0; 1741a1f6d91cSDavid Greenman vm_object_t object; 1742a1f6d91cSDavid Greenman vm_object_hash_entry_t entry; 1743a1f6d91cSDavid Greenman 1744a1f6d91cSDavid Greenman /* 1745a1f6d91cSDavid Greenman * make sure that no internal objs are hashed 1746a1f6d91cSDavid Greenman */ 1747a1f6d91cSDavid Greenman for (i=0; i<VM_OBJECT_HASH_COUNT;i++) { 1748a1f6d91cSDavid Greenman int lsize = 0; 1749a1f6d91cSDavid Greenman for (entry = vm_object_hashtable[i].tqh_first; 1750a1f6d91cSDavid Greenman entry != NULL; 1751a1f6d91cSDavid Greenman entry = entry->hash_links.tqe_next) { 1752a1f6d91cSDavid Greenman if( entry->object->flags & OBJ_INTERNAL) { 1753a1f6d91cSDavid Greenman printf("vmochk: internal obj on hash: size: %d\n", entry->object->size); 1754a1f6d91cSDavid Greenman } 1755a1f6d91cSDavid Greenman ++lsize; 1756a1f6d91cSDavid Greenman } 1757a1f6d91cSDavid Greenman if( lsize > maxhash) 1758a1f6d91cSDavid Greenman maxhash = lsize; 1759a1f6d91cSDavid Greenman } 1760a1f6d91cSDavid Greenman 1761a1f6d91cSDavid Greenman printf("maximum object hash queue size: %d\n", maxhash); 1762a1f6d91cSDavid Greenman 1763a1f6d91cSDavid Greenman /* 1764a1f6d91cSDavid Greenman * make sure that internal objs are in a map somewhere 1765a1f6d91cSDavid Greenman * and none have zero ref counts. 1766a1f6d91cSDavid Greenman */ 1767a1f6d91cSDavid Greenman for (object = vm_object_list.tqh_first; 1768a1f6d91cSDavid Greenman object != NULL; 1769a1f6d91cSDavid Greenman object = object->object_list.tqe_next) { 1770a1f6d91cSDavid Greenman if( object->flags & OBJ_INTERNAL) { 1771a1f6d91cSDavid Greenman if( object->ref_count == 0) { 1772a1f6d91cSDavid Greenman printf("vmochk: internal obj has zero ref count: %d\n", 1773a1f6d91cSDavid Greenman object->size); 1774a1f6d91cSDavid Greenman } 1775a1f6d91cSDavid Greenman if( !vm_object_in_map(object)) { 1776a1f6d91cSDavid Greenman printf("vmochk: internal obj is not in a map: ref: %d, size: %d, pager: 0x%x, shadow: 0x%x\n", object->ref_count, object->size, object->pager, object->shadow); 1777a1f6d91cSDavid Greenman } 1778a1f6d91cSDavid Greenman } 1779a1f6d91cSDavid Greenman } 1780a1f6d91cSDavid Greenman } 1781a1f6d91cSDavid Greenman 178226f9a767SRodney W. Grimes #define DEBUG 17838a129caeSDavid Greenman #if defined(DEBUG) || defined(DDB) 178426f9a767SRodney W. Grimes /* 1785df8bae1dSRodney W. Grimes * vm_object_print: [ debug ] 1786df8bae1dSRodney W. Grimes */ 17870d94caffSDavid Greenman void 17880d94caffSDavid Greenman vm_object_print(object, full) 1789df8bae1dSRodney W. Grimes vm_object_t object; 1790df8bae1dSRodney W. Grimes boolean_t full; 1791df8bae1dSRodney W. Grimes { 1792df8bae1dSRodney W. Grimes register vm_page_t p; 1793df8bae1dSRodney W. Grimes 1794df8bae1dSRodney W. Grimes register int count; 1795df8bae1dSRodney W. Grimes 1796df8bae1dSRodney W. Grimes if (object == NULL) 1797df8bae1dSRodney W. Grimes return; 1798df8bae1dSRodney W. Grimes 1799df8bae1dSRodney W. Grimes iprintf("Object 0x%x: size=0x%x, res=%d, ref=%d, ", 1800df8bae1dSRodney W. Grimes (int) object, (int) object->size, 1801df8bae1dSRodney W. Grimes object->resident_page_count, object->ref_count); 1802df8bae1dSRodney W. Grimes printf("pager=0x%x+0x%x, shadow=(0x%x)+0x%x\n", 1803df8bae1dSRodney W. Grimes (int) object->pager, (int) object->paging_offset, 1804df8bae1dSRodney W. Grimes (int) object->shadow, (int) object->shadow_offset); 180505f0fdd2SPoul-Henning Kamp printf("cache: next=%p, prev=%p\n", 1806df8bae1dSRodney W. Grimes object->cached_list.tqe_next, object->cached_list.tqe_prev); 1807df8bae1dSRodney W. Grimes 1808df8bae1dSRodney W. Grimes if (!full) 1809df8bae1dSRodney W. Grimes return; 1810df8bae1dSRodney W. Grimes 1811df8bae1dSRodney W. Grimes indent += 2; 1812df8bae1dSRodney W. Grimes count = 0; 1813df8bae1dSRodney W. Grimes for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 1814df8bae1dSRodney W. Grimes if (count == 0) 1815df8bae1dSRodney W. Grimes iprintf("memory:="); 1816df8bae1dSRodney W. Grimes else if (count == 6) { 1817df8bae1dSRodney W. Grimes printf("\n"); 1818df8bae1dSRodney W. Grimes iprintf(" ..."); 1819df8bae1dSRodney W. Grimes count = 0; 1820df8bae1dSRodney W. Grimes } else 1821df8bae1dSRodney W. Grimes printf(","); 1822df8bae1dSRodney W. Grimes count++; 1823df8bae1dSRodney W. Grimes 182405f0fdd2SPoul-Henning Kamp printf("(off=0x%lx,page=0x%lx)", 182505f0fdd2SPoul-Henning Kamp (u_long) p->offset, (u_long) VM_PAGE_TO_PHYS(p)); 1826df8bae1dSRodney W. Grimes } 1827df8bae1dSRodney W. Grimes if (count != 0) 1828df8bae1dSRodney W. Grimes printf("\n"); 1829df8bae1dSRodney W. Grimes indent -= 2; 1830df8bae1dSRodney W. Grimes } 18318a129caeSDavid Greenman #endif /* defined(DEBUG) || defined(DDB) */ 1832