1df8bae1dSRodney W. Grimes /* 2df8bae1dSRodney W. Grimes * Copyright (c) 1991, 1993 3df8bae1dSRodney W. Grimes * The Regents of the University of California. All rights reserved. 4df8bae1dSRodney W. Grimes * 5df8bae1dSRodney W. Grimes * This code is derived from software contributed to Berkeley by 6df8bae1dSRodney W. Grimes * The Mach Operating System project at Carnegie-Mellon University. 7df8bae1dSRodney W. Grimes * 8df8bae1dSRodney W. Grimes * Redistribution and use in source and binary forms, with or without 9df8bae1dSRodney W. Grimes * modification, are permitted provided that the following conditions 10df8bae1dSRodney W. Grimes * are met: 11df8bae1dSRodney W. Grimes * 1. Redistributions of source code must retain the above copyright 12df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer. 13df8bae1dSRodney W. Grimes * 2. Redistributions in binary form must reproduce the above copyright 14df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer in the 15df8bae1dSRodney W. Grimes * documentation and/or other materials provided with the distribution. 16df8bae1dSRodney W. Grimes * 3. All advertising materials mentioning features or use of this software 175929bcfaSPhilippe Charnier * must display the following acknowledgement: 18df8bae1dSRodney W. Grimes * This product includes software developed by the University of 19df8bae1dSRodney W. Grimes * California, Berkeley and its contributors. 20df8bae1dSRodney W. Grimes * 4. Neither the name of the University nor the names of its contributors 21df8bae1dSRodney W. Grimes * may be used to endorse or promote products derived from this software 22df8bae1dSRodney W. Grimes * without specific prior written permission. 23df8bae1dSRodney W. Grimes * 24df8bae1dSRodney W. Grimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25df8bae1dSRodney W. Grimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26df8bae1dSRodney W. Grimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27df8bae1dSRodney W. Grimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28df8bae1dSRodney W. Grimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29df8bae1dSRodney W. Grimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30df8bae1dSRodney W. Grimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31df8bae1dSRodney W. Grimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32df8bae1dSRodney W. Grimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33df8bae1dSRodney W. Grimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34df8bae1dSRodney W. Grimes * SUCH DAMAGE. 35df8bae1dSRodney W. Grimes * 363c4dd356SDavid Greenman * from: @(#)vm_object.c 8.5 (Berkeley) 3/22/94 37df8bae1dSRodney W. Grimes * 38df8bae1dSRodney W. Grimes * 39df8bae1dSRodney W. Grimes * Copyright (c) 1987, 1990 Carnegie-Mellon University. 40df8bae1dSRodney W. Grimes * All rights reserved. 41df8bae1dSRodney W. Grimes * 42df8bae1dSRodney W. Grimes * Authors: Avadis Tevanian, Jr., Michael Wayne Young 43df8bae1dSRodney W. Grimes * 44df8bae1dSRodney W. Grimes * Permission to use, copy, modify and distribute this software and 45df8bae1dSRodney W. Grimes * its documentation is hereby granted, provided that both the copyright 46df8bae1dSRodney W. Grimes * notice and this permission notice appear in all copies of the 47df8bae1dSRodney W. Grimes * software, derivative works or modified versions, and any portions 48df8bae1dSRodney W. Grimes * thereof, and that both notices appear in supporting documentation. 49df8bae1dSRodney W. Grimes * 50df8bae1dSRodney W. Grimes * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 51df8bae1dSRodney W. Grimes * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 52df8bae1dSRodney W. Grimes * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 53df8bae1dSRodney W. Grimes * 54df8bae1dSRodney W. Grimes * Carnegie Mellon requests users of this software to return to 55df8bae1dSRodney W. Grimes * 56df8bae1dSRodney W. Grimes * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 57df8bae1dSRodney W. Grimes * School of Computer Science 58df8bae1dSRodney W. Grimes * Carnegie Mellon University 59df8bae1dSRodney W. Grimes * Pittsburgh PA 15213-3890 60df8bae1dSRodney W. Grimes * 61df8bae1dSRodney W. Grimes * any improvements or extensions that they make and grant Carnegie the 62df8bae1dSRodney W. Grimes * rights to redistribute these changes. 633c4dd356SDavid Greenman * 64c3aac50fSPeter Wemm * $FreeBSD$ 65df8bae1dSRodney W. Grimes */ 66df8bae1dSRodney W. Grimes 67df8bae1dSRodney W. Grimes /* 68df8bae1dSRodney W. Grimes * Virtual memory object module. 69df8bae1dSRodney W. Grimes */ 70df8bae1dSRodney W. Grimes 71df8bae1dSRodney W. Grimes #include <sys/param.h> 72df8bae1dSRodney W. Grimes #include <sys/systm.h> 73fb919e4dSMark Murray #include <sys/lock.h> 74867a482dSJohn Dyson #include <sys/mman.h> 75cf2819ccSJohn Dyson #include <sys/mount.h> 76b9b7a4beSMatthew Dillon #include <sys/kernel.h> 77b9b7a4beSMatthew Dillon #include <sys/sysctl.h> 781b367556SJason Evans #include <sys/mutex.h> 79fb919e4dSMark Murray #include <sys/proc.h> /* for curproc, pageproc */ 80fb919e4dSMark Murray #include <sys/socket.h> 81fb919e4dSMark Murray #include <sys/vnode.h> 82fb919e4dSMark Murray #include <sys/vmmeter.h> 831005a129SJohn Baldwin #include <sys/sx.h> 84df8bae1dSRodney W. Grimes 85df8bae1dSRodney W. Grimes #include <vm/vm.h> 86efeaf95aSDavid Greenman #include <vm/vm_param.h> 87efeaf95aSDavid Greenman #include <vm/pmap.h> 88efeaf95aSDavid Greenman #include <vm/vm_map.h> 89efeaf95aSDavid Greenman #include <vm/vm_object.h> 90df8bae1dSRodney W. Grimes #include <vm/vm_page.h> 9126f9a767SRodney W. Grimes #include <vm/vm_pageout.h> 920d94caffSDavid Greenman #include <vm/vm_pager.h> 9305f0fdd2SPoul-Henning Kamp #include <vm/swap_pager.h> 94a1f6d91cSDavid Greenman #include <vm/vm_kern.h> 95efeaf95aSDavid Greenman #include <vm/vm_extern.h> 96670d17b5SJeff Roberson #include <vm/uma.h> 9726f9a767SRodney W. Grimes 98b9b7a4beSMatthew Dillon #define EASY_SCAN_FACTOR 8 99b9b7a4beSMatthew Dillon 100b9b7a4beSMatthew Dillon #define MSYNC_FLUSH_HARDSEQ 0x01 101b9b7a4beSMatthew Dillon #define MSYNC_FLUSH_SOFTSEQ 0x02 102b9b7a4beSMatthew Dillon 103b9b7a4beSMatthew Dillon /* 104b9b7a4beSMatthew Dillon * msync / VM object flushing optimizations 105b9b7a4beSMatthew Dillon */ 106b9b7a4beSMatthew Dillon static int msync_flush_flags = MSYNC_FLUSH_HARDSEQ | MSYNC_FLUSH_SOFTSEQ; 107b9b7a4beSMatthew Dillon SYSCTL_INT(_vm, OID_AUTO, msync_flush_flags, 108b9b7a4beSMatthew Dillon CTLFLAG_RW, &msync_flush_flags, 0, ""); 109b9b7a4beSMatthew Dillon 110b9b7a4beSMatthew Dillon static void vm_object_qcollapse(vm_object_t object); 111b9b7a4beSMatthew Dillon static int vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int curgeneration, int pagerflags); 1121ca58953SAlan Cox static void vm_object_pip_sleep(vm_object_t object, char *waitid); 113f6b04d2bSDavid Greenman 114df8bae1dSRodney W. Grimes /* 115df8bae1dSRodney W. Grimes * Virtual memory objects maintain the actual data 116df8bae1dSRodney W. Grimes * associated with allocated virtual memory. A given 117df8bae1dSRodney W. Grimes * page of memory exists within exactly one object. 118df8bae1dSRodney W. Grimes * 119df8bae1dSRodney W. Grimes * An object is only deallocated when all "references" 120df8bae1dSRodney W. Grimes * are given up. Only one "reference" to a given 121df8bae1dSRodney W. Grimes * region of an object should be writeable. 122df8bae1dSRodney W. Grimes * 123df8bae1dSRodney W. Grimes * Associated with each object is a list of all resident 124df8bae1dSRodney W. Grimes * memory pages belonging to that object; this list is 125df8bae1dSRodney W. Grimes * maintained by the "vm_page" module, and locked by the object's 126df8bae1dSRodney W. Grimes * lock. 127df8bae1dSRodney W. Grimes * 128df8bae1dSRodney W. Grimes * Each object also records a "pager" routine which is 129df8bae1dSRodney W. Grimes * used to retrieve (and store) pages to the proper backing 130df8bae1dSRodney W. Grimes * storage. In addition, objects may be backed by other 131df8bae1dSRodney W. Grimes * objects from which they were virtual-copied. 132df8bae1dSRodney W. Grimes * 133df8bae1dSRodney W. Grimes * The only items within the object structure which are 134df8bae1dSRodney W. Grimes * modified after time of creation are: 135df8bae1dSRodney W. Grimes * reference count locked by object's lock 136df8bae1dSRodney W. Grimes * pager routine locked by object's lock 137df8bae1dSRodney W. Grimes * 138df8bae1dSRodney W. Grimes */ 139df8bae1dSRodney W. Grimes 14028f8db14SBruce Evans struct object_q vm_object_list; 141a5698387SAlan Cox struct mtx vm_object_list_mtx; /* lock for object list and count */ 14228f8db14SBruce Evans vm_object_t kernel_object; 14328f8db14SBruce Evans vm_object_t kmem_object; 144f708ef1bSPoul-Henning Kamp static struct vm_object kernel_object_store; 145f708ef1bSPoul-Henning Kamp static struct vm_object kmem_object_store; 146aef922f5SJohn Dyson extern int vm_pageout_page_count; 147df8bae1dSRodney W. Grimes 148f708ef1bSPoul-Henning Kamp static long object_collapses; 149f708ef1bSPoul-Henning Kamp static long object_bypasses; 150dcc5840eSAlan Cox static int next_index; 151670d17b5SJeff Roberson static uma_zone_t obj_zone; 15299448ed1SJohn Dyson #define VM_OBJECTS_INIT 256 1538355f576SJeff Roberson 1548355f576SJeff Roberson static void vm_object_zinit(void *mem, int size); 1558355f576SJeff Roberson 1568355f576SJeff Roberson #ifdef INVARIANTS 1578355f576SJeff Roberson static void vm_object_zdtor(void *mem, int size, void *arg); 1588355f576SJeff Roberson 1598355f576SJeff Roberson static void 1608355f576SJeff Roberson vm_object_zdtor(void *mem, int size, void *arg) 1618355f576SJeff Roberson { 1628355f576SJeff Roberson vm_object_t object; 1638355f576SJeff Roberson 1648355f576SJeff Roberson object = (vm_object_t)mem; 1658355f576SJeff Roberson KASSERT(object->paging_in_progress == 0, 1668355f576SJeff Roberson ("object %p paging_in_progress = %d", 1678355f576SJeff Roberson object, object->paging_in_progress)); 1688355f576SJeff Roberson KASSERT(object->resident_page_count == 0, 1698355f576SJeff Roberson ("object %p resident_page_count = %d", 1708355f576SJeff Roberson object, object->resident_page_count)); 1718355f576SJeff Roberson KASSERT(object->shadow_count == 0, 1728355f576SJeff Roberson ("object %p shadow_count = %d", 1738355f576SJeff Roberson object, object->shadow_count)); 1748355f576SJeff Roberson } 1758355f576SJeff Roberson #endif 1768355f576SJeff Roberson 1778355f576SJeff Roberson static void 1788355f576SJeff Roberson vm_object_zinit(void *mem, int size) 1798355f576SJeff Roberson { 1808355f576SJeff Roberson vm_object_t object; 1818355f576SJeff Roberson 1828355f576SJeff Roberson object = (vm_object_t)mem; 183e3a9e1b2SAlan Cox bzero(&object->mtx, sizeof(object->mtx)); 1848355f576SJeff Roberson 1858355f576SJeff Roberson /* These are true for any object that has been freed */ 1868355f576SJeff Roberson object->paging_in_progress = 0; 1878355f576SJeff Roberson object->resident_page_count = 0; 1888355f576SJeff Roberson object->shadow_count = 0; 1898355f576SJeff Roberson } 190df8bae1dSRodney W. Grimes 1913075778bSJohn Dyson void 1926395da54SIan Dowse _vm_object_allocate(objtype_t type, vm_pindex_t size, vm_object_t object) 193df8bae1dSRodney W. Grimes { 194cf3e6e48SAlan Cox int incr; 1950cddd8f0SMatthew Dillon 196c9917419SAlan Cox mtx_init(&object->mtx, "vm object", NULL, MTX_DEF | MTX_DUPOK); 1974b420d50SAlan Cox 198df8bae1dSRodney W. Grimes TAILQ_INIT(&object->memq); 19924a1cce3SDavid Greenman TAILQ_INIT(&object->shadow_head); 200a1f6d91cSDavid Greenman 201b86ec922SMatthew Dillon object->root = NULL; 20224a1cce3SDavid Greenman object->type = type; 203df8bae1dSRodney W. Grimes object->size = size; 204a1f6d91cSDavid Greenman object->ref_count = 1; 20524a1cce3SDavid Greenman object->flags = 0; 20660517fd1SJohn Baldwin if ((object->type == OBJT_DEFAULT) || (object->type == OBJT_SWAP)) 20760517fd1SJohn Baldwin vm_object_set_flag(object, OBJ_ONEMAPPING); 20899448ed1SJohn Dyson if (size > (PQ_L2_SIZE / 3 + PQ_PRIME1)) 20999448ed1SJohn Dyson incr = PQ_L2_SIZE / 3 + PQ_PRIME1; 21099448ed1SJohn Dyson else 21199448ed1SJohn Dyson incr = size; 212dcc5840eSAlan Cox do 213dcc5840eSAlan Cox object->pg_color = next_index; 214dcc5840eSAlan Cox while (!atomic_cmpset_int(&next_index, object->pg_color, 21579660d83SAlan Cox (object->pg_color + incr) & PQ_L2_MASK)); 21624a1cce3SDavid Greenman object->handle = NULL; 21724a1cce3SDavid Greenman object->backing_object = NULL; 218a316d390SJohn Dyson object->backing_object_offset = (vm_ooffset_t) 0; 219a1f6d91cSDavid Greenman 22081f71edaSMatt Jacob atomic_add_int(&object->generation, 1); 22160517fd1SJohn Baldwin 222a5698387SAlan Cox mtx_lock(&vm_object_list_mtx); 22360517fd1SJohn Baldwin TAILQ_INSERT_TAIL(&vm_object_list, object, object_list); 224a5698387SAlan Cox mtx_unlock(&vm_object_list_mtx); 225df8bae1dSRodney W. Grimes } 226df8bae1dSRodney W. Grimes 227df8bae1dSRodney W. Grimes /* 22826f9a767SRodney W. Grimes * vm_object_init: 22926f9a767SRodney W. Grimes * 23026f9a767SRodney W. Grimes * Initialize the VM objects module. 23126f9a767SRodney W. Grimes */ 23226f9a767SRodney W. Grimes void 2331b40f8c0SMatthew Dillon vm_object_init(void) 23426f9a767SRodney W. Grimes { 23526f9a767SRodney W. Grimes TAILQ_INIT(&vm_object_list); 2366008862bSJohn Baldwin mtx_init(&vm_object_list_mtx, "vm object_list", NULL, MTX_DEF); 2370217125fSDavid Greenman 23826f9a767SRodney W. Grimes kernel_object = &kernel_object_store; 239a316d390SJohn Dyson _vm_object_allocate(OBJT_DEFAULT, OFF_TO_IDX(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS), 24026f9a767SRodney W. Grimes kernel_object); 24126f9a767SRodney W. Grimes 24226f9a767SRodney W. Grimes kmem_object = &kmem_object_store; 243a316d390SJohn Dyson _vm_object_allocate(OBJT_DEFAULT, OFF_TO_IDX(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS), 24426f9a767SRodney W. Grimes kmem_object); 2458355f576SJeff Roberson obj_zone = uma_zcreate("VM OBJECT", sizeof (struct vm_object), NULL, 2468355f576SJeff Roberson #ifdef INVARIANTS 2478355f576SJeff Roberson vm_object_zdtor, 2488355f576SJeff Roberson #else 2498355f576SJeff Roberson NULL, 2508355f576SJeff Roberson #endif 2518355f576SJeff Roberson vm_object_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); 2528355f576SJeff Roberson uma_prealloc(obj_zone, VM_OBJECTS_INIT); 25399448ed1SJohn Dyson } 25499448ed1SJohn Dyson 25599448ed1SJohn Dyson void 2561b40f8c0SMatthew Dillon vm_object_set_flag(vm_object_t object, u_short bits) 2571b40f8c0SMatthew Dillon { 258b06805adSJake Burkholder object->flags |= bits; 2591b40f8c0SMatthew Dillon } 2601b40f8c0SMatthew Dillon 2611b40f8c0SMatthew Dillon void 2621b40f8c0SMatthew Dillon vm_object_clear_flag(vm_object_t object, u_short bits) 2631b40f8c0SMatthew Dillon { 2645440b5a9SAlan Cox 265d647a0edSAlan Cox VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 266b06805adSJake Burkholder object->flags &= ~bits; 2671b40f8c0SMatthew Dillon } 2681b40f8c0SMatthew Dillon 2691b40f8c0SMatthew Dillon void 2701b40f8c0SMatthew Dillon vm_object_pip_add(vm_object_t object, short i) 2711b40f8c0SMatthew Dillon { 272f279b88dSAlan Cox 273d647a0edSAlan Cox VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 274b06805adSJake Burkholder object->paging_in_progress += i; 2751b40f8c0SMatthew Dillon } 2761b40f8c0SMatthew Dillon 2771b40f8c0SMatthew Dillon void 2781b40f8c0SMatthew Dillon vm_object_pip_subtract(vm_object_t object, short i) 2791b40f8c0SMatthew Dillon { 280d647a0edSAlan Cox 2810fa05eaeSAlan Cox VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 282b06805adSJake Burkholder object->paging_in_progress -= i; 2831b40f8c0SMatthew Dillon } 2841b40f8c0SMatthew Dillon 2851b40f8c0SMatthew Dillon void 2861b40f8c0SMatthew Dillon vm_object_pip_wakeup(vm_object_t object) 2871b40f8c0SMatthew Dillon { 288f279b88dSAlan Cox 289d647a0edSAlan Cox VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 290b06805adSJake Burkholder object->paging_in_progress--; 2911b40f8c0SMatthew Dillon if ((object->flags & OBJ_PIPWNT) && object->paging_in_progress == 0) { 2921b40f8c0SMatthew Dillon vm_object_clear_flag(object, OBJ_PIPWNT); 2931b40f8c0SMatthew Dillon wakeup(object); 2941b40f8c0SMatthew Dillon } 2951b40f8c0SMatthew Dillon } 2961b40f8c0SMatthew Dillon 2971b40f8c0SMatthew Dillon void 2981b40f8c0SMatthew Dillon vm_object_pip_wakeupn(vm_object_t object, short i) 2991b40f8c0SMatthew Dillon { 300d647a0edSAlan Cox 3010d420ad3SAlan Cox VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 3021b40f8c0SMatthew Dillon if (i) 303b06805adSJake Burkholder object->paging_in_progress -= i; 3041b40f8c0SMatthew Dillon if ((object->flags & OBJ_PIPWNT) && object->paging_in_progress == 0) { 3051b40f8c0SMatthew Dillon vm_object_clear_flag(object, OBJ_PIPWNT); 3061b40f8c0SMatthew Dillon wakeup(object); 3071b40f8c0SMatthew Dillon } 3081b40f8c0SMatthew Dillon } 3091b40f8c0SMatthew Dillon 3101ca58953SAlan Cox static void 3111b40f8c0SMatthew Dillon vm_object_pip_sleep(vm_object_t object, char *waitid) 3121b40f8c0SMatthew Dillon { 3131b40f8c0SMatthew Dillon GIANT_REQUIRED; 3141b40f8c0SMatthew Dillon if (object->paging_in_progress) { 3151b40f8c0SMatthew Dillon int s = splvm(); 3161b40f8c0SMatthew Dillon if (object->paging_in_progress) { 3171b40f8c0SMatthew Dillon vm_object_set_flag(object, OBJ_PIPWNT); 3181b40f8c0SMatthew Dillon tsleep(object, PVM, waitid, 0); 3191b40f8c0SMatthew Dillon } 3201b40f8c0SMatthew Dillon splx(s); 3211b40f8c0SMatthew Dillon } 3221b40f8c0SMatthew Dillon } 3231b40f8c0SMatthew Dillon 3241b40f8c0SMatthew Dillon void 3251b40f8c0SMatthew Dillon vm_object_pip_wait(vm_object_t object, char *waitid) 3261b40f8c0SMatthew Dillon { 3271ca58953SAlan Cox 3281ca58953SAlan Cox VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 3291ca58953SAlan Cox while (object->paging_in_progress) { 3301ca58953SAlan Cox object->flags |= OBJ_PIPWNT; 3311ca58953SAlan Cox msleep(object, VM_OBJECT_MTX(object), PVM, waitid, 0); 3321ca58953SAlan Cox } 3331b40f8c0SMatthew Dillon } 3341b40f8c0SMatthew Dillon 33526f9a767SRodney W. Grimes /* 33698cb733cSKenneth D. Merry * vm_object_allocate_wait 33798cb733cSKenneth D. Merry * 33898cb733cSKenneth D. Merry * Return a new object with the given size, and give the user the 33998cb733cSKenneth D. Merry * option of waiting for it to complete or failing if the needed 34098cb733cSKenneth D. Merry * memory isn't available. 34198cb733cSKenneth D. Merry */ 34298cb733cSKenneth D. Merry vm_object_t 34398cb733cSKenneth D. Merry vm_object_allocate_wait(objtype_t type, vm_pindex_t size, int flags) 34498cb733cSKenneth D. Merry { 34598cb733cSKenneth D. Merry vm_object_t result; 34698cb733cSKenneth D. Merry 34798cb733cSKenneth D. Merry result = (vm_object_t) uma_zalloc(obj_zone, flags); 34898cb733cSKenneth D. Merry 34998cb733cSKenneth D. Merry if (result != NULL) 35098cb733cSKenneth D. Merry _vm_object_allocate(type, size, result); 35198cb733cSKenneth D. Merry 35298cb733cSKenneth D. Merry return (result); 35398cb733cSKenneth D. Merry } 35498cb733cSKenneth D. Merry 35598cb733cSKenneth D. Merry /* 35626f9a767SRodney W. Grimes * vm_object_allocate: 35726f9a767SRodney W. Grimes * 35826f9a767SRodney W. Grimes * Returns a new object with the given size. 35926f9a767SRodney W. Grimes */ 36026f9a767SRodney W. Grimes vm_object_t 3616395da54SIan Dowse vm_object_allocate(objtype_t type, vm_pindex_t size) 36226f9a767SRodney W. Grimes { 363a163d034SWarner Losh return(vm_object_allocate_wait(type, size, M_WAITOK)); 36426f9a767SRodney W. Grimes } 36526f9a767SRodney W. Grimes 36626f9a767SRodney W. Grimes 36726f9a767SRodney W. Grimes /* 368df8bae1dSRodney W. Grimes * vm_object_reference: 369df8bae1dSRodney W. Grimes * 370df8bae1dSRodney W. Grimes * Gets another reference to the given object. 371df8bae1dSRodney W. Grimes */ 3726476c0d2SJohn Dyson void 3731b40f8c0SMatthew Dillon vm_object_reference(vm_object_t object) 374df8bae1dSRodney W. Grimes { 375df8bae1dSRodney W. Grimes if (object == NULL) 376df8bae1dSRodney W. Grimes return; 37795e5e988SJohn Dyson 37815c176c1SAlan Cox vm_object_lock(object); 379245df27cSMatthew Dillon #if 0 380245df27cSMatthew Dillon /* object can be re-referenced during final cleaning */ 3815526d2d9SEivind Eklund KASSERT(!(object->flags & OBJ_DEAD), 3825526d2d9SEivind Eklund ("vm_object_reference: attempting to reference dead obj")); 383245df27cSMatthew Dillon #endif 38495e5e988SJohn Dyson 385df8bae1dSRodney W. Grimes object->ref_count++; 38647221757SJohn Dyson if (object->type == OBJT_VNODE) { 3873ef3e7c4SJeff Roberson while (vget((struct vnode *) object->handle, LK_RETRY, curthread)) { 38847221757SJohn Dyson printf("vm_object_reference: delay in getting object\n"); 38947221757SJohn Dyson } 39047221757SJohn Dyson } 39115c176c1SAlan Cox vm_object_unlock(object); 39295e5e988SJohn Dyson } 39395e5e988SJohn Dyson 39423955314SAlfred Perlstein /* 3959d5abbddSJens Schweikhardt * Handle deallocating an object of type OBJT_VNODE. 39623955314SAlfred Perlstein */ 397bf27292bSJohn Dyson void 3981b40f8c0SMatthew Dillon vm_object_vndeallocate(vm_object_t object) 39995e5e988SJohn Dyson { 40095e5e988SJohn Dyson struct vnode *vp = (struct vnode *) object->handle; 401219cbf59SEivind Eklund 4020cddd8f0SMatthew Dillon GIANT_REQUIRED; 4035526d2d9SEivind Eklund KASSERT(object->type == OBJT_VNODE, 4045526d2d9SEivind Eklund ("vm_object_vndeallocate: not a vnode object")); 405219cbf59SEivind Eklund KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp")); 406219cbf59SEivind Eklund #ifdef INVARIANTS 40795e5e988SJohn Dyson if (object->ref_count == 0) { 40895e5e988SJohn Dyson vprint("vm_object_vndeallocate", vp); 40995e5e988SJohn Dyson panic("vm_object_vndeallocate: bad object reference count"); 41095e5e988SJohn Dyson } 41195e5e988SJohn Dyson #endif 41295e5e988SJohn Dyson 41395e5e988SJohn Dyson object->ref_count--; 41447221757SJohn Dyson if (object->ref_count == 0) { 415e6e370a7SJeff Roberson mp_fixme("Unlocked vflag access."); 416e6e370a7SJeff Roberson vp->v_vflag &= ~VV_TEXT; 4172be70f79SJohn Dyson } 41823955314SAlfred Perlstein /* 41923955314SAlfred Perlstein * vrele may need a vop lock 42023955314SAlfred Perlstein */ 42147221757SJohn Dyson vrele(vp); 422df8bae1dSRodney W. Grimes } 423df8bae1dSRodney W. Grimes 424df8bae1dSRodney W. Grimes /* 425df8bae1dSRodney W. Grimes * vm_object_deallocate: 426df8bae1dSRodney W. Grimes * 427df8bae1dSRodney W. Grimes * Release a reference to the specified object, 428df8bae1dSRodney W. Grimes * gained either through a vm_object_allocate 429df8bae1dSRodney W. Grimes * or a vm_object_reference call. When all references 430df8bae1dSRodney W. Grimes * are gone, storage associated with this object 431df8bae1dSRodney W. Grimes * may be relinquished. 432df8bae1dSRodney W. Grimes * 433df8bae1dSRodney W. Grimes * No object may be locked. 434df8bae1dSRodney W. Grimes */ 43526f9a767SRodney W. Grimes void 4361b40f8c0SMatthew Dillon vm_object_deallocate(vm_object_t object) 437df8bae1dSRodney W. Grimes { 438df8bae1dSRodney W. Grimes vm_object_t temp; 439df8bae1dSRodney W. Grimes 4404dbeceeeSAlan Cox vm_object_lock(object); 441df8bae1dSRodney W. Grimes while (object != NULL) { 442df8bae1dSRodney W. Grimes 44395e5e988SJohn Dyson if (object->type == OBJT_VNODE) { 44495e5e988SJohn Dyson vm_object_vndeallocate(object); 4454dbeceeeSAlan Cox vm_object_unlock(object); 44695e5e988SJohn Dyson return; 44795e5e988SJohn Dyson } 44895e5e988SJohn Dyson 4498125b1e6SAlfred Perlstein KASSERT(object->ref_count != 0, 4508125b1e6SAlfred Perlstein ("vm_object_deallocate: object deallocated too many times: %d", object->type)); 4512be70f79SJohn Dyson 4522be70f79SJohn Dyson /* 4538125b1e6SAlfred Perlstein * If the reference count goes to 0 we start calling 4548125b1e6SAlfred Perlstein * vm_object_terminate() on the object chain. 4558125b1e6SAlfred Perlstein * A ref count of 1 may be a special case depending on the 4568125b1e6SAlfred Perlstein * shadow count being 0 or 1. 4572be70f79SJohn Dyson */ 458c0877f10SJohn Dyson object->ref_count--; 4598125b1e6SAlfred Perlstein if (object->ref_count > 1) { 4604dbeceeeSAlan Cox vm_object_unlock(object); 461c0877f10SJohn Dyson return; 4628125b1e6SAlfred Perlstein } else if (object->ref_count == 1) { 4638125b1e6SAlfred Perlstein if (object->shadow_count == 0) { 4648125b1e6SAlfred Perlstein vm_object_set_flag(object, OBJ_ONEMAPPING); 4658125b1e6SAlfred Perlstein } else if ((object->shadow_count == 1) && 4668125b1e6SAlfred Perlstein (object->handle == NULL) && 46724a1cce3SDavid Greenman (object->type == OBJT_DEFAULT || 46824a1cce3SDavid Greenman object->type == OBJT_SWAP)) { 469a1f6d91cSDavid Greenman vm_object_t robject; 47095e5e988SJohn Dyson 471b18bfc3dSJohn Dyson robject = TAILQ_FIRST(&object->shadow_head); 4725526d2d9SEivind Eklund KASSERT(robject != NULL, 473219cbf59SEivind Eklund ("vm_object_deallocate: ref_count: %d, shadow_count: %d", 4745526d2d9SEivind Eklund object->ref_count, 4755526d2d9SEivind Eklund object->shadow_count)); 47695e5e988SJohn Dyson if ((robject->handle == NULL) && 47724a1cce3SDavid Greenman (robject->type == OBJT_DEFAULT || 47824a1cce3SDavid Greenman robject->type == OBJT_SWAP)) { 479a1f6d91cSDavid Greenman 48095e5e988SJohn Dyson robject->ref_count++; 48195e5e988SJohn Dyson 4821c7c3c6aSMatthew Dillon while ( 4831c7c3c6aSMatthew Dillon robject->paging_in_progress || 4841c7c3c6aSMatthew Dillon object->paging_in_progress 4851c7c3c6aSMatthew Dillon ) { 48666095752SJohn Dyson vm_object_pip_sleep(robject, "objde1"); 48766095752SJohn Dyson vm_object_pip_sleep(object, "objde2"); 488a1f6d91cSDavid Greenman } 489a1f6d91cSDavid Greenman 49095e5e988SJohn Dyson if (robject->ref_count == 1) { 49195e5e988SJohn Dyson robject->ref_count--; 492ba8da839SDavid Greenman object = robject; 49395e5e988SJohn Dyson goto doterm; 49495e5e988SJohn Dyson } 49595e5e988SJohn Dyson 49695e5e988SJohn Dyson object = robject; 49795e5e988SJohn Dyson vm_object_collapse(object); 498ba8da839SDavid Greenman continue; 499a1f6d91cSDavid Greenman } 50095e5e988SJohn Dyson } 5014dbeceeeSAlan Cox vm_object_unlock(object); 502a1f6d91cSDavid Greenman return; 50395e5e988SJohn Dyson } 50495e5e988SJohn Dyson doterm: 505c829b9d0SAlan Cox VM_OBJECT_LOCK(object); 50624a1cce3SDavid Greenman temp = object->backing_object; 507c9917419SAlan Cox if (temp != NULL) { 508c9917419SAlan Cox VM_OBJECT_LOCK(temp); 50924a1cce3SDavid Greenman TAILQ_REMOVE(&temp->shadow_head, object, shadow_list); 51095e5e988SJohn Dyson temp->shadow_count--; 511eaf13dd7SJohn Dyson temp->generation++; 512c9917419SAlan Cox VM_OBJECT_UNLOCK(temp); 51395461b45SJohn Dyson object->backing_object = NULL; 514de5f6a77SJohn Dyson } 515245df27cSMatthew Dillon /* 516245df27cSMatthew Dillon * Don't double-terminate, we could be in a termination 517245df27cSMatthew Dillon * recursion due to the terminate having to sync data 518245df27cSMatthew Dillon * to disk. 519245df27cSMatthew Dillon */ 520245df27cSMatthew Dillon if ((object->flags & OBJ_DEAD) == 0) 521df8bae1dSRodney W. Grimes vm_object_terminate(object); 522c829b9d0SAlan Cox else 523c829b9d0SAlan Cox VM_OBJECT_UNLOCK(object); 524df8bae1dSRodney W. Grimes object = temp; 525df8bae1dSRodney W. Grimes } 5264dbeceeeSAlan Cox vm_object_unlock(object); 527df8bae1dSRodney W. Grimes } 528df8bae1dSRodney W. Grimes 529df8bae1dSRodney W. Grimes /* 530df8bae1dSRodney W. Grimes * vm_object_terminate actually destroys the specified object, freeing 531df8bae1dSRodney W. Grimes * up all previously used resources. 532df8bae1dSRodney W. Grimes * 533df8bae1dSRodney W. Grimes * The object must be locked. 5341c7c3c6aSMatthew Dillon * This routine may block. 535df8bae1dSRodney W. Grimes */ 53695e5e988SJohn Dyson void 5371b40f8c0SMatthew Dillon vm_object_terminate(vm_object_t object) 538df8bae1dSRodney W. Grimes { 539d031cff1SMatthew Dillon vm_page_t p; 540e4b7635dSDavid Greenman int s; 541df8bae1dSRodney W. Grimes 542c829b9d0SAlan Cox VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 5430cddd8f0SMatthew Dillon 54495e5e988SJohn Dyson /* 54595e5e988SJohn Dyson * Make sure no one uses us. 54695e5e988SJohn Dyson */ 547069e9bc1SDoug Rabson vm_object_set_flag(object, OBJ_DEAD); 5483c631446SJohn Dyson 549df8bae1dSRodney W. Grimes /* 550f6b04d2bSDavid Greenman * wait for the pageout daemon to be done with the object 551df8bae1dSRodney W. Grimes */ 55266095752SJohn Dyson vm_object_pip_wait(object, "objtrm"); 5531ca58953SAlan Cox VM_OBJECT_UNLOCK(object); 554df8bae1dSRodney W. Grimes 5555526d2d9SEivind Eklund KASSERT(!object->paging_in_progress, 5565526d2d9SEivind Eklund ("vm_object_terminate: pageout in progress")); 55726f9a767SRodney W. Grimes 55826f9a767SRodney W. Grimes /* 5590d94caffSDavid Greenman * Clean and free the pages, as appropriate. All references to the 5600d94caffSDavid Greenman * object are gone, so we don't need to lock it. 56126f9a767SRodney W. Grimes */ 56224a1cce3SDavid Greenman if (object->type == OBJT_VNODE) { 56395e5e988SJohn Dyson struct vnode *vp; 56495e5e988SJohn Dyson 56595e5e988SJohn Dyson /* 56695e5e988SJohn Dyson * Clean pages and flush buffers. 56795e5e988SJohn Dyson */ 568b6e48e03SAlan Cox VM_OBJECT_LOCK(object); 5698f9110f6SJohn Dyson vm_object_page_clean(object, 0, 0, OBJPC_SYNC); 570b6e48e03SAlan Cox VM_OBJECT_UNLOCK(object); 57195e5e988SJohn Dyson 57295e5e988SJohn Dyson vp = (struct vnode *) object->handle; 573f6b04d2bSDavid Greenman vinvalbuf(vp, V_SAVE, NOCRED, NULL, 0, 0); 574bef608bdSJohn Dyson } 575bef608bdSJohn Dyson 576971dd342SAlfred Perlstein KASSERT(object->ref_count == 0, 577971dd342SAlfred Perlstein ("vm_object_terminate: object with references, ref_count=%d", 578971dd342SAlfred Perlstein object->ref_count)); 579996c772fSJohn Dyson 5800d94caffSDavid Greenman /* 581356863ebSDavid Greenman * Now free any remaining pages. For internal objects, this also 582356863ebSDavid Greenman * removes them from paging queues. Don't free wired pages, just 583356863ebSDavid Greenman * remove them from the object. 584df8bae1dSRodney W. Grimes */ 585e4b7635dSDavid Greenman s = splvm(); 58656030358SAlan Cox vm_page_lock_queues(); 587b18bfc3dSJohn Dyson while ((p = TAILQ_FIRST(&object->memq)) != NULL) { 588971dd342SAlfred Perlstein KASSERT(!p->busy && (p->flags & PG_BUSY) == 0, 589971dd342SAlfred Perlstein ("vm_object_terminate: freeing busy page %p " 590971dd342SAlfred Perlstein "p->busy = %d, p->flags %x\n", p, p->busy, p->flags)); 5910b10ba98SDavid Greenman if (p->wire_count == 0) { 592e69763a3SDoug Rabson vm_page_busy(p); 593df8bae1dSRodney W. Grimes vm_page_free(p); 594df8bae1dSRodney W. Grimes cnt.v_pfree++; 5950b10ba98SDavid Greenman } else { 596356863ebSDavid Greenman vm_page_busy(p); 5970b10ba98SDavid Greenman vm_page_remove(p); 5980b10ba98SDavid Greenman } 599df8bae1dSRodney W. Grimes } 60056030358SAlan Cox vm_page_unlock_queues(); 601e4b7635dSDavid Greenman splx(s); 602bef608bdSJohn Dyson 6032d8acc0fSJohn Dyson /* 6049fcfb650SDavid Greenman * Let the pager know object is dead. 6059fcfb650SDavid Greenman */ 6069fcfb650SDavid Greenman vm_pager_deallocate(object); 6079fcfb650SDavid Greenman 6089fcfb650SDavid Greenman /* 6090b10ba98SDavid Greenman * Remove the object from the global object list. 6102d8acc0fSJohn Dyson */ 6119ed346baSBosko Milekic mtx_lock(&vm_object_list_mtx); 612df8bae1dSRodney W. Grimes TAILQ_REMOVE(&vm_object_list, object, object_list); 6139ed346baSBosko Milekic mtx_unlock(&vm_object_list_mtx); 6140b10ba98SDavid Greenman 6154b420d50SAlan Cox mtx_destroy(&object->mtx); 6160b10ba98SDavid Greenman wakeup(object); 6170b10ba98SDavid Greenman 618df8bae1dSRodney W. Grimes /* 619df8bae1dSRodney W. Grimes * Free the space for the object. 620df8bae1dSRodney W. Grimes */ 621670d17b5SJeff Roberson uma_zfree(obj_zone, object); 62247221757SJohn Dyson } 623df8bae1dSRodney W. Grimes 624df8bae1dSRodney W. Grimes /* 625df8bae1dSRodney W. Grimes * vm_object_page_clean 626df8bae1dSRodney W. Grimes * 6274f79d873SMatthew Dillon * Clean all dirty pages in the specified range of object. Leaves page 6284f79d873SMatthew Dillon * on whatever queue it is currently on. If NOSYNC is set then do not 6294f79d873SMatthew Dillon * write out pages with PG_NOSYNC set (originally comes from MAP_NOSYNC), 6304f79d873SMatthew Dillon * leaving the object dirty. 63126f9a767SRodney W. Grimes * 63243b7990eSMatthew Dillon * When stuffing pages asynchronously, allow clustering. XXX we need a 63343b7990eSMatthew Dillon * synchronous clustering mode implementation. 63443b7990eSMatthew Dillon * 63526f9a767SRodney W. Grimes * Odd semantics: if start == end, we clean everything. 63626f9a767SRodney W. Grimes * 63726f9a767SRodney W. Grimes * The object must be locked. 63826f9a767SRodney W. Grimes */ 639f6b04d2bSDavid Greenman void 6401b40f8c0SMatthew Dillon vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end, int flags) 641f6b04d2bSDavid Greenman { 642b9b7a4beSMatthew Dillon vm_page_t p, np; 6436395da54SIan Dowse vm_pindex_t tstart, tend; 644bd7e5f99SJohn Dyson vm_pindex_t pi; 64524a1cce3SDavid Greenman struct vnode *vp; 6464f79d873SMatthew Dillon int clearobjflags; 6478f9110f6SJohn Dyson int pagerflags; 6482d8acc0fSJohn Dyson int curgeneration; 649f6b04d2bSDavid Greenman 6500cddd8f0SMatthew Dillon GIANT_REQUIRED; 651b6e48e03SAlan Cox VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 652aef922f5SJohn Dyson if (object->type != OBJT_VNODE || 653aef922f5SJohn Dyson (object->flags & OBJ_MIGHTBEDIRTY) == 0) 654f6b04d2bSDavid Greenman return; 655f6b04d2bSDavid Greenman 65643b7990eSMatthew Dillon pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) ? VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK; 6578f9110f6SJohn Dyson pagerflags |= (flags & OBJPC_INVAL) ? VM_PAGER_PUT_INVAL : 0; 6588f9110f6SJohn Dyson 65924a1cce3SDavid Greenman vp = object->handle; 66024a1cce3SDavid Greenman 661069e9bc1SDoug Rabson vm_object_set_flag(object, OBJ_CLEANING); 66224a1cce3SDavid Greenman 663f6b04d2bSDavid Greenman tstart = start; 664f6b04d2bSDavid Greenman if (end == 0) { 665f6b04d2bSDavid Greenman tend = object->size; 666f6b04d2bSDavid Greenman } else { 667f6b04d2bSDavid Greenman tend = end; 668f6b04d2bSDavid Greenman } 669eaf13dd7SJohn Dyson 67035c01631SAlan Cox vm_page_lock_queues(); 6714f79d873SMatthew Dillon /* 672b9b7a4beSMatthew Dillon * If the caller is smart and only msync()s a range he knows is 673b9b7a4beSMatthew Dillon * dirty, we may be able to avoid an object scan. This results in 674b9b7a4beSMatthew Dillon * a phenominal improvement in performance. We cannot do this 675b9b7a4beSMatthew Dillon * as a matter of course because the object may be huge - e.g. 676b9b7a4beSMatthew Dillon * the size might be in the gigabytes or terrabytes. 677b9b7a4beSMatthew Dillon */ 678b9b7a4beSMatthew Dillon if (msync_flush_flags & MSYNC_FLUSH_HARDSEQ) { 679300b96acSIan Dowse vm_pindex_t tscan; 680b9b7a4beSMatthew Dillon int scanlimit; 681b9b7a4beSMatthew Dillon int scanreset; 682b9b7a4beSMatthew Dillon 683b9b7a4beSMatthew Dillon scanreset = object->resident_page_count / EASY_SCAN_FACTOR; 684b9b7a4beSMatthew Dillon if (scanreset < 16) 685b9b7a4beSMatthew Dillon scanreset = 16; 68643b7990eSMatthew Dillon pagerflags |= VM_PAGER_IGNORE_CLEANCHK; 687b9b7a4beSMatthew Dillon 688b9b7a4beSMatthew Dillon scanlimit = scanreset; 689b9b7a4beSMatthew Dillon tscan = tstart; 690b9b7a4beSMatthew Dillon while (tscan < tend) { 691b9b7a4beSMatthew Dillon curgeneration = object->generation; 692b9b7a4beSMatthew Dillon p = vm_page_lookup(object, tscan); 693b9b7a4beSMatthew Dillon if (p == NULL || p->valid == 0 || 694b9b7a4beSMatthew Dillon (p->queue - p->pc) == PQ_CACHE) { 695b9b7a4beSMatthew Dillon if (--scanlimit == 0) 696b9b7a4beSMatthew Dillon break; 697b9b7a4beSMatthew Dillon ++tscan; 698b9b7a4beSMatthew Dillon continue; 699b9b7a4beSMatthew Dillon } 700b9b7a4beSMatthew Dillon vm_page_test_dirty(p); 701b9b7a4beSMatthew Dillon if ((p->dirty & p->valid) == 0) { 702b9b7a4beSMatthew Dillon if (--scanlimit == 0) 703b9b7a4beSMatthew Dillon break; 704b9b7a4beSMatthew Dillon ++tscan; 705b9b7a4beSMatthew Dillon continue; 706b9b7a4beSMatthew Dillon } 707b9b7a4beSMatthew Dillon /* 708b9b7a4beSMatthew Dillon * If we have been asked to skip nosync pages and 709b9b7a4beSMatthew Dillon * this is a nosync page, we can't continue. 710b9b7a4beSMatthew Dillon */ 711b9b7a4beSMatthew Dillon if ((flags & OBJPC_NOSYNC) && (p->flags & PG_NOSYNC)) { 712b9b7a4beSMatthew Dillon if (--scanlimit == 0) 713b9b7a4beSMatthew Dillon break; 714b9b7a4beSMatthew Dillon ++tscan; 715b9b7a4beSMatthew Dillon continue; 716b9b7a4beSMatthew Dillon } 717b9b7a4beSMatthew Dillon scanlimit = scanreset; 718b9b7a4beSMatthew Dillon 719b9b7a4beSMatthew Dillon /* 720b9b7a4beSMatthew Dillon * This returns 0 if it was unable to busy the first 721b9b7a4beSMatthew Dillon * page (i.e. had to sleep). 722b9b7a4beSMatthew Dillon */ 723b9b7a4beSMatthew Dillon tscan += vm_object_page_collect_flush(object, p, curgeneration, pagerflags); 724b9b7a4beSMatthew Dillon } 725b9b7a4beSMatthew Dillon 726b9b7a4beSMatthew Dillon /* 727b9b7a4beSMatthew Dillon * If everything was dirty and we flushed it successfully, 728b9b7a4beSMatthew Dillon * and the requested range is not the entire object, we 729b9b7a4beSMatthew Dillon * don't have to mess with CLEANCHK or MIGHTBEDIRTY and can 730b9b7a4beSMatthew Dillon * return immediately. 731b9b7a4beSMatthew Dillon */ 732b9b7a4beSMatthew Dillon if (tscan >= tend && (tstart || tend < object->size)) { 73335c01631SAlan Cox vm_page_unlock_queues(); 734b9b7a4beSMatthew Dillon vm_object_clear_flag(object, OBJ_CLEANING); 735b9b7a4beSMatthew Dillon return; 736b9b7a4beSMatthew Dillon } 73743b7990eSMatthew Dillon pagerflags &= ~VM_PAGER_IGNORE_CLEANCHK; 738b9b7a4beSMatthew Dillon } 739b9b7a4beSMatthew Dillon 740b9b7a4beSMatthew Dillon /* 7414f79d873SMatthew Dillon * Generally set CLEANCHK interlock and make the page read-only so 7424f79d873SMatthew Dillon * we can then clear the object flags. 7434f79d873SMatthew Dillon * 7444f79d873SMatthew Dillon * However, if this is a nosync mmap then the object is likely to 7454f79d873SMatthew Dillon * stay dirty so do not mess with the page and do not clear the 7464f79d873SMatthew Dillon * object flags. 7474f79d873SMatthew Dillon */ 7484f79d873SMatthew Dillon clearobjflags = 1; 749fc2ffbe6SPoul-Henning Kamp TAILQ_FOREACH(p, &object->memq, listq) { 750e69763a3SDoug Rabson vm_page_flag_set(p, PG_CLEANCHK); 7514f79d873SMatthew Dillon if ((flags & OBJPC_NOSYNC) && (p->flags & PG_NOSYNC)) 7524f79d873SMatthew Dillon clearobjflags = 0; 7534f79d873SMatthew Dillon else 754a12cc0e4SAlan Cox pmap_page_protect(p, VM_PROT_READ); 755eaf13dd7SJohn Dyson } 756eaf13dd7SJohn Dyson 7574f79d873SMatthew Dillon if (clearobjflags && (tstart == 0) && (tend == object->size)) { 758245df27cSMatthew Dillon struct vnode *vp; 759245df27cSMatthew Dillon 760069e9bc1SDoug Rabson vm_object_clear_flag(object, OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY); 761245df27cSMatthew Dillon if (object->type == OBJT_VNODE && 762245df27cSMatthew Dillon (vp = (struct vnode *)object->handle) != NULL) { 763e6e370a7SJeff Roberson VI_LOCK(vp); 764e6e370a7SJeff Roberson if (vp->v_iflag & VI_OBJDIRTY) 765e6e370a7SJeff Roberson vp->v_iflag &= ~VI_OBJDIRTY; 766e6e370a7SJeff Roberson VI_UNLOCK(vp); 767245df27cSMatthew Dillon } 768ec4f9fb0SDavid Greenman } 769f6b04d2bSDavid Greenman 770bd7e5f99SJohn Dyson rescan: 7712d8acc0fSJohn Dyson curgeneration = object->generation; 7722d8acc0fSJohn Dyson 773b18bfc3dSJohn Dyson for (p = TAILQ_FIRST(&object->memq); p; p = np) { 774b9b7a4beSMatthew Dillon int n; 775b9b7a4beSMatthew Dillon 776b18bfc3dSJohn Dyson np = TAILQ_NEXT(p, listq); 777bd7e5f99SJohn Dyson 778b9b7a4beSMatthew Dillon again: 779bd7e5f99SJohn Dyson pi = p->pindex; 780bd7e5f99SJohn Dyson if (((p->flags & PG_CLEANCHK) == 0) || 781bd7e5f99SJohn Dyson (pi < tstart) || (pi >= tend) || 7825070c7f8SJohn Dyson (p->valid == 0) || 7835070c7f8SJohn Dyson ((p->queue - p->pc) == PQ_CACHE)) { 784e69763a3SDoug Rabson vm_page_flag_clear(p, PG_CLEANCHK); 785aef922f5SJohn Dyson continue; 786f6b04d2bSDavid Greenman } 787f6b04d2bSDavid Greenman 788bd7e5f99SJohn Dyson vm_page_test_dirty(p); 789bd7e5f99SJohn Dyson if ((p->dirty & p->valid) == 0) { 790e69763a3SDoug Rabson vm_page_flag_clear(p, PG_CLEANCHK); 791bd7e5f99SJohn Dyson continue; 792bd7e5f99SJohn Dyson } 793ec4f9fb0SDavid Greenman 7944f79d873SMatthew Dillon /* 7954f79d873SMatthew Dillon * If we have been asked to skip nosync pages and this is a 7964f79d873SMatthew Dillon * nosync page, skip it. Note that the object flags were 7974f79d873SMatthew Dillon * not cleared in this case so we do not have to set them. 7984f79d873SMatthew Dillon */ 7994f79d873SMatthew Dillon if ((flags & OBJPC_NOSYNC) && (p->flags & PG_NOSYNC)) { 8004f79d873SMatthew Dillon vm_page_flag_clear(p, PG_CLEANCHK); 8014f79d873SMatthew Dillon continue; 8024f79d873SMatthew Dillon } 8034f79d873SMatthew Dillon 804b9b7a4beSMatthew Dillon n = vm_object_page_collect_flush(object, p, 805b9b7a4beSMatthew Dillon curgeneration, pagerflags); 806b9b7a4beSMatthew Dillon if (n == 0) 807b9b7a4beSMatthew Dillon goto rescan; 808b9b7a4beSMatthew Dillon 809b9b7a4beSMatthew Dillon if (object->generation != curgeneration) 810b9b7a4beSMatthew Dillon goto rescan; 811b9b7a4beSMatthew Dillon 812b9b7a4beSMatthew Dillon /* 813b9b7a4beSMatthew Dillon * Try to optimize the next page. If we can't we pick up 814b9b7a4beSMatthew Dillon * our (random) scan where we left off. 815b9b7a4beSMatthew Dillon */ 816b9b7a4beSMatthew Dillon if (msync_flush_flags & MSYNC_FLUSH_SOFTSEQ) { 817b9b7a4beSMatthew Dillon if ((p = vm_page_lookup(object, pi + n)) != NULL) 818b9b7a4beSMatthew Dillon goto again; 819b9b7a4beSMatthew Dillon } 820b9b7a4beSMatthew Dillon } 82135c01631SAlan Cox vm_page_unlock_queues(); 822b9b7a4beSMatthew Dillon #if 0 823b9b7a4beSMatthew Dillon VOP_FSYNC(vp, NULL, (pagerflags & VM_PAGER_PUT_SYNC)?MNT_WAIT:0, curproc); 824b9b7a4beSMatthew Dillon #endif 825b9b7a4beSMatthew Dillon 826b9b7a4beSMatthew Dillon vm_object_clear_flag(object, OBJ_CLEANING); 827b9b7a4beSMatthew Dillon return; 828b9b7a4beSMatthew Dillon } 829b9b7a4beSMatthew Dillon 830b9b7a4beSMatthew Dillon static int 831b9b7a4beSMatthew Dillon vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int curgeneration, int pagerflags) 832b9b7a4beSMatthew Dillon { 833b9b7a4beSMatthew Dillon int runlen; 834b9b7a4beSMatthew Dillon int s; 835b9b7a4beSMatthew Dillon int maxf; 836b9b7a4beSMatthew Dillon int chkb; 837b9b7a4beSMatthew Dillon int maxb; 838b9b7a4beSMatthew Dillon int i; 839b9b7a4beSMatthew Dillon vm_pindex_t pi; 840b9b7a4beSMatthew Dillon vm_page_t maf[vm_pageout_page_count]; 841b9b7a4beSMatthew Dillon vm_page_t mab[vm_pageout_page_count]; 842b9b7a4beSMatthew Dillon vm_page_t ma[vm_pageout_page_count]; 843b9b7a4beSMatthew Dillon 844b18bfc3dSJohn Dyson s = splvm(); 84535c01631SAlan Cox mtx_assert(&vm_page_queue_mtx, MA_OWNED); 846b9b7a4beSMatthew Dillon pi = p->pindex; 847bd82dc74SAlan Cox while (vm_page_sleep_if_busy(p, TRUE, "vpcwai")) { 84835c01631SAlan Cox vm_page_lock_queues(); 8492d8acc0fSJohn Dyson if (object->generation != curgeneration) { 850f6b04d2bSDavid Greenman splx(s); 851b9b7a4beSMatthew Dillon return(0); 852f6b04d2bSDavid Greenman } 853bd82dc74SAlan Cox } 854bd7e5f99SJohn Dyson maxf = 0; 855bd7e5f99SJohn Dyson for(i = 1; i < vm_pageout_page_count; i++) { 856b9b7a4beSMatthew Dillon vm_page_t tp; 857b9b7a4beSMatthew Dillon 8588aef1712SMatthew Dillon if ((tp = vm_page_lookup(object, pi + i)) != NULL) { 859bd7e5f99SJohn Dyson if ((tp->flags & PG_BUSY) || 86043b7990eSMatthew Dillon ((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 && 86143b7990eSMatthew Dillon (tp->flags & PG_CLEANCHK) == 0) || 862ffc82b0aSJohn Dyson (tp->busy != 0)) 863bd7e5f99SJohn Dyson break; 8645070c7f8SJohn Dyson if((tp->queue - tp->pc) == PQ_CACHE) { 865e69763a3SDoug Rabson vm_page_flag_clear(tp, PG_CLEANCHK); 8663077a9c2SJohn Dyson break; 8673077a9c2SJohn Dyson } 868bd7e5f99SJohn Dyson vm_page_test_dirty(tp); 869bd7e5f99SJohn Dyson if ((tp->dirty & tp->valid) == 0) { 870e69763a3SDoug Rabson vm_page_flag_clear(tp, PG_CLEANCHK); 871bd7e5f99SJohn Dyson break; 872bd7e5f99SJohn Dyson } 873bd7e5f99SJohn Dyson maf[ i - 1 ] = tp; 874bd7e5f99SJohn Dyson maxf++; 875bd7e5f99SJohn Dyson continue; 876bd7e5f99SJohn Dyson } 877bd7e5f99SJohn Dyson break; 878bd7e5f99SJohn Dyson } 879aef922f5SJohn Dyson 880bd7e5f99SJohn Dyson maxb = 0; 881bd7e5f99SJohn Dyson chkb = vm_pageout_page_count - maxf; 882bd7e5f99SJohn Dyson if (chkb) { 883bd7e5f99SJohn Dyson for(i = 1; i < chkb;i++) { 884b9b7a4beSMatthew Dillon vm_page_t tp; 885b9b7a4beSMatthew Dillon 8868aef1712SMatthew Dillon if ((tp = vm_page_lookup(object, pi - i)) != NULL) { 887bd7e5f99SJohn Dyson if ((tp->flags & PG_BUSY) || 88843b7990eSMatthew Dillon ((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 && 88943b7990eSMatthew Dillon (tp->flags & PG_CLEANCHK) == 0) || 890ffc82b0aSJohn Dyson (tp->busy != 0)) 891bd7e5f99SJohn Dyson break; 8925070c7f8SJohn Dyson if ((tp->queue - tp->pc) == PQ_CACHE) { 893e69763a3SDoug Rabson vm_page_flag_clear(tp, PG_CLEANCHK); 8943077a9c2SJohn Dyson break; 8953077a9c2SJohn Dyson } 896bd7e5f99SJohn Dyson vm_page_test_dirty(tp); 897bd7e5f99SJohn Dyson if ((tp->dirty & tp->valid) == 0) { 898e69763a3SDoug Rabson vm_page_flag_clear(tp, PG_CLEANCHK); 899bd7e5f99SJohn Dyson break; 900bd7e5f99SJohn Dyson } 901bd7e5f99SJohn Dyson mab[ i - 1 ] = tp; 902bd7e5f99SJohn Dyson maxb++; 903bd7e5f99SJohn Dyson continue; 904bd7e5f99SJohn Dyson } 905bd7e5f99SJohn Dyson break; 906bd7e5f99SJohn Dyson } 907bd7e5f99SJohn Dyson } 908bd7e5f99SJohn Dyson 909bd7e5f99SJohn Dyson for(i = 0; i < maxb; i++) { 910bd7e5f99SJohn Dyson int index = (maxb - i) - 1; 911bd7e5f99SJohn Dyson ma[index] = mab[i]; 912e69763a3SDoug Rabson vm_page_flag_clear(ma[index], PG_CLEANCHK); 913bd7e5f99SJohn Dyson } 914e69763a3SDoug Rabson vm_page_flag_clear(p, PG_CLEANCHK); 915bd7e5f99SJohn Dyson ma[maxb] = p; 916bd7e5f99SJohn Dyson for(i = 0; i < maxf; i++) { 917bd7e5f99SJohn Dyson int index = (maxb + i) + 1; 918bd7e5f99SJohn Dyson ma[index] = maf[i]; 919e69763a3SDoug Rabson vm_page_flag_clear(ma[index], PG_CLEANCHK); 920f6b04d2bSDavid Greenman } 921bd7e5f99SJohn Dyson runlen = maxb + maxf + 1; 922cf2819ccSJohn Dyson 923f35329acSJohn Dyson splx(s); 924b6e48e03SAlan Cox vm_pageout_flush(ma, runlen, pagerflags, TRUE); 925cf2819ccSJohn Dyson for (i = 0; i < runlen; i++) { 926cf2819ccSJohn Dyson if (ma[i]->valid & ma[i]->dirty) { 927a12cc0e4SAlan Cox pmap_page_protect(ma[i], VM_PROT_READ); 928e69763a3SDoug Rabson vm_page_flag_set(ma[i], PG_CLEANCHK); 929aef922f5SJohn Dyson 930b9b7a4beSMatthew Dillon /* 931b9b7a4beSMatthew Dillon * maxf will end up being the actual number of pages 932b9b7a4beSMatthew Dillon * we wrote out contiguously, non-inclusive of the 933b9b7a4beSMatthew Dillon * first page. We do not count look-behind pages. 934b9b7a4beSMatthew Dillon */ 935b9b7a4beSMatthew Dillon if (i >= maxb + 1 && (maxf > i - maxb - 1)) 936b9b7a4beSMatthew Dillon maxf = i - maxb - 1; 937b9b7a4beSMatthew Dillon } 938b9b7a4beSMatthew Dillon } 939b9b7a4beSMatthew Dillon return(maxf + 1); 94026f9a767SRodney W. Grimes } 941df8bae1dSRodney W. Grimes 9421efb74fbSJohn Dyson /* 943867a482dSJohn Dyson * vm_object_madvise: 944867a482dSJohn Dyson * 945867a482dSJohn Dyson * Implements the madvise function at the object/page level. 9461c7c3c6aSMatthew Dillon * 947193b9358SAlan Cox * MADV_WILLNEED (any object) 948193b9358SAlan Cox * 949193b9358SAlan Cox * Activate the specified pages if they are resident. 950193b9358SAlan Cox * 951193b9358SAlan Cox * MADV_DONTNEED (any object) 952193b9358SAlan Cox * 953193b9358SAlan Cox * Deactivate the specified pages if they are resident. 954193b9358SAlan Cox * 955193b9358SAlan Cox * MADV_FREE (OBJT_DEFAULT/OBJT_SWAP objects, 956193b9358SAlan Cox * OBJ_ONEMAPPING only) 957193b9358SAlan Cox * 958193b9358SAlan Cox * Deactivate and clean the specified pages if they are 959193b9358SAlan Cox * resident. This permits the process to reuse the pages 960193b9358SAlan Cox * without faulting or the kernel to reclaim the pages 961193b9358SAlan Cox * without I/O. 962867a482dSJohn Dyson */ 963867a482dSJohn Dyson void 9641b40f8c0SMatthew Dillon vm_object_madvise(vm_object_t object, vm_pindex_t pindex, int count, int advise) 965867a482dSJohn Dyson { 9666e20a165SJohn Dyson vm_pindex_t end, tpindex; 9676e20a165SJohn Dyson vm_object_t tobject; 968867a482dSJohn Dyson vm_page_t m; 969867a482dSJohn Dyson 970867a482dSJohn Dyson if (object == NULL) 971867a482dSJohn Dyson return; 972867a482dSJohn Dyson 97315c176c1SAlan Cox vm_object_lock(object); 974094f6d26SAlan Cox 975867a482dSJohn Dyson end = pindex + count; 976867a482dSJohn Dyson 9771c7c3c6aSMatthew Dillon /* 9781c7c3c6aSMatthew Dillon * Locate and adjust resident pages 9791c7c3c6aSMatthew Dillon */ 9801c7c3c6aSMatthew Dillon for (; pindex < end; pindex += 1) { 9816e20a165SJohn Dyson relookup: 9826e20a165SJohn Dyson tobject = object; 9836e20a165SJohn Dyson tpindex = pindex; 9846e20a165SJohn Dyson shadowlookup: 98558b4e6ccSAlan Cox /* 98658b4e6ccSAlan Cox * MADV_FREE only operates on OBJT_DEFAULT or OBJT_SWAP pages 98758b4e6ccSAlan Cox * and those pages must be OBJ_ONEMAPPING. 98858b4e6ccSAlan Cox */ 98958b4e6ccSAlan Cox if (advise == MADV_FREE) { 99058b4e6ccSAlan Cox if ((tobject->type != OBJT_DEFAULT && 99158b4e6ccSAlan Cox tobject->type != OBJT_SWAP) || 99258b4e6ccSAlan Cox (tobject->flags & OBJ_ONEMAPPING) == 0) { 9936e20a165SJohn Dyson continue; 9946e20a165SJohn Dyson } 99558b4e6ccSAlan Cox } 9961c7c3c6aSMatthew Dillon 9971c7c3c6aSMatthew Dillon m = vm_page_lookup(tobject, tpindex); 9981c7c3c6aSMatthew Dillon 9991c7c3c6aSMatthew Dillon if (m == NULL) { 10001ce137beSMatthew Dillon /* 10011ce137beSMatthew Dillon * There may be swap even if there is no backing page 10021ce137beSMatthew Dillon */ 10031ce137beSMatthew Dillon if (advise == MADV_FREE && tobject->type == OBJT_SWAP) 10041ce137beSMatthew Dillon swap_pager_freespace(tobject, tpindex, 1); 10051ce137beSMatthew Dillon 10061ce137beSMatthew Dillon /* 10071ce137beSMatthew Dillon * next object 10081ce137beSMatthew Dillon */ 10096e20a165SJohn Dyson tobject = tobject->backing_object; 10101c7c3c6aSMatthew Dillon if (tobject == NULL) 10111c7c3c6aSMatthew Dillon continue; 10126e20a165SJohn Dyson tpindex += OFF_TO_IDX(tobject->backing_object_offset); 10136e20a165SJohn Dyson goto shadowlookup; 10146e20a165SJohn Dyson } 1015867a482dSJohn Dyson 1016867a482dSJohn Dyson /* 1017867a482dSJohn Dyson * If the page is busy or not in a normal active state, 10188b03c8edSMatthew Dillon * we skip it. If the page is not managed there are no 10198b03c8edSMatthew Dillon * page queues to mess with. Things can break if we mess 10208b03c8edSMatthew Dillon * with pages in any of the below states. 1021867a482dSJohn Dyson */ 102232585dd6SAlan Cox vm_page_lock_queues(); 102332585dd6SAlan Cox if (m->hold_count || 10241c7c3c6aSMatthew Dillon m->wire_count || 10258b03c8edSMatthew Dillon (m->flags & PG_UNMANAGED) || 102632585dd6SAlan Cox m->valid != VM_PAGE_BITS_ALL) { 102732585dd6SAlan Cox vm_page_unlock_queues(); 1028867a482dSJohn Dyson continue; 10296e20a165SJohn Dyson } 103032585dd6SAlan Cox if (vm_page_sleep_if_busy(m, TRUE, "madvpo")) 10316e20a165SJohn Dyson goto relookup; 1032867a482dSJohn Dyson if (advise == MADV_WILLNEED) { 1033867a482dSJohn Dyson vm_page_activate(m); 10346e20a165SJohn Dyson } else if (advise == MADV_DONTNEED) { 1035479112dfSMatthew Dillon vm_page_dontneed(m); 10360a47b48bSJohn Dyson } else if (advise == MADV_FREE) { 10371c7c3c6aSMatthew Dillon /* 10382aaeadf8SMatthew Dillon * Mark the page clean. This will allow the page 10392aaeadf8SMatthew Dillon * to be freed up by the system. However, such pages 10402aaeadf8SMatthew Dillon * are often reused quickly by malloc()/free() 10412aaeadf8SMatthew Dillon * so we do not do anything that would cause 10422aaeadf8SMatthew Dillon * a page fault if we can help it. 10432aaeadf8SMatthew Dillon * 10442aaeadf8SMatthew Dillon * Specifically, we do not try to actually free 10452aaeadf8SMatthew Dillon * the page now nor do we try to put it in the 10462aaeadf8SMatthew Dillon * cache (which would cause a page fault on reuse). 104741c67e12SMatthew Dillon * 104841c67e12SMatthew Dillon * But we do make the page is freeable as we 104941c67e12SMatthew Dillon * can without actually taking the step of unmapping 105041c67e12SMatthew Dillon * it. 10511c7c3c6aSMatthew Dillon */ 10520385347cSPeter Wemm pmap_clear_modify(m); 10536e20a165SJohn Dyson m->dirty = 0; 105441c67e12SMatthew Dillon m->act_count = 0; 1055479112dfSMatthew Dillon vm_page_dontneed(m); 1056867a482dSJohn Dyson } 10572999e9faSAlan Cox vm_page_unlock_queues(); 10582999e9faSAlan Cox if (advise == MADV_FREE && tobject->type == OBJT_SWAP) 10592999e9faSAlan Cox swap_pager_freespace(tobject, tpindex, 1); 1060867a482dSJohn Dyson } 106115c176c1SAlan Cox vm_object_unlock(object); 1062867a482dSJohn Dyson } 1063867a482dSJohn Dyson 1064867a482dSJohn Dyson /* 1065df8bae1dSRodney W. Grimes * vm_object_shadow: 1066df8bae1dSRodney W. Grimes * 1067df8bae1dSRodney W. Grimes * Create a new object which is backed by the 1068df8bae1dSRodney W. Grimes * specified existing object range. The source 1069df8bae1dSRodney W. Grimes * object reference is deallocated. 1070df8bae1dSRodney W. Grimes * 1071df8bae1dSRodney W. Grimes * The new object and offset into that object 1072df8bae1dSRodney W. Grimes * are returned in the source parameters. 1073df8bae1dSRodney W. Grimes */ 107426f9a767SRodney W. Grimes void 10751b40f8c0SMatthew Dillon vm_object_shadow( 10761b40f8c0SMatthew Dillon vm_object_t *object, /* IN/OUT */ 10771b40f8c0SMatthew Dillon vm_ooffset_t *offset, /* IN/OUT */ 10781b40f8c0SMatthew Dillon vm_size_t length) 1079df8bae1dSRodney W. Grimes { 1080d031cff1SMatthew Dillon vm_object_t source; 1081d031cff1SMatthew Dillon vm_object_t result; 1082df8bae1dSRodney W. Grimes 1083570a2f4aSAlan Cox GIANT_REQUIRED; 1084570a2f4aSAlan Cox 1085df8bae1dSRodney W. Grimes source = *object; 1086df8bae1dSRodney W. Grimes 1087df8bae1dSRodney W. Grimes /* 10889a2f6362SAlan Cox * Don't create the new object if the old object isn't shared. 10899a2f6362SAlan Cox */ 1090570a2f4aSAlan Cox if (source != NULL) { 1091570a2f4aSAlan Cox VM_OBJECT_LOCK(source); 1092570a2f4aSAlan Cox if (source->ref_count == 1 && 10939a2f6362SAlan Cox source->handle == NULL && 10949a2f6362SAlan Cox (source->type == OBJT_DEFAULT || 10959917e010SAlan Cox source->type == OBJT_SWAP)) { 1096570a2f4aSAlan Cox VM_OBJECT_UNLOCK(source); 10979a2f6362SAlan Cox return; 10989917e010SAlan Cox } 1099570a2f4aSAlan Cox VM_OBJECT_UNLOCK(source); 1100570a2f4aSAlan Cox } 11019a2f6362SAlan Cox 11029a2f6362SAlan Cox /* 1103570a2f4aSAlan Cox * Allocate a new object with the given length. 1104df8bae1dSRodney W. Grimes */ 1105971dd342SAlfred Perlstein result = vm_object_allocate(OBJT_DEFAULT, length); 1106df8bae1dSRodney W. Grimes 1107df8bae1dSRodney W. Grimes /* 11080d94caffSDavid Greenman * The new object shadows the source object, adding a reference to it. 11090d94caffSDavid Greenman * Our caller changes his reference to point to the new object, 11100d94caffSDavid Greenman * removing a reference to the source object. Net result: no change 11110d94caffSDavid Greenman * of reference count. 11129b09fe24SMatthew Dillon * 11139b09fe24SMatthew Dillon * Try to optimize the result object's page color when shadowing 1114956f3135SPhilippe Charnier * in order to maintain page coloring consistency in the combined 11159b09fe24SMatthew Dillon * shadowed object. 1116df8bae1dSRodney W. Grimes */ 111724a1cce3SDavid Greenman result->backing_object = source; 1118570a2f4aSAlan Cox if (source != NULL) { 1119570a2f4aSAlan Cox VM_OBJECT_LOCK(source); 1120de5f6a77SJohn Dyson TAILQ_INSERT_TAIL(&source->shadow_head, result, shadow_list); 1121eaf13dd7SJohn Dyson source->shadow_count++; 1122eaf13dd7SJohn Dyson source->generation++; 1123dcc5840eSAlan Cox if (length < source->size) 1124dcc5840eSAlan Cox length = source->size; 1125dcc5840eSAlan Cox if (length > PQ_L2_SIZE / 3 + PQ_PRIME1 || 1126dcc5840eSAlan Cox source->generation > 1) 1127dcc5840eSAlan Cox length = PQ_L2_SIZE / 3 + PQ_PRIME1; 1128dcc5840eSAlan Cox result->pg_color = (source->pg_color + 1129dcc5840eSAlan Cox length * source->generation) & PQ_L2_MASK; 1130570a2f4aSAlan Cox VM_OBJECT_UNLOCK(source); 1131dcc5840eSAlan Cox next_index = (result->pg_color + PQ_L2_SIZE / 3 + PQ_PRIME1) & 1132dcc5840eSAlan Cox PQ_L2_MASK; 1133de5f6a77SJohn Dyson } 1134df8bae1dSRodney W. Grimes 1135df8bae1dSRodney W. Grimes /* 11360d94caffSDavid Greenman * Store the offset into the source object, and fix up the offset into 11370d94caffSDavid Greenman * the new object. 1138df8bae1dSRodney W. Grimes */ 113924a1cce3SDavid Greenman result->backing_object_offset = *offset; 1140df8bae1dSRodney W. Grimes 1141df8bae1dSRodney W. Grimes /* 1142df8bae1dSRodney W. Grimes * Return the new things 1143df8bae1dSRodney W. Grimes */ 1144df8bae1dSRodney W. Grimes *offset = 0; 1145df8bae1dSRodney W. Grimes *object = result; 1146df8bae1dSRodney W. Grimes } 1147df8bae1dSRodney W. Grimes 1148c5aaa06dSAlan Cox /* 1149c5aaa06dSAlan Cox * vm_object_split: 1150c5aaa06dSAlan Cox * 1151c5aaa06dSAlan Cox * Split the pages in a map entry into a new object. This affords 1152c5aaa06dSAlan Cox * easier removal of unused pages, and keeps object inheritance from 1153c5aaa06dSAlan Cox * being a negative impact on memory usage. 1154c5aaa06dSAlan Cox */ 1155c5aaa06dSAlan Cox void 1156c5aaa06dSAlan Cox vm_object_split(vm_map_entry_t entry) 1157c5aaa06dSAlan Cox { 1158c5aaa06dSAlan Cox vm_page_t m; 1159c5aaa06dSAlan Cox vm_object_t orig_object, new_object, source; 1160c5aaa06dSAlan Cox vm_offset_t s, e; 116123f09d50SIan Dowse vm_pindex_t offidxstart, offidxend; 116223f09d50SIan Dowse vm_size_t idx, size; 1163c5aaa06dSAlan Cox vm_ooffset_t offset; 1164c5aaa06dSAlan Cox 1165c5aaa06dSAlan Cox GIANT_REQUIRED; 1166c5aaa06dSAlan Cox 1167c5aaa06dSAlan Cox orig_object = entry->object.vm_object; 1168c5aaa06dSAlan Cox if (orig_object->type != OBJT_DEFAULT && orig_object->type != OBJT_SWAP) 1169c5aaa06dSAlan Cox return; 1170c5aaa06dSAlan Cox if (orig_object->ref_count <= 1) 1171c5aaa06dSAlan Cox return; 1172c5aaa06dSAlan Cox 1173c5aaa06dSAlan Cox offset = entry->offset; 1174c5aaa06dSAlan Cox s = entry->start; 1175c5aaa06dSAlan Cox e = entry->end; 1176c5aaa06dSAlan Cox 1177c5aaa06dSAlan Cox offidxstart = OFF_TO_IDX(offset); 1178c5aaa06dSAlan Cox offidxend = offidxstart + OFF_TO_IDX(e - s); 1179c5aaa06dSAlan Cox size = offidxend - offidxstart; 1180c5aaa06dSAlan Cox 1181c5aaa06dSAlan Cox new_object = vm_pager_allocate(orig_object->type, 1182c5aaa06dSAlan Cox NULL, IDX_TO_OFF(size), VM_PROT_ALL, 0LL); 1183c5aaa06dSAlan Cox if (new_object == NULL) 1184c5aaa06dSAlan Cox return; 1185c5aaa06dSAlan Cox 1186c5aaa06dSAlan Cox source = orig_object->backing_object; 1187c5aaa06dSAlan Cox if (source != NULL) { 1188c5aaa06dSAlan Cox vm_object_reference(source); /* Referenced by new_object */ 1189c5aaa06dSAlan Cox TAILQ_INSERT_TAIL(&source->shadow_head, 1190c5aaa06dSAlan Cox new_object, shadow_list); 1191e2479b4fSAlan Cox VM_OBJECT_LOCK(source); 1192c5aaa06dSAlan Cox vm_object_clear_flag(source, OBJ_ONEMAPPING); 1193e2479b4fSAlan Cox VM_OBJECT_UNLOCK(source); 1194c5aaa06dSAlan Cox new_object->backing_object_offset = 119556030358SAlan Cox orig_object->backing_object_offset + offset; 1196c5aaa06dSAlan Cox new_object->backing_object = source; 1197c5aaa06dSAlan Cox source->shadow_count++; 1198c5aaa06dSAlan Cox source->generation++; 1199c5aaa06dSAlan Cox } 1200c5aaa06dSAlan Cox for (idx = 0; idx < size; idx++) { 1201c5aaa06dSAlan Cox retry: 1202c5aaa06dSAlan Cox m = vm_page_lookup(orig_object, offidxstart + idx); 1203c5aaa06dSAlan Cox if (m == NULL) 1204c5aaa06dSAlan Cox continue; 1205c5aaa06dSAlan Cox 1206c5aaa06dSAlan Cox /* 1207c5aaa06dSAlan Cox * We must wait for pending I/O to complete before we can 1208c5aaa06dSAlan Cox * rename the page. 1209c5aaa06dSAlan Cox * 1210c5aaa06dSAlan Cox * We do not have to VM_PROT_NONE the page as mappings should 1211c5aaa06dSAlan Cox * not be changed by this operation. 1212c5aaa06dSAlan Cox */ 121300f9e8b4SAlan Cox vm_page_lock_queues(); 121400f9e8b4SAlan Cox if (vm_page_sleep_if_busy(m, TRUE, "spltwt")) 1215c5aaa06dSAlan Cox goto retry; 1216c5aaa06dSAlan Cox 1217c5aaa06dSAlan Cox vm_page_busy(m); 1218c5aaa06dSAlan Cox vm_page_rename(m, new_object, idx); 1219c5aaa06dSAlan Cox /* page automatically made dirty by rename and cache handled */ 1220c5aaa06dSAlan Cox vm_page_busy(m); 1221bd82dc74SAlan Cox vm_page_unlock_queues(); 1222c5aaa06dSAlan Cox } 1223d22bc710SAlan Cox VM_OBJECT_LOCK(orig_object); 1224d7a013c3SAlan Cox if (orig_object->type == OBJT_SWAP) { 1225c5aaa06dSAlan Cox vm_object_pip_add(orig_object, 1); 1226d22bc710SAlan Cox VM_OBJECT_UNLOCK(orig_object); 1227c5aaa06dSAlan Cox /* 1228c5aaa06dSAlan Cox * copy orig_object pages into new_object 1229c5aaa06dSAlan Cox * and destroy unneeded pages in 1230c5aaa06dSAlan Cox * shadow object. 1231c5aaa06dSAlan Cox */ 1232c5aaa06dSAlan Cox swap_pager_copy(orig_object, new_object, offidxstart, 0); 1233d7a013c3SAlan Cox VM_OBJECT_LOCK(orig_object); 1234c5aaa06dSAlan Cox vm_object_pip_wakeup(orig_object); 1235c5aaa06dSAlan Cox } 1236d7a013c3SAlan Cox VM_OBJECT_UNLOCK(orig_object); 1237dc907f66SAlan Cox vm_page_lock_queues(); 1238c7118ed6SAlan Cox TAILQ_FOREACH(m, &new_object->memq, listq) 1239c5aaa06dSAlan Cox vm_page_wakeup(m); 1240dc907f66SAlan Cox vm_page_unlock_queues(); 1241c5aaa06dSAlan Cox entry->object.vm_object = new_object; 1242c5aaa06dSAlan Cox entry->offset = 0LL; 1243c5aaa06dSAlan Cox vm_object_deallocate(orig_object); 1244c5aaa06dSAlan Cox } 1245c5aaa06dSAlan Cox 12462ad1a3f7SMatthew Dillon #define OBSC_TEST_ALL_SHADOWED 0x0001 12472ad1a3f7SMatthew Dillon #define OBSC_COLLAPSE_NOWAIT 0x0002 12482ad1a3f7SMatthew Dillon #define OBSC_COLLAPSE_WAIT 0x0004 12492ad1a3f7SMatthew Dillon 12502ad1a3f7SMatthew Dillon static __inline int 12512ad1a3f7SMatthew Dillon vm_object_backing_scan(vm_object_t object, int op) 12522ad1a3f7SMatthew Dillon { 12532ad1a3f7SMatthew Dillon int s; 12542ad1a3f7SMatthew Dillon int r = 1; 12552ad1a3f7SMatthew Dillon vm_page_t p; 12562ad1a3f7SMatthew Dillon vm_object_t backing_object; 12572ad1a3f7SMatthew Dillon vm_pindex_t backing_offset_index; 12582ad1a3f7SMatthew Dillon 12592ad1a3f7SMatthew Dillon s = splvm(); 12600cddd8f0SMatthew Dillon GIANT_REQUIRED; 12612ad1a3f7SMatthew Dillon 12622ad1a3f7SMatthew Dillon backing_object = object->backing_object; 12632ad1a3f7SMatthew Dillon backing_offset_index = OFF_TO_IDX(object->backing_object_offset); 12642ad1a3f7SMatthew Dillon 12652ad1a3f7SMatthew Dillon /* 12662ad1a3f7SMatthew Dillon * Initial conditions 12672ad1a3f7SMatthew Dillon */ 12682ad1a3f7SMatthew Dillon if (op & OBSC_TEST_ALL_SHADOWED) { 12692ad1a3f7SMatthew Dillon /* 1270956f3135SPhilippe Charnier * We do not want to have to test for the existence of 12712ad1a3f7SMatthew Dillon * swap pages in the backing object. XXX but with the 12722ad1a3f7SMatthew Dillon * new swapper this would be pretty easy to do. 12732ad1a3f7SMatthew Dillon * 12742ad1a3f7SMatthew Dillon * XXX what about anonymous MAP_SHARED memory that hasn't 12752ad1a3f7SMatthew Dillon * been ZFOD faulted yet? If we do not test for this, the 12762ad1a3f7SMatthew Dillon * shadow test may succeed! XXX 12772ad1a3f7SMatthew Dillon */ 12782ad1a3f7SMatthew Dillon if (backing_object->type != OBJT_DEFAULT) { 12792ad1a3f7SMatthew Dillon splx(s); 12802ad1a3f7SMatthew Dillon return (0); 12812ad1a3f7SMatthew Dillon } 12822ad1a3f7SMatthew Dillon } 12832ad1a3f7SMatthew Dillon if (op & OBSC_COLLAPSE_WAIT) { 12842ad1a3f7SMatthew Dillon vm_object_set_flag(backing_object, OBJ_DEAD); 12852ad1a3f7SMatthew Dillon } 12862ad1a3f7SMatthew Dillon 12872ad1a3f7SMatthew Dillon /* 12882ad1a3f7SMatthew Dillon * Our scan 12892ad1a3f7SMatthew Dillon */ 12902ad1a3f7SMatthew Dillon p = TAILQ_FIRST(&backing_object->memq); 12912ad1a3f7SMatthew Dillon while (p) { 12922ad1a3f7SMatthew Dillon vm_page_t next = TAILQ_NEXT(p, listq); 12932ad1a3f7SMatthew Dillon vm_pindex_t new_pindex = p->pindex - backing_offset_index; 12942ad1a3f7SMatthew Dillon 12952ad1a3f7SMatthew Dillon if (op & OBSC_TEST_ALL_SHADOWED) { 12962ad1a3f7SMatthew Dillon vm_page_t pp; 12972ad1a3f7SMatthew Dillon 12982ad1a3f7SMatthew Dillon /* 12992ad1a3f7SMatthew Dillon * Ignore pages outside the parent object's range 13002ad1a3f7SMatthew Dillon * and outside the parent object's mapping of the 13012ad1a3f7SMatthew Dillon * backing object. 13022ad1a3f7SMatthew Dillon * 13032ad1a3f7SMatthew Dillon * note that we do not busy the backing object's 13042ad1a3f7SMatthew Dillon * page. 13052ad1a3f7SMatthew Dillon */ 13062ad1a3f7SMatthew Dillon if ( 13072ad1a3f7SMatthew Dillon p->pindex < backing_offset_index || 13082ad1a3f7SMatthew Dillon new_pindex >= object->size 13092ad1a3f7SMatthew Dillon ) { 13102ad1a3f7SMatthew Dillon p = next; 13112ad1a3f7SMatthew Dillon continue; 13122ad1a3f7SMatthew Dillon } 13132ad1a3f7SMatthew Dillon 13142ad1a3f7SMatthew Dillon /* 13152ad1a3f7SMatthew Dillon * See if the parent has the page or if the parent's 13162ad1a3f7SMatthew Dillon * object pager has the page. If the parent has the 13172ad1a3f7SMatthew Dillon * page but the page is not valid, the parent's 13182ad1a3f7SMatthew Dillon * object pager must have the page. 13192ad1a3f7SMatthew Dillon * 13202ad1a3f7SMatthew Dillon * If this fails, the parent does not completely shadow 13212ad1a3f7SMatthew Dillon * the object and we might as well give up now. 13222ad1a3f7SMatthew Dillon */ 13232ad1a3f7SMatthew Dillon 13242ad1a3f7SMatthew Dillon pp = vm_page_lookup(object, new_pindex); 13252ad1a3f7SMatthew Dillon if ( 13262ad1a3f7SMatthew Dillon (pp == NULL || pp->valid == 0) && 13272ad1a3f7SMatthew Dillon !vm_pager_has_page(object, new_pindex, NULL, NULL) 13282ad1a3f7SMatthew Dillon ) { 13292ad1a3f7SMatthew Dillon r = 0; 13302ad1a3f7SMatthew Dillon break; 13312ad1a3f7SMatthew Dillon } 13322ad1a3f7SMatthew Dillon } 13332ad1a3f7SMatthew Dillon 13342ad1a3f7SMatthew Dillon /* 13352ad1a3f7SMatthew Dillon * Check for busy page 13362ad1a3f7SMatthew Dillon */ 13372ad1a3f7SMatthew Dillon if (op & (OBSC_COLLAPSE_WAIT | OBSC_COLLAPSE_NOWAIT)) { 13382ad1a3f7SMatthew Dillon vm_page_t pp; 13392ad1a3f7SMatthew Dillon 134000f9e8b4SAlan Cox vm_page_lock_queues(); 13412ad1a3f7SMatthew Dillon if (op & OBSC_COLLAPSE_NOWAIT) { 134200f9e8b4SAlan Cox if ((p->flags & PG_BUSY) || 13432ad1a3f7SMatthew Dillon !p->valid || 13442ad1a3f7SMatthew Dillon p->hold_count || 13452ad1a3f7SMatthew Dillon p->wire_count || 134600f9e8b4SAlan Cox p->busy) { 134700f9e8b4SAlan Cox vm_page_unlock_queues(); 13482ad1a3f7SMatthew Dillon p = next; 13492ad1a3f7SMatthew Dillon continue; 13502ad1a3f7SMatthew Dillon } 13512ad1a3f7SMatthew Dillon } else if (op & OBSC_COLLAPSE_WAIT) { 135200f9e8b4SAlan Cox if (vm_page_sleep_if_busy(p, TRUE, "vmocol")) { 13532ad1a3f7SMatthew Dillon /* 13542ad1a3f7SMatthew Dillon * If we slept, anything could have 13552ad1a3f7SMatthew Dillon * happened. Since the object is 13562ad1a3f7SMatthew Dillon * marked dead, the backing offset 13572ad1a3f7SMatthew Dillon * should not have changed so we 13582ad1a3f7SMatthew Dillon * just restart our scan. 13592ad1a3f7SMatthew Dillon */ 13602ad1a3f7SMatthew Dillon p = TAILQ_FIRST(&backing_object->memq); 13612ad1a3f7SMatthew Dillon continue; 13622ad1a3f7SMatthew Dillon } 13632ad1a3f7SMatthew Dillon } 13642ad1a3f7SMatthew Dillon 13652ad1a3f7SMatthew Dillon /* 13662ad1a3f7SMatthew Dillon * Busy the page 13672ad1a3f7SMatthew Dillon */ 13682ad1a3f7SMatthew Dillon vm_page_busy(p); 136900f9e8b4SAlan Cox vm_page_unlock_queues(); 13702ad1a3f7SMatthew Dillon 13712ad1a3f7SMatthew Dillon KASSERT( 13722ad1a3f7SMatthew Dillon p->object == backing_object, 13732ad1a3f7SMatthew Dillon ("vm_object_qcollapse(): object mismatch") 13742ad1a3f7SMatthew Dillon ); 13752ad1a3f7SMatthew Dillon 13762ad1a3f7SMatthew Dillon /* 13772ad1a3f7SMatthew Dillon * Destroy any associated swap 13782ad1a3f7SMatthew Dillon */ 13792ad1a3f7SMatthew Dillon if (backing_object->type == OBJT_SWAP) { 13802ad1a3f7SMatthew Dillon swap_pager_freespace( 13812ad1a3f7SMatthew Dillon backing_object, 13822ad1a3f7SMatthew Dillon p->pindex, 13832ad1a3f7SMatthew Dillon 1 13842ad1a3f7SMatthew Dillon ); 13852ad1a3f7SMatthew Dillon } 13862ad1a3f7SMatthew Dillon 13872ad1a3f7SMatthew Dillon if ( 13882ad1a3f7SMatthew Dillon p->pindex < backing_offset_index || 13892ad1a3f7SMatthew Dillon new_pindex >= object->size 13902ad1a3f7SMatthew Dillon ) { 13912ad1a3f7SMatthew Dillon /* 13922ad1a3f7SMatthew Dillon * Page is out of the parent object's range, we 13932ad1a3f7SMatthew Dillon * can simply destroy it. 13942ad1a3f7SMatthew Dillon */ 13956a684ecfSAlan Cox vm_page_lock_queues(); 13964fec79beSAlan Cox pmap_remove_all(p); 13972ad1a3f7SMatthew Dillon vm_page_free(p); 13986a684ecfSAlan Cox vm_page_unlock_queues(); 13992ad1a3f7SMatthew Dillon p = next; 14002ad1a3f7SMatthew Dillon continue; 14012ad1a3f7SMatthew Dillon } 14022ad1a3f7SMatthew Dillon 14032ad1a3f7SMatthew Dillon pp = vm_page_lookup(object, new_pindex); 14042ad1a3f7SMatthew Dillon if ( 14052ad1a3f7SMatthew Dillon pp != NULL || 14062ad1a3f7SMatthew Dillon vm_pager_has_page(object, new_pindex, NULL, NULL) 14072ad1a3f7SMatthew Dillon ) { 14082ad1a3f7SMatthew Dillon /* 14092ad1a3f7SMatthew Dillon * page already exists in parent OR swap exists 14102ad1a3f7SMatthew Dillon * for this location in the parent. Destroy 14112ad1a3f7SMatthew Dillon * the original page from the backing object. 14122ad1a3f7SMatthew Dillon * 14132ad1a3f7SMatthew Dillon * Leave the parent's page alone 14142ad1a3f7SMatthew Dillon */ 14156a684ecfSAlan Cox vm_page_lock_queues(); 14164fec79beSAlan Cox pmap_remove_all(p); 14172ad1a3f7SMatthew Dillon vm_page_free(p); 14186a684ecfSAlan Cox vm_page_unlock_queues(); 14192ad1a3f7SMatthew Dillon p = next; 14202ad1a3f7SMatthew Dillon continue; 14212ad1a3f7SMatthew Dillon } 14222ad1a3f7SMatthew Dillon 14232ad1a3f7SMatthew Dillon /* 14242ad1a3f7SMatthew Dillon * Page does not exist in parent, rename the 14252ad1a3f7SMatthew Dillon * page from the backing object to the main object. 1426d1bf5d56SMatthew Dillon * 1427d1bf5d56SMatthew Dillon * If the page was mapped to a process, it can remain 1428d1bf5d56SMatthew Dillon * mapped through the rename. 14292ad1a3f7SMatthew Dillon */ 1430a28cc55eSAlan Cox vm_page_lock_queues(); 14312ad1a3f7SMatthew Dillon vm_page_rename(p, object, new_pindex); 1432a28cc55eSAlan Cox vm_page_unlock_queues(); 14332ad1a3f7SMatthew Dillon /* page automatically made dirty by rename */ 14342ad1a3f7SMatthew Dillon } 14352ad1a3f7SMatthew Dillon p = next; 14362ad1a3f7SMatthew Dillon } 14372ad1a3f7SMatthew Dillon splx(s); 14382ad1a3f7SMatthew Dillon return (r); 14392ad1a3f7SMatthew Dillon } 14402ad1a3f7SMatthew Dillon 1441df8bae1dSRodney W. Grimes 1442df8bae1dSRodney W. Grimes /* 14432fe6e4d7SDavid Greenman * this version of collapse allows the operation to occur earlier and 14442fe6e4d7SDavid Greenman * when paging_in_progress is true for an object... This is not a complete 14452fe6e4d7SDavid Greenman * operation, but should plug 99.9% of the rest of the leaks. 14462fe6e4d7SDavid Greenman */ 14472fe6e4d7SDavid Greenman static void 14481b40f8c0SMatthew Dillon vm_object_qcollapse(vm_object_t object) 14492fe6e4d7SDavid Greenman { 14502ad1a3f7SMatthew Dillon vm_object_t backing_object = object->backing_object; 14512fe6e4d7SDavid Greenman 14521b40f8c0SMatthew Dillon GIANT_REQUIRED; 14531b40f8c0SMatthew Dillon 14542fe6e4d7SDavid Greenman if (backing_object->ref_count != 1) 14552fe6e4d7SDavid Greenman return; 14562fe6e4d7SDavid Greenman 1457010cf3b9SDavid Greenman backing_object->ref_count += 2; 1458010cf3b9SDavid Greenman 14592ad1a3f7SMatthew Dillon vm_object_backing_scan(object, OBSC_COLLAPSE_NOWAIT); 14601c7c3c6aSMatthew Dillon 1461010cf3b9SDavid Greenman backing_object->ref_count -= 2; 14622fe6e4d7SDavid Greenman } 14632fe6e4d7SDavid Greenman 1464df8bae1dSRodney W. Grimes /* 1465df8bae1dSRodney W. Grimes * vm_object_collapse: 1466df8bae1dSRodney W. Grimes * 1467df8bae1dSRodney W. Grimes * Collapse an object with the object backing it. 1468df8bae1dSRodney W. Grimes * Pages in the backing object are moved into the 1469df8bae1dSRodney W. Grimes * parent, and the backing object is deallocated. 1470df8bae1dSRodney W. Grimes */ 147126f9a767SRodney W. Grimes void 14721b40f8c0SMatthew Dillon vm_object_collapse(vm_object_t object) 1473df8bae1dSRodney W. Grimes { 14740cddd8f0SMatthew Dillon GIANT_REQUIRED; 147523955314SAlfred Perlstein 1476df8bae1dSRodney W. Grimes while (TRUE) { 14772ad1a3f7SMatthew Dillon vm_object_t backing_object; 14782ad1a3f7SMatthew Dillon 1479df8bae1dSRodney W. Grimes /* 1480df8bae1dSRodney W. Grimes * Verify that the conditions are right for collapse: 1481df8bae1dSRodney W. Grimes * 14822ad1a3f7SMatthew Dillon * The object exists and the backing object exists. 1483df8bae1dSRodney W. Grimes */ 14842fe6e4d7SDavid Greenman if (object == NULL) 14852ad1a3f7SMatthew Dillon break; 1486df8bae1dSRodney W. Grimes 148724a1cce3SDavid Greenman if ((backing_object = object->backing_object) == NULL) 14882ad1a3f7SMatthew Dillon break; 1489df8bae1dSRodney W. Grimes 1490f919ebdeSDavid Greenman /* 1491f919ebdeSDavid Greenman * we check the backing object first, because it is most likely 149224a1cce3SDavid Greenman * not collapsable. 1493f919ebdeSDavid Greenman */ 149424a1cce3SDavid Greenman if (backing_object->handle != NULL || 149524a1cce3SDavid Greenman (backing_object->type != OBJT_DEFAULT && 149624a1cce3SDavid Greenman backing_object->type != OBJT_SWAP) || 1497f919ebdeSDavid Greenman (backing_object->flags & OBJ_DEAD) || 149824a1cce3SDavid Greenman object->handle != NULL || 149924a1cce3SDavid Greenman (object->type != OBJT_DEFAULT && 150024a1cce3SDavid Greenman object->type != OBJT_SWAP) || 150124a1cce3SDavid Greenman (object->flags & OBJ_DEAD)) { 15022ad1a3f7SMatthew Dillon break; 150324a1cce3SDavid Greenman } 15049b4814bbSDavid Greenman 15052ad1a3f7SMatthew Dillon if ( 15062ad1a3f7SMatthew Dillon object->paging_in_progress != 0 || 15072ad1a3f7SMatthew Dillon backing_object->paging_in_progress != 0 15082ad1a3f7SMatthew Dillon ) { 1509b9921222SDavid Greenman vm_object_qcollapse(object); 15102ad1a3f7SMatthew Dillon break; 1511df8bae1dSRodney W. Grimes } 1512f919ebdeSDavid Greenman 151326f9a767SRodney W. Grimes /* 15140d94caffSDavid Greenman * We know that we can either collapse the backing object (if 15152ad1a3f7SMatthew Dillon * the parent is the only reference to it) or (perhaps) have 15162ad1a3f7SMatthew Dillon * the parent bypass the object if the parent happens to shadow 15172ad1a3f7SMatthew Dillon * all the resident pages in the entire backing object. 15182ad1a3f7SMatthew Dillon * 15192ad1a3f7SMatthew Dillon * This is ignoring pager-backed pages such as swap pages. 15202ad1a3f7SMatthew Dillon * vm_object_backing_scan fails the shadowing test in this 15212ad1a3f7SMatthew Dillon * case. 1522df8bae1dSRodney W. Grimes */ 1523df8bae1dSRodney W. Grimes if (backing_object->ref_count == 1) { 1524df8bae1dSRodney W. Grimes /* 15252ad1a3f7SMatthew Dillon * If there is exactly one reference to the backing 15262ad1a3f7SMatthew Dillon * object, we can collapse it into the parent. 1527df8bae1dSRodney W. Grimes */ 15282ad1a3f7SMatthew Dillon vm_object_backing_scan(object, OBSC_COLLAPSE_WAIT); 1529df8bae1dSRodney W. Grimes 1530df8bae1dSRodney W. Grimes /* 1531df8bae1dSRodney W. Grimes * Move the pager from backing_object to object. 1532df8bae1dSRodney W. Grimes */ 153324a1cce3SDavid Greenman if (backing_object->type == OBJT_SWAP) { 1534d22bc710SAlan Cox VM_OBJECT_LOCK(backing_object); 1535d474eaaaSDoug Rabson vm_object_pip_add(backing_object, 1); 1536d22bc710SAlan Cox VM_OBJECT_UNLOCK(backing_object); 153724a1cce3SDavid Greenman /* 15381c7c3c6aSMatthew Dillon * scrap the paging_offset junk and do a 15391c7c3c6aSMatthew Dillon * discrete copy. This also removes major 15401c7c3c6aSMatthew Dillon * assumptions about how the swap-pager 15411c7c3c6aSMatthew Dillon * works from where it doesn't belong. The 15421c7c3c6aSMatthew Dillon * new swapper is able to optimize the 15431c7c3c6aSMatthew Dillon * destroy-source case. 154424a1cce3SDavid Greenman */ 1545d22bc710SAlan Cox VM_OBJECT_LOCK(object); 15461c7c3c6aSMatthew Dillon vm_object_pip_add(object, 1); 1547d22bc710SAlan Cox VM_OBJECT_UNLOCK(object); 15481c7c3c6aSMatthew Dillon swap_pager_copy( 15491c7c3c6aSMatthew Dillon backing_object, 15501c7c3c6aSMatthew Dillon object, 15511c7c3c6aSMatthew Dillon OFF_TO_IDX(object->backing_object_offset), TRUE); 1552d7a013c3SAlan Cox VM_OBJECT_LOCK(object); 1553f919ebdeSDavid Greenman vm_object_pip_wakeup(object); 1554d7a013c3SAlan Cox VM_OBJECT_UNLOCK(object); 1555c0503609SDavid Greenman 1556d7a013c3SAlan Cox VM_OBJECT_LOCK(backing_object); 1557f919ebdeSDavid Greenman vm_object_pip_wakeup(backing_object); 1558d7a013c3SAlan Cox VM_OBJECT_UNLOCK(backing_object); 1559c0503609SDavid Greenman } 1560df8bae1dSRodney W. Grimes /* 1561df8bae1dSRodney W. Grimes * Object now shadows whatever backing_object did. 15622ad1a3f7SMatthew Dillon * Note that the reference to 15632ad1a3f7SMatthew Dillon * backing_object->backing_object moves from within 15642ad1a3f7SMatthew Dillon * backing_object to within object. 1565df8bae1dSRodney W. Grimes */ 15662ad1a3f7SMatthew Dillon TAILQ_REMOVE( 15672ad1a3f7SMatthew Dillon &object->backing_object->shadow_head, 15682ad1a3f7SMatthew Dillon object, 15692ad1a3f7SMatthew Dillon shadow_list 15702ad1a3f7SMatthew Dillon ); 1571eaf13dd7SJohn Dyson object->backing_object->shadow_count--; 1572eaf13dd7SJohn Dyson object->backing_object->generation++; 1573de5f6a77SJohn Dyson if (backing_object->backing_object) { 15742ad1a3f7SMatthew Dillon TAILQ_REMOVE( 15752ad1a3f7SMatthew Dillon &backing_object->backing_object->shadow_head, 15762ad1a3f7SMatthew Dillon backing_object, 15772ad1a3f7SMatthew Dillon shadow_list 15782ad1a3f7SMatthew Dillon ); 1579eaf13dd7SJohn Dyson backing_object->backing_object->shadow_count--; 1580eaf13dd7SJohn Dyson backing_object->backing_object->generation++; 1581de5f6a77SJohn Dyson } 158224a1cce3SDavid Greenman object->backing_object = backing_object->backing_object; 1583de5f6a77SJohn Dyson if (object->backing_object) { 15842ad1a3f7SMatthew Dillon TAILQ_INSERT_TAIL( 15852ad1a3f7SMatthew Dillon &object->backing_object->shadow_head, 15862ad1a3f7SMatthew Dillon object, 15872ad1a3f7SMatthew Dillon shadow_list 15882ad1a3f7SMatthew Dillon ); 1589eaf13dd7SJohn Dyson object->backing_object->shadow_count++; 1590eaf13dd7SJohn Dyson object->backing_object->generation++; 1591de5f6a77SJohn Dyson } 15922fe6e4d7SDavid Greenman 15932ad1a3f7SMatthew Dillon object->backing_object_offset += 15942ad1a3f7SMatthew Dillon backing_object->backing_object_offset; 15952ad1a3f7SMatthew Dillon 1596df8bae1dSRodney W. Grimes /* 1597df8bae1dSRodney W. Grimes * Discard backing_object. 1598df8bae1dSRodney W. Grimes * 15990d94caffSDavid Greenman * Since the backing object has no pages, no pager left, 16000d94caffSDavid Greenman * and no object references within it, all that is 16010d94caffSDavid Greenman * necessary is to dispose of it. 1602df8bae1dSRodney W. Grimes */ 1603245df27cSMatthew Dillon KASSERT(backing_object->ref_count == 1, ("backing_object %p was somehow re-referenced during collapse!", backing_object)); 1604245df27cSMatthew Dillon KASSERT(TAILQ_FIRST(&backing_object->memq) == NULL, ("backing_object %p somehow has left over pages during collapse!", backing_object)); 1605df8bae1dSRodney W. Grimes 1606a5698387SAlan Cox mtx_lock(&vm_object_list_mtx); 16072ad1a3f7SMatthew Dillon TAILQ_REMOVE( 16082ad1a3f7SMatthew Dillon &vm_object_list, 16092ad1a3f7SMatthew Dillon backing_object, 16102ad1a3f7SMatthew Dillon object_list 16112ad1a3f7SMatthew Dillon ); 1612a5698387SAlan Cox mtx_unlock(&vm_object_list_mtx); 1613df8bae1dSRodney W. Grimes 1614e3a9e1b2SAlan Cox mtx_destroy(&backing_object->mtx); 1615e3a9e1b2SAlan Cox 1616670d17b5SJeff Roberson uma_zfree(obj_zone, backing_object); 1617df8bae1dSRodney W. Grimes 1618df8bae1dSRodney W. Grimes object_collapses++; 16190d94caffSDavid Greenman } else { 162095e5e988SJohn Dyson vm_object_t new_backing_object; 1621df8bae1dSRodney W. Grimes 1622df8bae1dSRodney W. Grimes /* 16232ad1a3f7SMatthew Dillon * If we do not entirely shadow the backing object, 16242ad1a3f7SMatthew Dillon * there is nothing we can do so we give up. 1625df8bae1dSRodney W. Grimes */ 16262ad1a3f7SMatthew Dillon if (vm_object_backing_scan(object, OBSC_TEST_ALL_SHADOWED) == 0) { 16272ad1a3f7SMatthew Dillon break; 162824a1cce3SDavid Greenman } 1629df8bae1dSRodney W. Grimes 1630df8bae1dSRodney W. Grimes /* 16310d94caffSDavid Greenman * Make the parent shadow the next object in the 16320d94caffSDavid Greenman * chain. Deallocating backing_object will not remove 16330d94caffSDavid Greenman * it, since its reference count is at least 2. 1634df8bae1dSRodney W. Grimes */ 16352ad1a3f7SMatthew Dillon TAILQ_REMOVE( 16362ad1a3f7SMatthew Dillon &backing_object->shadow_head, 16372ad1a3f7SMatthew Dillon object, 16382ad1a3f7SMatthew Dillon shadow_list 16392ad1a3f7SMatthew Dillon ); 1640eaf13dd7SJohn Dyson backing_object->shadow_count--; 1641eaf13dd7SJohn Dyson backing_object->generation++; 164295e5e988SJohn Dyson 164395e5e988SJohn Dyson new_backing_object = backing_object->backing_object; 16448aef1712SMatthew Dillon if ((object->backing_object = new_backing_object) != NULL) { 164595e5e988SJohn Dyson vm_object_reference(new_backing_object); 16462ad1a3f7SMatthew Dillon TAILQ_INSERT_TAIL( 16472ad1a3f7SMatthew Dillon &new_backing_object->shadow_head, 16482ad1a3f7SMatthew Dillon object, 16492ad1a3f7SMatthew Dillon shadow_list 16502ad1a3f7SMatthew Dillon ); 1651eaf13dd7SJohn Dyson new_backing_object->shadow_count++; 1652eaf13dd7SJohn Dyson new_backing_object->generation++; 165395e5e988SJohn Dyson object->backing_object_offset += 165495e5e988SJohn Dyson backing_object->backing_object_offset; 1655de5f6a77SJohn Dyson } 1656df8bae1dSRodney W. Grimes 1657df8bae1dSRodney W. Grimes /* 16580d94caffSDavid Greenman * Drop the reference count on backing_object. Since 16590d94caffSDavid Greenman * its ref_count was at least 2, it will not vanish; 1660eaf13dd7SJohn Dyson * so we don't need to call vm_object_deallocate, but 1661eaf13dd7SJohn Dyson * we do anyway. 1662df8bae1dSRodney W. Grimes */ 166395e5e988SJohn Dyson vm_object_deallocate(backing_object); 1664df8bae1dSRodney W. Grimes object_bypasses++; 1665df8bae1dSRodney W. Grimes } 1666df8bae1dSRodney W. Grimes 1667df8bae1dSRodney W. Grimes /* 1668df8bae1dSRodney W. Grimes * Try again with this object's new backing object. 1669df8bae1dSRodney W. Grimes */ 1670df8bae1dSRodney W. Grimes } 1671df8bae1dSRodney W. Grimes } 1672df8bae1dSRodney W. Grimes 1673df8bae1dSRodney W. Grimes /* 1674df8bae1dSRodney W. Grimes * vm_object_page_remove: [internal] 1675df8bae1dSRodney W. Grimes * 1676df8bae1dSRodney W. Grimes * Removes all physical pages in the specified 1677df8bae1dSRodney W. Grimes * object range from the object's list of pages. 1678df8bae1dSRodney W. Grimes * 1679df8bae1dSRodney W. Grimes * The object must be locked. 1680df8bae1dSRodney W. Grimes */ 168126f9a767SRodney W. Grimes void 1682ecde4b32SAlan Cox vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end, 1683ecde4b32SAlan Cox boolean_t clean_only) 1684df8bae1dSRodney W. Grimes { 1685d031cff1SMatthew Dillon vm_page_t p, next; 1686df8bae1dSRodney W. Grimes 1687ecde4b32SAlan Cox VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1688ecde4b32SAlan Cox if (object->resident_page_count == 0) 1689df8bae1dSRodney W. Grimes return; 169095e5e988SJohn Dyson 16918b03c8edSMatthew Dillon /* 16928b03c8edSMatthew Dillon * Since physically-backed objects do not use managed pages, we can't 16938b03c8edSMatthew Dillon * remove pages from the object (we must instead remove the page 16948b03c8edSMatthew Dillon * references, and then destroy the object). 16958b03c8edSMatthew Dillon */ 1696ecde4b32SAlan Cox KASSERT(object->type != OBJT_PHYS, 1697ecde4b32SAlan Cox ("attempt to remove pages from a physical object")); 16988b03c8edSMatthew Dillon 1699d474eaaaSDoug Rabson vm_object_pip_add(object, 1); 170026f9a767SRodney W. Grimes again: 170132585dd6SAlan Cox vm_page_lock_queues(); 170275741c04SAlan Cox if ((p = TAILQ_FIRST(&object->memq)) != NULL) { 170375741c04SAlan Cox if (p->pindex < start) { 170475741c04SAlan Cox p = vm_page_splay(start, object->root); 170575741c04SAlan Cox if ((object->root = p)->pindex < start) 170675741c04SAlan Cox p = TAILQ_NEXT(p, listq); 170775741c04SAlan Cox } 170875741c04SAlan Cox } 170975741c04SAlan Cox /* 171075741c04SAlan Cox * Assert: the variable p is either (1) the page with the 171175741c04SAlan Cox * least pindex greater than or equal to the parameter pindex 171275741c04SAlan Cox * or (2) NULL. 171375741c04SAlan Cox */ 171475741c04SAlan Cox for (; 1715ecde4b32SAlan Cox p != NULL && p->pindex < end; 171675741c04SAlan Cox p = next) { 1717b18bfc3dSJohn Dyson next = TAILQ_NEXT(p, listq); 171875741c04SAlan Cox 1719bd7e5f99SJohn Dyson if (p->wire_count != 0) { 17204fec79beSAlan Cox pmap_remove_all(p); 1721ce65e68cSDavid Greenman if (!clean_only) 1722bd7e5f99SJohn Dyson p->valid = 0; 17230d94caffSDavid Greenman continue; 17240d94caffSDavid Greenman } 172532585dd6SAlan Cox if (vm_page_sleep_if_busy(p, TRUE, "vmopar")) 172626f9a767SRodney W. Grimes goto again; 17278f9110f6SJohn Dyson if (clean_only && p->valid) { 17287c1f6cedSDavid Greenman vm_page_test_dirty(p); 17297c1f6cedSDavid Greenman if (p->valid & p->dirty) 17307c1f6cedSDavid Greenman continue; 17317c1f6cedSDavid Greenman } 1732e69763a3SDoug Rabson vm_page_busy(p); 17334fec79beSAlan Cox pmap_remove_all(p); 1734df8bae1dSRodney W. Grimes vm_page_free(p); 173526f9a767SRodney W. Grimes } 173632585dd6SAlan Cox vm_page_unlock_queues(); 1737f919ebdeSDavid Greenman vm_object_pip_wakeup(object); 1738c0503609SDavid Greenman } 1739df8bae1dSRodney W. Grimes 1740df8bae1dSRodney W. Grimes /* 1741df8bae1dSRodney W. Grimes * Routine: vm_object_coalesce 1742df8bae1dSRodney W. Grimes * Function: Coalesces two objects backing up adjoining 1743df8bae1dSRodney W. Grimes * regions of memory into a single object. 1744df8bae1dSRodney W. Grimes * 1745df8bae1dSRodney W. Grimes * returns TRUE if objects were combined. 1746df8bae1dSRodney W. Grimes * 1747df8bae1dSRodney W. Grimes * NOTE: Only works at the moment if the second object is NULL - 1748df8bae1dSRodney W. Grimes * if it's not, which object do we lock first? 1749df8bae1dSRodney W. Grimes * 1750df8bae1dSRodney W. Grimes * Parameters: 1751df8bae1dSRodney W. Grimes * prev_object First object to coalesce 1752df8bae1dSRodney W. Grimes * prev_offset Offset into prev_object 1753df8bae1dSRodney W. Grimes * next_object Second object into coalesce 1754df8bae1dSRodney W. Grimes * next_offset Offset into next_object 1755df8bae1dSRodney W. Grimes * 1756df8bae1dSRodney W. Grimes * prev_size Size of reference to prev_object 1757df8bae1dSRodney W. Grimes * next_size Size of reference to next_object 1758df8bae1dSRodney W. Grimes * 1759df8bae1dSRodney W. Grimes * Conditions: 1760df8bae1dSRodney W. Grimes * The object must *not* be locked. 1761df8bae1dSRodney W. Grimes */ 17620d94caffSDavid Greenman boolean_t 176300e1854aSAlan Cox vm_object_coalesce(vm_object_t prev_object, vm_pindex_t prev_pindex, 176400e1854aSAlan Cox vm_size_t prev_size, vm_size_t next_size) 1765df8bae1dSRodney W. Grimes { 1766ea41812fSAlan Cox vm_pindex_t next_pindex; 1767df8bae1dSRodney W. Grimes 176800e1854aSAlan Cox if (prev_object == NULL) 1769df8bae1dSRodney W. Grimes return (TRUE); 177015c176c1SAlan Cox vm_object_lock(prev_object); 17714112823fSMatthew Dillon if (prev_object->type != OBJT_DEFAULT && 17724112823fSMatthew Dillon prev_object->type != OBJT_SWAP) { 177315c176c1SAlan Cox vm_object_unlock(prev_object); 177430dcfc09SJohn Dyson return (FALSE); 177530dcfc09SJohn Dyson } 177630dcfc09SJohn Dyson 1777df8bae1dSRodney W. Grimes /* 1778df8bae1dSRodney W. Grimes * Try to collapse the object first 1779df8bae1dSRodney W. Grimes */ 1780df8bae1dSRodney W. Grimes vm_object_collapse(prev_object); 1781df8bae1dSRodney W. Grimes 1782df8bae1dSRodney W. Grimes /* 17830d94caffSDavid Greenman * Can't coalesce if: . more than one reference . paged out . shadows 17840d94caffSDavid Greenman * another object . has a copy elsewhere (any of which mean that the 17850d94caffSDavid Greenman * pages not mapped to prev_entry may be in use anyway) 1786df8bae1dSRodney W. Grimes */ 17878cc7e047SJohn Dyson if (prev_object->backing_object != NULL) { 178815c176c1SAlan Cox vm_object_unlock(prev_object); 1789df8bae1dSRodney W. Grimes return (FALSE); 1790df8bae1dSRodney W. Grimes } 1791a316d390SJohn Dyson 1792a316d390SJohn Dyson prev_size >>= PAGE_SHIFT; 1793a316d390SJohn Dyson next_size >>= PAGE_SHIFT; 1794ea41812fSAlan Cox next_pindex = prev_pindex + prev_size; 17958cc7e047SJohn Dyson 17968cc7e047SJohn Dyson if ((prev_object->ref_count > 1) && 1797ea41812fSAlan Cox (prev_object->size != next_pindex)) { 179815c176c1SAlan Cox vm_object_unlock(prev_object); 17998cc7e047SJohn Dyson return (FALSE); 18008cc7e047SJohn Dyson } 18018cc7e047SJohn Dyson 1802df8bae1dSRodney W. Grimes /* 18030d94caffSDavid Greenman * Remove any pages that may still be in the object from a previous 18040d94caffSDavid Greenman * deallocation. 1805df8bae1dSRodney W. Grimes */ 1806ea41812fSAlan Cox if (next_pindex < prev_object->size) { 1807d647a0edSAlan Cox VM_OBJECT_LOCK(prev_object); 1808df8bae1dSRodney W. Grimes vm_object_page_remove(prev_object, 1809ea41812fSAlan Cox next_pindex, 1810ea41812fSAlan Cox next_pindex + next_size, FALSE); 1811ea41812fSAlan Cox if (prev_object->type == OBJT_SWAP) 1812ea41812fSAlan Cox swap_pager_freespace(prev_object, 1813ea41812fSAlan Cox next_pindex, next_size); 1814155080d3SAlan Cox VM_OBJECT_UNLOCK(prev_object); 1815ea41812fSAlan Cox } 1816df8bae1dSRodney W. Grimes 1817df8bae1dSRodney W. Grimes /* 1818df8bae1dSRodney W. Grimes * Extend the object if necessary. 1819df8bae1dSRodney W. Grimes */ 1820ea41812fSAlan Cox if (next_pindex + next_size > prev_object->size) 1821ea41812fSAlan Cox prev_object->size = next_pindex + next_size; 1822df8bae1dSRodney W. Grimes 182315c176c1SAlan Cox vm_object_unlock(prev_object); 1824df8bae1dSRodney W. Grimes return (TRUE); 1825df8bae1dSRodney W. Grimes } 1826df8bae1dSRodney W. Grimes 18277a5a6352SMatthew Dillon void 18287a5a6352SMatthew Dillon vm_object_set_writeable_dirty(vm_object_t object) 18297a5a6352SMatthew Dillon { 18307a5a6352SMatthew Dillon struct vnode *vp; 18317a5a6352SMatthew Dillon 18327a5a6352SMatthew Dillon vm_object_set_flag(object, OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY); 18337a5a6352SMatthew Dillon if (object->type == OBJT_VNODE && 18347a5a6352SMatthew Dillon (vp = (struct vnode *)object->handle) != NULL) { 1835e6e370a7SJeff Roberson VI_LOCK(vp); 1836e6e370a7SJeff Roberson if ((vp->v_iflag & VI_OBJDIRTY) == 0) 1837e6e370a7SJeff Roberson vp->v_iflag |= VI_OBJDIRTY; 1838e6e370a7SJeff Roberson VI_UNLOCK(vp); 18397a5a6352SMatthew Dillon } 18407a5a6352SMatthew Dillon } 18417a5a6352SMatthew Dillon 1842c7c34a24SBruce Evans #include "opt_ddb.h" 1843c3cb3e12SDavid Greenman #ifdef DDB 1844c7c34a24SBruce Evans #include <sys/kernel.h> 1845c7c34a24SBruce Evans 1846ce9edcf5SPoul-Henning Kamp #include <sys/cons.h> 1847c7c34a24SBruce Evans 1848c7c34a24SBruce Evans #include <ddb/ddb.h> 1849c7c34a24SBruce Evans 1850cac597e4SBruce Evans static int 18511b40f8c0SMatthew Dillon _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry) 1852a1f6d91cSDavid Greenman { 1853a1f6d91cSDavid Greenman vm_map_t tmpm; 1854a1f6d91cSDavid Greenman vm_map_entry_t tmpe; 1855a1f6d91cSDavid Greenman vm_object_t obj; 1856a1f6d91cSDavid Greenman int entcount; 1857a1f6d91cSDavid Greenman 1858a1f6d91cSDavid Greenman if (map == 0) 1859a1f6d91cSDavid Greenman return 0; 1860a1f6d91cSDavid Greenman 1861a1f6d91cSDavid Greenman if (entry == 0) { 1862a1f6d91cSDavid Greenman tmpe = map->header.next; 1863a1f6d91cSDavid Greenman entcount = map->nentries; 1864a1f6d91cSDavid Greenman while (entcount-- && (tmpe != &map->header)) { 1865a1f6d91cSDavid Greenman if (_vm_object_in_map(map, object, tmpe)) { 1866a1f6d91cSDavid Greenman return 1; 1867a1f6d91cSDavid Greenman } 1868a1f6d91cSDavid Greenman tmpe = tmpe->next; 1869a1f6d91cSDavid Greenman } 18709fdfe602SMatthew Dillon } else if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) { 18719fdfe602SMatthew Dillon tmpm = entry->object.sub_map; 1872a1f6d91cSDavid Greenman tmpe = tmpm->header.next; 1873a1f6d91cSDavid Greenman entcount = tmpm->nentries; 1874a1f6d91cSDavid Greenman while (entcount-- && tmpe != &tmpm->header) { 1875a1f6d91cSDavid Greenman if (_vm_object_in_map(tmpm, object, tmpe)) { 1876a1f6d91cSDavid Greenman return 1; 1877a1f6d91cSDavid Greenman } 1878a1f6d91cSDavid Greenman tmpe = tmpe->next; 1879a1f6d91cSDavid Greenman } 18808aef1712SMatthew Dillon } else if ((obj = entry->object.vm_object) != NULL) { 188124a1cce3SDavid Greenman for (; obj; obj = obj->backing_object) 1882a1f6d91cSDavid Greenman if (obj == object) { 1883a1f6d91cSDavid Greenman return 1; 1884a1f6d91cSDavid Greenman } 1885a1f6d91cSDavid Greenman } 1886a1f6d91cSDavid Greenman return 0; 1887a1f6d91cSDavid Greenman } 1888a1f6d91cSDavid Greenman 1889cac597e4SBruce Evans static int 18901b40f8c0SMatthew Dillon vm_object_in_map(vm_object_t object) 1891a1f6d91cSDavid Greenman { 1892a1f6d91cSDavid Greenman struct proc *p; 18931005a129SJohn Baldwin 189460517fd1SJohn Baldwin /* sx_slock(&allproc_lock); */ 1895fc2ffbe6SPoul-Henning Kamp LIST_FOREACH(p, &allproc, p_list) { 1896a1f6d91cSDavid Greenman if (!p->p_vmspace /* || (p->p_flag & (P_SYSTEM|P_WEXIT)) */) 1897a1f6d91cSDavid Greenman continue; 1898553629ebSJake Burkholder if (_vm_object_in_map(&p->p_vmspace->vm_map, object, 0)) { 189960517fd1SJohn Baldwin /* sx_sunlock(&allproc_lock); */ 1900a1f6d91cSDavid Greenman return 1; 1901a1f6d91cSDavid Greenman } 1902553629ebSJake Burkholder } 190360517fd1SJohn Baldwin /* sx_sunlock(&allproc_lock); */ 1904a1f6d91cSDavid Greenman if (_vm_object_in_map(kernel_map, object, 0)) 1905a1f6d91cSDavid Greenman return 1; 1906a1f6d91cSDavid Greenman if (_vm_object_in_map(kmem_map, object, 0)) 1907a1f6d91cSDavid Greenman return 1; 1908a1f6d91cSDavid Greenman if (_vm_object_in_map(pager_map, object, 0)) 1909a1f6d91cSDavid Greenman return 1; 1910a1f6d91cSDavid Greenman if (_vm_object_in_map(buffer_map, object, 0)) 1911a1f6d91cSDavid Greenman return 1; 1912a1f6d91cSDavid Greenman return 0; 1913a1f6d91cSDavid Greenman } 1914a1f6d91cSDavid Greenman 1915c7c34a24SBruce Evans DB_SHOW_COMMAND(vmochk, vm_object_check) 1916f708ef1bSPoul-Henning Kamp { 1917a1f6d91cSDavid Greenman vm_object_t object; 1918a1f6d91cSDavid Greenman 1919a1f6d91cSDavid Greenman /* 1920a1f6d91cSDavid Greenman * make sure that internal objs are in a map somewhere 1921a1f6d91cSDavid Greenman * and none have zero ref counts. 1922a1f6d91cSDavid Greenman */ 1923cc64b484SAlfred Perlstein TAILQ_FOREACH(object, &vm_object_list, object_list) { 192424a1cce3SDavid Greenman if (object->handle == NULL && 192524a1cce3SDavid Greenman (object->type == OBJT_DEFAULT || object->type == OBJT_SWAP)) { 1926a1f6d91cSDavid Greenman if (object->ref_count == 0) { 19273efc015bSPeter Wemm db_printf("vmochk: internal obj has zero ref count: %ld\n", 19283efc015bSPeter Wemm (long)object->size); 1929a1f6d91cSDavid Greenman } 1930a1f6d91cSDavid Greenman if (!vm_object_in_map(object)) { 1931fc62ef1fSBruce Evans db_printf( 1932fc62ef1fSBruce Evans "vmochk: internal obj is not in a map: " 1933fc62ef1fSBruce Evans "ref: %d, size: %lu: 0x%lx, backing_object: %p\n", 1934fc62ef1fSBruce Evans object->ref_count, (u_long)object->size, 1935fc62ef1fSBruce Evans (u_long)object->size, 1936fc62ef1fSBruce Evans (void *)object->backing_object); 1937a1f6d91cSDavid Greenman } 1938a1f6d91cSDavid Greenman } 1939a1f6d91cSDavid Greenman } 1940a1f6d91cSDavid Greenman } 1941a1f6d91cSDavid Greenman 194226f9a767SRodney W. Grimes /* 1943df8bae1dSRodney W. Grimes * vm_object_print: [ debug ] 1944df8bae1dSRodney W. Grimes */ 1945c7c34a24SBruce Evans DB_SHOW_COMMAND(object, vm_object_print_static) 1946df8bae1dSRodney W. Grimes { 1947c7c34a24SBruce Evans /* XXX convert args. */ 1948c7c34a24SBruce Evans vm_object_t object = (vm_object_t)addr; 1949c7c34a24SBruce Evans boolean_t full = have_addr; 1950c7c34a24SBruce Evans 1951d031cff1SMatthew Dillon vm_page_t p; 1952df8bae1dSRodney W. Grimes 1953c7c34a24SBruce Evans /* XXX count is an (unused) arg. Avoid shadowing it. */ 1954c7c34a24SBruce Evans #define count was_count 1955c7c34a24SBruce Evans 1956d031cff1SMatthew Dillon int count; 1957df8bae1dSRodney W. Grimes 1958df8bae1dSRodney W. Grimes if (object == NULL) 1959df8bae1dSRodney W. Grimes return; 1960df8bae1dSRodney W. Grimes 1961eb95adefSBruce Evans db_iprintf( 1962e47cd172SMaxime Henrion "Object %p: type=%d, size=0x%jx, res=%d, ref=%d, flags=0x%x\n", 1963e47cd172SMaxime Henrion object, (int)object->type, (uintmax_t)object->size, 1964eb95adefSBruce Evans object->resident_page_count, object->ref_count, object->flags); 1965e47cd172SMaxime Henrion db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%jx\n", 19661c7c3c6aSMatthew Dillon object->shadow_count, 1967eb95adefSBruce Evans object->backing_object ? object->backing_object->ref_count : 0, 1968e47cd172SMaxime Henrion object->backing_object, (uintmax_t)object->backing_object_offset); 1969df8bae1dSRodney W. Grimes 1970df8bae1dSRodney W. Grimes if (!full) 1971df8bae1dSRodney W. Grimes return; 1972df8bae1dSRodney W. Grimes 1973c7c34a24SBruce Evans db_indent += 2; 1974df8bae1dSRodney W. Grimes count = 0; 1975fc2ffbe6SPoul-Henning Kamp TAILQ_FOREACH(p, &object->memq, listq) { 1976df8bae1dSRodney W. Grimes if (count == 0) 1977c7c34a24SBruce Evans db_iprintf("memory:="); 1978df8bae1dSRodney W. Grimes else if (count == 6) { 1979c7c34a24SBruce Evans db_printf("\n"); 1980c7c34a24SBruce Evans db_iprintf(" ..."); 1981df8bae1dSRodney W. Grimes count = 0; 1982df8bae1dSRodney W. Grimes } else 1983c7c34a24SBruce Evans db_printf(","); 1984df8bae1dSRodney W. Grimes count++; 1985df8bae1dSRodney W. Grimes 1986e47cd172SMaxime Henrion db_printf("(off=0x%jx,page=0x%jx)", 1987e47cd172SMaxime Henrion (uintmax_t)p->pindex, (uintmax_t)VM_PAGE_TO_PHYS(p)); 1988df8bae1dSRodney W. Grimes } 1989df8bae1dSRodney W. Grimes if (count != 0) 1990c7c34a24SBruce Evans db_printf("\n"); 1991c7c34a24SBruce Evans db_indent -= 2; 1992df8bae1dSRodney W. Grimes } 19935070c7f8SJohn Dyson 1994c7c34a24SBruce Evans /* XXX. */ 1995c7c34a24SBruce Evans #undef count 1996c7c34a24SBruce Evans 1997c7c34a24SBruce Evans /* XXX need this non-static entry for calling from vm_map_print. */ 19985070c7f8SJohn Dyson void 19991b40f8c0SMatthew Dillon vm_object_print( 20001b40f8c0SMatthew Dillon /* db_expr_t */ long addr, 20011b40f8c0SMatthew Dillon boolean_t have_addr, 20021b40f8c0SMatthew Dillon /* db_expr_t */ long count, 20031b40f8c0SMatthew Dillon char *modif) 2004c7c34a24SBruce Evans { 2005c7c34a24SBruce Evans vm_object_print_static(addr, have_addr, count, modif); 2006c7c34a24SBruce Evans } 2007c7c34a24SBruce Evans 2008c7c34a24SBruce Evans DB_SHOW_COMMAND(vmopag, vm_object_print_pages) 20095070c7f8SJohn Dyson { 20105070c7f8SJohn Dyson vm_object_t object; 20115070c7f8SJohn Dyson int nl = 0; 20125070c7f8SJohn Dyson int c; 2013cc64b484SAlfred Perlstein 2014cc64b484SAlfred Perlstein TAILQ_FOREACH(object, &vm_object_list, object_list) { 20155070c7f8SJohn Dyson vm_pindex_t idx, fidx; 20165070c7f8SJohn Dyson vm_pindex_t osize; 2017227f9a1cSJake Burkholder vm_paddr_t pa = -1, padiff; 20185070c7f8SJohn Dyson int rcount; 20195070c7f8SJohn Dyson vm_page_t m; 20205070c7f8SJohn Dyson 2021fc62ef1fSBruce Evans db_printf("new object: %p\n", (void *)object); 20225070c7f8SJohn Dyson if (nl > 18) { 20235070c7f8SJohn Dyson c = cngetc(); 20245070c7f8SJohn Dyson if (c != ' ') 20255070c7f8SJohn Dyson return; 20265070c7f8SJohn Dyson nl = 0; 20275070c7f8SJohn Dyson } 20285070c7f8SJohn Dyson nl++; 20295070c7f8SJohn Dyson rcount = 0; 20305070c7f8SJohn Dyson fidx = 0; 20315070c7f8SJohn Dyson osize = object->size; 20325070c7f8SJohn Dyson if (osize > 128) 20335070c7f8SJohn Dyson osize = 128; 20345070c7f8SJohn Dyson for (idx = 0; idx < osize; idx++) { 20355070c7f8SJohn Dyson m = vm_page_lookup(object, idx); 20365070c7f8SJohn Dyson if (m == NULL) { 20375070c7f8SJohn Dyson if (rcount) { 20383efc015bSPeter Wemm db_printf(" index(%ld)run(%d)pa(0x%lx)\n", 20393efc015bSPeter Wemm (long)fidx, rcount, (long)pa); 20405070c7f8SJohn Dyson if (nl > 18) { 20415070c7f8SJohn Dyson c = cngetc(); 20425070c7f8SJohn Dyson if (c != ' ') 20435070c7f8SJohn Dyson return; 20445070c7f8SJohn Dyson nl = 0; 20455070c7f8SJohn Dyson } 20465070c7f8SJohn Dyson nl++; 20475070c7f8SJohn Dyson rcount = 0; 20485070c7f8SJohn Dyson } 20495070c7f8SJohn Dyson continue; 20505070c7f8SJohn Dyson } 20515070c7f8SJohn Dyson 20525070c7f8SJohn Dyson 20535070c7f8SJohn Dyson if (rcount && 20545070c7f8SJohn Dyson (VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) { 20555070c7f8SJohn Dyson ++rcount; 20565070c7f8SJohn Dyson continue; 20575070c7f8SJohn Dyson } 20585070c7f8SJohn Dyson if (rcount) { 20595070c7f8SJohn Dyson padiff = pa + rcount * PAGE_SIZE - VM_PAGE_TO_PHYS(m); 20605070c7f8SJohn Dyson padiff >>= PAGE_SHIFT; 20615070c7f8SJohn Dyson padiff &= PQ_L2_MASK; 20625070c7f8SJohn Dyson if (padiff == 0) { 20635070c7f8SJohn Dyson pa = VM_PAGE_TO_PHYS(m) - rcount * PAGE_SIZE; 20645070c7f8SJohn Dyson ++rcount; 20655070c7f8SJohn Dyson continue; 20665070c7f8SJohn Dyson } 20673efc015bSPeter Wemm db_printf(" index(%ld)run(%d)pa(0x%lx)", 20683efc015bSPeter Wemm (long)fidx, rcount, (long)pa); 20693efc015bSPeter Wemm db_printf("pd(%ld)\n", (long)padiff); 20705070c7f8SJohn Dyson if (nl > 18) { 20715070c7f8SJohn Dyson c = cngetc(); 20725070c7f8SJohn Dyson if (c != ' ') 20735070c7f8SJohn Dyson return; 20745070c7f8SJohn Dyson nl = 0; 20755070c7f8SJohn Dyson } 20765070c7f8SJohn Dyson nl++; 20775070c7f8SJohn Dyson } 20785070c7f8SJohn Dyson fidx = idx; 20795070c7f8SJohn Dyson pa = VM_PAGE_TO_PHYS(m); 20805070c7f8SJohn Dyson rcount = 1; 20815070c7f8SJohn Dyson } 20825070c7f8SJohn Dyson if (rcount) { 20833efc015bSPeter Wemm db_printf(" index(%ld)run(%d)pa(0x%lx)\n", 20843efc015bSPeter Wemm (long)fidx, rcount, (long)pa); 20855070c7f8SJohn Dyson if (nl > 18) { 20865070c7f8SJohn Dyson c = cngetc(); 20875070c7f8SJohn Dyson if (c != ' ') 20885070c7f8SJohn Dyson return; 20895070c7f8SJohn Dyson nl = 0; 20905070c7f8SJohn Dyson } 20915070c7f8SJohn Dyson nl++; 20925070c7f8SJohn Dyson } 20935070c7f8SJohn Dyson } 20945070c7f8SJohn Dyson } 2095c3cb3e12SDavid Greenman #endif /* DDB */ 2096