160727d8bSWarner Losh /*- 2796df753SPedro F. Giffuni * SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU) 351369649SPedro F. Giffuni * 4df8bae1dSRodney W. Grimes * Copyright (c) 1991, 1993 5df8bae1dSRodney W. Grimes * The Regents of the University of California. All rights reserved. 6df8bae1dSRodney W. Grimes * 7df8bae1dSRodney W. Grimes * This code is derived from software contributed to Berkeley by 8df8bae1dSRodney W. Grimes * The Mach Operating System project at Carnegie-Mellon University. 9df8bae1dSRodney W. Grimes * 10df8bae1dSRodney W. Grimes * Redistribution and use in source and binary forms, with or without 11df8bae1dSRodney W. Grimes * modification, are permitted provided that the following conditions 12df8bae1dSRodney W. Grimes * are met: 13df8bae1dSRodney W. Grimes * 1. Redistributions of source code must retain the above copyright 14df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer. 15df8bae1dSRodney W. Grimes * 2. Redistributions in binary form must reproduce the above copyright 16df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer in the 17df8bae1dSRodney W. Grimes * documentation and/or other materials provided with the distribution. 18fbbd9655SWarner Losh * 3. Neither the name of the University nor the names of its contributors 19df8bae1dSRodney W. Grimes * may be used to endorse or promote products derived from this software 20df8bae1dSRodney W. Grimes * without specific prior written permission. 21df8bae1dSRodney W. Grimes * 22df8bae1dSRodney W. Grimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23df8bae1dSRodney W. Grimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24df8bae1dSRodney W. Grimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25df8bae1dSRodney W. Grimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26df8bae1dSRodney W. Grimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27df8bae1dSRodney W. Grimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28df8bae1dSRodney W. Grimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29df8bae1dSRodney W. Grimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30df8bae1dSRodney W. Grimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31df8bae1dSRodney W. Grimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32df8bae1dSRodney W. Grimes * SUCH DAMAGE. 33df8bae1dSRodney W. Grimes * 343c4dd356SDavid Greenman * from: @(#)vm_object.c 8.5 (Berkeley) 3/22/94 35df8bae1dSRodney W. Grimes * 36df8bae1dSRodney W. Grimes * 37df8bae1dSRodney W. Grimes * Copyright (c) 1987, 1990 Carnegie-Mellon University. 38df8bae1dSRodney W. Grimes * All rights reserved. 39df8bae1dSRodney W. Grimes * 40df8bae1dSRodney W. Grimes * Authors: Avadis Tevanian, Jr., Michael Wayne Young 41df8bae1dSRodney W. Grimes * 42df8bae1dSRodney W. Grimes * Permission to use, copy, modify and distribute this software and 43df8bae1dSRodney W. Grimes * its documentation is hereby granted, provided that both the copyright 44df8bae1dSRodney W. Grimes * notice and this permission notice appear in all copies of the 45df8bae1dSRodney W. Grimes * software, derivative works or modified versions, and any portions 46df8bae1dSRodney W. Grimes * thereof, and that both notices appear in supporting documentation. 47df8bae1dSRodney W. Grimes * 48df8bae1dSRodney W. Grimes * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 49df8bae1dSRodney W. Grimes * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 50df8bae1dSRodney W. Grimes * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 51df8bae1dSRodney W. Grimes * 52df8bae1dSRodney W. Grimes * Carnegie Mellon requests users of this software to return to 53df8bae1dSRodney W. Grimes * 54df8bae1dSRodney W. Grimes * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 55df8bae1dSRodney W. Grimes * School of Computer Science 56df8bae1dSRodney W. Grimes * Carnegie Mellon University 57df8bae1dSRodney W. Grimes * Pittsburgh PA 15213-3890 58df8bae1dSRodney W. Grimes * 59df8bae1dSRodney W. Grimes * any improvements or extensions that they make and grant Carnegie the 60df8bae1dSRodney W. Grimes * rights to redistribute these changes. 61df8bae1dSRodney W. Grimes */ 62df8bae1dSRodney W. Grimes 63df8bae1dSRodney W. Grimes /* 64df8bae1dSRodney W. Grimes * Virtual memory object module. 65df8bae1dSRodney W. Grimes */ 66df8bae1dSRodney W. Grimes 67874651b1SDavid E. O'Brien #include <sys/cdefs.h> 68874651b1SDavid E. O'Brien __FBSDID("$FreeBSD$"); 69874651b1SDavid E. O'Brien 70f8a47341SAlan Cox #include "opt_vm.h" 71f8a47341SAlan Cox 72df8bae1dSRodney W. Grimes #include <sys/param.h> 73df8bae1dSRodney W. Grimes #include <sys/systm.h> 743f289c3fSJeff Roberson #include <sys/cpuset.h> 75fb919e4dSMark Murray #include <sys/lock.h> 76867a482dSJohn Dyson #include <sys/mman.h> 77cf2819ccSJohn Dyson #include <sys/mount.h> 78b9b7a4beSMatthew Dillon #include <sys/kernel.h> 79f425ab8eSKonstantin Belousov #include <sys/pctrie.h> 80b9b7a4beSMatthew Dillon #include <sys/sysctl.h> 811b367556SJason Evans #include <sys/mutex.h> 82fb919e4dSMark Murray #include <sys/proc.h> /* for curproc, pageproc */ 83cf27e0d1SJeff Roberson #include <sys/refcount.h> 84fb919e4dSMark Murray #include <sys/socket.h> 853364c323SKonstantin Belousov #include <sys/resourcevar.h> 86205be21dSJeff Roberson #include <sys/refcount.h> 8789f6b863SAttilio Rao #include <sys/rwlock.h> 88ff87ae35SJohn Baldwin #include <sys/user.h> 89fb919e4dSMark Murray #include <sys/vnode.h> 90fb919e4dSMark Murray #include <sys/vmmeter.h> 911005a129SJohn Baldwin #include <sys/sx.h> 92df8bae1dSRodney W. Grimes 93df8bae1dSRodney W. Grimes #include <vm/vm.h> 94efeaf95aSDavid Greenman #include <vm/vm_param.h> 95efeaf95aSDavid Greenman #include <vm/pmap.h> 96efeaf95aSDavid Greenman #include <vm/vm_map.h> 97efeaf95aSDavid Greenman #include <vm/vm_object.h> 98df8bae1dSRodney W. Grimes #include <vm/vm_page.h> 9926f9a767SRodney W. Grimes #include <vm/vm_pageout.h> 1000d94caffSDavid Greenman #include <vm/vm_pager.h> 101e2068d0bSJeff Roberson #include <vm/vm_phys.h> 102e2068d0bSJeff Roberson #include <vm/vm_pagequeue.h> 10305f0fdd2SPoul-Henning Kamp #include <vm/swap_pager.h> 104a1f6d91cSDavid Greenman #include <vm/vm_kern.h> 105efeaf95aSDavid Greenman #include <vm/vm_extern.h> 106774d251dSAttilio Rao #include <vm/vm_radix.h> 107f8a47341SAlan Cox #include <vm/vm_reserv.h> 108670d17b5SJeff Roberson #include <vm/uma.h> 10926f9a767SRodney W. Grimes 110c53f7aceSDag-Erling Smørgrav static int old_msync; 111c53f7aceSDag-Erling Smørgrav SYSCTL_INT(_vm, OID_AUTO, old_msync, CTLFLAG_RW, &old_msync, 0, 112c53f7aceSDag-Erling Smørgrav "Use old (insecure) msync behavior"); 113c53f7aceSDag-Erling Smørgrav 114757216f3SKonstantin Belousov static int vm_object_page_collect_flush(vm_object_t object, vm_page_t p, 115126d6082SKonstantin Belousov int pagerflags, int flags, boolean_t *clearobjflags, 116126d6082SKonstantin Belousov boolean_t *eio); 1173280870dSKonstantin Belousov static boolean_t vm_object_page_remove_write(vm_page_t p, int flags, 118126d6082SKonstantin Belousov boolean_t *clearobjflags); 119b9b7a4beSMatthew Dillon static void vm_object_qcollapse(vm_object_t object); 12002dd8331SAlan Cox static void vm_object_vndeallocate(vm_object_t object); 121f6b04d2bSDavid Greenman 122df8bae1dSRodney W. Grimes /* 123df8bae1dSRodney W. Grimes * Virtual memory objects maintain the actual data 124df8bae1dSRodney W. Grimes * associated with allocated virtual memory. A given 125df8bae1dSRodney W. Grimes * page of memory exists within exactly one object. 126df8bae1dSRodney W. Grimes * 127df8bae1dSRodney W. Grimes * An object is only deallocated when all "references" 128df8bae1dSRodney W. Grimes * are given up. Only one "reference" to a given 129df8bae1dSRodney W. Grimes * region of an object should be writeable. 130df8bae1dSRodney W. Grimes * 131df8bae1dSRodney W. Grimes * Associated with each object is a list of all resident 132df8bae1dSRodney W. Grimes * memory pages belonging to that object; this list is 133df8bae1dSRodney W. Grimes * maintained by the "vm_page" module, and locked by the object's 134df8bae1dSRodney W. Grimes * lock. 135df8bae1dSRodney W. Grimes * 136df8bae1dSRodney W. Grimes * Each object also records a "pager" routine which is 137df8bae1dSRodney W. Grimes * used to retrieve (and store) pages to the proper backing 138df8bae1dSRodney W. Grimes * storage. In addition, objects may be backed by other 139df8bae1dSRodney W. Grimes * objects from which they were virtual-copied. 140df8bae1dSRodney W. Grimes * 141df8bae1dSRodney W. Grimes * The only items within the object structure which are 142df8bae1dSRodney W. Grimes * modified after time of creation are: 143df8bae1dSRodney W. Grimes * reference count locked by object's lock 144df8bae1dSRodney W. Grimes * pager routine locked by object's lock 145df8bae1dSRodney W. Grimes * 146df8bae1dSRodney W. Grimes */ 147df8bae1dSRodney W. Grimes 14828f8db14SBruce Evans struct object_q vm_object_list; 149a5698387SAlan Cox struct mtx vm_object_list_mtx; /* lock for object list and count */ 150cccf11b8SAlan Cox 151cccf11b8SAlan Cox struct vm_object kernel_object_store; 152df8bae1dSRodney W. Grimes 1536472ac3dSEd Schouten static SYSCTL_NODE(_vm_stats, OID_AUTO, object, CTLFLAG_RD, 0, 1546472ac3dSEd Schouten "VM object stats"); 155604c2bbcSAlan Cox 15611542376SAlan Cox static counter_u64_t object_collapses = EARLY_COUNTER; 15711542376SAlan Cox SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, collapses, CTLFLAG_RD, 15811542376SAlan Cox &object_collapses, 15911542376SAlan Cox "VM object collapses"); 160604c2bbcSAlan Cox 16111542376SAlan Cox static counter_u64_t object_bypasses = EARLY_COUNTER; 16211542376SAlan Cox SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, bypasses, CTLFLAG_RD, 16311542376SAlan Cox &object_bypasses, 16411542376SAlan Cox "VM object bypasses"); 16511542376SAlan Cox 16611542376SAlan Cox static void 16711542376SAlan Cox counter_startup(void) 16811542376SAlan Cox { 16911542376SAlan Cox 17011542376SAlan Cox object_collapses = counter_u64_alloc(M_WAITOK); 17111542376SAlan Cox object_bypasses = counter_u64_alloc(M_WAITOK); 17211542376SAlan Cox } 17311542376SAlan Cox SYSINIT(object_counters, SI_SUB_CPU, SI_ORDER_ANY, counter_startup, NULL); 174dad740e9SAlan Cox 175670d17b5SJeff Roberson static uma_zone_t obj_zone; 1768355f576SJeff Roberson 177b23f72e9SBrian Feldman static int vm_object_zinit(void *mem, int size, int flags); 1788355f576SJeff Roberson 1798355f576SJeff Roberson #ifdef INVARIANTS 1808355f576SJeff Roberson static void vm_object_zdtor(void *mem, int size, void *arg); 1818355f576SJeff Roberson 1828355f576SJeff Roberson static void 1838355f576SJeff Roberson vm_object_zdtor(void *mem, int size, void *arg) 1848355f576SJeff Roberson { 1858355f576SJeff Roberson vm_object_t object; 1868355f576SJeff Roberson 1878355f576SJeff Roberson object = (vm_object_t)mem; 188e735691bSJohn Baldwin KASSERT(object->ref_count == 0, 189e735691bSJohn Baldwin ("object %p ref_count = %d", object, object->ref_count)); 19043186e53SAlan Cox KASSERT(TAILQ_EMPTY(&object->memq), 191198da1b2SAttilio Rao ("object %p has resident pages in its memq", object)); 192774d251dSAttilio Rao KASSERT(vm_radix_is_empty(&object->rtree), 193774d251dSAttilio Rao ("object %p has resident pages in its trie", object)); 194f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0 195f8a47341SAlan Cox KASSERT(LIST_EMPTY(&object->rvq), 196f8a47341SAlan Cox ("object %p has reservations", 197f8a47341SAlan Cox object)); 198f8a47341SAlan Cox #endif 19911b57401SHans Petter Selasky KASSERT(REFCOUNT_COUNT(object->paging_in_progress) == 0, 2008355f576SJeff Roberson ("object %p paging_in_progress = %d", 20111b57401SHans Petter Selasky object, REFCOUNT_COUNT(object->paging_in_progress))); 202205be21dSJeff Roberson KASSERT(object->busy == 0, 203205be21dSJeff Roberson ("object %p busy = %d", 204205be21dSJeff Roberson object, object->busy)); 2058355f576SJeff Roberson KASSERT(object->resident_page_count == 0, 2068355f576SJeff Roberson ("object %p resident_page_count = %d", 2078355f576SJeff Roberson object, object->resident_page_count)); 2088355f576SJeff Roberson KASSERT(object->shadow_count == 0, 2098355f576SJeff Roberson ("object %p shadow_count = %d", 2108355f576SJeff Roberson object, object->shadow_count)); 211e735691bSJohn Baldwin KASSERT(object->type == OBJT_DEAD, 212e735691bSJohn Baldwin ("object %p has non-dead type %d", 213e735691bSJohn Baldwin object, object->type)); 2148355f576SJeff Roberson } 2158355f576SJeff Roberson #endif 2168355f576SJeff Roberson 217b23f72e9SBrian Feldman static int 218b23f72e9SBrian Feldman vm_object_zinit(void *mem, int size, int flags) 2198355f576SJeff Roberson { 2208355f576SJeff Roberson vm_object_t object; 2218355f576SJeff Roberson 2228355f576SJeff Roberson object = (vm_object_t)mem; 223777a36c5SAlan Cox rw_init_flags(&object->lock, "vm object", RW_DUPOK | RW_NEW); 2248355f576SJeff Roberson 2258355f576SJeff Roberson /* These are true for any object that has been freed */ 226e735691bSJohn Baldwin object->type = OBJT_DEAD; 227cd1241fbSKonstantin Belousov vm_radix_init(&object->rtree); 228*51df5321SJeff Roberson refcount_init(&object->ref_count, 0); 229cf27e0d1SJeff Roberson refcount_init(&object->paging_in_progress, 0); 230205be21dSJeff Roberson refcount_init(&object->busy, 0); 2318355f576SJeff Roberson object->resident_page_count = 0; 2328355f576SJeff Roberson object->shadow_count = 0; 233f425ab8eSKonstantin Belousov object->flags = OBJ_DEAD; 234e735691bSJohn Baldwin 235e735691bSJohn Baldwin mtx_lock(&vm_object_list_mtx); 236e735691bSJohn Baldwin TAILQ_INSERT_TAIL(&vm_object_list, object, object_list); 237e735691bSJohn Baldwin mtx_unlock(&vm_object_list_mtx); 238b23f72e9SBrian Feldman return (0); 2398355f576SJeff Roberson } 240df8bae1dSRodney W. Grimes 241a4915c21SAttilio Rao static void 2426395da54SIan Dowse _vm_object_allocate(objtype_t type, vm_pindex_t size, vm_object_t object) 243df8bae1dSRodney W. Grimes { 2440cddd8f0SMatthew Dillon 245df8bae1dSRodney W. Grimes TAILQ_INIT(&object->memq); 2461c500307SAlan Cox LIST_INIT(&object->shadow_head); 247a1f6d91cSDavid Greenman 24824a1cce3SDavid Greenman object->type = type; 249f425ab8eSKonstantin Belousov if (type == OBJT_SWAP) 250f425ab8eSKonstantin Belousov pctrie_init(&object->un_pager.swp.swp_blks); 251f425ab8eSKonstantin Belousov 252f425ab8eSKonstantin Belousov /* 253f425ab8eSKonstantin Belousov * Ensure that swap_pager_swapoff() iteration over object_list 254f425ab8eSKonstantin Belousov * sees up to date type and pctrie head if it observed 255f425ab8eSKonstantin Belousov * non-dead object. 256f425ab8eSKonstantin Belousov */ 257f425ab8eSKonstantin Belousov atomic_thread_fence_rel(); 258f425ab8eSKonstantin Belousov 25928634820SAlan Cox switch (type) { 26028634820SAlan Cox case OBJT_DEAD: 26128634820SAlan Cox panic("_vm_object_allocate: can't create OBJT_DEAD"); 26228634820SAlan Cox case OBJT_DEFAULT: 26328634820SAlan Cox case OBJT_SWAP: 26428634820SAlan Cox object->flags = OBJ_ONEMAPPING; 26528634820SAlan Cox break; 26628634820SAlan Cox case OBJT_DEVICE: 26728634820SAlan Cox case OBJT_SG: 26828634820SAlan Cox object->flags = OBJ_FICTITIOUS | OBJ_UNMANAGED; 26928634820SAlan Cox break; 27028634820SAlan Cox case OBJT_MGTDEVICE: 27128634820SAlan Cox object->flags = OBJ_FICTITIOUS; 27228634820SAlan Cox break; 27328634820SAlan Cox case OBJT_PHYS: 27428634820SAlan Cox object->flags = OBJ_UNMANAGED; 27528634820SAlan Cox break; 27628634820SAlan Cox case OBJT_VNODE: 27728634820SAlan Cox object->flags = 0; 27828634820SAlan Cox break; 27928634820SAlan Cox default: 28028634820SAlan Cox panic("_vm_object_allocate: type %d is undefined", type); 28128634820SAlan Cox } 282df8bae1dSRodney W. Grimes object->size = size; 2834c29d2deSMark Johnston object->domain.dr_policy = NULL; 284b881da26SAlan Cox object->generation = 1; 285*51df5321SJeff Roberson refcount_init(&object->ref_count, 1); 2863153e878SAlan Cox object->memattr = VM_MEMATTR_DEFAULT; 287ef694c1aSEdward Tomasz Napierala object->cred = NULL; 2883364c323SKonstantin Belousov object->charge = 0; 28924a1cce3SDavid Greenman object->handle = NULL; 29024a1cce3SDavid Greenman object->backing_object = NULL; 291a316d390SJohn Dyson object->backing_object_offset = (vm_ooffset_t) 0; 292f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0 293f8a47341SAlan Cox LIST_INIT(&object->rvq); 294f8a47341SAlan Cox #endif 2951bdbd705SKonstantin Belousov umtx_shm_object_init(object); 296df8bae1dSRodney W. Grimes } 297df8bae1dSRodney W. Grimes 298df8bae1dSRodney W. Grimes /* 29926f9a767SRodney W. Grimes * vm_object_init: 30026f9a767SRodney W. Grimes * 30126f9a767SRodney W. Grimes * Initialize the VM objects module. 30226f9a767SRodney W. Grimes */ 30326f9a767SRodney W. Grimes void 3041b40f8c0SMatthew Dillon vm_object_init(void) 30526f9a767SRodney W. Grimes { 30626f9a767SRodney W. Grimes TAILQ_INIT(&vm_object_list); 3076008862bSJohn Baldwin mtx_init(&vm_object_list_mtx, "vm object_list", NULL, MTX_DEF); 3080217125fSDavid Greenman 30989f6b863SAttilio Rao rw_init(&kernel_object->lock, "kernel vm object"); 310d1780e8dSKonstantin Belousov _vm_object_allocate(OBJT_PHYS, atop(VM_MAX_KERNEL_ADDRESS - 311d1780e8dSKonstantin Belousov VM_MIN_KERNEL_ADDRESS), kernel_object); 312f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0 313f8a47341SAlan Cox kernel_object->flags |= OBJ_COLORED; 314f8a47341SAlan Cox kernel_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS); 315f8a47341SAlan Cox #endif 31626f9a767SRodney W. Grimes 3178dbca793STor Egge /* 3188dbca793STor Egge * The lock portion of struct vm_object must be type stable due 3198dbca793STor Egge * to vm_pageout_fallback_object_lock locking a vm object 3208dbca793STor Egge * without holding any references to it. 3218dbca793STor Egge */ 3228355f576SJeff Roberson obj_zone = uma_zcreate("VM OBJECT", sizeof (struct vm_object), NULL, 3238355f576SJeff Roberson #ifdef INVARIANTS 3248355f576SJeff Roberson vm_object_zdtor, 3258355f576SJeff Roberson #else 3268355f576SJeff Roberson NULL, 3278355f576SJeff Roberson #endif 3285df87b21SJeff Roberson vm_object_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); 329774d251dSAttilio Rao 330cd1241fbSKonstantin Belousov vm_radix_zinit(); 33199448ed1SJohn Dyson } 33299448ed1SJohn Dyson 33399448ed1SJohn Dyson void 3341b40f8c0SMatthew Dillon vm_object_clear_flag(vm_object_t object, u_short bits) 3351b40f8c0SMatthew Dillon { 3365440b5a9SAlan Cox 33789f6b863SAttilio Rao VM_OBJECT_ASSERT_WLOCKED(object); 338b06805adSJake Burkholder object->flags &= ~bits; 3391b40f8c0SMatthew Dillon } 3401b40f8c0SMatthew Dillon 3413153e878SAlan Cox /* 3423153e878SAlan Cox * Sets the default memory attribute for the specified object. Pages 3433153e878SAlan Cox * that are allocated to this object are by default assigned this memory 3443153e878SAlan Cox * attribute. 3453153e878SAlan Cox * 3463153e878SAlan Cox * Presently, this function must be called before any pages are allocated 3473153e878SAlan Cox * to the object. In the future, this requirement may be relaxed for 3483153e878SAlan Cox * "default" and "swap" objects. 3493153e878SAlan Cox */ 3503153e878SAlan Cox int 3513153e878SAlan Cox vm_object_set_memattr(vm_object_t object, vm_memattr_t memattr) 3523153e878SAlan Cox { 3533153e878SAlan Cox 35489f6b863SAttilio Rao VM_OBJECT_ASSERT_WLOCKED(object); 3553153e878SAlan Cox switch (object->type) { 3563153e878SAlan Cox case OBJT_DEFAULT: 3573153e878SAlan Cox case OBJT_DEVICE: 35896b0b92aSAlan Cox case OBJT_MGTDEVICE: 3593153e878SAlan Cox case OBJT_PHYS: 36001381811SJohn Baldwin case OBJT_SG: 3613153e878SAlan Cox case OBJT_SWAP: 3623153e878SAlan Cox case OBJT_VNODE: 3633153e878SAlan Cox if (!TAILQ_EMPTY(&object->memq)) 3643153e878SAlan Cox return (KERN_FAILURE); 3653153e878SAlan Cox break; 3663153e878SAlan Cox case OBJT_DEAD: 3673153e878SAlan Cox return (KERN_INVALID_ARGUMENT); 36896b0b92aSAlan Cox default: 36996b0b92aSAlan Cox panic("vm_object_set_memattr: object %p is of undefined type", 37096b0b92aSAlan Cox object); 3713153e878SAlan Cox } 3723153e878SAlan Cox object->memattr = memattr; 3733153e878SAlan Cox return (KERN_SUCCESS); 3743153e878SAlan Cox } 3753153e878SAlan Cox 3761b40f8c0SMatthew Dillon void 3771b40f8c0SMatthew Dillon vm_object_pip_add(vm_object_t object, short i) 3781b40f8c0SMatthew Dillon { 379f279b88dSAlan Cox 380cf27e0d1SJeff Roberson refcount_acquiren(&object->paging_in_progress, i); 3811b40f8c0SMatthew Dillon } 3821b40f8c0SMatthew Dillon 3831b40f8c0SMatthew Dillon void 3841b40f8c0SMatthew Dillon vm_object_pip_wakeup(vm_object_t object) 3851b40f8c0SMatthew Dillon { 386f279b88dSAlan Cox 387cf27e0d1SJeff Roberson refcount_release(&object->paging_in_progress); 3881b40f8c0SMatthew Dillon } 3891b40f8c0SMatthew Dillon 3901b40f8c0SMatthew Dillon void 3911b40f8c0SMatthew Dillon vm_object_pip_wakeupn(vm_object_t object, short i) 3921b40f8c0SMatthew Dillon { 393d647a0edSAlan Cox 394cf27e0d1SJeff Roberson refcount_releasen(&object->paging_in_progress, i); 3951b40f8c0SMatthew Dillon } 3961b40f8c0SMatthew Dillon 3971b40f8c0SMatthew Dillon void 3981b40f8c0SMatthew Dillon vm_object_pip_wait(vm_object_t object, char *waitid) 3991b40f8c0SMatthew Dillon { 4001ca58953SAlan Cox 40189f6b863SAttilio Rao VM_OBJECT_ASSERT_WLOCKED(object); 402cf27e0d1SJeff Roberson 40311b57401SHans Petter Selasky while (REFCOUNT_COUNT(object->paging_in_progress) > 0) { 404cf27e0d1SJeff Roberson VM_OBJECT_WUNLOCK(object); 405cf27e0d1SJeff Roberson refcount_wait(&object->paging_in_progress, waitid, PVM); 406cf27e0d1SJeff Roberson VM_OBJECT_WLOCK(object); 4071ca58953SAlan Cox } 4081b40f8c0SMatthew Dillon } 4091b40f8c0SMatthew Dillon 410cf27e0d1SJeff Roberson void 411cf27e0d1SJeff Roberson vm_object_pip_wait_unlocked(vm_object_t object, char *waitid) 412cf27e0d1SJeff Roberson { 413cf27e0d1SJeff Roberson 414cf27e0d1SJeff Roberson VM_OBJECT_ASSERT_UNLOCKED(object); 415cf27e0d1SJeff Roberson 41611b57401SHans Petter Selasky while (REFCOUNT_COUNT(object->paging_in_progress) > 0) 417cf27e0d1SJeff Roberson refcount_wait(&object->paging_in_progress, waitid, PVM); 418cf27e0d1SJeff Roberson } 419cf27e0d1SJeff Roberson 42026f9a767SRodney W. Grimes /* 42126f9a767SRodney W. Grimes * vm_object_allocate: 42226f9a767SRodney W. Grimes * 42326f9a767SRodney W. Grimes * Returns a new object with the given size. 42426f9a767SRodney W. Grimes */ 42526f9a767SRodney W. Grimes vm_object_t 4266395da54SIan Dowse vm_object_allocate(objtype_t type, vm_pindex_t size) 42726f9a767SRodney W. Grimes { 42890688d13SAlan Cox vm_object_t object; 42990688d13SAlan Cox 43090688d13SAlan Cox object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK); 43190688d13SAlan Cox _vm_object_allocate(type, size, object); 43290688d13SAlan Cox return (object); 43326f9a767SRodney W. Grimes } 43426f9a767SRodney W. Grimes 43526f9a767SRodney W. Grimes 43626f9a767SRodney W. Grimes /* 437df8bae1dSRodney W. Grimes * vm_object_reference: 438df8bae1dSRodney W. Grimes * 43915347817SAlan Cox * Gets another reference to the given object. Note: OBJ_DEAD 44015347817SAlan Cox * objects can be referenced during final cleaning. 441df8bae1dSRodney W. Grimes */ 4426476c0d2SJohn Dyson void 4431b40f8c0SMatthew Dillon vm_object_reference(vm_object_t object) 444df8bae1dSRodney W. Grimes { 445df8bae1dSRodney W. Grimes if (object == NULL) 446df8bae1dSRodney W. Grimes return; 447*51df5321SJeff Roberson VM_OBJECT_RLOCK(object); 44852481a9aSJeff Roberson vm_object_reference_locked(object); 449*51df5321SJeff Roberson VM_OBJECT_RUNLOCK(object); 45095e5e988SJohn Dyson } 45195e5e988SJohn Dyson 45223955314SAlfred Perlstein /* 453b921a12bSAlan Cox * vm_object_reference_locked: 454b921a12bSAlan Cox * 455b921a12bSAlan Cox * Gets another reference to the given object. 456b921a12bSAlan Cox * 457b921a12bSAlan Cox * The object must be locked. 458b921a12bSAlan Cox */ 459b921a12bSAlan Cox void 460b921a12bSAlan Cox vm_object_reference_locked(vm_object_t object) 461b921a12bSAlan Cox { 462b921a12bSAlan Cox struct vnode *vp; 463b921a12bSAlan Cox 464*51df5321SJeff Roberson VM_OBJECT_ASSERT_LOCKED(object); 465*51df5321SJeff Roberson refcount_acquire(&object->ref_count); 466b921a12bSAlan Cox if (object->type == OBJT_VNODE) { 467b921a12bSAlan Cox vp = object->handle; 468b921a12bSAlan Cox vref(vp); 469b921a12bSAlan Cox } 470b921a12bSAlan Cox } 471b921a12bSAlan Cox 472b921a12bSAlan Cox /* 4739d5abbddSJens Schweikhardt * Handle deallocating an object of type OBJT_VNODE. 47423955314SAlfred Perlstein */ 47502dd8331SAlan Cox static void 4761b40f8c0SMatthew Dillon vm_object_vndeallocate(vm_object_t object) 47795e5e988SJohn Dyson { 47895e5e988SJohn Dyson struct vnode *vp = (struct vnode *) object->handle; 479219cbf59SEivind Eklund 4805526d2d9SEivind Eklund KASSERT(object->type == OBJT_VNODE, 4815526d2d9SEivind Eklund ("vm_object_vndeallocate: not a vnode object")); 482219cbf59SEivind Eklund KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp")); 48395e5e988SJohn Dyson 484*51df5321SJeff Roberson if (refcount_release(&object->ref_count) && 485*51df5321SJeff Roberson !umtx_shm_vnobj_persistent) 4861bdbd705SKonstantin Belousov umtx_shm_object_terminated(object); 4871bdbd705SKonstantin Belousov 488*51df5321SJeff Roberson VM_OBJECT_RUNLOCK(object); 48903fa5b34SKonstantin Belousov /* vrele may need the vnode lock. */ 49047221757SJohn Dyson vrele(vp); 49186769ac0SKonstantin Belousov } 492df8bae1dSRodney W. Grimes 493df8bae1dSRodney W. Grimes /* 494df8bae1dSRodney W. Grimes * vm_object_deallocate: 495df8bae1dSRodney W. Grimes * 496df8bae1dSRodney W. Grimes * Release a reference to the specified object, 497df8bae1dSRodney W. Grimes * gained either through a vm_object_allocate 498df8bae1dSRodney W. Grimes * or a vm_object_reference call. When all references 499df8bae1dSRodney W. Grimes * are gone, storage associated with this object 500df8bae1dSRodney W. Grimes * may be relinquished. 501df8bae1dSRodney W. Grimes * 502df8bae1dSRodney W. Grimes * No object may be locked. 503df8bae1dSRodney W. Grimes */ 50426f9a767SRodney W. Grimes void 5051b40f8c0SMatthew Dillon vm_object_deallocate(vm_object_t object) 506df8bae1dSRodney W. Grimes { 507df8bae1dSRodney W. Grimes vm_object_t temp; 508*51df5321SJeff Roberson bool released; 509df8bae1dSRodney W. Grimes 510df8bae1dSRodney W. Grimes while (object != NULL) { 511*51df5321SJeff Roberson VM_OBJECT_RLOCK(object); 5123b68228cSAlan Cox if (object->type == OBJT_VNODE) { 51395e5e988SJohn Dyson vm_object_vndeallocate(object); 51423b186d3SAlan Cox return; 5155050aa86SKonstantin Belousov } 51695e5e988SJohn Dyson 517*51df5321SJeff Roberson /* 518*51df5321SJeff Roberson * If the reference count goes to 0 we start calling 519*51df5321SJeff Roberson * vm_object_terminate() on the object chain. A ref count 520*51df5321SJeff Roberson * of 1 may be a special case depending on the shadow count 521*51df5321SJeff Roberson * being 0 or 1. These cases require a write lock on the 522*51df5321SJeff Roberson * object. 523*51df5321SJeff Roberson */ 524*51df5321SJeff Roberson released = refcount_release_if_gt(&object->ref_count, 2); 525*51df5321SJeff Roberson VM_OBJECT_RUNLOCK(object); 526*51df5321SJeff Roberson if (released) 527*51df5321SJeff Roberson return; 528*51df5321SJeff Roberson 529*51df5321SJeff Roberson VM_OBJECT_WLOCK(object); 5308125b1e6SAlfred Perlstein KASSERT(object->ref_count != 0, 5318125b1e6SAlfred Perlstein ("vm_object_deallocate: object deallocated too many times: %d", object->type)); 5322be70f79SJohn Dyson 533*51df5321SJeff Roberson refcount_release(&object->ref_count); 5348125b1e6SAlfred Perlstein if (object->ref_count > 1) { 53589f6b863SAttilio Rao VM_OBJECT_WUNLOCK(object); 53623b186d3SAlan Cox return; 5378125b1e6SAlfred Perlstein } else if (object->ref_count == 1) { 5384c8e0452SAlan Cox if (object->shadow_count == 0 && 5394c8e0452SAlan Cox object->handle == NULL && 5404c8e0452SAlan Cox (object->type == OBJT_DEFAULT || 5416f2af3fcSKonstantin Belousov (object->type == OBJT_SWAP && 542f08f7dcaSKonstantin Belousov (object->flags & OBJ_TMPFS_NODE) == 0))) { 5438125b1e6SAlfred Perlstein vm_object_set_flag(object, OBJ_ONEMAPPING); 5448125b1e6SAlfred Perlstein } else if ((object->shadow_count == 1) && 5458125b1e6SAlfred Perlstein (object->handle == NULL) && 54624a1cce3SDavid Greenman (object->type == OBJT_DEFAULT || 54724a1cce3SDavid Greenman object->type == OBJT_SWAP)) { 548a1f6d91cSDavid Greenman vm_object_t robject; 54995e5e988SJohn Dyson 5501c500307SAlan Cox robject = LIST_FIRST(&object->shadow_head); 5515526d2d9SEivind Eklund KASSERT(robject != NULL, 552219cbf59SEivind Eklund ("vm_object_deallocate: ref_count: %d, shadow_count: %d", 5535526d2d9SEivind Eklund object->ref_count, 5545526d2d9SEivind Eklund object->shadow_count)); 5554bace8e7SKonstantin Belousov KASSERT((robject->flags & OBJ_TMPFS_NODE) == 0, 5564bace8e7SKonstantin Belousov ("shadowed tmpfs v_object %p", object)); 55789f6b863SAttilio Rao if (!VM_OBJECT_TRYWLOCK(robject)) { 558b72b0115SAlan Cox /* 559b72b0115SAlan Cox * Avoid a potential deadlock. 560b72b0115SAlan Cox */ 561*51df5321SJeff Roberson refcount_acquire(&object->ref_count); 56289f6b863SAttilio Rao VM_OBJECT_WUNLOCK(object); 563a7d86121SAlan Cox /* 564a7d86121SAlan Cox * More likely than not the thread 565a7d86121SAlan Cox * holding robject's lock has lower 566a7d86121SAlan Cox * priority than the current thread. 567a7d86121SAlan Cox * Let the lower priority thread run. 568a7d86121SAlan Cox */ 5698db5fc58SJohn Baldwin pause("vmo_de", 1); 570b72b0115SAlan Cox continue; 571b72b0115SAlan Cox } 572d936694fSAlan Cox /* 573d936694fSAlan Cox * Collapse object into its shadow unless its 574d936694fSAlan Cox * shadow is dead. In that case, object will 575d936694fSAlan Cox * be deallocated by the thread that is 576d936694fSAlan Cox * deallocating its shadow. 577d936694fSAlan Cox */ 578d936694fSAlan Cox if ((robject->flags & OBJ_DEAD) == 0 && 579d936694fSAlan Cox (robject->handle == NULL) && 58024a1cce3SDavid Greenman (robject->type == OBJT_DEFAULT || 58124a1cce3SDavid Greenman robject->type == OBJT_SWAP)) { 582a1f6d91cSDavid Greenman 583*51df5321SJeff Roberson refcount_acquire(&robject->ref_count); 584138449dcSAlan Cox retry: 58511b57401SHans Petter Selasky if (REFCOUNT_COUNT(robject->paging_in_progress) > 0) { 58689f6b863SAttilio Rao VM_OBJECT_WUNLOCK(object); 587138449dcSAlan Cox vm_object_pip_wait(robject, 588138449dcSAlan Cox "objde1"); 5892e9f4a69SAlan Cox temp = robject->backing_object; 5902e9f4a69SAlan Cox if (object == temp) { 59189f6b863SAttilio Rao VM_OBJECT_WLOCK(object); 592138449dcSAlan Cox goto retry; 5932e9f4a69SAlan Cox } 59411b57401SHans Petter Selasky } else if (REFCOUNT_COUNT(object->paging_in_progress) > 0) { 59589f6b863SAttilio Rao VM_OBJECT_WUNLOCK(robject); 596cf27e0d1SJeff Roberson VM_OBJECT_WUNLOCK(object); 597cf27e0d1SJeff Roberson refcount_wait( 598cf27e0d1SJeff Roberson &object->paging_in_progress, 599cf27e0d1SJeff Roberson "objde2", PVM); 60089f6b863SAttilio Rao VM_OBJECT_WLOCK(robject); 6012e9f4a69SAlan Cox temp = robject->backing_object; 6022e9f4a69SAlan Cox if (object == temp) { 60389f6b863SAttilio Rao VM_OBJECT_WLOCK(object); 604138449dcSAlan Cox goto retry; 605a1f6d91cSDavid Greenman } 6062e9f4a69SAlan Cox } else 60789f6b863SAttilio Rao VM_OBJECT_WUNLOCK(object); 6082e9f4a69SAlan Cox 60995e5e988SJohn Dyson if (robject->ref_count == 1) { 61095e5e988SJohn Dyson robject->ref_count--; 611ba8da839SDavid Greenman object = robject; 61295e5e988SJohn Dyson goto doterm; 61395e5e988SJohn Dyson } 61495e5e988SJohn Dyson object = robject; 61595e5e988SJohn Dyson vm_object_collapse(object); 61689f6b863SAttilio Rao VM_OBJECT_WUNLOCK(object); 617ba8da839SDavid Greenman continue; 618a1f6d91cSDavid Greenman } 61989f6b863SAttilio Rao VM_OBJECT_WUNLOCK(robject); 62095e5e988SJohn Dyson } 62189f6b863SAttilio Rao VM_OBJECT_WUNLOCK(object); 62223b186d3SAlan Cox return; 62395e5e988SJohn Dyson } 62495e5e988SJohn Dyson doterm: 6251bdbd705SKonstantin Belousov umtx_shm_object_terminated(object); 62624a1cce3SDavid Greenman temp = object->backing_object; 627c9917419SAlan Cox if (temp != NULL) { 6284bace8e7SKonstantin Belousov KASSERT((object->flags & OBJ_TMPFS_NODE) == 0, 6294bace8e7SKonstantin Belousov ("shadowed tmpfs v_object 2 %p", object)); 63089f6b863SAttilio Rao VM_OBJECT_WLOCK(temp); 6311c500307SAlan Cox LIST_REMOVE(object, shadow_list); 63295e5e988SJohn Dyson temp->shadow_count--; 63389f6b863SAttilio Rao VM_OBJECT_WUNLOCK(temp); 63495461b45SJohn Dyson object->backing_object = NULL; 635de5f6a77SJohn Dyson } 636245df27cSMatthew Dillon /* 637245df27cSMatthew Dillon * Don't double-terminate, we could be in a termination 638245df27cSMatthew Dillon * recursion due to the terminate having to sync data 639245df27cSMatthew Dillon * to disk. 640245df27cSMatthew Dillon */ 641783a68aaSKonstantin Belousov if ((object->flags & OBJ_DEAD) == 0) { 642783a68aaSKonstantin Belousov vm_object_set_flag(object, OBJ_DEAD); 643df8bae1dSRodney W. Grimes vm_object_terminate(object); 644783a68aaSKonstantin Belousov } else 64589f6b863SAttilio Rao VM_OBJECT_WUNLOCK(object); 646df8bae1dSRodney W. Grimes object = temp; 647df8bae1dSRodney W. Grimes } 648df8bae1dSRodney W. Grimes } 649df8bae1dSRodney W. Grimes 650df8bae1dSRodney W. Grimes /* 6512ac78f0eSStephan Uphoff * vm_object_destroy removes the object from the global object list 6522ac78f0eSStephan Uphoff * and frees the space for the object. 6532ac78f0eSStephan Uphoff */ 6542ac78f0eSStephan Uphoff void 6552ac78f0eSStephan Uphoff vm_object_destroy(vm_object_t object) 6562ac78f0eSStephan Uphoff { 6572ac78f0eSStephan Uphoff 6582ac78f0eSStephan Uphoff /* 6593364c323SKonstantin Belousov * Release the allocation charge. 6603364c323SKonstantin Belousov */ 661ef694c1aSEdward Tomasz Napierala if (object->cred != NULL) { 662ef694c1aSEdward Tomasz Napierala swap_release_by_cred(object->charge, object->cred); 6633364c323SKonstantin Belousov object->charge = 0; 664ef694c1aSEdward Tomasz Napierala crfree(object->cred); 665ef694c1aSEdward Tomasz Napierala object->cred = NULL; 6663364c323SKonstantin Belousov } 6673364c323SKonstantin Belousov 6683364c323SKonstantin Belousov /* 6692ac78f0eSStephan Uphoff * Free the space for the object. 6702ac78f0eSStephan Uphoff */ 6712ac78f0eSStephan Uphoff uma_zfree(obj_zone, object); 6722ac78f0eSStephan Uphoff } 6732ac78f0eSStephan Uphoff 6742ac78f0eSStephan Uphoff /* 6757bbdb843SRuslan Bukin * vm_object_terminate_pages removes any remaining pageable pages 6767bbdb843SRuslan Bukin * from the object and resets the object to an empty state. 6777bbdb843SRuslan Bukin */ 6787bbdb843SRuslan Bukin static void 6797bbdb843SRuslan Bukin vm_object_terminate_pages(vm_object_t object) 6807bbdb843SRuslan Bukin { 6817bbdb843SRuslan Bukin vm_page_t p, p_next; 6827bbdb843SRuslan Bukin 6837bbdb843SRuslan Bukin VM_OBJECT_ASSERT_WLOCKED(object); 6847bbdb843SRuslan Bukin 6857bbdb843SRuslan Bukin /* 6867bbdb843SRuslan Bukin * Free any remaining pageable pages. This also removes them from the 6877bbdb843SRuslan Bukin * paging queues. However, don't free wired pages, just remove them 6887bbdb843SRuslan Bukin * from the object. Rather than incrementally removing each page from 6897bbdb843SRuslan Bukin * the object, the page and object are reset to any empty state. 6907bbdb843SRuslan Bukin */ 6917bbdb843SRuslan Bukin TAILQ_FOREACH_SAFE(p, &object->memq, listq, p_next) { 6927bbdb843SRuslan Bukin vm_page_assert_unbusied(p); 693fee2a2faSMark Johnston KASSERT(p->object == object && 694fee2a2faSMark Johnston (p->ref_count & VPRC_OBJREF) != 0, 695fee2a2faSMark Johnston ("vm_object_terminate_pages: page %p is inconsistent", p)); 696fee2a2faSMark Johnston 6977bbdb843SRuslan Bukin p->object = NULL; 698fee2a2faSMark Johnston if (vm_page_drop(p, VPRC_OBJREF) == VPRC_OBJREF) { 6995cd29d0fSMark Johnston VM_CNT_INC(v_pfree); 7005cd29d0fSMark Johnston vm_page_free(p); 7014074d642SAlan Cox } 702fee2a2faSMark Johnston } 7032fcd1ff6SKonstantin Belousov 7047bbdb843SRuslan Bukin /* 7057bbdb843SRuslan Bukin * If the object contained any pages, then reset it to an empty state. 7067bbdb843SRuslan Bukin * None of the object's fields, including "resident_page_count", were 7077bbdb843SRuslan Bukin * modified by the preceding loop. 7087bbdb843SRuslan Bukin */ 7097bbdb843SRuslan Bukin if (object->resident_page_count != 0) { 7107bbdb843SRuslan Bukin vm_radix_reclaim_allnodes(&object->rtree); 7117bbdb843SRuslan Bukin TAILQ_INIT(&object->memq); 7127bbdb843SRuslan Bukin object->resident_page_count = 0; 7137bbdb843SRuslan Bukin if (object->type == OBJT_VNODE) 7147bbdb843SRuslan Bukin vdrop(object->handle); 7157bbdb843SRuslan Bukin } 7167bbdb843SRuslan Bukin } 7177bbdb843SRuslan Bukin 7187bbdb843SRuslan Bukin /* 719df8bae1dSRodney W. Grimes * vm_object_terminate actually destroys the specified object, freeing 720df8bae1dSRodney W. Grimes * up all previously used resources. 721df8bae1dSRodney W. Grimes * 722df8bae1dSRodney W. Grimes * The object must be locked. 7231c7c3c6aSMatthew Dillon * This routine may block. 724df8bae1dSRodney W. Grimes */ 72595e5e988SJohn Dyson void 7261b40f8c0SMatthew Dillon vm_object_terminate(vm_object_t object) 727df8bae1dSRodney W. Grimes { 72889f6b863SAttilio Rao VM_OBJECT_ASSERT_WLOCKED(object); 729783a68aaSKonstantin Belousov KASSERT((object->flags & OBJ_DEAD) != 0, 730783a68aaSKonstantin Belousov ("terminating non-dead obj %p", object)); 731bef608bdSJohn Dyson 732cf27e0d1SJeff Roberson /* 733cf27e0d1SJeff Roberson * wait for the pageout daemon to be done with the object 734cf27e0d1SJeff Roberson */ 735cf27e0d1SJeff Roberson vm_object_pip_wait(object, "objtrm"); 736cf27e0d1SJeff Roberson 73711b57401SHans Petter Selasky KASSERT(!REFCOUNT_COUNT(object->paging_in_progress), 738cf27e0d1SJeff Roberson ("vm_object_terminate: pageout in progress")); 739cf27e0d1SJeff Roberson 740971dd342SAlfred Perlstein KASSERT(object->ref_count == 0, 741971dd342SAlfred Perlstein ("vm_object_terminate: object with references, ref_count=%d", 742971dd342SAlfred Perlstein object->ref_count)); 743996c772fSJohn Dyson 7447bbdb843SRuslan Bukin if ((object->flags & OBJ_PG_DTOR) == 0) 7457bbdb843SRuslan Bukin vm_object_terminate_pages(object); 746bef608bdSJohn Dyson 747f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0 748f8a47341SAlan Cox if (__predict_false(!LIST_EMPTY(&object->rvq))) 749f8a47341SAlan Cox vm_reserv_break_all(object); 750f8a47341SAlan Cox #endif 7517bfda801SAlan Cox 752e735691bSJohn Baldwin KASSERT(object->cred == NULL || object->type == OBJT_DEFAULT || 753e735691bSJohn Baldwin object->type == OBJT_SWAP, 754e735691bSJohn Baldwin ("%s: non-swap obj %p has cred", __func__, object)); 755e735691bSJohn Baldwin 7562d8acc0fSJohn Dyson /* 7579fcfb650SDavid Greenman * Let the pager know object is dead. 7589fcfb650SDavid Greenman */ 7599fcfb650SDavid Greenman vm_pager_deallocate(object); 76089f6b863SAttilio Rao VM_OBJECT_WUNLOCK(object); 7619fcfb650SDavid Greenman 7622ac78f0eSStephan Uphoff vm_object_destroy(object); 76347221757SJohn Dyson } 764df8bae1dSRodney W. Grimes 765edf93b25SAlan Cox /* 766edf93b25SAlan Cox * Make the page read-only so that we can clear the object flags. However, if 767edf93b25SAlan Cox * this is a nosync mmap then the object is likely to stay dirty so do not 768edf93b25SAlan Cox * mess with the page and do not clear the object flags. Returns TRUE if the 769edf93b25SAlan Cox * page should be flushed, and FALSE otherwise. 770edf93b25SAlan Cox */ 7713280870dSKonstantin Belousov static boolean_t 772126d6082SKonstantin Belousov vm_object_page_remove_write(vm_page_t p, int flags, boolean_t *clearobjflags) 7733280870dSKonstantin Belousov { 7743280870dSKonstantin Belousov 775fff5403fSJeff Roberson vm_page_assert_busied(p); 776fff5403fSJeff Roberson 7773280870dSKonstantin Belousov /* 7783280870dSKonstantin Belousov * If we have been asked to skip nosync pages and this is a 7793280870dSKonstantin Belousov * nosync page, skip it. Note that the object flags were not 7803280870dSKonstantin Belousov * cleared in this case so we do not have to set them. 7813280870dSKonstantin Belousov */ 782fff5403fSJeff Roberson if ((flags & OBJPC_NOSYNC) != 0 && (p->aflags & PGA_NOSYNC) != 0) { 783126d6082SKonstantin Belousov *clearobjflags = FALSE; 7843280870dSKonstantin Belousov return (FALSE); 7853280870dSKonstantin Belousov } else { 7863280870dSKonstantin Belousov pmap_remove_write(p); 7873280870dSKonstantin Belousov return (p->dirty != 0); 7883280870dSKonstantin Belousov } 7893280870dSKonstantin Belousov } 7903280870dSKonstantin Belousov 791df8bae1dSRodney W. Grimes /* 792df8bae1dSRodney W. Grimes * vm_object_page_clean 793df8bae1dSRodney W. Grimes * 7944f79d873SMatthew Dillon * Clean all dirty pages in the specified range of object. Leaves page 7954f79d873SMatthew Dillon * on whatever queue it is currently on. If NOSYNC is set then do not 796fff5403fSJeff Roberson * write out pages with PGA_NOSYNC set (originally comes from MAP_NOSYNC), 7974f79d873SMatthew Dillon * leaving the object dirty. 79826f9a767SRodney W. Grimes * 79943b7990eSMatthew Dillon * When stuffing pages asynchronously, allow clustering. XXX we need a 80043b7990eSMatthew Dillon * synchronous clustering mode implementation. 80143b7990eSMatthew Dillon * 80226f9a767SRodney W. Grimes * Odd semantics: if start == end, we clean everything. 80326f9a767SRodney W. Grimes * 80426f9a767SRodney W. Grimes * The object must be locked. 805126d6082SKonstantin Belousov * 806126d6082SKonstantin Belousov * Returns FALSE if some page from the range was not written, as 807126d6082SKonstantin Belousov * reported by the pager, and TRUE otherwise. 80826f9a767SRodney W. Grimes */ 809126d6082SKonstantin Belousov boolean_t 81017f3095dSAlan Cox vm_object_page_clean(vm_object_t object, vm_ooffset_t start, vm_ooffset_t end, 811e239bb97SKonstantin Belousov int flags) 812f6b04d2bSDavid Greenman { 813e239bb97SKonstantin Belousov vm_page_t np, p; 81417f3095dSAlan Cox vm_pindex_t pi, tend, tstart; 815126d6082SKonstantin Belousov int curgeneration, n, pagerflags; 816126d6082SKonstantin Belousov boolean_t clearobjflags, eio, res; 817f6b04d2bSDavid Greenman 81889f6b863SAttilio Rao VM_OBJECT_ASSERT_WLOCKED(object); 819e5f299ffSKonstantin Belousov 820e5f299ffSKonstantin Belousov /* 821e5f299ffSKonstantin Belousov * The OBJ_MIGHTBEDIRTY flag is only set for OBJT_VNODE 822e5f299ffSKonstantin Belousov * objects. The check below prevents the function from 823e5f299ffSKonstantin Belousov * operating on non-vnode objects. 824e5f299ffSKonstantin Belousov */ 825e239bb97SKonstantin Belousov if ((object->flags & OBJ_MIGHTBEDIRTY) == 0 || 826e239bb97SKonstantin Belousov object->resident_page_count == 0) 827126d6082SKonstantin Belousov return (TRUE); 828f6b04d2bSDavid Greenman 829e239bb97SKonstantin Belousov pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) != 0 ? 830e239bb97SKonstantin Belousov VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK; 831e239bb97SKonstantin Belousov pagerflags |= (flags & OBJPC_INVAL) != 0 ? VM_PAGER_PUT_INVAL : 0; 832e239bb97SKonstantin Belousov 83317f3095dSAlan Cox tstart = OFF_TO_IDX(start); 83417f3095dSAlan Cox tend = (end == 0) ? object->size : OFF_TO_IDX(end + PAGE_MASK); 83517f3095dSAlan Cox clearobjflags = tstart == 0 && tend >= object->size; 836126d6082SKonstantin Belousov res = TRUE; 837f6b04d2bSDavid Greenman 838bd7e5f99SJohn Dyson rescan: 8392d8acc0fSJohn Dyson curgeneration = object->generation; 8402d8acc0fSJohn Dyson 84117f3095dSAlan Cox for (p = vm_page_find_least(object, tstart); p != NULL; p = np) { 842bd7e5f99SJohn Dyson pi = p->pindex; 843e239bb97SKonstantin Belousov if (pi >= tend) 844e239bb97SKonstantin Belousov break; 845e239bb97SKonstantin Belousov np = TAILQ_NEXT(p, listq); 8460012f373SJeff Roberson if (vm_page_none_valid(p)) 847aef922f5SJohn Dyson continue; 84863e97555SJeff Roberson if (vm_page_busy_acquire(p, VM_ALLOC_WAITFAIL) == 0) { 849e65919f9SKonstantin Belousov if (object->generation != curgeneration) { 850e65919f9SKonstantin Belousov if ((flags & OBJPC_SYNC) != 0) 851e239bb97SKonstantin Belousov goto rescan; 852e65919f9SKonstantin Belousov else 853126d6082SKonstantin Belousov clearobjflags = FALSE; 854e65919f9SKonstantin Belousov } 855780636b7SKonstantin Belousov np = vm_page_find_least(object, pi); 856780636b7SKonstantin Belousov continue; 857f6b04d2bSDavid Greenman } 85863e97555SJeff Roberson if (!vm_object_page_remove_write(p, flags, &clearobjflags)) { 85963e97555SJeff Roberson vm_page_xunbusy(p); 860bd7e5f99SJohn Dyson continue; 86163e97555SJeff Roberson } 862e239bb97SKonstantin Belousov 8633280870dSKonstantin Belousov n = vm_object_page_collect_flush(object, p, pagerflags, 864126d6082SKonstantin Belousov flags, &clearobjflags, &eio); 865126d6082SKonstantin Belousov if (eio) { 866126d6082SKonstantin Belousov res = FALSE; 867126d6082SKonstantin Belousov clearobjflags = FALSE; 868126d6082SKonstantin Belousov } 869e65919f9SKonstantin Belousov if (object->generation != curgeneration) { 870e65919f9SKonstantin Belousov if ((flags & OBJPC_SYNC) != 0) 871b9b7a4beSMatthew Dillon goto rescan; 872e65919f9SKonstantin Belousov else 873126d6082SKonstantin Belousov clearobjflags = FALSE; 874e65919f9SKonstantin Belousov } 875031ec8c1SKonstantin Belousov 876031ec8c1SKonstantin Belousov /* 877031ec8c1SKonstantin Belousov * If the VOP_PUTPAGES() did a truncated write, so 878031ec8c1SKonstantin Belousov * that even the first page of the run is not fully 879031ec8c1SKonstantin Belousov * written, vm_pageout_flush() returns 0 as the run 880031ec8c1SKonstantin Belousov * length. Since the condition that caused truncated 881031ec8c1SKonstantin Belousov * write may be permanent, e.g. exhausted free space, 882031ec8c1SKonstantin Belousov * accepting n == 0 would cause an infinite loop. 883031ec8c1SKonstantin Belousov * 884031ec8c1SKonstantin Belousov * Forwarding the iterator leaves the unwritten page 885031ec8c1SKonstantin Belousov * behind, but there is not much we can do there if 886031ec8c1SKonstantin Belousov * filesystem refuses to write it. 887031ec8c1SKonstantin Belousov */ 888126d6082SKonstantin Belousov if (n == 0) { 889031ec8c1SKonstantin Belousov n = 1; 890126d6082SKonstantin Belousov clearobjflags = FALSE; 891126d6082SKonstantin Belousov } 892e239bb97SKonstantin Belousov np = vm_page_find_least(object, pi + n); 893b9b7a4beSMatthew Dillon } 894b9b7a4beSMatthew Dillon #if 0 895e239bb97SKonstantin Belousov VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC) ? MNT_WAIT : 0); 896b9b7a4beSMatthew Dillon #endif 897b9b7a4beSMatthew Dillon 898edf93b25SAlan Cox if (clearobjflags) 8993280870dSKonstantin Belousov vm_object_clear_flag(object, OBJ_MIGHTBEDIRTY); 900126d6082SKonstantin Belousov return (res); 901b9b7a4beSMatthew Dillon } 902b9b7a4beSMatthew Dillon 903b9b7a4beSMatthew Dillon static int 9043280870dSKonstantin Belousov vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags, 905126d6082SKonstantin Belousov int flags, boolean_t *clearobjflags, boolean_t *eio) 906b9b7a4beSMatthew Dillon { 9073157c503SKonstantin Belousov vm_page_t ma[vm_pageout_page_count], p_first, tp; 9083157c503SKonstantin Belousov int count, i, mreq, runlen; 909b9b7a4beSMatthew Dillon 9107bec141bSKip Macy vm_page_lock_assert(p, MA_NOTOWNED); 91163e97555SJeff Roberson vm_page_assert_xbusied(p); 91289f6b863SAttilio Rao VM_OBJECT_ASSERT_WLOCKED(object); 9133157c503SKonstantin Belousov 9143157c503SKonstantin Belousov count = 1; 9153157c503SKonstantin Belousov mreq = 0; 9163157c503SKonstantin Belousov 9173157c503SKonstantin Belousov for (tp = p; count < vm_pageout_page_count; count++) { 9183157c503SKonstantin Belousov tp = vm_page_next(tp); 91963e97555SJeff Roberson if (tp == NULL || vm_page_tryxbusy(tp) == 0) 920bd7e5f99SJohn Dyson break; 92163e97555SJeff Roberson if (!vm_object_page_remove_write(tp, flags, clearobjflags)) { 92263e97555SJeff Roberson vm_page_xunbusy(tp); 923bd7e5f99SJohn Dyson break; 924bd7e5f99SJohn Dyson } 92563e97555SJeff Roberson } 926aef922f5SJohn Dyson 9273157c503SKonstantin Belousov for (p_first = p; count < vm_pageout_page_count; count++) { 9283157c503SKonstantin Belousov tp = vm_page_prev(p_first); 92963e97555SJeff Roberson if (tp == NULL || vm_page_tryxbusy(tp) == 0) 930bd7e5f99SJohn Dyson break; 93163e97555SJeff Roberson if (!vm_object_page_remove_write(tp, flags, clearobjflags)) { 93263e97555SJeff Roberson vm_page_xunbusy(tp); 933bd7e5f99SJohn Dyson break; 93463e97555SJeff Roberson } 9353157c503SKonstantin Belousov p_first = tp; 9363157c503SKonstantin Belousov mreq++; 937bd7e5f99SJohn Dyson } 938bd7e5f99SJohn Dyson 9393157c503SKonstantin Belousov for (tp = p_first, i = 0; i < count; tp = TAILQ_NEXT(tp, listq), i++) 9403157c503SKonstantin Belousov ma[i] = tp; 941cf2819ccSJohn Dyson 942126d6082SKonstantin Belousov vm_pageout_flush(ma, count, pagerflags, mreq, &runlen, eio); 9431e8a675cSKonstantin Belousov return (runlen); 94426f9a767SRodney W. Grimes } 945df8bae1dSRodney W. Grimes 9461efb74fbSJohn Dyson /* 947950f8459SAlan Cox * Note that there is absolutely no sense in writing out 948950f8459SAlan Cox * anonymous objects, so we track down the vnode object 949950f8459SAlan Cox * to write out. 950950f8459SAlan Cox * We invalidate (remove) all pages from the address space 951950f8459SAlan Cox * for semantic correctness. 952950f8459SAlan Cox * 9536bbee8e2SAlan Cox * If the backing object is a device object with unmanaged pages, then any 9546bbee8e2SAlan Cox * mappings to the specified range of pages must be removed before this 9556bbee8e2SAlan Cox * function is called. 9566bbee8e2SAlan Cox * 957950f8459SAlan Cox * Note: certain anonymous maps, such as MAP_NOSYNC maps, 958950f8459SAlan Cox * may start out with a NULL object. 959950f8459SAlan Cox */ 960126d6082SKonstantin Belousov boolean_t 961950f8459SAlan Cox vm_object_sync(vm_object_t object, vm_ooffset_t offset, vm_size_t size, 962950f8459SAlan Cox boolean_t syncio, boolean_t invalidate) 963950f8459SAlan Cox { 964950f8459SAlan Cox vm_object_t backing_object; 965950f8459SAlan Cox struct vnode *vp; 9663b582b4eSTor Egge struct mount *mp; 967126d6082SKonstantin Belousov int error, flags, fsync_after; 968126d6082SKonstantin Belousov boolean_t res; 969950f8459SAlan Cox 970950f8459SAlan Cox if (object == NULL) 971126d6082SKonstantin Belousov return (TRUE); 972126d6082SKonstantin Belousov res = TRUE; 973126d6082SKonstantin Belousov error = 0; 97489f6b863SAttilio Rao VM_OBJECT_WLOCK(object); 975950f8459SAlan Cox while ((backing_object = object->backing_object) != NULL) { 97689f6b863SAttilio Rao VM_OBJECT_WLOCK(backing_object); 97756e0670fSAlan Cox offset += object->backing_object_offset; 97889f6b863SAttilio Rao VM_OBJECT_WUNLOCK(object); 979950f8459SAlan Cox object = backing_object; 980950f8459SAlan Cox if (object->size < OFF_TO_IDX(offset + size)) 981950f8459SAlan Cox size = IDX_TO_OFF(object->size) - offset; 982950f8459SAlan Cox } 983950f8459SAlan Cox /* 984950f8459SAlan Cox * Flush pages if writing is allowed, invalidate them 985950f8459SAlan Cox * if invalidation requested. Pages undergoing I/O 986950f8459SAlan Cox * will be ignored by vm_object_page_remove(). 987950f8459SAlan Cox * 988950f8459SAlan Cox * We cannot lock the vnode and then wait for paging 989950f8459SAlan Cox * to complete without deadlocking against vm_fault. 990950f8459SAlan Cox * Instead we simply call vm_object_page_remove() and 991950f8459SAlan Cox * allow it to block internally on a page-by-page 992950f8459SAlan Cox * basis when it encounters pages undergoing async 993950f8459SAlan Cox * I/O. 994950f8459SAlan Cox */ 995950f8459SAlan Cox if (object->type == OBJT_VNODE && 9965bf94937SKonstantin Belousov (object->flags & OBJ_MIGHTBEDIRTY) != 0 && 9975bf94937SKonstantin Belousov ((vp = object->handle)->v_vflag & VV_NOSYNC) == 0) { 99889f6b863SAttilio Rao VM_OBJECT_WUNLOCK(object); 9993b582b4eSTor Egge (void) vn_start_write(vp, &mp, V_WAIT); 1000cb05b60aSAttilio Rao vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 100175ff604aSKonstantin Belousov if (syncio && !invalidate && offset == 0 && 1002d1780e8dSKonstantin Belousov atop(size) == object->size) { 100375ff604aSKonstantin Belousov /* 100475ff604aSKonstantin Belousov * If syncing the whole mapping of the file, 100575ff604aSKonstantin Belousov * it is faster to schedule all the writes in 100675ff604aSKonstantin Belousov * async mode, also allowing the clustering, 100775ff604aSKonstantin Belousov * and then wait for i/o to complete. 100875ff604aSKonstantin Belousov */ 100975ff604aSKonstantin Belousov flags = 0; 101075ff604aSKonstantin Belousov fsync_after = TRUE; 101175ff604aSKonstantin Belousov } else { 1012950f8459SAlan Cox flags = (syncio || invalidate) ? OBJPC_SYNC : 0; 101375ff604aSKonstantin Belousov flags |= invalidate ? (OBJPC_SYNC | OBJPC_INVAL) : 0; 101475ff604aSKonstantin Belousov fsync_after = FALSE; 101575ff604aSKonstantin Belousov } 101689f6b863SAttilio Rao VM_OBJECT_WLOCK(object); 1017126d6082SKonstantin Belousov res = vm_object_page_clean(object, offset, offset + size, 1018126d6082SKonstantin Belousov flags); 101989f6b863SAttilio Rao VM_OBJECT_WUNLOCK(object); 102075ff604aSKonstantin Belousov if (fsync_after) 1021126d6082SKonstantin Belousov error = VOP_FSYNC(vp, MNT_WAIT, curthread); 102222db15c0SAttilio Rao VOP_UNLOCK(vp, 0); 10233b582b4eSTor Egge vn_finished_write(mp); 1024126d6082SKonstantin Belousov if (error != 0) 1025126d6082SKonstantin Belousov res = FALSE; 102689f6b863SAttilio Rao VM_OBJECT_WLOCK(object); 1027950f8459SAlan Cox } 1028950f8459SAlan Cox if ((object->type == OBJT_VNODE || 1029950f8459SAlan Cox object->type == OBJT_DEVICE) && invalidate) { 10306bbee8e2SAlan Cox if (object->type == OBJT_DEVICE) 10316bbee8e2SAlan Cox /* 10326bbee8e2SAlan Cox * The option OBJPR_NOTMAPPED must be passed here 10336bbee8e2SAlan Cox * because vm_object_page_remove() cannot remove 10346bbee8e2SAlan Cox * unmanaged mappings. 10356bbee8e2SAlan Cox */ 10366bbee8e2SAlan Cox flags = OBJPR_NOTMAPPED; 10376bbee8e2SAlan Cox else if (old_msync) 10386195b24aSKonstantin Belousov flags = 0; 10396bbee8e2SAlan Cox else 10406195b24aSKonstantin Belousov flags = OBJPR_CLEANONLY; 10416bbee8e2SAlan Cox vm_object_page_remove(object, OFF_TO_IDX(offset), 10426bbee8e2SAlan Cox OFF_TO_IDX(offset + size + PAGE_MASK), flags); 1043950f8459SAlan Cox } 104489f6b863SAttilio Rao VM_OBJECT_WUNLOCK(object); 1045126d6082SKonstantin Belousov return (res); 1046950f8459SAlan Cox } 1047950f8459SAlan Cox 1048950f8459SAlan Cox /* 1049aa3650eaSMark Johnston * Determine whether the given advice can be applied to the object. Advice is 1050aa3650eaSMark Johnston * not applied to unmanaged pages since they never belong to page queues, and 1051aa3650eaSMark Johnston * since MADV_FREE is destructive, it can apply only to anonymous pages that 1052aa3650eaSMark Johnston * have been mapped at most once. 1053aa3650eaSMark Johnston */ 1054aa3650eaSMark Johnston static bool 1055aa3650eaSMark Johnston vm_object_advice_applies(vm_object_t object, int advice) 1056aa3650eaSMark Johnston { 1057aa3650eaSMark Johnston 1058aa3650eaSMark Johnston if ((object->flags & OBJ_UNMANAGED) != 0) 1059aa3650eaSMark Johnston return (false); 1060aa3650eaSMark Johnston if (advice != MADV_FREE) 1061aa3650eaSMark Johnston return (true); 1062aa3650eaSMark Johnston return ((object->type == OBJT_DEFAULT || object->type == OBJT_SWAP) && 1063aa3650eaSMark Johnston (object->flags & OBJ_ONEMAPPING) != 0); 1064aa3650eaSMark Johnston } 1065aa3650eaSMark Johnston 1066aa3650eaSMark Johnston static void 1067aa3650eaSMark Johnston vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex, 1068aa3650eaSMark Johnston vm_size_t size) 1069aa3650eaSMark Johnston { 1070aa3650eaSMark Johnston 1071aa3650eaSMark Johnston if (advice == MADV_FREE && object->type == OBJT_SWAP) 1072aa3650eaSMark Johnston swap_pager_freespace(object, pindex, size); 1073aa3650eaSMark Johnston } 1074aa3650eaSMark Johnston 1075aa3650eaSMark Johnston /* 1076867a482dSJohn Dyson * vm_object_madvise: 1077867a482dSJohn Dyson * 1078867a482dSJohn Dyson * Implements the madvise function at the object/page level. 10791c7c3c6aSMatthew Dillon * 1080193b9358SAlan Cox * MADV_WILLNEED (any object) 1081193b9358SAlan Cox * 1082193b9358SAlan Cox * Activate the specified pages if they are resident. 1083193b9358SAlan Cox * 1084193b9358SAlan Cox * MADV_DONTNEED (any object) 1085193b9358SAlan Cox * 1086193b9358SAlan Cox * Deactivate the specified pages if they are resident. 1087193b9358SAlan Cox * 1088193b9358SAlan Cox * MADV_FREE (OBJT_DEFAULT/OBJT_SWAP objects, 1089193b9358SAlan Cox * OBJ_ONEMAPPING only) 1090193b9358SAlan Cox * 1091193b9358SAlan Cox * Deactivate and clean the specified pages if they are 1092193b9358SAlan Cox * resident. This permits the process to reuse the pages 1093193b9358SAlan Cox * without faulting or the kernel to reclaim the pages 1094193b9358SAlan Cox * without I/O. 1095867a482dSJohn Dyson */ 1096867a482dSJohn Dyson void 109792a59946SJohn Baldwin vm_object_madvise(vm_object_t object, vm_pindex_t pindex, vm_pindex_t end, 1098c2655a40SMark Johnston int advice) 1099867a482dSJohn Dyson { 110092a59946SJohn Baldwin vm_pindex_t tpindex; 110134567de7SAlan Cox vm_object_t backing_object, tobject; 1102aa3650eaSMark Johnston vm_page_t m, tm; 1103867a482dSJohn Dyson 1104867a482dSJohn Dyson if (object == NULL) 1105867a482dSJohn Dyson return; 1106c2655a40SMark Johnston 11076e20a165SJohn Dyson relookup: 1108aa3650eaSMark Johnston VM_OBJECT_WLOCK(object); 1109aa3650eaSMark Johnston if (!vm_object_advice_applies(object, advice)) { 1110aa3650eaSMark Johnston VM_OBJECT_WUNLOCK(object); 1111aa3650eaSMark Johnston return; 11126e20a165SJohn Dyson } 1113aa3650eaSMark Johnston for (m = vm_page_find_least(object, pindex); pindex < end; pindex++) { 1114aa3650eaSMark Johnston tobject = object; 1115c2655a40SMark Johnston 11161ce137beSMatthew Dillon /* 1117aa3650eaSMark Johnston * If the next page isn't resident in the top-level object, we 1118aa3650eaSMark Johnston * need to search the shadow chain. When applying MADV_FREE, we 1119aa3650eaSMark Johnston * take care to release any swap space used to store 1120aa3650eaSMark Johnston * non-resident pages. 1121aa3650eaSMark Johnston */ 1122aa3650eaSMark Johnston if (m == NULL || pindex < m->pindex) { 1123aa3650eaSMark Johnston /* 1124aa3650eaSMark Johnston * Optimize a common case: if the top-level object has 1125aa3650eaSMark Johnston * no backing object, we can skip over the non-resident 1126aa3650eaSMark Johnston * range in constant time. 11271ce137beSMatthew Dillon */ 1128c2655a40SMark Johnston if (object->backing_object == NULL) { 1129c2655a40SMark Johnston tpindex = (m != NULL && m->pindex < end) ? 1130c2655a40SMark Johnston m->pindex : end; 1131aa3650eaSMark Johnston vm_object_madvise_freespace(object, advice, 1132aa3650eaSMark Johnston pindex, tpindex - pindex); 1133c2655a40SMark Johnston if ((pindex = tpindex) == end) 1134c2655a40SMark Johnston break; 1135aa3650eaSMark Johnston goto next_page; 1136aa3650eaSMark Johnston } 1137aa3650eaSMark Johnston 1138aa3650eaSMark Johnston tpindex = pindex; 1139aa3650eaSMark Johnston do { 1140aa3650eaSMark Johnston vm_object_madvise_freespace(tobject, advice, 1141aa3650eaSMark Johnston tpindex, 1); 11421ce137beSMatthew Dillon /* 1143aa3650eaSMark Johnston * Prepare to search the next object in the 1144aa3650eaSMark Johnston * chain. 11451ce137beSMatthew Dillon */ 114634567de7SAlan Cox backing_object = tobject->backing_object; 114734567de7SAlan Cox if (backing_object == NULL) 1148aa3650eaSMark Johnston goto next_pindex; 114989f6b863SAttilio Rao VM_OBJECT_WLOCK(backing_object); 1150aa3650eaSMark Johnston tpindex += 1151aa3650eaSMark Johnston OFF_TO_IDX(tobject->backing_object_offset); 11529b98b796SAlan Cox if (tobject != object) 115389f6b863SAttilio Rao VM_OBJECT_WUNLOCK(tobject); 115434567de7SAlan Cox tobject = backing_object; 1155aa3650eaSMark Johnston if (!vm_object_advice_applies(tobject, advice)) 1156aa3650eaSMark Johnston goto next_pindex; 1157aa3650eaSMark Johnston } while ((tm = vm_page_lookup(tobject, tpindex)) == 1158aa3650eaSMark Johnston NULL); 1159aa3650eaSMark Johnston } else { 1160aa3650eaSMark Johnston next_page: 1161aa3650eaSMark Johnston tm = m; 1162aa3650eaSMark Johnston m = TAILQ_NEXT(m, listq); 1163c2655a40SMark Johnston } 1164c2655a40SMark Johnston 1165867a482dSJohn Dyson /* 11660012f373SJeff Roberson * If the page is not in a normal state, skip it. The page 11670012f373SJeff Roberson * can not be invalidated while the object lock is held. 1168867a482dSJohn Dyson */ 11690012f373SJeff Roberson if (!vm_page_all_valid(tm) || vm_page_wired(tm)) 1170aa3650eaSMark Johnston goto next_pindex; 1171aa3650eaSMark Johnston KASSERT((tm->flags & PG_FICTITIOUS) == 0, 1172aa3650eaSMark Johnston ("vm_object_madvise: page %p is fictitious", tm)); 1173aa3650eaSMark Johnston KASSERT((tm->oflags & VPO_UNMANAGED) == 0, 1174aa3650eaSMark Johnston ("vm_object_madvise: page %p is not managed", tm)); 117563e97555SJeff Roberson if (vm_page_tryxbusy(tm) == 0) { 1176aa3650eaSMark Johnston if (object != tobject) 1177aa3650eaSMark Johnston VM_OBJECT_WUNLOCK(object); 1178c2655a40SMark Johnston if (advice == MADV_WILLNEED) { 1179b11b56b5SAlan Cox /* 1180b11b56b5SAlan Cox * Reference the page before unlocking and 1181b11b56b5SAlan Cox * sleeping so that the page daemon is less 1182b11b56b5SAlan Cox * likely to reclaim it. 1183b11b56b5SAlan Cox */ 1184aa3650eaSMark Johnston vm_page_aflag_set(tm, PGA_REFERENCED); 1185567e51e1SAlan Cox } 1186aa3650eaSMark Johnston vm_page_busy_sleep(tm, "madvpo", false); 11876e20a165SJohn Dyson goto relookup; 118834567de7SAlan Cox } 1189fee2a2faSMark Johnston vm_page_lock(tm); 1190aa3650eaSMark Johnston vm_page_advise(tm, advice); 1191aa3650eaSMark Johnston vm_page_unlock(tm); 119263e97555SJeff Roberson vm_page_xunbusy(tm); 1193aa3650eaSMark Johnston vm_object_madvise_freespace(tobject, advice, tm->pindex, 1); 1194aa3650eaSMark Johnston next_pindex: 11959b98b796SAlan Cox if (tobject != object) 119689f6b863SAttilio Rao VM_OBJECT_WUNLOCK(tobject); 1197867a482dSJohn Dyson } 119889f6b863SAttilio Rao VM_OBJECT_WUNLOCK(object); 1199867a482dSJohn Dyson } 1200867a482dSJohn Dyson 1201867a482dSJohn Dyson /* 1202df8bae1dSRodney W. Grimes * vm_object_shadow: 1203df8bae1dSRodney W. Grimes * 1204df8bae1dSRodney W. Grimes * Create a new object which is backed by the 1205df8bae1dSRodney W. Grimes * specified existing object range. The source 1206df8bae1dSRodney W. Grimes * object reference is deallocated. 1207df8bae1dSRodney W. Grimes * 1208df8bae1dSRodney W. Grimes * The new object and offset into that object 1209df8bae1dSRodney W. Grimes * are returned in the source parameters. 1210df8bae1dSRodney W. Grimes */ 121126f9a767SRodney W. Grimes void 12121b40f8c0SMatthew Dillon vm_object_shadow( 12131b40f8c0SMatthew Dillon vm_object_t *object, /* IN/OUT */ 12141b40f8c0SMatthew Dillon vm_ooffset_t *offset, /* IN/OUT */ 12151b40f8c0SMatthew Dillon vm_size_t length) 1216df8bae1dSRodney W. Grimes { 1217d031cff1SMatthew Dillon vm_object_t source; 1218d031cff1SMatthew Dillon vm_object_t result; 1219df8bae1dSRodney W. Grimes 1220df8bae1dSRodney W. Grimes source = *object; 1221df8bae1dSRodney W. Grimes 1222df8bae1dSRodney W. Grimes /* 12239a2f6362SAlan Cox * Don't create the new object if the old object isn't shared. 12249a2f6362SAlan Cox */ 1225570a2f4aSAlan Cox if (source != NULL) { 1226*51df5321SJeff Roberson VM_OBJECT_RLOCK(source); 1227570a2f4aSAlan Cox if (source->ref_count == 1 && 12289a2f6362SAlan Cox source->handle == NULL && 12299a2f6362SAlan Cox (source->type == OBJT_DEFAULT || 12309917e010SAlan Cox source->type == OBJT_SWAP)) { 1231*51df5321SJeff Roberson VM_OBJECT_RUNLOCK(source); 12329a2f6362SAlan Cox return; 12339917e010SAlan Cox } 1234*51df5321SJeff Roberson VM_OBJECT_RUNLOCK(source); 1235570a2f4aSAlan Cox } 12369a2f6362SAlan Cox 12379a2f6362SAlan Cox /* 1238570a2f4aSAlan Cox * Allocate a new object with the given length. 1239df8bae1dSRodney W. Grimes */ 12400cc74f14SAlan Cox result = vm_object_allocate(OBJT_DEFAULT, atop(length)); 1241df8bae1dSRodney W. Grimes 1242df8bae1dSRodney W. Grimes /* 12430d94caffSDavid Greenman * The new object shadows the source object, adding a reference to it. 12440d94caffSDavid Greenman * Our caller changes his reference to point to the new object, 12450d94caffSDavid Greenman * removing a reference to the source object. Net result: no change 12460d94caffSDavid Greenman * of reference count. 12479b09fe24SMatthew Dillon * 12489b09fe24SMatthew Dillon * Try to optimize the result object's page color when shadowing 1249956f3135SPhilippe Charnier * in order to maintain page coloring consistency in the combined 12509b09fe24SMatthew Dillon * shadowed object. 1251df8bae1dSRodney W. Grimes */ 125224a1cce3SDavid Greenman result->backing_object = source; 12539174ca7bSTor Egge /* 12549174ca7bSTor Egge * Store the offset into the source object, and fix up the offset into 12559174ca7bSTor Egge * the new object. 12569174ca7bSTor Egge */ 12579174ca7bSTor Egge result->backing_object_offset = *offset; 1258570a2f4aSAlan Cox if (source != NULL) { 125989f6b863SAttilio Rao VM_OBJECT_WLOCK(source); 12603f289c3fSJeff Roberson result->domain = source->domain; 12611c500307SAlan Cox LIST_INSERT_HEAD(&source->shadow_head, result, shadow_list); 1262eaf13dd7SJohn Dyson source->shadow_count++; 1263f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0 12647b54b1a9SAlan Cox result->flags |= source->flags & OBJ_COLORED; 1265f8a47341SAlan Cox result->pg_color = (source->pg_color + OFF_TO_IDX(*offset)) & 1266f8a47341SAlan Cox ((1 << (VM_NFREEORDER - 1)) - 1); 1267f8a47341SAlan Cox #endif 126889f6b863SAttilio Rao VM_OBJECT_WUNLOCK(source); 1269de5f6a77SJohn Dyson } 1270df8bae1dSRodney W. Grimes 1271df8bae1dSRodney W. Grimes 1272df8bae1dSRodney W. Grimes /* 1273df8bae1dSRodney W. Grimes * Return the new things 1274df8bae1dSRodney W. Grimes */ 1275df8bae1dSRodney W. Grimes *offset = 0; 1276df8bae1dSRodney W. Grimes *object = result; 1277df8bae1dSRodney W. Grimes } 1278df8bae1dSRodney W. Grimes 1279c5aaa06dSAlan Cox /* 1280c5aaa06dSAlan Cox * vm_object_split: 1281c5aaa06dSAlan Cox * 1282c5aaa06dSAlan Cox * Split the pages in a map entry into a new object. This affords 1283c5aaa06dSAlan Cox * easier removal of unused pages, and keeps object inheritance from 1284c5aaa06dSAlan Cox * being a negative impact on memory usage. 1285c5aaa06dSAlan Cox */ 1286c5aaa06dSAlan Cox void 1287c5aaa06dSAlan Cox vm_object_split(vm_map_entry_t entry) 1288c5aaa06dSAlan Cox { 128973000556SAlan Cox vm_page_t m, m_next; 1290c5aaa06dSAlan Cox vm_object_t orig_object, new_object, source; 129173000556SAlan Cox vm_pindex_t idx, offidxstart; 129273000556SAlan Cox vm_size_t size; 1293c5aaa06dSAlan Cox 1294c5aaa06dSAlan Cox orig_object = entry->object.vm_object; 1295c5aaa06dSAlan Cox if (orig_object->type != OBJT_DEFAULT && orig_object->type != OBJT_SWAP) 1296c5aaa06dSAlan Cox return; 1297c5aaa06dSAlan Cox if (orig_object->ref_count <= 1) 1298c5aaa06dSAlan Cox return; 129989f6b863SAttilio Rao VM_OBJECT_WUNLOCK(orig_object); 1300c5aaa06dSAlan Cox 13014da9f125SAlan Cox offidxstart = OFF_TO_IDX(entry->offset); 130295442adfSAlan Cox size = atop(entry->end - entry->start); 1303c5aaa06dSAlan Cox 13044da9f125SAlan Cox /* 13054da9f125SAlan Cox * If swap_pager_copy() is later called, it will convert new_object 13064da9f125SAlan Cox * into a swap object. 13074da9f125SAlan Cox */ 13084da9f125SAlan Cox new_object = vm_object_allocate(OBJT_DEFAULT, size); 1309c5aaa06dSAlan Cox 1310c5474b8fSAlan Cox /* 1311c5474b8fSAlan Cox * At this point, the new object is still private, so the order in 1312c5474b8fSAlan Cox * which the original and new objects are locked does not matter. 1313c5474b8fSAlan Cox */ 131489f6b863SAttilio Rao VM_OBJECT_WLOCK(new_object); 131589f6b863SAttilio Rao VM_OBJECT_WLOCK(orig_object); 13163f289c3fSJeff Roberson new_object->domain = orig_object->domain; 1317c5aaa06dSAlan Cox source = orig_object->backing_object; 1318c5aaa06dSAlan Cox if (source != NULL) { 131989f6b863SAttilio Rao VM_OBJECT_WLOCK(source); 132019c244d0SAlan Cox if ((source->flags & OBJ_DEAD) != 0) { 132189f6b863SAttilio Rao VM_OBJECT_WUNLOCK(source); 132289f6b863SAttilio Rao VM_OBJECT_WUNLOCK(orig_object); 132389f6b863SAttilio Rao VM_OBJECT_WUNLOCK(new_object); 132419c244d0SAlan Cox vm_object_deallocate(new_object); 132589f6b863SAttilio Rao VM_OBJECT_WLOCK(orig_object); 132619c244d0SAlan Cox return; 132719c244d0SAlan Cox } 13281c500307SAlan Cox LIST_INSERT_HEAD(&source->shadow_head, 1329c5aaa06dSAlan Cox new_object, shadow_list); 13308e3a76fbSAlan Cox source->shadow_count++; 1331b921a12bSAlan Cox vm_object_reference_locked(source); /* for new_object */ 1332c5aaa06dSAlan Cox vm_object_clear_flag(source, OBJ_ONEMAPPING); 133389f6b863SAttilio Rao VM_OBJECT_WUNLOCK(source); 1334c5aaa06dSAlan Cox new_object->backing_object_offset = 13354da9f125SAlan Cox orig_object->backing_object_offset + entry->offset; 1336c5aaa06dSAlan Cox new_object->backing_object = source; 1337c5aaa06dSAlan Cox } 1338ef694c1aSEdward Tomasz Napierala if (orig_object->cred != NULL) { 1339ef694c1aSEdward Tomasz Napierala new_object->cred = orig_object->cred; 1340ef694c1aSEdward Tomasz Napierala crhold(orig_object->cred); 13413364c323SKonstantin Belousov new_object->charge = ptoa(size); 13423364c323SKonstantin Belousov KASSERT(orig_object->charge >= ptoa(size), 13433364c323SKonstantin Belousov ("orig_object->charge < 0")); 13443364c323SKonstantin Belousov orig_object->charge -= ptoa(size); 13453364c323SKonstantin Belousov } 1346c5aaa06dSAlan Cox retry: 1347b382c10aSKonstantin Belousov m = vm_page_find_least(orig_object, offidxstart); 134873000556SAlan Cox for (; m != NULL && (idx = m->pindex - offidxstart) < size; 134973000556SAlan Cox m = m_next) { 135073000556SAlan Cox m_next = TAILQ_NEXT(m, listq); 1351c5aaa06dSAlan Cox 1352c5aaa06dSAlan Cox /* 1353c5aaa06dSAlan Cox * We must wait for pending I/O to complete before we can 1354c5aaa06dSAlan Cox * rename the page. 1355c5aaa06dSAlan Cox * 1356c5aaa06dSAlan Cox * We do not have to VM_PROT_NONE the page as mappings should 1357c5aaa06dSAlan Cox * not be changed by this operation. 1358c5aaa06dSAlan Cox */ 135963e97555SJeff Roberson if (vm_page_tryxbusy(m) == 0) { 136089f6b863SAttilio Rao VM_OBJECT_WUNLOCK(new_object); 13614cdea4a8SJeff Roberson vm_page_sleep_if_busy(m, "spltwt"); 136289f6b863SAttilio Rao VM_OBJECT_WLOCK(new_object); 1363c5aaa06dSAlan Cox goto retry; 1364de33beddSAlan Cox } 1365e946b949SAttilio Rao 13663453bca8SAlan Cox /* vm_page_rename() will dirty the page. */ 1367e946b949SAttilio Rao if (vm_page_rename(m, new_object, idx)) { 136863e97555SJeff Roberson vm_page_xunbusy(m); 1369e946b949SAttilio Rao VM_OBJECT_WUNLOCK(new_object); 1370e946b949SAttilio Rao VM_OBJECT_WUNLOCK(orig_object); 13718d6fbbb8SJeff Roberson vm_radix_wait(); 1372e946b949SAttilio Rao VM_OBJECT_WLOCK(orig_object); 1373e946b949SAttilio Rao VM_OBJECT_WLOCK(new_object); 1374e946b949SAttilio Rao goto retry; 1375e946b949SAttilio Rao } 137663e97555SJeff Roberson 1377b5f359b7SAlan Cox #if VM_NRESERVLEVEL > 0 1378b5f359b7SAlan Cox /* 1379b5f359b7SAlan Cox * If some of the reservation's allocated pages remain with 1380b5f359b7SAlan Cox * the original object, then transferring the reservation to 1381b5f359b7SAlan Cox * the new object is neither particularly beneficial nor 1382b5f359b7SAlan Cox * particularly harmful as compared to leaving the reservation 1383b5f359b7SAlan Cox * with the original object. If, however, all of the 1384b5f359b7SAlan Cox * reservation's allocated pages are transferred to the new 1385b5f359b7SAlan Cox * object, then transferring the reservation is typically 1386b5f359b7SAlan Cox * beneficial. Determining which of these two cases applies 1387b5f359b7SAlan Cox * would be more costly than unconditionally renaming the 1388b5f359b7SAlan Cox * reservation. 1389b5f359b7SAlan Cox */ 1390b5f359b7SAlan Cox vm_reserv_rename(m, new_object, orig_object, offidxstart); 1391b5f359b7SAlan Cox #endif 13928da1c098SJeff Roberson if (orig_object->type != OBJT_SWAP) 13938da1c098SJeff Roberson vm_page_xunbusy(m); 1394c5aaa06dSAlan Cox } 1395d7a013c3SAlan Cox if (orig_object->type == OBJT_SWAP) { 1396c5aaa06dSAlan Cox /* 1397c7c8dd7eSAlan Cox * swap_pager_copy() can sleep, in which case the orig_object's 1398c7c8dd7eSAlan Cox * and new_object's locks are released and reacquired. 1399c5aaa06dSAlan Cox */ 1400c5aaa06dSAlan Cox swap_pager_copy(orig_object, new_object, offidxstart, 0); 1401dfd55c0cSAttilio Rao TAILQ_FOREACH(m, &new_object->memq, listq) 1402c7aebda8SAttilio Rao vm_page_xunbusy(m); 1403c5aaa06dSAlan Cox } 140489f6b863SAttilio Rao VM_OBJECT_WUNLOCK(orig_object); 140589f6b863SAttilio Rao VM_OBJECT_WUNLOCK(new_object); 1406c5aaa06dSAlan Cox entry->object.vm_object = new_object; 1407c5aaa06dSAlan Cox entry->offset = 0LL; 1408c5aaa06dSAlan Cox vm_object_deallocate(orig_object); 140989f6b863SAttilio Rao VM_OBJECT_WLOCK(new_object); 1410c5aaa06dSAlan Cox } 1411c5aaa06dSAlan Cox 14122ad1a3f7SMatthew Dillon #define OBSC_COLLAPSE_NOWAIT 0x0002 14132ad1a3f7SMatthew Dillon #define OBSC_COLLAPSE_WAIT 0x0004 14142ad1a3f7SMatthew Dillon 141599a1570aSKonstantin Belousov static vm_page_t 14164cc8daf7SConrad Meyer vm_object_collapse_scan_wait(vm_object_t object, vm_page_t p, vm_page_t next, 141799a1570aSKonstantin Belousov int op) 141899a1570aSKonstantin Belousov { 141999a1570aSKonstantin Belousov vm_object_t backing_object; 142099a1570aSKonstantin Belousov 142199a1570aSKonstantin Belousov VM_OBJECT_ASSERT_WLOCKED(object); 142299a1570aSKonstantin Belousov backing_object = object->backing_object; 142399a1570aSKonstantin Belousov VM_OBJECT_ASSERT_WLOCKED(backing_object); 142499a1570aSKonstantin Belousov 142599a1570aSKonstantin Belousov KASSERT(p == NULL || p->object == object || p->object == backing_object, 142699a1570aSKonstantin Belousov ("invalid ownership %p %p %p", p, object, backing_object)); 142799a1570aSKonstantin Belousov if ((op & OBSC_COLLAPSE_NOWAIT) != 0) 142899a1570aSKonstantin Belousov return (next); 14298d6fbbb8SJeff Roberson /* The page is only NULL when rename fails. */ 14304cdea4a8SJeff Roberson if (p == NULL) { 14318d6fbbb8SJeff Roberson vm_radix_wait(); 14324cdea4a8SJeff Roberson } else { 14334cdea4a8SJeff Roberson if (p->object == object) 14344cdea4a8SJeff Roberson VM_OBJECT_WUNLOCK(backing_object); 143599a1570aSKonstantin Belousov else 14364cdea4a8SJeff Roberson VM_OBJECT_WUNLOCK(object); 14375975e53dSKonstantin Belousov vm_page_busy_sleep(p, "vmocol", false); 14384cdea4a8SJeff Roberson } 143999a1570aSKonstantin Belousov VM_OBJECT_WLOCK(object); 144099a1570aSKonstantin Belousov VM_OBJECT_WLOCK(backing_object); 144199a1570aSKonstantin Belousov return (TAILQ_FIRST(&backing_object->memq)); 144299a1570aSKonstantin Belousov } 144399a1570aSKonstantin Belousov 144499a1570aSKonstantin Belousov static bool 14454cc8daf7SConrad Meyer vm_object_scan_all_shadowed(vm_object_t object) 14464cc8daf7SConrad Meyer { 14474cc8daf7SConrad Meyer vm_object_t backing_object; 14484cc8daf7SConrad Meyer vm_page_t p, pp; 144977d6fd97SKonstantin Belousov vm_pindex_t backing_offset_index, new_pindex, pi, ps; 14504cc8daf7SConrad Meyer 14514cc8daf7SConrad Meyer VM_OBJECT_ASSERT_WLOCKED(object); 14524cc8daf7SConrad Meyer VM_OBJECT_ASSERT_WLOCKED(object->backing_object); 14534cc8daf7SConrad Meyer 14544cc8daf7SConrad Meyer backing_object = object->backing_object; 14554cc8daf7SConrad Meyer 145677d6fd97SKonstantin Belousov if (backing_object->type != OBJT_DEFAULT && 145777d6fd97SKonstantin Belousov backing_object->type != OBJT_SWAP) 14584cc8daf7SConrad Meyer return (false); 14594cc8daf7SConrad Meyer 146077d6fd97SKonstantin Belousov pi = backing_offset_index = OFF_TO_IDX(object->backing_object_offset); 146177d6fd97SKonstantin Belousov p = vm_page_find_least(backing_object, pi); 146277d6fd97SKonstantin Belousov ps = swap_pager_find_least(backing_object, pi); 14634cc8daf7SConrad Meyer 14644cc8daf7SConrad Meyer /* 146577d6fd97SKonstantin Belousov * Only check pages inside the parent object's range and 146677d6fd97SKonstantin Belousov * inside the parent object's mapping of the backing object. 14674cc8daf7SConrad Meyer */ 146877d6fd97SKonstantin Belousov for (;; pi++) { 146977d6fd97SKonstantin Belousov if (p != NULL && p->pindex < pi) 147077d6fd97SKonstantin Belousov p = TAILQ_NEXT(p, listq); 147177d6fd97SKonstantin Belousov if (ps < pi) 147277d6fd97SKonstantin Belousov ps = swap_pager_find_least(backing_object, pi); 147377d6fd97SKonstantin Belousov if (p == NULL && ps >= backing_object->size) 147477d6fd97SKonstantin Belousov break; 147577d6fd97SKonstantin Belousov else if (p == NULL) 147677d6fd97SKonstantin Belousov pi = ps; 147777d6fd97SKonstantin Belousov else 147877d6fd97SKonstantin Belousov pi = MIN(p->pindex, ps); 147977d6fd97SKonstantin Belousov 148077d6fd97SKonstantin Belousov new_pindex = pi - backing_offset_index; 148177d6fd97SKonstantin Belousov if (new_pindex >= object->size) 148277d6fd97SKonstantin Belousov break; 14834cc8daf7SConrad Meyer 14844cc8daf7SConrad Meyer /* 14854cc8daf7SConrad Meyer * See if the parent has the page or if the parent's object 14864cc8daf7SConrad Meyer * pager has the page. If the parent has the page but the page 14874cc8daf7SConrad Meyer * is not valid, the parent's object pager must have the page. 14884cc8daf7SConrad Meyer * 14894cc8daf7SConrad Meyer * If this fails, the parent does not completely shadow the 14904cc8daf7SConrad Meyer * object and we might as well give up now. 14914cc8daf7SConrad Meyer */ 14924cc8daf7SConrad Meyer pp = vm_page_lookup(object, new_pindex); 14930012f373SJeff Roberson /* 14940012f373SJeff Roberson * The valid check here is stable due to object lock being 14950012f373SJeff Roberson * required to clear valid and initiate paging. 14960012f373SJeff Roberson */ 14970012f373SJeff Roberson if ((pp == NULL || vm_page_none_valid(pp)) && 14984cc8daf7SConrad Meyer !vm_pager_has_page(object, new_pindex, NULL, NULL)) 14994cc8daf7SConrad Meyer return (false); 15004cc8daf7SConrad Meyer } 15014cc8daf7SConrad Meyer return (true); 15024cc8daf7SConrad Meyer } 15034cc8daf7SConrad Meyer 15044cc8daf7SConrad Meyer static bool 15054cc8daf7SConrad Meyer vm_object_collapse_scan(vm_object_t object, int op) 15062ad1a3f7SMatthew Dillon { 15072ad1a3f7SMatthew Dillon vm_object_t backing_object; 150899a1570aSKonstantin Belousov vm_page_t next, p, pp; 150999a1570aSKonstantin Belousov vm_pindex_t backing_offset_index, new_pindex; 15102ad1a3f7SMatthew Dillon 151189f6b863SAttilio Rao VM_OBJECT_ASSERT_WLOCKED(object); 151289f6b863SAttilio Rao VM_OBJECT_ASSERT_WLOCKED(object->backing_object); 15132ad1a3f7SMatthew Dillon 15142ad1a3f7SMatthew Dillon backing_object = object->backing_object; 15152ad1a3f7SMatthew Dillon backing_offset_index = OFF_TO_IDX(object->backing_object_offset); 15162ad1a3f7SMatthew Dillon 15172ad1a3f7SMatthew Dillon /* 15182ad1a3f7SMatthew Dillon * Initial conditions 15192ad1a3f7SMatthew Dillon */ 15204cc8daf7SConrad Meyer if ((op & OBSC_COLLAPSE_WAIT) != 0) 15212ad1a3f7SMatthew Dillon vm_object_set_flag(backing_object, OBJ_DEAD); 15222ad1a3f7SMatthew Dillon 15232ad1a3f7SMatthew Dillon /* 15242ad1a3f7SMatthew Dillon * Our scan 15252ad1a3f7SMatthew Dillon */ 15264cc8daf7SConrad Meyer for (p = TAILQ_FIRST(&backing_object->memq); p != NULL; p = next) { 152799a1570aSKonstantin Belousov next = TAILQ_NEXT(p, listq); 152899a1570aSKonstantin Belousov new_pindex = p->pindex - backing_offset_index; 15292ad1a3f7SMatthew Dillon 15302ad1a3f7SMatthew Dillon /* 15312ad1a3f7SMatthew Dillon * Check for busy page 15322ad1a3f7SMatthew Dillon */ 153363e97555SJeff Roberson if (vm_page_tryxbusy(p) == 0) { 15344cc8daf7SConrad Meyer next = vm_object_collapse_scan_wait(object, p, next, op); 15352ad1a3f7SMatthew Dillon continue; 15362ad1a3f7SMatthew Dillon } 15372ad1a3f7SMatthew Dillon 153899a1570aSKonstantin Belousov KASSERT(p->object == backing_object, 15394cc8daf7SConrad Meyer ("vm_object_collapse_scan: object mismatch")); 15402ad1a3f7SMatthew Dillon 154199a1570aSKonstantin Belousov if (p->pindex < backing_offset_index || 154299a1570aSKonstantin Belousov new_pindex >= object->size) { 1543e946b949SAttilio Rao if (backing_object->type == OBJT_SWAP) 15444cc8daf7SConrad Meyer swap_pager_freespace(backing_object, p->pindex, 15454cc8daf7SConrad Meyer 1); 1546e946b949SAttilio Rao 1547f6d89838SAlan Cox KASSERT(!pmap_page_is_mapped(p), 1548f6d89838SAlan Cox ("freeing mapped page %p", p)); 15490fd977b3SMark Johnston if (vm_page_remove(p)) 15502ad1a3f7SMatthew Dillon vm_page_free(p); 15518da1c098SJeff Roberson else 15528da1c098SJeff Roberson vm_page_xunbusy(p); 15532ad1a3f7SMatthew Dillon continue; 15542ad1a3f7SMatthew Dillon } 15552ad1a3f7SMatthew Dillon 15562ad1a3f7SMatthew Dillon pp = vm_page_lookup(object, new_pindex); 155763e97555SJeff Roberson if (pp != NULL && vm_page_tryxbusy(pp) == 0) { 155863e97555SJeff Roberson vm_page_xunbusy(p); 1559e18cc7bfSMax Laier /* 15604cc8daf7SConrad Meyer * The page in the parent is busy and possibly not 15614cc8daf7SConrad Meyer * (yet) valid. Until its state is finalized by the 15624cc8daf7SConrad Meyer * busy bit owner, we can't tell whether it shadows the 15634cc8daf7SConrad Meyer * original page. Therefore, we must either skip it 15644cc8daf7SConrad Meyer * and the original (backing_object) page or wait for 15654cc8daf7SConrad Meyer * its state to be finalized. 1566e18cc7bfSMax Laier * 15674cc8daf7SConrad Meyer * This is due to a race with vm_fault() where we must 15684cc8daf7SConrad Meyer * unbusy the original (backing_obj) page before we can 15694cc8daf7SConrad Meyer * (re)lock the parent. Hence we can get here. 1570e18cc7bfSMax Laier */ 15714cc8daf7SConrad Meyer next = vm_object_collapse_scan_wait(object, pp, next, 15724cc8daf7SConrad Meyer op); 1573e18cc7bfSMax Laier continue; 1574e18cc7bfSMax Laier } 157599a1570aSKonstantin Belousov 15760012f373SJeff Roberson KASSERT(pp == NULL || !vm_page_none_valid(pp), 157799a1570aSKonstantin Belousov ("unbusy invalid page %p", pp)); 157899a1570aSKonstantin Belousov 15794cc8daf7SConrad Meyer if (pp != NULL || vm_pager_has_page(object, new_pindex, NULL, 15804cc8daf7SConrad Meyer NULL)) { 158199a1570aSKonstantin Belousov /* 15824cc8daf7SConrad Meyer * The page already exists in the parent OR swap exists 15834cc8daf7SConrad Meyer * for this location in the parent. Leave the parent's 15844cc8daf7SConrad Meyer * page alone. Destroy the original page from the 15854cc8daf7SConrad Meyer * backing object. 158699a1570aSKonstantin Belousov */ 1587e946b949SAttilio Rao if (backing_object->type == OBJT_SWAP) 15884cc8daf7SConrad Meyer swap_pager_freespace(backing_object, p->pindex, 15894cc8daf7SConrad Meyer 1); 1590f6d89838SAlan Cox KASSERT(!pmap_page_is_mapped(p), 1591f6d89838SAlan Cox ("freeing mapped page %p", p)); 15920fd977b3SMark Johnston if (vm_page_remove(p)) 15932ad1a3f7SMatthew Dillon vm_page_free(p); 15948da1c098SJeff Roberson else 15958da1c098SJeff Roberson vm_page_xunbusy(p); 159663e97555SJeff Roberson if (pp != NULL) 159763e97555SJeff Roberson vm_page_xunbusy(pp); 15982ad1a3f7SMatthew Dillon continue; 15992ad1a3f7SMatthew Dillon } 16002ad1a3f7SMatthew Dillon 1601e946b949SAttilio Rao /* 16024cc8daf7SConrad Meyer * Page does not exist in parent, rename the page from the 16034cc8daf7SConrad Meyer * backing object to the main object. 1604e946b949SAttilio Rao * 16054cc8daf7SConrad Meyer * If the page was mapped to a process, it can remain mapped 16063453bca8SAlan Cox * through the rename. vm_page_rename() will dirty the page. 1607e946b949SAttilio Rao */ 1608e946b949SAttilio Rao if (vm_page_rename(p, object, new_pindex)) { 160963e97555SJeff Roberson vm_page_xunbusy(p); 161063e97555SJeff Roberson if (pp != NULL) 161163e97555SJeff Roberson vm_page_xunbusy(pp); 16124cc8daf7SConrad Meyer next = vm_object_collapse_scan_wait(object, NULL, next, 16134cc8daf7SConrad Meyer op); 1614e946b949SAttilio Rao continue; 1615e946b949SAttilio Rao } 161614a5dc17SAttilio Rao 161714a5dc17SAttilio Rao /* Use the old pindex to free the right page. */ 1618e946b949SAttilio Rao if (backing_object->type == OBJT_SWAP) 161914a5dc17SAttilio Rao swap_pager_freespace(backing_object, 162014a5dc17SAttilio Rao new_pindex + backing_offset_index, 1); 1621e946b949SAttilio Rao 1622f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0 1623f8a47341SAlan Cox /* 1624f8a47341SAlan Cox * Rename the reservation. 1625f8a47341SAlan Cox */ 1626f8a47341SAlan Cox vm_reserv_rename(p, object, backing_object, 1627f8a47341SAlan Cox backing_offset_index); 1628f8a47341SAlan Cox #endif 16298da1c098SJeff Roberson vm_page_xunbusy(p); 16302ad1a3f7SMatthew Dillon } 163199a1570aSKonstantin Belousov return (true); 16322ad1a3f7SMatthew Dillon } 16332ad1a3f7SMatthew Dillon 1634df8bae1dSRodney W. Grimes 1635df8bae1dSRodney W. Grimes /* 16362fe6e4d7SDavid Greenman * this version of collapse allows the operation to occur earlier and 16372fe6e4d7SDavid Greenman * when paging_in_progress is true for an object... This is not a complete 16382fe6e4d7SDavid Greenman * operation, but should plug 99.9% of the rest of the leaks. 16392fe6e4d7SDavid Greenman */ 16402fe6e4d7SDavid Greenman static void 16411b40f8c0SMatthew Dillon vm_object_qcollapse(vm_object_t object) 16422fe6e4d7SDavid Greenman { 16432ad1a3f7SMatthew Dillon vm_object_t backing_object = object->backing_object; 16442fe6e4d7SDavid Greenman 164589f6b863SAttilio Rao VM_OBJECT_ASSERT_WLOCKED(object); 164689f6b863SAttilio Rao VM_OBJECT_ASSERT_WLOCKED(backing_object); 16471b40f8c0SMatthew Dillon 16482fe6e4d7SDavid Greenman if (backing_object->ref_count != 1) 16492fe6e4d7SDavid Greenman return; 16502fe6e4d7SDavid Greenman 16514cc8daf7SConrad Meyer vm_object_collapse_scan(object, OBSC_COLLAPSE_NOWAIT); 16522fe6e4d7SDavid Greenman } 16532fe6e4d7SDavid Greenman 1654df8bae1dSRodney W. Grimes /* 1655df8bae1dSRodney W. Grimes * vm_object_collapse: 1656df8bae1dSRodney W. Grimes * 1657df8bae1dSRodney W. Grimes * Collapse an object with the object backing it. 1658df8bae1dSRodney W. Grimes * Pages in the backing object are moved into the 1659df8bae1dSRodney W. Grimes * parent, and the backing object is deallocated. 1660df8bae1dSRodney W. Grimes */ 166126f9a767SRodney W. Grimes void 16621b40f8c0SMatthew Dillon vm_object_collapse(vm_object_t object) 1663df8bae1dSRodney W. Grimes { 166498f139daSKonstantin Belousov vm_object_t backing_object, new_backing_object; 166598f139daSKonstantin Belousov 166689f6b863SAttilio Rao VM_OBJECT_ASSERT_WLOCKED(object); 166723955314SAlfred Perlstein 1668df8bae1dSRodney W. Grimes while (TRUE) { 1669df8bae1dSRodney W. Grimes /* 1670df8bae1dSRodney W. Grimes * Verify that the conditions are right for collapse: 1671df8bae1dSRodney W. Grimes * 16722ad1a3f7SMatthew Dillon * The object exists and the backing object exists. 1673df8bae1dSRodney W. Grimes */ 167424a1cce3SDavid Greenman if ((backing_object = object->backing_object) == NULL) 16752ad1a3f7SMatthew Dillon break; 1676df8bae1dSRodney W. Grimes 1677f919ebdeSDavid Greenman /* 1678f919ebdeSDavid Greenman * we check the backing object first, because it is most likely 167924a1cce3SDavid Greenman * not collapsable. 1680f919ebdeSDavid Greenman */ 168189f6b863SAttilio Rao VM_OBJECT_WLOCK(backing_object); 168224a1cce3SDavid Greenman if (backing_object->handle != NULL || 168324a1cce3SDavid Greenman (backing_object->type != OBJT_DEFAULT && 168412487941SKonstantin Belousov backing_object->type != OBJT_SWAP) || 168512487941SKonstantin Belousov (backing_object->flags & (OBJ_DEAD | OBJ_NOSPLIT)) != 0 || 168624a1cce3SDavid Greenman object->handle != NULL || 168724a1cce3SDavid Greenman (object->type != OBJT_DEFAULT && 168824a1cce3SDavid Greenman object->type != OBJT_SWAP) || 168924a1cce3SDavid Greenman (object->flags & OBJ_DEAD)) { 169089f6b863SAttilio Rao VM_OBJECT_WUNLOCK(backing_object); 16912ad1a3f7SMatthew Dillon break; 169224a1cce3SDavid Greenman } 16939b4814bbSDavid Greenman 169411b57401SHans Petter Selasky if (REFCOUNT_COUNT(object->paging_in_progress) > 0 || 169511b57401SHans Petter Selasky REFCOUNT_COUNT(backing_object->paging_in_progress) > 0) { 1696b9921222SDavid Greenman vm_object_qcollapse(object); 169789f6b863SAttilio Rao VM_OBJECT_WUNLOCK(backing_object); 16982ad1a3f7SMatthew Dillon break; 1699df8bae1dSRodney W. Grimes } 170098f139daSKonstantin Belousov 170126f9a767SRodney W. Grimes /* 17020d94caffSDavid Greenman * We know that we can either collapse the backing object (if 17032ad1a3f7SMatthew Dillon * the parent is the only reference to it) or (perhaps) have 17042ad1a3f7SMatthew Dillon * the parent bypass the object if the parent happens to shadow 17052ad1a3f7SMatthew Dillon * all the resident pages in the entire backing object. 17062ad1a3f7SMatthew Dillon * 17072ad1a3f7SMatthew Dillon * This is ignoring pager-backed pages such as swap pages. 17084cc8daf7SConrad Meyer * vm_object_collapse_scan fails the shadowing test in this 17092ad1a3f7SMatthew Dillon * case. 1710df8bae1dSRodney W. Grimes */ 1711df8bae1dSRodney W. Grimes if (backing_object->ref_count == 1) { 1712aa9bc3b1SKonstantin Belousov vm_object_pip_add(object, 1); 1713aa9bc3b1SKonstantin Belousov vm_object_pip_add(backing_object, 1); 1714aa9bc3b1SKonstantin Belousov 1715df8bae1dSRodney W. Grimes /* 17162ad1a3f7SMatthew Dillon * If there is exactly one reference to the backing 17172ad1a3f7SMatthew Dillon * object, we can collapse it into the parent. 1718df8bae1dSRodney W. Grimes */ 17194cc8daf7SConrad Meyer vm_object_collapse_scan(object, OBSC_COLLAPSE_WAIT); 1720df8bae1dSRodney W. Grimes 1721f8a47341SAlan Cox #if VM_NRESERVLEVEL > 0 1722f8a47341SAlan Cox /* 1723f8a47341SAlan Cox * Break any reservations from backing_object. 1724f8a47341SAlan Cox */ 1725f8a47341SAlan Cox if (__predict_false(!LIST_EMPTY(&backing_object->rvq))) 1726f8a47341SAlan Cox vm_reserv_break_all(backing_object); 1727f8a47341SAlan Cox #endif 1728f8a47341SAlan Cox 1729df8bae1dSRodney W. Grimes /* 1730df8bae1dSRodney W. Grimes * Move the pager from backing_object to object. 1731df8bae1dSRodney W. Grimes */ 17326be36525SAlan Cox if (backing_object->type == OBJT_SWAP) { 173324a1cce3SDavid Greenman /* 1734c7c8dd7eSAlan Cox * swap_pager_copy() can sleep, in which case 1735c7c8dd7eSAlan Cox * the backing_object's and object's locks are 1736c7c8dd7eSAlan Cox * released and reacquired. 1737571a1e92SAttilio Rao * Since swap_pager_copy() is being asked to 1738571a1e92SAttilio Rao * destroy the source, it will change the 1739571a1e92SAttilio Rao * backing_object's type to OBJT_DEFAULT. 174024a1cce3SDavid Greenman */ 17411c7c3c6aSMatthew Dillon swap_pager_copy( 17421c7c3c6aSMatthew Dillon backing_object, 17431c7c3c6aSMatthew Dillon object, 17441c7c3c6aSMatthew Dillon OFF_TO_IDX(object->backing_object_offset), TRUE); 1745c0503609SDavid Greenman } 1746df8bae1dSRodney W. Grimes /* 1747df8bae1dSRodney W. Grimes * Object now shadows whatever backing_object did. 17482ad1a3f7SMatthew Dillon * Note that the reference to 17492ad1a3f7SMatthew Dillon * backing_object->backing_object moves from within 17502ad1a3f7SMatthew Dillon * backing_object to within object. 1751df8bae1dSRodney W. Grimes */ 17521c500307SAlan Cox LIST_REMOVE(object, shadow_list); 17534f7c7f6eSAlan Cox backing_object->shadow_count--; 1754de5f6a77SJohn Dyson if (backing_object->backing_object) { 175589f6b863SAttilio Rao VM_OBJECT_WLOCK(backing_object->backing_object); 17561c500307SAlan Cox LIST_REMOVE(backing_object, shadow_list); 175743186e53SAlan Cox LIST_INSERT_HEAD( 175843186e53SAlan Cox &backing_object->backing_object->shadow_head, 175943186e53SAlan Cox object, shadow_list); 176043186e53SAlan Cox /* 176143186e53SAlan Cox * The shadow_count has not changed. 176243186e53SAlan Cox */ 176389f6b863SAttilio Rao VM_OBJECT_WUNLOCK(backing_object->backing_object); 1764de5f6a77SJohn Dyson } 176524a1cce3SDavid Greenman object->backing_object = backing_object->backing_object; 17662ad1a3f7SMatthew Dillon object->backing_object_offset += 17672ad1a3f7SMatthew Dillon backing_object->backing_object_offset; 17682ad1a3f7SMatthew Dillon 1769df8bae1dSRodney W. Grimes /* 1770df8bae1dSRodney W. Grimes * Discard backing_object. 1771df8bae1dSRodney W. Grimes * 17720d94caffSDavid Greenman * Since the backing object has no pages, no pager left, 17730d94caffSDavid Greenman * and no object references within it, all that is 17740d94caffSDavid Greenman * necessary is to dispose of it. 1775df8bae1dSRodney W. Grimes */ 17769b4d473aSKonstantin Belousov KASSERT(backing_object->ref_count == 1, ( 17779b4d473aSKonstantin Belousov "backing_object %p was somehow re-referenced during collapse!", 17789b4d473aSKonstantin Belousov backing_object)); 1779aa9bc3b1SKonstantin Belousov vm_object_pip_wakeup(backing_object); 1780e735691bSJohn Baldwin backing_object->type = OBJT_DEAD; 1781e735691bSJohn Baldwin backing_object->ref_count = 0; 178289f6b863SAttilio Rao VM_OBJECT_WUNLOCK(backing_object); 17839b4d473aSKonstantin Belousov vm_object_destroy(backing_object); 1784df8bae1dSRodney W. Grimes 1785aa9bc3b1SKonstantin Belousov vm_object_pip_wakeup(object); 178611542376SAlan Cox counter_u64_add(object_collapses, 1); 17870d94caffSDavid Greenman } else { 1788df8bae1dSRodney W. Grimes /* 17892ad1a3f7SMatthew Dillon * If we do not entirely shadow the backing object, 17902ad1a3f7SMatthew Dillon * there is nothing we can do so we give up. 1791df8bae1dSRodney W. Grimes */ 1792df59a0feSJeff Roberson if (object->resident_page_count != object->size && 17934cc8daf7SConrad Meyer !vm_object_scan_all_shadowed(object)) { 179489f6b863SAttilio Rao VM_OBJECT_WUNLOCK(backing_object); 17952ad1a3f7SMatthew Dillon break; 179624a1cce3SDavid Greenman } 1797df8bae1dSRodney W. Grimes 1798df8bae1dSRodney W. Grimes /* 17990d94caffSDavid Greenman * Make the parent shadow the next object in the 18000d94caffSDavid Greenman * chain. Deallocating backing_object will not remove 18010d94caffSDavid Greenman * it, since its reference count is at least 2. 1802df8bae1dSRodney W. Grimes */ 18031c500307SAlan Cox LIST_REMOVE(object, shadow_list); 1804eaf13dd7SJohn Dyson backing_object->shadow_count--; 180595e5e988SJohn Dyson 180695e5e988SJohn Dyson new_backing_object = backing_object->backing_object; 18078aef1712SMatthew Dillon if ((object->backing_object = new_backing_object) != NULL) { 180889f6b863SAttilio Rao VM_OBJECT_WLOCK(new_backing_object); 18091c500307SAlan Cox LIST_INSERT_HEAD( 18102ad1a3f7SMatthew Dillon &new_backing_object->shadow_head, 18112ad1a3f7SMatthew Dillon object, 18122ad1a3f7SMatthew Dillon shadow_list 18132ad1a3f7SMatthew Dillon ); 1814eaf13dd7SJohn Dyson new_backing_object->shadow_count++; 1815b921a12bSAlan Cox vm_object_reference_locked(new_backing_object); 181689f6b863SAttilio Rao VM_OBJECT_WUNLOCK(new_backing_object); 181795e5e988SJohn Dyson object->backing_object_offset += 181895e5e988SJohn Dyson backing_object->backing_object_offset; 1819de5f6a77SJohn Dyson } 1820df8bae1dSRodney W. Grimes 1821df8bae1dSRodney W. Grimes /* 18220d94caffSDavid Greenman * Drop the reference count on backing_object. Since 182322ec553fSAlan Cox * its ref_count was at least 2, it will not vanish. 1824df8bae1dSRodney W. Grimes */ 1825*51df5321SJeff Roberson refcount_release(&backing_object->ref_count); 182689f6b863SAttilio Rao VM_OBJECT_WUNLOCK(backing_object); 182711542376SAlan Cox counter_u64_add(object_bypasses, 1); 1828df8bae1dSRodney W. Grimes } 1829df8bae1dSRodney W. Grimes 1830df8bae1dSRodney W. Grimes /* 1831df8bae1dSRodney W. Grimes * Try again with this object's new backing object. 1832df8bae1dSRodney W. Grimes */ 1833df8bae1dSRodney W. Grimes } 1834df8bae1dSRodney W. Grimes } 1835df8bae1dSRodney W. Grimes 1836df8bae1dSRodney W. Grimes /* 1837bff99f0dSAlan Cox * vm_object_page_remove: 1838df8bae1dSRodney W. Grimes * 183968855966SAlan Cox * For the given object, either frees or invalidates each of the 18406bbee8e2SAlan Cox * specified pages. In general, a page is freed. However, if a page is 18416bbee8e2SAlan Cox * wired for any reason other than the existence of a managed, wired 18426bbee8e2SAlan Cox * mapping, then it may be invalidated but not removed from the object. 18436bbee8e2SAlan Cox * Pages are specified by the given range ["start", "end") and the option 18446bbee8e2SAlan Cox * OBJPR_CLEANONLY. As a special case, if "end" is zero, then the range 18456bbee8e2SAlan Cox * extends from "start" to the end of the object. If the option 18466bbee8e2SAlan Cox * OBJPR_CLEANONLY is specified, then only the non-dirty pages within the 18476bbee8e2SAlan Cox * specified range are affected. If the option OBJPR_NOTMAPPED is 18486bbee8e2SAlan Cox * specified, then the pages within the specified range must have no 18496bbee8e2SAlan Cox * mappings. Otherwise, if this option is not specified, any mappings to 18506bbee8e2SAlan Cox * the specified pages are removed before the pages are freed or 18516bbee8e2SAlan Cox * invalidated. 185268855966SAlan Cox * 18536bbee8e2SAlan Cox * In general, this operation should only be performed on objects that 18546bbee8e2SAlan Cox * contain managed pages. There are, however, two exceptions. First, it 18556bbee8e2SAlan Cox * is performed on the kernel and kmem objects by vm_map_entry_delete(). 18566bbee8e2SAlan Cox * Second, it is used by msync(..., MS_INVALIDATE) to invalidate device- 18576bbee8e2SAlan Cox * backed pages. In both of these cases, the option OBJPR_CLEANONLY must 18586bbee8e2SAlan Cox * not be specified and the option OBJPR_NOTMAPPED must be specified. 1859df8bae1dSRodney W. Grimes * 1860df8bae1dSRodney W. Grimes * The object must be locked. 1861df8bae1dSRodney W. Grimes */ 186226f9a767SRodney W. Grimes void 1863ecde4b32SAlan Cox vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end, 18646bbee8e2SAlan Cox int options) 1865df8bae1dSRodney W. Grimes { 1866d031cff1SMatthew Dillon vm_page_t p, next; 1867df8bae1dSRodney W. Grimes 186889f6b863SAttilio Rao VM_OBJECT_ASSERT_WLOCKED(object); 186928634820SAlan Cox KASSERT((object->flags & OBJ_UNMANAGED) == 0 || 18706bbee8e2SAlan Cox (options & (OBJPR_CLEANONLY | OBJPR_NOTMAPPED)) == OBJPR_NOTMAPPED, 18716bbee8e2SAlan Cox ("vm_object_page_remove: illegal options for object %p", object)); 1872ecde4b32SAlan Cox if (object->resident_page_count == 0) 18737667839aSAlan Cox return; 1874d474eaaaSDoug Rabson vm_object_pip_add(object, 1); 187526f9a767SRodney W. Grimes again: 1876b382c10aSKonstantin Belousov p = vm_page_find_least(object, start); 18772965a453SKip Macy 187875741c04SAlan Cox /* 18796bbee8e2SAlan Cox * Here, the variable "p" is either (1) the page with the least pindex 18806bbee8e2SAlan Cox * greater than or equal to the parameter "start" or (2) NULL. 188175741c04SAlan Cox */ 18826bbee8e2SAlan Cox for (; p != NULL && (p->pindex < end || end == 0); p = next) { 1883b18bfc3dSJohn Dyson next = TAILQ_NEXT(p, listq); 188475741c04SAlan Cox 188559677d3cSAlan Cox /* 18866bbee8e2SAlan Cox * If the page is wired for any reason besides the existence 18876bbee8e2SAlan Cox * of managed, wired mappings, then it cannot be freed. For 18886bbee8e2SAlan Cox * example, fictitious pages, which represent device memory, 18896bbee8e2SAlan Cox * are inherently wired and cannot be freed. They can, 18906bbee8e2SAlan Cox * however, be invalidated if the option OBJPR_CLEANONLY is 18916bbee8e2SAlan Cox * not specified. 189259677d3cSAlan Cox */ 189363e97555SJeff Roberson if (vm_page_tryxbusy(p) == 0) { 18944cdea4a8SJeff Roberson vm_page_sleep_if_busy(p, "vmopar"); 1895fee2a2faSMark Johnston goto again; 1896fee2a2faSMark Johnston } 1897d842aa51SMark Johnston if (vm_page_wired(p)) { 1898fee2a2faSMark Johnston wired: 1899cf060942SAlan Cox if ((options & OBJPR_NOTMAPPED) == 0 && 1900cf060942SAlan Cox object->ref_count != 0) 19014fec79beSAlan Cox pmap_remove_all(p); 19026bbee8e2SAlan Cox if ((options & OBJPR_CLEANONLY) == 0) { 19030012f373SJeff Roberson vm_page_invalid(p); 1904a28042d1SAlan Cox vm_page_undirty(p); 1905a28042d1SAlan Cox } 190663e97555SJeff Roberson vm_page_xunbusy(p); 190793c5d3a4SKonstantin Belousov continue; 19080d94caffSDavid Greenman } 190968855966SAlan Cox KASSERT((p->flags & PG_FICTITIOUS) == 0, 191068855966SAlan Cox ("vm_object_page_remove: page %p is fictitious", p)); 19110012f373SJeff Roberson if ((options & OBJPR_CLEANONLY) != 0 && 19120012f373SJeff Roberson !vm_page_none_valid(p)) { 1913cf060942SAlan Cox if ((options & OBJPR_NOTMAPPED) == 0 && 1914fee2a2faSMark Johnston object->ref_count != 0 && 1915fee2a2faSMark Johnston !vm_page_try_remove_write(p)) 1916fee2a2faSMark Johnston goto wired; 191763e97555SJeff Roberson if (p->dirty != 0) { 191863e97555SJeff Roberson vm_page_xunbusy(p); 191993c5d3a4SKonstantin Belousov continue; 19202965a453SKip Macy } 192163e97555SJeff Roberson } 1922fee2a2faSMark Johnston if ((options & OBJPR_NOTMAPPED) == 0 && 1923fee2a2faSMark Johnston object->ref_count != 0 && !vm_page_try_remove_all(p)) 1924fee2a2faSMark Johnston goto wired; 19255cd29d0fSMark Johnston vm_page_free(p); 19262965a453SKip Macy } 1927f919ebdeSDavid Greenman vm_object_pip_wakeup(object); 1928c0503609SDavid Greenman } 1929df8bae1dSRodney W. Grimes 1930df8bae1dSRodney W. Grimes /* 19313138cd36SMark Johnston * vm_object_page_noreuse: 1932936c09acSJohn Baldwin * 19333138cd36SMark Johnston * For the given object, attempt to move the specified pages to 19343138cd36SMark Johnston * the head of the inactive queue. This bypasses regular LRU 19353138cd36SMark Johnston * operation and allows the pages to be reused quickly under memory 19363138cd36SMark Johnston * pressure. If a page is wired for any reason, then it will not 19373138cd36SMark Johnston * be queued. Pages are specified by the range ["start", "end"). 19383138cd36SMark Johnston * As a special case, if "end" is zero, then the range extends from 19393138cd36SMark Johnston * "start" to the end of the object. 1940936c09acSJohn Baldwin * 1941936c09acSJohn Baldwin * This operation should only be performed on objects that 194228634820SAlan Cox * contain non-fictitious, managed pages. 1943936c09acSJohn Baldwin * 1944936c09acSJohn Baldwin * The object must be locked. 1945936c09acSJohn Baldwin */ 1946936c09acSJohn Baldwin void 19473138cd36SMark Johnston vm_object_page_noreuse(vm_object_t object, vm_pindex_t start, vm_pindex_t end) 1948936c09acSJohn Baldwin { 194993c5d3a4SKonstantin Belousov struct mtx *mtx; 1950936c09acSJohn Baldwin vm_page_t p, next; 1951936c09acSJohn Baldwin 195252d1addaSAlan Cox VM_OBJECT_ASSERT_LOCKED(object); 195328634820SAlan Cox KASSERT((object->flags & (OBJ_FICTITIOUS | OBJ_UNMANAGED)) == 0, 19543138cd36SMark Johnston ("vm_object_page_noreuse: illegal object %p", object)); 1955936c09acSJohn Baldwin if (object->resident_page_count == 0) 1956936c09acSJohn Baldwin return; 1957936c09acSJohn Baldwin p = vm_page_find_least(object, start); 1958936c09acSJohn Baldwin 1959936c09acSJohn Baldwin /* 1960936c09acSJohn Baldwin * Here, the variable "p" is either (1) the page with the least pindex 1961936c09acSJohn Baldwin * greater than or equal to the parameter "start" or (2) NULL. 1962936c09acSJohn Baldwin */ 1963936c09acSJohn Baldwin mtx = NULL; 1964936c09acSJohn Baldwin for (; p != NULL && (p->pindex < end || end == 0); p = next) { 1965936c09acSJohn Baldwin next = TAILQ_NEXT(p, listq); 196693c5d3a4SKonstantin Belousov vm_page_change_lock(p, &mtx); 19673138cd36SMark Johnston vm_page_deactivate_noreuse(p); 1968936c09acSJohn Baldwin } 1969936c09acSJohn Baldwin if (mtx != NULL) 1970936c09acSJohn Baldwin mtx_unlock(mtx); 1971936c09acSJohn Baldwin } 1972936c09acSJohn Baldwin 1973936c09acSJohn Baldwin /* 1974387aabc5SAlan Cox * Populate the specified range of the object with valid pages. Returns 1975387aabc5SAlan Cox * TRUE if the range is successfully populated and FALSE otherwise. 1976387aabc5SAlan Cox * 1977387aabc5SAlan Cox * Note: This function should be optimized to pass a larger array of 1978387aabc5SAlan Cox * pages to vm_pager_get_pages() before it is applied to a non- 1979387aabc5SAlan Cox * OBJT_DEVICE object. 1980387aabc5SAlan Cox * 1981387aabc5SAlan Cox * The object must be locked. 1982387aabc5SAlan Cox */ 1983387aabc5SAlan Cox boolean_t 1984387aabc5SAlan Cox vm_object_populate(vm_object_t object, vm_pindex_t start, vm_pindex_t end) 1985387aabc5SAlan Cox { 1986093c7f39SGleb Smirnoff vm_page_t m; 1987387aabc5SAlan Cox vm_pindex_t pindex; 1988387aabc5SAlan Cox int rv; 1989387aabc5SAlan Cox 199089f6b863SAttilio Rao VM_OBJECT_ASSERT_WLOCKED(object); 1991387aabc5SAlan Cox for (pindex = start; pindex < end; pindex++) { 1992c7575748SJeff Roberson rv = vm_page_grab_valid(&m, object, pindex, VM_ALLOC_NORMAL); 1993c7575748SJeff Roberson if (rv != VM_PAGER_OK) 1994387aabc5SAlan Cox break; 1995c7575748SJeff Roberson 1996387aabc5SAlan Cox /* 1997387aabc5SAlan Cox * Keep "m" busy because a subsequent iteration may unlock 1998387aabc5SAlan Cox * the object. 1999387aabc5SAlan Cox */ 2000387aabc5SAlan Cox } 2001387aabc5SAlan Cox if (pindex > start) { 2002387aabc5SAlan Cox m = vm_page_lookup(object, start); 2003387aabc5SAlan Cox while (m != NULL && m->pindex < pindex) { 2004c7aebda8SAttilio Rao vm_page_xunbusy(m); 2005387aabc5SAlan Cox m = TAILQ_NEXT(m, listq); 2006387aabc5SAlan Cox } 2007387aabc5SAlan Cox } 2008387aabc5SAlan Cox return (pindex == end); 2009387aabc5SAlan Cox } 2010387aabc5SAlan Cox 2011387aabc5SAlan Cox /* 2012df8bae1dSRodney W. Grimes * Routine: vm_object_coalesce 2013df8bae1dSRodney W. Grimes * Function: Coalesces two objects backing up adjoining 2014df8bae1dSRodney W. Grimes * regions of memory into a single object. 2015df8bae1dSRodney W. Grimes * 2016df8bae1dSRodney W. Grimes * returns TRUE if objects were combined. 2017df8bae1dSRodney W. Grimes * 2018df8bae1dSRodney W. Grimes * NOTE: Only works at the moment if the second object is NULL - 2019df8bae1dSRodney W. Grimes * if it's not, which object do we lock first? 2020df8bae1dSRodney W. Grimes * 2021df8bae1dSRodney W. Grimes * Parameters: 2022df8bae1dSRodney W. Grimes * prev_object First object to coalesce 2023df8bae1dSRodney W. Grimes * prev_offset Offset into prev_object 2024df8bae1dSRodney W. Grimes * prev_size Size of reference to prev_object 202557a21abaSAlan Cox * next_size Size of reference to the second object 20263364c323SKonstantin Belousov * reserved Indicator that extension region has 20273364c323SKonstantin Belousov * swap accounted for 2028df8bae1dSRodney W. Grimes * 2029df8bae1dSRodney W. Grimes * Conditions: 2030df8bae1dSRodney W. Grimes * The object must *not* be locked. 2031df8bae1dSRodney W. Grimes */ 20320d94caffSDavid Greenman boolean_t 203357a21abaSAlan Cox vm_object_coalesce(vm_object_t prev_object, vm_ooffset_t prev_offset, 20343364c323SKonstantin Belousov vm_size_t prev_size, vm_size_t next_size, boolean_t reserved) 2035df8bae1dSRodney W. Grimes { 2036ea41812fSAlan Cox vm_pindex_t next_pindex; 2037df8bae1dSRodney W. Grimes 203800e1854aSAlan Cox if (prev_object == NULL) 2039df8bae1dSRodney W. Grimes return (TRUE); 204089f6b863SAttilio Rao VM_OBJECT_WLOCK(prev_object); 20419ded9474SKonstantin Belousov if ((prev_object->type != OBJT_DEFAULT && 20429ded9474SKonstantin Belousov prev_object->type != OBJT_SWAP) || 20434f49b435SKonstantin Belousov (prev_object->flags & OBJ_NOSPLIT) != 0) { 204489f6b863SAttilio Rao VM_OBJECT_WUNLOCK(prev_object); 204530dcfc09SJohn Dyson return (FALSE); 204630dcfc09SJohn Dyson } 204730dcfc09SJohn Dyson 2048df8bae1dSRodney W. Grimes /* 2049df8bae1dSRodney W. Grimes * Try to collapse the object first 2050df8bae1dSRodney W. Grimes */ 2051df8bae1dSRodney W. Grimes vm_object_collapse(prev_object); 2052df8bae1dSRodney W. Grimes 2053df8bae1dSRodney W. Grimes /* 20540d94caffSDavid Greenman * Can't coalesce if: . more than one reference . paged out . shadows 20550d94caffSDavid Greenman * another object . has a copy elsewhere (any of which mean that the 20560d94caffSDavid Greenman * pages not mapped to prev_entry may be in use anyway) 2057df8bae1dSRodney W. Grimes */ 20588cc7e047SJohn Dyson if (prev_object->backing_object != NULL) { 205989f6b863SAttilio Rao VM_OBJECT_WUNLOCK(prev_object); 2060df8bae1dSRodney W. Grimes return (FALSE); 2061df8bae1dSRodney W. Grimes } 2062a316d390SJohn Dyson 2063a316d390SJohn Dyson prev_size >>= PAGE_SHIFT; 2064a316d390SJohn Dyson next_size >>= PAGE_SHIFT; 206557a21abaSAlan Cox next_pindex = OFF_TO_IDX(prev_offset) + prev_size; 20668cc7e047SJohn Dyson 20670e48e068SMark Johnston if (prev_object->ref_count > 1 && 20680e48e068SMark Johnston prev_object->size != next_pindex && 20690e48e068SMark Johnston (prev_object->flags & OBJ_ONEMAPPING) == 0) { 207089f6b863SAttilio Rao VM_OBJECT_WUNLOCK(prev_object); 20718cc7e047SJohn Dyson return (FALSE); 20728cc7e047SJohn Dyson } 20738cc7e047SJohn Dyson 2074df8bae1dSRodney W. Grimes /* 20753364c323SKonstantin Belousov * Account for the charge. 20763364c323SKonstantin Belousov */ 2077ef694c1aSEdward Tomasz Napierala if (prev_object->cred != NULL) { 20783364c323SKonstantin Belousov 20793364c323SKonstantin Belousov /* 20803364c323SKonstantin Belousov * If prev_object was charged, then this mapping, 2081763df3ecSPedro F. Giffuni * although not charged now, may become writable 2082ef694c1aSEdward Tomasz Napierala * later. Non-NULL cred in the object would prevent 20833364c323SKonstantin Belousov * swap reservation during enabling of the write 20843364c323SKonstantin Belousov * access, so reserve swap now. Failed reservation 20853364c323SKonstantin Belousov * cause allocation of the separate object for the map 20863364c323SKonstantin Belousov * entry, and swap reservation for this entry is 20873364c323SKonstantin Belousov * managed in appropriate time. 20883364c323SKonstantin Belousov */ 2089ef694c1aSEdward Tomasz Napierala if (!reserved && !swap_reserve_by_cred(ptoa(next_size), 2090ef694c1aSEdward Tomasz Napierala prev_object->cred)) { 20919f790a17SKonstantin Belousov VM_OBJECT_WUNLOCK(prev_object); 20923364c323SKonstantin Belousov return (FALSE); 20933364c323SKonstantin Belousov } 20943364c323SKonstantin Belousov prev_object->charge += ptoa(next_size); 20953364c323SKonstantin Belousov } 20963364c323SKonstantin Belousov 20973364c323SKonstantin Belousov /* 20980d94caffSDavid Greenman * Remove any pages that may still be in the object from a previous 20990d94caffSDavid Greenman * deallocation. 2100df8bae1dSRodney W. Grimes */ 2101ea41812fSAlan Cox if (next_pindex < prev_object->size) { 21026bbee8e2SAlan Cox vm_object_page_remove(prev_object, next_pindex, next_pindex + 21036bbee8e2SAlan Cox next_size, 0); 2104ea41812fSAlan Cox if (prev_object->type == OBJT_SWAP) 2105ea41812fSAlan Cox swap_pager_freespace(prev_object, 2106ea41812fSAlan Cox next_pindex, next_size); 21073364c323SKonstantin Belousov #if 0 2108ef694c1aSEdward Tomasz Napierala if (prev_object->cred != NULL) { 21093364c323SKonstantin Belousov KASSERT(prev_object->charge >= 21103364c323SKonstantin Belousov ptoa(prev_object->size - next_pindex), 21113364c323SKonstantin Belousov ("object %p overcharged 1 %jx %jx", prev_object, 21123364c323SKonstantin Belousov (uintmax_t)next_pindex, (uintmax_t)next_size)); 21133364c323SKonstantin Belousov prev_object->charge -= ptoa(prev_object->size - 21143364c323SKonstantin Belousov next_pindex); 21153364c323SKonstantin Belousov } 21163364c323SKonstantin Belousov #endif 2117ea41812fSAlan Cox } 2118df8bae1dSRodney W. Grimes 2119df8bae1dSRodney W. Grimes /* 2120df8bae1dSRodney W. Grimes * Extend the object if necessary. 2121df8bae1dSRodney W. Grimes */ 2122ea41812fSAlan Cox if (next_pindex + next_size > prev_object->size) 2123ea41812fSAlan Cox prev_object->size = next_pindex + next_size; 2124df8bae1dSRodney W. Grimes 212589f6b863SAttilio Rao VM_OBJECT_WUNLOCK(prev_object); 2126df8bae1dSRodney W. Grimes return (TRUE); 2127df8bae1dSRodney W. Grimes } 2128df8bae1dSRodney W. Grimes 21297a5a6352SMatthew Dillon void 21307a5a6352SMatthew Dillon vm_object_set_writeable_dirty(vm_object_t object) 21317a5a6352SMatthew Dillon { 21327a5a6352SMatthew Dillon 213389f6b863SAttilio Rao VM_OBJECT_ASSERT_WLOCKED(object); 2134f40cb1c6SKonstantin Belousov if (object->type != OBJT_VNODE) { 2135f40cb1c6SKonstantin Belousov if ((object->flags & OBJ_TMPFS_NODE) != 0) { 2136f40cb1c6SKonstantin Belousov KASSERT(object->type == OBJT_SWAP, ("non-swap tmpfs")); 2137f40cb1c6SKonstantin Belousov vm_object_set_flag(object, OBJ_TMPFS_DIRTY); 2138f40cb1c6SKonstantin Belousov } 21393280870dSKonstantin Belousov return; 2140f40cb1c6SKonstantin Belousov } 21413280870dSKonstantin Belousov object->generation++; 21423280870dSKonstantin Belousov if ((object->flags & OBJ_MIGHTBEDIRTY) != 0) 2143ee39666aSJeff Roberson return; 2144af51d7bfSAlan Cox vm_object_set_flag(object, OBJ_MIGHTBEDIRTY); 21457a5a6352SMatthew Dillon } 21467a5a6352SMatthew Dillon 214703462509SAlan Cox /* 214803462509SAlan Cox * vm_object_unwire: 214903462509SAlan Cox * 215003462509SAlan Cox * For each page offset within the specified range of the given object, 215103462509SAlan Cox * find the highest-level page in the shadow chain and unwire it. A page 215203462509SAlan Cox * must exist at every page offset, and the highest-level page must be 215303462509SAlan Cox * wired. 215403462509SAlan Cox */ 215503462509SAlan Cox void 215603462509SAlan Cox vm_object_unwire(vm_object_t object, vm_ooffset_t offset, vm_size_t length, 215703462509SAlan Cox uint8_t queue) 215803462509SAlan Cox { 215920e4afbfSKonstantin Belousov vm_object_t tobject, t1object; 216003462509SAlan Cox vm_page_t m, tm; 216103462509SAlan Cox vm_pindex_t end_pindex, pindex, tpindex; 216203462509SAlan Cox int depth, locked_depth; 216303462509SAlan Cox 216403462509SAlan Cox KASSERT((offset & PAGE_MASK) == 0, 216503462509SAlan Cox ("vm_object_unwire: offset is not page aligned")); 216603462509SAlan Cox KASSERT((length & PAGE_MASK) == 0, 216703462509SAlan Cox ("vm_object_unwire: length is not a multiple of PAGE_SIZE")); 216803462509SAlan Cox /* The wired count of a fictitious page never changes. */ 216903462509SAlan Cox if ((object->flags & OBJ_FICTITIOUS) != 0) 217003462509SAlan Cox return; 217103462509SAlan Cox pindex = OFF_TO_IDX(offset); 217203462509SAlan Cox end_pindex = pindex + atop(length); 217320e4afbfSKonstantin Belousov again: 217403462509SAlan Cox locked_depth = 1; 217503462509SAlan Cox VM_OBJECT_RLOCK(object); 217603462509SAlan Cox m = vm_page_find_least(object, pindex); 217703462509SAlan Cox while (pindex < end_pindex) { 217803462509SAlan Cox if (m == NULL || pindex < m->pindex) { 217903462509SAlan Cox /* 218003462509SAlan Cox * The first object in the shadow chain doesn't 218103462509SAlan Cox * contain a page at the current index. Therefore, 218203462509SAlan Cox * the page must exist in a backing object. 218303462509SAlan Cox */ 218403462509SAlan Cox tobject = object; 218503462509SAlan Cox tpindex = pindex; 218603462509SAlan Cox depth = 0; 218703462509SAlan Cox do { 218803462509SAlan Cox tpindex += 218903462509SAlan Cox OFF_TO_IDX(tobject->backing_object_offset); 219003462509SAlan Cox tobject = tobject->backing_object; 219103462509SAlan Cox KASSERT(tobject != NULL, 219203462509SAlan Cox ("vm_object_unwire: missing page")); 219303462509SAlan Cox if ((tobject->flags & OBJ_FICTITIOUS) != 0) 219403462509SAlan Cox goto next_page; 219503462509SAlan Cox depth++; 219603462509SAlan Cox if (depth == locked_depth) { 219703462509SAlan Cox locked_depth++; 219803462509SAlan Cox VM_OBJECT_RLOCK(tobject); 219903462509SAlan Cox } 220003462509SAlan Cox } while ((tm = vm_page_lookup(tobject, tpindex)) == 220103462509SAlan Cox NULL); 220203462509SAlan Cox } else { 220303462509SAlan Cox tm = m; 220403462509SAlan Cox m = TAILQ_NEXT(m, listq); 220503462509SAlan Cox } 220663e97555SJeff Roberson if (vm_page_trysbusy(tm) == 0) { 220787e93ea6SMark Johnston for (tobject = object; locked_depth >= 1; 220820e4afbfSKonstantin Belousov locked_depth--) { 220920e4afbfSKonstantin Belousov t1object = tobject->backing_object; 221087e93ea6SMark Johnston if (tm->object != tobject) 221120e4afbfSKonstantin Belousov VM_OBJECT_RUNLOCK(tobject); 221220e4afbfSKonstantin Belousov tobject = t1object; 221320e4afbfSKonstantin Belousov } 221420e4afbfSKonstantin Belousov vm_page_busy_sleep(tm, "unwbo", true); 221520e4afbfSKonstantin Belousov goto again; 221620e4afbfSKonstantin Belousov } 221703462509SAlan Cox vm_page_unwire(tm, queue); 221863e97555SJeff Roberson vm_page_sunbusy(tm); 221903462509SAlan Cox next_page: 222003462509SAlan Cox pindex++; 222103462509SAlan Cox } 222203462509SAlan Cox /* Release the accumulated object locks. */ 222320e4afbfSKonstantin Belousov for (tobject = object; locked_depth >= 1; locked_depth--) { 222420e4afbfSKonstantin Belousov t1object = tobject->backing_object; 222520e4afbfSKonstantin Belousov VM_OBJECT_RUNLOCK(tobject); 222620e4afbfSKonstantin Belousov tobject = t1object; 222703462509SAlan Cox } 222803462509SAlan Cox } 222903462509SAlan Cox 22300951bd36SEric van Gyzen /* 22310951bd36SEric van Gyzen * Return the vnode for the given object, or NULL if none exists. 22320951bd36SEric van Gyzen * For tmpfs objects, the function may return NULL if there is 22330951bd36SEric van Gyzen * no vnode allocated at the time of the call. 22340951bd36SEric van Gyzen */ 223563e4c6cdSEric van Gyzen struct vnode * 223663e4c6cdSEric van Gyzen vm_object_vnode(vm_object_t object) 223763e4c6cdSEric van Gyzen { 22380951bd36SEric van Gyzen struct vnode *vp; 223963e4c6cdSEric van Gyzen 224063e4c6cdSEric van Gyzen VM_OBJECT_ASSERT_LOCKED(object); 22410951bd36SEric van Gyzen if (object->type == OBJT_VNODE) { 22420951bd36SEric van Gyzen vp = object->handle; 22430951bd36SEric van Gyzen KASSERT(vp != NULL, ("%s: OBJT_VNODE has no vnode", __func__)); 22440951bd36SEric van Gyzen } else if (object->type == OBJT_SWAP && 22450951bd36SEric van Gyzen (object->flags & OBJ_TMPFS) != 0) { 22460951bd36SEric van Gyzen vp = object->un_pager.swp.swp_tmpfs; 22470951bd36SEric van Gyzen KASSERT(vp != NULL, ("%s: OBJT_TMPFS has no vnode", __func__)); 22480951bd36SEric van Gyzen } else { 22490951bd36SEric van Gyzen vp = NULL; 22500951bd36SEric van Gyzen } 22510951bd36SEric van Gyzen return (vp); 225263e4c6cdSEric van Gyzen } 225363e4c6cdSEric van Gyzen 2254205be21dSJeff Roberson 2255205be21dSJeff Roberson /* 2256205be21dSJeff Roberson * Busy the vm object. This prevents new pages belonging to the object from 2257205be21dSJeff Roberson * becoming busy. Existing pages persist as busy. Callers are responsible 2258205be21dSJeff Roberson * for checking page state before proceeding. 2259205be21dSJeff Roberson */ 2260205be21dSJeff Roberson void 2261205be21dSJeff Roberson vm_object_busy(vm_object_t obj) 2262205be21dSJeff Roberson { 2263205be21dSJeff Roberson 2264205be21dSJeff Roberson VM_OBJECT_ASSERT_LOCKED(obj); 2265205be21dSJeff Roberson 2266205be21dSJeff Roberson refcount_acquire(&obj->busy); 2267205be21dSJeff Roberson /* The fence is required to order loads of page busy. */ 2268205be21dSJeff Roberson atomic_thread_fence_acq_rel(); 2269205be21dSJeff Roberson } 2270205be21dSJeff Roberson 2271205be21dSJeff Roberson void 2272205be21dSJeff Roberson vm_object_unbusy(vm_object_t obj) 2273205be21dSJeff Roberson { 2274205be21dSJeff Roberson 2275205be21dSJeff Roberson 2276205be21dSJeff Roberson refcount_release(&obj->busy); 2277205be21dSJeff Roberson } 2278205be21dSJeff Roberson 2279205be21dSJeff Roberson void 2280205be21dSJeff Roberson vm_object_busy_wait(vm_object_t obj, const char *wmesg) 2281205be21dSJeff Roberson { 2282205be21dSJeff Roberson 2283205be21dSJeff Roberson VM_OBJECT_ASSERT_UNLOCKED(obj); 2284205be21dSJeff Roberson 2285205be21dSJeff Roberson if (obj->busy) 2286205be21dSJeff Roberson refcount_sleep(&obj->busy, wmesg, PVM); 2287205be21dSJeff Roberson } 2288205be21dSJeff Roberson 22895e38e3f5SEric van Gyzen /* 22905e38e3f5SEric van Gyzen * Return the kvme type of the given object. 22915e38e3f5SEric van Gyzen * If vpp is not NULL, set it to the object's vm_object_vnode() or NULL. 22925e38e3f5SEric van Gyzen */ 22935e38e3f5SEric van Gyzen int 22945e38e3f5SEric van Gyzen vm_object_kvme_type(vm_object_t object, struct vnode **vpp) 22955e38e3f5SEric van Gyzen { 22965e38e3f5SEric van Gyzen 22975e38e3f5SEric van Gyzen VM_OBJECT_ASSERT_LOCKED(object); 22985e38e3f5SEric van Gyzen if (vpp != NULL) 22995e38e3f5SEric van Gyzen *vpp = vm_object_vnode(object); 23005e38e3f5SEric van Gyzen switch (object->type) { 23015e38e3f5SEric van Gyzen case OBJT_DEFAULT: 23025e38e3f5SEric van Gyzen return (KVME_TYPE_DEFAULT); 23035e38e3f5SEric van Gyzen case OBJT_VNODE: 23045e38e3f5SEric van Gyzen return (KVME_TYPE_VNODE); 23055e38e3f5SEric van Gyzen case OBJT_SWAP: 23065e38e3f5SEric van Gyzen if ((object->flags & OBJ_TMPFS_NODE) != 0) 23075e38e3f5SEric van Gyzen return (KVME_TYPE_VNODE); 23085e38e3f5SEric van Gyzen return (KVME_TYPE_SWAP); 23095e38e3f5SEric van Gyzen case OBJT_DEVICE: 23105e38e3f5SEric van Gyzen return (KVME_TYPE_DEVICE); 23115e38e3f5SEric van Gyzen case OBJT_PHYS: 23125e38e3f5SEric van Gyzen return (KVME_TYPE_PHYS); 23135e38e3f5SEric van Gyzen case OBJT_DEAD: 23145e38e3f5SEric van Gyzen return (KVME_TYPE_DEAD); 23155e38e3f5SEric van Gyzen case OBJT_SG: 23165e38e3f5SEric van Gyzen return (KVME_TYPE_SG); 23175e38e3f5SEric van Gyzen case OBJT_MGTDEVICE: 23185e38e3f5SEric van Gyzen return (KVME_TYPE_MGTDEVICE); 23195e38e3f5SEric van Gyzen default: 23205e38e3f5SEric van Gyzen return (KVME_TYPE_UNKNOWN); 23215e38e3f5SEric van Gyzen } 23225e38e3f5SEric van Gyzen } 23235e38e3f5SEric van Gyzen 2324ff87ae35SJohn Baldwin static int 2325ff87ae35SJohn Baldwin sysctl_vm_object_list(SYSCTL_HANDLER_ARGS) 2326ff87ae35SJohn Baldwin { 23270ecee546SKonstantin Belousov struct kinfo_vmobject *kvo; 2328ff87ae35SJohn Baldwin char *fullpath, *freepath; 2329ff87ae35SJohn Baldwin struct vnode *vp; 2330ff87ae35SJohn Baldwin struct vattr va; 2331ff87ae35SJohn Baldwin vm_object_t obj; 2332ff87ae35SJohn Baldwin vm_page_t m; 2333ff87ae35SJohn Baldwin int count, error; 2334ff87ae35SJohn Baldwin 2335ff87ae35SJohn Baldwin if (req->oldptr == NULL) { 2336ff87ae35SJohn Baldwin /* 2337ff87ae35SJohn Baldwin * If an old buffer has not been provided, generate an 2338ff87ae35SJohn Baldwin * estimate of the space needed for a subsequent call. 2339ff87ae35SJohn Baldwin */ 2340ff87ae35SJohn Baldwin mtx_lock(&vm_object_list_mtx); 2341ff87ae35SJohn Baldwin count = 0; 2342ff87ae35SJohn Baldwin TAILQ_FOREACH(obj, &vm_object_list, object_list) { 2343ff87ae35SJohn Baldwin if (obj->type == OBJT_DEAD) 2344ff87ae35SJohn Baldwin continue; 2345ff87ae35SJohn Baldwin count++; 2346ff87ae35SJohn Baldwin } 2347ff87ae35SJohn Baldwin mtx_unlock(&vm_object_list_mtx); 2348ff87ae35SJohn Baldwin return (SYSCTL_OUT(req, NULL, sizeof(struct kinfo_vmobject) * 2349ff87ae35SJohn Baldwin count * 11 / 10)); 2350ff87ae35SJohn Baldwin } 2351ff87ae35SJohn Baldwin 23520ecee546SKonstantin Belousov kvo = malloc(sizeof(*kvo), M_TEMP, M_WAITOK); 2353ff87ae35SJohn Baldwin error = 0; 2354ff87ae35SJohn Baldwin 2355ff87ae35SJohn Baldwin /* 2356ff87ae35SJohn Baldwin * VM objects are type stable and are never removed from the 2357ff87ae35SJohn Baldwin * list once added. This allows us to safely read obj->object_list 2358ff87ae35SJohn Baldwin * after reacquiring the VM object lock. 2359ff87ae35SJohn Baldwin */ 2360ff87ae35SJohn Baldwin mtx_lock(&vm_object_list_mtx); 2361ff87ae35SJohn Baldwin TAILQ_FOREACH(obj, &vm_object_list, object_list) { 2362ff87ae35SJohn Baldwin if (obj->type == OBJT_DEAD) 2363ff87ae35SJohn Baldwin continue; 2364ff87ae35SJohn Baldwin VM_OBJECT_RLOCK(obj); 2365ff87ae35SJohn Baldwin if (obj->type == OBJT_DEAD) { 2366ff87ae35SJohn Baldwin VM_OBJECT_RUNLOCK(obj); 2367ff87ae35SJohn Baldwin continue; 2368ff87ae35SJohn Baldwin } 2369ff87ae35SJohn Baldwin mtx_unlock(&vm_object_list_mtx); 23700ecee546SKonstantin Belousov kvo->kvo_size = ptoa(obj->size); 23710ecee546SKonstantin Belousov kvo->kvo_resident = obj->resident_page_count; 23720ecee546SKonstantin Belousov kvo->kvo_ref_count = obj->ref_count; 23730ecee546SKonstantin Belousov kvo->kvo_shadow_count = obj->shadow_count; 23740ecee546SKonstantin Belousov kvo->kvo_memattr = obj->memattr; 23750ecee546SKonstantin Belousov kvo->kvo_active = 0; 23760ecee546SKonstantin Belousov kvo->kvo_inactive = 0; 2377ff87ae35SJohn Baldwin TAILQ_FOREACH(m, &obj->memq, listq) { 2378ff87ae35SJohn Baldwin /* 2379ff87ae35SJohn Baldwin * A page may belong to the object but be 2380ff87ae35SJohn Baldwin * dequeued and set to PQ_NONE while the 2381ff87ae35SJohn Baldwin * object lock is not held. This makes the 2382ff87ae35SJohn Baldwin * reads of m->queue below racy, and we do not 2383ff87ae35SJohn Baldwin * count pages set to PQ_NONE. However, this 2384ff87ae35SJohn Baldwin * sysctl is only meant to give an 2385ff87ae35SJohn Baldwin * approximation of the system anyway. 2386ff87ae35SJohn Baldwin */ 2387e8bcf696SMark Johnston if (m->queue == PQ_ACTIVE) 23880ecee546SKonstantin Belousov kvo->kvo_active++; 2389e8bcf696SMark Johnston else if (m->queue == PQ_INACTIVE) 23900ecee546SKonstantin Belousov kvo->kvo_inactive++; 2391ff87ae35SJohn Baldwin } 2392ff87ae35SJohn Baldwin 23930ecee546SKonstantin Belousov kvo->kvo_vn_fileid = 0; 23940ecee546SKonstantin Belousov kvo->kvo_vn_fsid = 0; 23950ecee546SKonstantin Belousov kvo->kvo_vn_fsid_freebsd11 = 0; 2396ff87ae35SJohn Baldwin freepath = NULL; 2397ff87ae35SJohn Baldwin fullpath = ""; 23985e38e3f5SEric van Gyzen kvo->kvo_type = vm_object_kvme_type(obj, &vp); 23995e38e3f5SEric van Gyzen if (vp != NULL) 2400ff87ae35SJohn Baldwin vref(vp); 2401ff87ae35SJohn Baldwin VM_OBJECT_RUNLOCK(obj); 2402ff87ae35SJohn Baldwin if (vp != NULL) { 2403ff87ae35SJohn Baldwin vn_fullpath(curthread, vp, &fullpath, &freepath); 2404ff87ae35SJohn Baldwin vn_lock(vp, LK_SHARED | LK_RETRY); 2405ff87ae35SJohn Baldwin if (VOP_GETATTR(vp, &va, curthread->td_ucred) == 0) { 24060ecee546SKonstantin Belousov kvo->kvo_vn_fileid = va.va_fileid; 24070ecee546SKonstantin Belousov kvo->kvo_vn_fsid = va.va_fsid; 24080ecee546SKonstantin Belousov kvo->kvo_vn_fsid_freebsd11 = va.va_fsid; 240969921123SKonstantin Belousov /* truncate */ 2410ff87ae35SJohn Baldwin } 2411ff87ae35SJohn Baldwin vput(vp); 2412ff87ae35SJohn Baldwin } 2413ff87ae35SJohn Baldwin 24140ecee546SKonstantin Belousov strlcpy(kvo->kvo_path, fullpath, sizeof(kvo->kvo_path)); 2415ff87ae35SJohn Baldwin if (freepath != NULL) 2416ff87ae35SJohn Baldwin free(freepath, M_TEMP); 2417ff87ae35SJohn Baldwin 2418ff87ae35SJohn Baldwin /* Pack record size down */ 24190ecee546SKonstantin Belousov kvo->kvo_structsize = offsetof(struct kinfo_vmobject, kvo_path) 24200ecee546SKonstantin Belousov + strlen(kvo->kvo_path) + 1; 24210ecee546SKonstantin Belousov kvo->kvo_structsize = roundup(kvo->kvo_structsize, 2422ff87ae35SJohn Baldwin sizeof(uint64_t)); 24230ecee546SKonstantin Belousov error = SYSCTL_OUT(req, kvo, kvo->kvo_structsize); 2424ff87ae35SJohn Baldwin mtx_lock(&vm_object_list_mtx); 2425ff87ae35SJohn Baldwin if (error) 2426ff87ae35SJohn Baldwin break; 2427ff87ae35SJohn Baldwin } 2428ff87ae35SJohn Baldwin mtx_unlock(&vm_object_list_mtx); 24290ecee546SKonstantin Belousov free(kvo, M_TEMP); 2430ff87ae35SJohn Baldwin return (error); 2431ff87ae35SJohn Baldwin } 2432ff87ae35SJohn Baldwin SYSCTL_PROC(_vm, OID_AUTO, objects, CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_SKIP | 2433ff87ae35SJohn Baldwin CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_object_list, "S,kinfo_vmobject", 2434ff87ae35SJohn Baldwin "List of VM objects"); 2435ff87ae35SJohn Baldwin 2436c7c34a24SBruce Evans #include "opt_ddb.h" 2437c3cb3e12SDavid Greenman #ifdef DDB 2438c7c34a24SBruce Evans #include <sys/kernel.h> 2439c7c34a24SBruce Evans 2440ce9edcf5SPoul-Henning Kamp #include <sys/cons.h> 2441c7c34a24SBruce Evans 2442c7c34a24SBruce Evans #include <ddb/ddb.h> 2443c7c34a24SBruce Evans 2444cac597e4SBruce Evans static int 24451b40f8c0SMatthew Dillon _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry) 2446a1f6d91cSDavid Greenman { 2447a1f6d91cSDavid Greenman vm_map_t tmpm; 2448a1f6d91cSDavid Greenman vm_map_entry_t tmpe; 2449a1f6d91cSDavid Greenman vm_object_t obj; 2450a1f6d91cSDavid Greenman 2451a1f6d91cSDavid Greenman if (map == 0) 2452a1f6d91cSDavid Greenman return 0; 2453a1f6d91cSDavid Greenman 2454a1f6d91cSDavid Greenman if (entry == 0) { 24552288078cSDoug Moore VM_MAP_ENTRY_FOREACH(tmpe, map) { 2456a1f6d91cSDavid Greenman if (_vm_object_in_map(map, object, tmpe)) { 2457a1f6d91cSDavid Greenman return 1; 2458a1f6d91cSDavid Greenman } 2459a1f6d91cSDavid Greenman } 24609fdfe602SMatthew Dillon } else if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) { 24619fdfe602SMatthew Dillon tmpm = entry->object.sub_map; 24622288078cSDoug Moore VM_MAP_ENTRY_FOREACH(tmpe, tmpm) { 2463a1f6d91cSDavid Greenman if (_vm_object_in_map(tmpm, object, tmpe)) { 2464a1f6d91cSDavid Greenman return 1; 2465a1f6d91cSDavid Greenman } 2466a1f6d91cSDavid Greenman } 24678aef1712SMatthew Dillon } else if ((obj = entry->object.vm_object) != NULL) { 246824a1cce3SDavid Greenman for (; obj; obj = obj->backing_object) 2469a1f6d91cSDavid Greenman if (obj == object) { 2470a1f6d91cSDavid Greenman return 1; 2471a1f6d91cSDavid Greenman } 2472a1f6d91cSDavid Greenman } 2473a1f6d91cSDavid Greenman return 0; 2474a1f6d91cSDavid Greenman } 2475a1f6d91cSDavid Greenman 2476cac597e4SBruce Evans static int 24771b40f8c0SMatthew Dillon vm_object_in_map(vm_object_t object) 2478a1f6d91cSDavid Greenman { 2479a1f6d91cSDavid Greenman struct proc *p; 24801005a129SJohn Baldwin 248160517fd1SJohn Baldwin /* sx_slock(&allproc_lock); */ 2482f67af5c9SXin LI FOREACH_PROC_IN_SYSTEM(p) { 2483a1f6d91cSDavid Greenman if (!p->p_vmspace /* || (p->p_flag & (P_SYSTEM|P_WEXIT)) */) 2484a1f6d91cSDavid Greenman continue; 2485553629ebSJake Burkholder if (_vm_object_in_map(&p->p_vmspace->vm_map, object, 0)) { 248660517fd1SJohn Baldwin /* sx_sunlock(&allproc_lock); */ 2487a1f6d91cSDavid Greenman return 1; 2488a1f6d91cSDavid Greenman } 2489553629ebSJake Burkholder } 249060517fd1SJohn Baldwin /* sx_sunlock(&allproc_lock); */ 2491a1f6d91cSDavid Greenman if (_vm_object_in_map(kernel_map, object, 0)) 2492a1f6d91cSDavid Greenman return 1; 2493a1f6d91cSDavid Greenman return 0; 2494a1f6d91cSDavid Greenman } 2495a1f6d91cSDavid Greenman 2496c7c34a24SBruce Evans DB_SHOW_COMMAND(vmochk, vm_object_check) 2497f708ef1bSPoul-Henning Kamp { 2498a1f6d91cSDavid Greenman vm_object_t object; 2499a1f6d91cSDavid Greenman 2500a1f6d91cSDavid Greenman /* 2501a1f6d91cSDavid Greenman * make sure that internal objs are in a map somewhere 2502a1f6d91cSDavid Greenman * and none have zero ref counts. 2503a1f6d91cSDavid Greenman */ 2504cc64b484SAlfred Perlstein TAILQ_FOREACH(object, &vm_object_list, object_list) { 250524a1cce3SDavid Greenman if (object->handle == NULL && 250624a1cce3SDavid Greenman (object->type == OBJT_DEFAULT || object->type == OBJT_SWAP)) { 2507a1f6d91cSDavid Greenman if (object->ref_count == 0) { 25083efc015bSPeter Wemm db_printf("vmochk: internal obj has zero ref count: %ld\n", 25093efc015bSPeter Wemm (long)object->size); 2510a1f6d91cSDavid Greenman } 2511a1f6d91cSDavid Greenman if (!vm_object_in_map(object)) { 2512fc62ef1fSBruce Evans db_printf( 2513fc62ef1fSBruce Evans "vmochk: internal obj is not in a map: " 2514fc62ef1fSBruce Evans "ref: %d, size: %lu: 0x%lx, backing_object: %p\n", 2515fc62ef1fSBruce Evans object->ref_count, (u_long)object->size, 2516fc62ef1fSBruce Evans (u_long)object->size, 2517fc62ef1fSBruce Evans (void *)object->backing_object); 2518a1f6d91cSDavid Greenman } 2519a1f6d91cSDavid Greenman } 2520a1f6d91cSDavid Greenman } 2521a1f6d91cSDavid Greenman } 2522a1f6d91cSDavid Greenman 252326f9a767SRodney W. Grimes /* 2524df8bae1dSRodney W. Grimes * vm_object_print: [ debug ] 2525df8bae1dSRodney W. Grimes */ 2526c7c34a24SBruce Evans DB_SHOW_COMMAND(object, vm_object_print_static) 2527df8bae1dSRodney W. Grimes { 2528c7c34a24SBruce Evans /* XXX convert args. */ 2529c7c34a24SBruce Evans vm_object_t object = (vm_object_t)addr; 2530c7c34a24SBruce Evans boolean_t full = have_addr; 2531c7c34a24SBruce Evans 2532d031cff1SMatthew Dillon vm_page_t p; 2533df8bae1dSRodney W. Grimes 2534c7c34a24SBruce Evans /* XXX count is an (unused) arg. Avoid shadowing it. */ 2535c7c34a24SBruce Evans #define count was_count 2536c7c34a24SBruce Evans 2537d031cff1SMatthew Dillon int count; 2538df8bae1dSRodney W. Grimes 2539df8bae1dSRodney W. Grimes if (object == NULL) 2540df8bae1dSRodney W. Grimes return; 2541df8bae1dSRodney W. Grimes 2542eb95adefSBruce Evans db_iprintf( 2543ef694c1aSEdward Tomasz Napierala "Object %p: type=%d, size=0x%jx, res=%d, ref=%d, flags=0x%x ruid %d charge %jx\n", 2544e47cd172SMaxime Henrion object, (int)object->type, (uintmax_t)object->size, 25453364c323SKonstantin Belousov object->resident_page_count, object->ref_count, object->flags, 2546ef694c1aSEdward Tomasz Napierala object->cred ? object->cred->cr_ruid : -1, (uintmax_t)object->charge); 2547e47cd172SMaxime Henrion db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%jx\n", 25481c7c3c6aSMatthew Dillon object->shadow_count, 2549eb95adefSBruce Evans object->backing_object ? object->backing_object->ref_count : 0, 2550e47cd172SMaxime Henrion object->backing_object, (uintmax_t)object->backing_object_offset); 2551df8bae1dSRodney W. Grimes 2552df8bae1dSRodney W. Grimes if (!full) 2553df8bae1dSRodney W. Grimes return; 2554df8bae1dSRodney W. Grimes 2555c7c34a24SBruce Evans db_indent += 2; 2556df8bae1dSRodney W. Grimes count = 0; 2557fc2ffbe6SPoul-Henning Kamp TAILQ_FOREACH(p, &object->memq, listq) { 2558df8bae1dSRodney W. Grimes if (count == 0) 2559c7c34a24SBruce Evans db_iprintf("memory:="); 2560df8bae1dSRodney W. Grimes else if (count == 6) { 2561c7c34a24SBruce Evans db_printf("\n"); 2562c7c34a24SBruce Evans db_iprintf(" ..."); 2563df8bae1dSRodney W. Grimes count = 0; 2564df8bae1dSRodney W. Grimes } else 2565c7c34a24SBruce Evans db_printf(","); 2566df8bae1dSRodney W. Grimes count++; 2567df8bae1dSRodney W. Grimes 2568e47cd172SMaxime Henrion db_printf("(off=0x%jx,page=0x%jx)", 2569e47cd172SMaxime Henrion (uintmax_t)p->pindex, (uintmax_t)VM_PAGE_TO_PHYS(p)); 2570df8bae1dSRodney W. Grimes } 2571df8bae1dSRodney W. Grimes if (count != 0) 2572c7c34a24SBruce Evans db_printf("\n"); 2573c7c34a24SBruce Evans db_indent -= 2; 2574df8bae1dSRodney W. Grimes } 25755070c7f8SJohn Dyson 2576c7c34a24SBruce Evans /* XXX. */ 2577c7c34a24SBruce Evans #undef count 2578c7c34a24SBruce Evans 2579c7c34a24SBruce Evans /* XXX need this non-static entry for calling from vm_map_print. */ 25805070c7f8SJohn Dyson void 25811b40f8c0SMatthew Dillon vm_object_print( 25821b40f8c0SMatthew Dillon /* db_expr_t */ long addr, 25831b40f8c0SMatthew Dillon boolean_t have_addr, 25841b40f8c0SMatthew Dillon /* db_expr_t */ long count, 25851b40f8c0SMatthew Dillon char *modif) 2586c7c34a24SBruce Evans { 2587c7c34a24SBruce Evans vm_object_print_static(addr, have_addr, count, modif); 2588c7c34a24SBruce Evans } 2589c7c34a24SBruce Evans 2590c7c34a24SBruce Evans DB_SHOW_COMMAND(vmopag, vm_object_print_pages) 25915070c7f8SJohn Dyson { 25925070c7f8SJohn Dyson vm_object_t object; 2593bb2ac86fSKonstantin Belousov vm_pindex_t fidx; 2594bb2ac86fSKonstantin Belousov vm_paddr_t pa; 2595bb2ac86fSKonstantin Belousov vm_page_t m, prev_m; 2596bb2ac86fSKonstantin Belousov int rcount, nl, c; 2597cc64b484SAlfred Perlstein 2598bb2ac86fSKonstantin Belousov nl = 0; 2599cc64b484SAlfred Perlstein TAILQ_FOREACH(object, &vm_object_list, object_list) { 2600fc62ef1fSBruce Evans db_printf("new object: %p\n", (void *)object); 26015070c7f8SJohn Dyson if (nl > 18) { 26025070c7f8SJohn Dyson c = cngetc(); 26035070c7f8SJohn Dyson if (c != ' ') 26045070c7f8SJohn Dyson return; 26055070c7f8SJohn Dyson nl = 0; 26065070c7f8SJohn Dyson } 26075070c7f8SJohn Dyson nl++; 26085070c7f8SJohn Dyson rcount = 0; 26095070c7f8SJohn Dyson fidx = 0; 2610bb2ac86fSKonstantin Belousov pa = -1; 2611bb2ac86fSKonstantin Belousov TAILQ_FOREACH(m, &object->memq, listq) { 2612bb2ac86fSKonstantin Belousov if (m->pindex > 128) 2613bb2ac86fSKonstantin Belousov break; 2614bb2ac86fSKonstantin Belousov if ((prev_m = TAILQ_PREV(m, pglist, listq)) != NULL && 2615bb2ac86fSKonstantin Belousov prev_m->pindex + 1 != m->pindex) { 26165070c7f8SJohn Dyson if (rcount) { 26173efc015bSPeter Wemm db_printf(" index(%ld)run(%d)pa(0x%lx)\n", 26183efc015bSPeter Wemm (long)fidx, rcount, (long)pa); 26195070c7f8SJohn Dyson if (nl > 18) { 26205070c7f8SJohn Dyson c = cngetc(); 26215070c7f8SJohn Dyson if (c != ' ') 26225070c7f8SJohn Dyson return; 26235070c7f8SJohn Dyson nl = 0; 26245070c7f8SJohn Dyson } 26255070c7f8SJohn Dyson nl++; 26265070c7f8SJohn Dyson rcount = 0; 26275070c7f8SJohn Dyson } 26285070c7f8SJohn Dyson } 26295070c7f8SJohn Dyson if (rcount && 26305070c7f8SJohn Dyson (VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) { 26315070c7f8SJohn Dyson ++rcount; 26325070c7f8SJohn Dyson continue; 26335070c7f8SJohn Dyson } 26345070c7f8SJohn Dyson if (rcount) { 26352446e4f0SAlan Cox db_printf(" index(%ld)run(%d)pa(0x%lx)\n", 26363efc015bSPeter Wemm (long)fidx, rcount, (long)pa); 26375070c7f8SJohn Dyson if (nl > 18) { 26385070c7f8SJohn Dyson c = cngetc(); 26395070c7f8SJohn Dyson if (c != ' ') 26405070c7f8SJohn Dyson return; 26415070c7f8SJohn Dyson nl = 0; 26425070c7f8SJohn Dyson } 26435070c7f8SJohn Dyson nl++; 26445070c7f8SJohn Dyson } 2645bb2ac86fSKonstantin Belousov fidx = m->pindex; 26465070c7f8SJohn Dyson pa = VM_PAGE_TO_PHYS(m); 26475070c7f8SJohn Dyson rcount = 1; 26485070c7f8SJohn Dyson } 26495070c7f8SJohn Dyson if (rcount) { 26503efc015bSPeter Wemm db_printf(" index(%ld)run(%d)pa(0x%lx)\n", 26513efc015bSPeter Wemm (long)fidx, rcount, (long)pa); 26525070c7f8SJohn Dyson if (nl > 18) { 26535070c7f8SJohn Dyson c = cngetc(); 26545070c7f8SJohn Dyson if (c != ' ') 26555070c7f8SJohn Dyson return; 26565070c7f8SJohn Dyson nl = 0; 26575070c7f8SJohn Dyson } 26585070c7f8SJohn Dyson nl++; 26595070c7f8SJohn Dyson } 26605070c7f8SJohn Dyson } 26615070c7f8SJohn Dyson } 2662c3cb3e12SDavid Greenman #endif /* DDB */ 2663