1df8bae1dSRodney W. Grimes /* 226f9a767SRodney W. Grimes * Copyright (c) 1994 John S. Dyson 3df8bae1dSRodney W. Grimes * Copyright (c) 1990 University of Utah. 4df8bae1dSRodney W. Grimes * Copyright (c) 1991, 1993 5df8bae1dSRodney W. Grimes * The Regents of the University of California. All rights reserved. 6df8bae1dSRodney W. Grimes * 7df8bae1dSRodney W. Grimes * This code is derived from software contributed to Berkeley by 8df8bae1dSRodney W. Grimes * the Systems Programming Group of the University of Utah Computer 9df8bae1dSRodney W. Grimes * Science Department. 10df8bae1dSRodney W. Grimes * 11df8bae1dSRodney W. Grimes * Redistribution and use in source and binary forms, with or without 12df8bae1dSRodney W. Grimes * modification, are permitted provided that the following conditions 13df8bae1dSRodney W. Grimes * are met: 14df8bae1dSRodney W. Grimes * 1. Redistributions of source code must retain the above copyright 15df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer. 16df8bae1dSRodney W. Grimes * 2. Redistributions in binary form must reproduce the above copyright 17df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer in the 18df8bae1dSRodney W. Grimes * documentation and/or other materials provided with the distribution. 19df8bae1dSRodney W. Grimes * 3. All advertising materials mentioning features or use of this software 20df8bae1dSRodney W. Grimes * must display the following acknowledgement: 21df8bae1dSRodney W. Grimes * This product includes software developed by the University of 22df8bae1dSRodney W. Grimes * California, Berkeley and its contributors. 23df8bae1dSRodney W. Grimes * 4. Neither the name of the University nor the names of its contributors 24df8bae1dSRodney W. Grimes * may be used to endorse or promote products derived from this software 25df8bae1dSRodney W. Grimes * without specific prior written permission. 26df8bae1dSRodney W. Grimes * 27df8bae1dSRodney W. Grimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 28df8bae1dSRodney W. Grimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29df8bae1dSRodney W. Grimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 30df8bae1dSRodney W. Grimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 31df8bae1dSRodney W. Grimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32df8bae1dSRodney W. Grimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 33df8bae1dSRodney W. Grimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 34df8bae1dSRodney W. Grimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 35df8bae1dSRodney W. Grimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 36df8bae1dSRodney W. Grimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37df8bae1dSRodney W. Grimes * SUCH DAMAGE. 38df8bae1dSRodney W. Grimes * 39df8bae1dSRodney W. Grimes * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$ 40df8bae1dSRodney W. Grimes * 41df8bae1dSRodney W. Grimes * @(#)swap_pager.c 8.9 (Berkeley) 3/21/94 42f5a12711SPoul-Henning Kamp * $Id: swap_pager.c,v 1.48 1995/11/02 06:42:45 davidg Exp $ 43df8bae1dSRodney W. Grimes */ 44df8bae1dSRodney W. Grimes 45df8bae1dSRodney W. Grimes /* 46df8bae1dSRodney W. Grimes * Quick hack to page to dedicated partition(s). 47df8bae1dSRodney W. Grimes * TODO: 48df8bae1dSRodney W. Grimes * Add multiprocessor locks 49df8bae1dSRodney W. Grimes * Deal with async writes in a better fashion 50df8bae1dSRodney W. Grimes */ 51df8bae1dSRodney W. Grimes 52df8bae1dSRodney W. Grimes #include <sys/param.h> 53df8bae1dSRodney W. Grimes #include <sys/systm.h> 5464abb5a5SDavid Greenman #include <sys/kernel.h> 55df8bae1dSRodney W. Grimes #include <sys/proc.h> 56df8bae1dSRodney W. Grimes #include <sys/buf.h> 57df8bae1dSRodney W. Grimes #include <sys/vnode.h> 58df8bae1dSRodney W. Grimes #include <sys/malloc.h> 59df8bae1dSRodney W. Grimes 60df8bae1dSRodney W. Grimes #include <miscfs/specfs/specdev.h> 6126f9a767SRodney W. Grimes #include <sys/rlist.h> 62df8bae1dSRodney W. Grimes 63df8bae1dSRodney W. Grimes #include <vm/vm.h> 6426f9a767SRodney W. Grimes #include <vm/vm_pager.h> 65df8bae1dSRodney W. Grimes #include <vm/vm_page.h> 66df8bae1dSRodney W. Grimes #include <vm/vm_pageout.h> 67df8bae1dSRodney W. Grimes #include <vm/swap_pager.h> 6864abb5a5SDavid Greenman #include <vm/vm_kern.h> 69df8bae1dSRodney W. Grimes 70df8bae1dSRodney W. Grimes #ifndef NPENDINGIO 710d94caffSDavid Greenman #define NPENDINGIO 10 72df8bae1dSRodney W. Grimes #endif 73df8bae1dSRodney W. Grimes 7426f9a767SRodney W. Grimes int nswiodone; 7526f9a767SRodney W. Grimes int swap_pager_full; 7626f9a767SRodney W. Grimes extern int vm_swap_size; 77f5a12711SPoul-Henning Kamp static int no_swap_space = 1; 7835c10d22SDavid Greenman struct rlist *swaplist; 7935c10d22SDavid Greenman int nswaplist; 8026f9a767SRodney W. Grimes 8126f9a767SRodney W. Grimes #define MAX_PAGEOUT_CLUSTER 8 82df8bae1dSRodney W. Grimes 83df8bae1dSRodney W. Grimes TAILQ_HEAD(swpclean, swpagerclean); 84df8bae1dSRodney W. Grimes 8526f9a767SRodney W. Grimes typedef struct swpagerclean *swp_clean_t; 8626f9a767SRodney W. Grimes 87df8bae1dSRodney W. Grimes struct swpagerclean { 88df8bae1dSRodney W. Grimes TAILQ_ENTRY(swpagerclean) spc_list; 89df8bae1dSRodney W. Grimes int spc_flags; 90df8bae1dSRodney W. Grimes struct buf *spc_bp; 912a4895f4SDavid Greenman vm_object_t spc_object; 92df8bae1dSRodney W. Grimes vm_offset_t spc_kva; 9326f9a767SRodney W. Grimes int spc_count; 9426f9a767SRodney W. Grimes vm_page_t spc_m[MAX_PAGEOUT_CLUSTER]; 95df8bae1dSRodney W. Grimes } swcleanlist[NPENDINGIO]; 9626f9a767SRodney W. Grimes 9726f9a767SRodney W. Grimes 98df8bae1dSRodney W. Grimes /* spc_flags values */ 9926f9a767SRodney W. Grimes #define SPC_ERROR 0x01 100df8bae1dSRodney W. Grimes 10126f9a767SRodney W. Grimes #define SWB_EMPTY (-1) 102df8bae1dSRodney W. Grimes 103a401ebbeSDavid Greenman struct swpclean swap_pager_done; /* list of completed page cleans */ 104df8bae1dSRodney W. Grimes struct swpclean swap_pager_inuse; /* list of pending page cleans */ 105df8bae1dSRodney W. Grimes struct swpclean swap_pager_free; /* list of free pager clean structs */ 10624a1cce3SDavid Greenman struct pagerlst swap_pager_object_list; /* list of "named" anon region objects */ 10724a1cce3SDavid Greenman struct pagerlst swap_pager_un_object_list; /* list of "unnamed" anon region objects */ 108df8bae1dSRodney W. Grimes 10926f9a767SRodney W. Grimes #define SWAP_FREE_NEEDED 0x1 /* need a swap block */ 110a1f6d91cSDavid Greenman #define SWAP_FREE_NEEDED_BY_PAGEOUT 0x2 11126f9a767SRodney W. Grimes int swap_pager_needflags; 11226f9a767SRodney W. Grimes 113f5a12711SPoul-Henning Kamp static struct pagerlst *swp_qs[] = { 11424a1cce3SDavid Greenman &swap_pager_object_list, &swap_pager_un_object_list, (struct pagerlst *) 0 11526f9a767SRodney W. Grimes }; 11626f9a767SRodney W. Grimes 11724a1cce3SDavid Greenman /* 11824a1cce3SDavid Greenman * pagerops for OBJT_SWAP - "swap pager". 11924a1cce3SDavid Greenman */ 120df8bae1dSRodney W. Grimes struct pagerops swappagerops = { 121df8bae1dSRodney W. Grimes swap_pager_init, 122df8bae1dSRodney W. Grimes swap_pager_alloc, 123df8bae1dSRodney W. Grimes swap_pager_dealloc, 12424a1cce3SDavid Greenman swap_pager_getpages, 12524a1cce3SDavid Greenman swap_pager_putpages, 12624a1cce3SDavid Greenman swap_pager_haspage, 12724a1cce3SDavid Greenman swap_pager_sync 128df8bae1dSRodney W. Grimes }; 129df8bae1dSRodney W. Grimes 130f5a12711SPoul-Henning Kamp static int npendingio = NPENDINGIO; 131f5a12711SPoul-Henning Kamp static void swap_pager_finish(); 13226f9a767SRodney W. Grimes int dmmin, dmmax; 13326f9a767SRodney W. Grimes 13424a1cce3SDavid Greenman 1350d94caffSDavid Greenman static inline void 1360d94caffSDavid Greenman swapsizecheck() 1370d94caffSDavid Greenman { 13826f9a767SRodney W. Grimes if (vm_swap_size < 128 * btodb(PAGE_SIZE)) { 139a1f6d91cSDavid Greenman if (swap_pager_full == 0) 14026f9a767SRodney W. Grimes printf("swap_pager: out of space\n"); 14126f9a767SRodney W. Grimes swap_pager_full = 1; 14226f9a767SRodney W. Grimes } else if (vm_swap_size > 192 * btodb(PAGE_SIZE)) 14326f9a767SRodney W. Grimes swap_pager_full = 0; 14426f9a767SRodney W. Grimes } 14526f9a767SRodney W. Grimes 146f5a12711SPoul-Henning Kamp static void 147df8bae1dSRodney W. Grimes swap_pager_init() 148df8bae1dSRodney W. Grimes { 14924a1cce3SDavid Greenman TAILQ_INIT(&swap_pager_object_list); 15024a1cce3SDavid Greenman TAILQ_INIT(&swap_pager_un_object_list); 151df8bae1dSRodney W. Grimes 152df8bae1dSRodney W. Grimes /* 153df8bae1dSRodney W. Grimes * Initialize clean lists 154df8bae1dSRodney W. Grimes */ 155df8bae1dSRodney W. Grimes TAILQ_INIT(&swap_pager_inuse); 15626f9a767SRodney W. Grimes TAILQ_INIT(&swap_pager_done); 157df8bae1dSRodney W. Grimes TAILQ_INIT(&swap_pager_free); 15826f9a767SRodney W. Grimes 159df8bae1dSRodney W. Grimes /* 160df8bae1dSRodney W. Grimes * Calculate the swap allocation constants. 161df8bae1dSRodney W. Grimes */ 16226f9a767SRodney W. Grimes dmmin = CLBYTES / DEV_BSIZE; 16326f9a767SRodney W. Grimes dmmax = btodb(SWB_NPAGES * PAGE_SIZE) * 2; 164df8bae1dSRodney W. Grimes } 165df8bae1dSRodney W. Grimes 16624a1cce3SDavid Greenman void 16724a1cce3SDavid Greenman swap_pager_swap_init() 168df8bae1dSRodney W. Grimes { 16926f9a767SRodney W. Grimes swp_clean_t spc; 17026f9a767SRodney W. Grimes struct buf *bp; 17124a1cce3SDavid Greenman int i; 1720d94caffSDavid Greenman 17326f9a767SRodney W. Grimes /* 1740d94caffSDavid Greenman * kva's are allocated here so that we dont need to keep doing 1750d94caffSDavid Greenman * kmem_alloc pageables at runtime 17626f9a767SRodney W. Grimes */ 17726f9a767SRodney W. Grimes for (i = 0, spc = swcleanlist; i < npendingio; i++, spc++) { 178fff93ab6SDavid Greenman spc->spc_kva = kmem_alloc_pageable(pager_map, PAGE_SIZE * MAX_PAGEOUT_CLUSTER); 17926f9a767SRodney W. Grimes if (!spc->spc_kva) { 18026f9a767SRodney W. Grimes break; 18126f9a767SRodney W. Grimes } 182a1f6d91cSDavid Greenman spc->spc_bp = malloc(sizeof(*bp), M_TEMP, M_KERNEL); 18326f9a767SRodney W. Grimes if (!spc->spc_bp) { 18426f9a767SRodney W. Grimes kmem_free_wakeup(pager_map, spc->spc_kva, PAGE_SIZE); 18526f9a767SRodney W. Grimes break; 18626f9a767SRodney W. Grimes } 18726f9a767SRodney W. Grimes spc->spc_flags = 0; 18826f9a767SRodney W. Grimes TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 18926f9a767SRodney W. Grimes } 19026f9a767SRodney W. Grimes } 19124a1cce3SDavid Greenman 19224a1cce3SDavid Greenman int 19324a1cce3SDavid Greenman swap_pager_swp_alloc(object, wait) 19424a1cce3SDavid Greenman vm_object_t object; 19524a1cce3SDavid Greenman int wait; 19624a1cce3SDavid Greenman { 1972a4895f4SDavid Greenman sw_blk_t swb; 1982a4895f4SDavid Greenman int nblocks; 19924a1cce3SDavid Greenman int i, j; 20024a1cce3SDavid Greenman 2012a4895f4SDavid Greenman nblocks = (btodb(object->size) + btodb(SWB_NPAGES * PAGE_SIZE) - 1) / 2022a4895f4SDavid Greenman btodb(SWB_NPAGES * PAGE_SIZE); 20324a1cce3SDavid Greenman 2042a4895f4SDavid Greenman swb = malloc(nblocks * sizeof(*swb), M_VMPGDATA, wait); 2052a4895f4SDavid Greenman if (swb == NULL) 20624a1cce3SDavid Greenman return 1; 20724a1cce3SDavid Greenman 2082a4895f4SDavid Greenman for (i = 0; i < nblocks; i++) { 2092a4895f4SDavid Greenman swb[i].swb_valid = 0; 2102a4895f4SDavid Greenman swb[i].swb_locked = 0; 21126f9a767SRodney W. Grimes for (j = 0; j < SWB_NPAGES; j++) 2122a4895f4SDavid Greenman swb[i].swb_block[j] = SWB_EMPTY; 21326f9a767SRodney W. Grimes } 21426f9a767SRodney W. Grimes 2152a4895f4SDavid Greenman object->un_pager.swp.swp_nblocks = nblocks; 2162a4895f4SDavid Greenman object->un_pager.swp.swp_allocsize = 0; 2172a4895f4SDavid Greenman object->un_pager.swp.swp_blocks = swb; 2182a4895f4SDavid Greenman object->un_pager.swp.swp_poip = 0; 21924a1cce3SDavid Greenman 22024a1cce3SDavid Greenman if (object->handle != NULL) { 22124a1cce3SDavid Greenman TAILQ_INSERT_TAIL(&swap_pager_object_list, object, pager_object_list); 222df8bae1dSRodney W. Grimes } else { 22324a1cce3SDavid Greenman TAILQ_INSERT_TAIL(&swap_pager_un_object_list, object, pager_object_list); 224df8bae1dSRodney W. Grimes } 225df8bae1dSRodney W. Grimes 22624a1cce3SDavid Greenman return 0; 22724a1cce3SDavid Greenman } 22824a1cce3SDavid Greenman 22924a1cce3SDavid Greenman /* 2302a4895f4SDavid Greenman * Allocate an object and associated resources. 23124a1cce3SDavid Greenman * Note that if we are called from the pageout daemon (handle == NULL) 23224a1cce3SDavid Greenman * we should not wait for memory as it could resulting in deadlock. 23324a1cce3SDavid Greenman */ 234f5a12711SPoul-Henning Kamp static vm_object_t 23524a1cce3SDavid Greenman swap_pager_alloc(handle, size, prot, offset) 23624a1cce3SDavid Greenman void *handle; 23724a1cce3SDavid Greenman register vm_size_t size; 23824a1cce3SDavid Greenman vm_prot_t prot; 23924a1cce3SDavid Greenman vm_offset_t offset; 24024a1cce3SDavid Greenman { 24124a1cce3SDavid Greenman vm_object_t object; 24224a1cce3SDavid Greenman int i; 24324a1cce3SDavid Greenman 24424a1cce3SDavid Greenman /* 24524a1cce3SDavid Greenman * If this is a "named" anonymous region, look it up and use the 24624a1cce3SDavid Greenman * object if it exists, otherwise allocate a new one. 24724a1cce3SDavid Greenman */ 24824a1cce3SDavid Greenman if (handle) { 24924a1cce3SDavid Greenman object = vm_pager_object_lookup(&swap_pager_object_list, handle); 25024a1cce3SDavid Greenman if (object != NULL) { 25124a1cce3SDavid Greenman vm_object_reference(object); 25224a1cce3SDavid Greenman } else { 25324a1cce3SDavid Greenman /* 25424a1cce3SDavid Greenman * XXX - there is a race condition here. Two processes 25524a1cce3SDavid Greenman * can request the same named object simultaneuously, 25624a1cce3SDavid Greenman * and if one blocks for memory, the result is a disaster. 25724a1cce3SDavid Greenman * Probably quite rare, but is yet another reason to just 25824a1cce3SDavid Greenman * rip support of "named anonymous regions" out altogether. 25924a1cce3SDavid Greenman */ 26024a1cce3SDavid Greenman object = vm_object_allocate(OBJT_SWAP, offset + size); 26124a1cce3SDavid Greenman object->handle = handle; 26224a1cce3SDavid Greenman (void) swap_pager_swp_alloc(object, M_WAITOK); 26324a1cce3SDavid Greenman } 26424a1cce3SDavid Greenman } else { 26524a1cce3SDavid Greenman object = vm_object_allocate(OBJT_SWAP, offset + size); 26624a1cce3SDavid Greenman (void) swap_pager_swp_alloc(object, M_WAITOK); 26724a1cce3SDavid Greenman } 26824a1cce3SDavid Greenman 26924a1cce3SDavid Greenman return (object); 270df8bae1dSRodney W. Grimes } 271df8bae1dSRodney W. Grimes 27226f9a767SRodney W. Grimes /* 27326f9a767SRodney W. Grimes * returns disk block associated with pager and offset 27426f9a767SRodney W. Grimes * additionally, as a side effect returns a flag indicating 27526f9a767SRodney W. Grimes * if the block has been written 27626f9a767SRodney W. Grimes */ 27726f9a767SRodney W. Grimes 278a1f6d91cSDavid Greenman inline static int * 27924a1cce3SDavid Greenman swap_pager_diskaddr(object, offset, valid) 28024a1cce3SDavid Greenman vm_object_t object; 28126f9a767SRodney W. Grimes vm_offset_t offset; 28226f9a767SRodney W. Grimes int *valid; 28326f9a767SRodney W. Grimes { 28426f9a767SRodney W. Grimes register sw_blk_t swb; 28526f9a767SRodney W. Grimes int ix; 28626f9a767SRodney W. Grimes 28726f9a767SRodney W. Grimes if (valid) 28826f9a767SRodney W. Grimes *valid = 0; 28926f9a767SRodney W. Grimes ix = offset / (SWB_NPAGES * PAGE_SIZE); 2902a4895f4SDavid Greenman if ((ix >= object->un_pager.swp.swp_nblocks) || 29124a1cce3SDavid Greenman (offset >= object->size)) { 29226f9a767SRodney W. Grimes return (FALSE); 29326f9a767SRodney W. Grimes } 2942a4895f4SDavid Greenman swb = &object->un_pager.swp.swp_blocks[ix]; 29526f9a767SRodney W. Grimes ix = (offset % (SWB_NPAGES * PAGE_SIZE)) / PAGE_SIZE; 29626f9a767SRodney W. Grimes if (valid) 29726f9a767SRodney W. Grimes *valid = swb->swb_valid & (1 << ix); 29826f9a767SRodney W. Grimes return &swb->swb_block[ix]; 29926f9a767SRodney W. Grimes } 30026f9a767SRodney W. Grimes 30126f9a767SRodney W. Grimes /* 30226f9a767SRodney W. Grimes * Utility routine to set the valid (written) bit for 30326f9a767SRodney W. Grimes * a block associated with a pager and offset 30426f9a767SRodney W. Grimes */ 305df8bae1dSRodney W. Grimes static void 3062a4895f4SDavid Greenman swap_pager_setvalid(object, offset, valid) 3072a4895f4SDavid Greenman vm_object_t object; 30826f9a767SRodney W. Grimes vm_offset_t offset; 30926f9a767SRodney W. Grimes int valid; 31026f9a767SRodney W. Grimes { 31126f9a767SRodney W. Grimes register sw_blk_t swb; 31226f9a767SRodney W. Grimes int ix; 31326f9a767SRodney W. Grimes 31426f9a767SRodney W. Grimes ix = offset / (SWB_NPAGES * PAGE_SIZE); 3152a4895f4SDavid Greenman if (ix >= object->un_pager.swp.swp_nblocks) 31626f9a767SRodney W. Grimes return; 31726f9a767SRodney W. Grimes 3182a4895f4SDavid Greenman swb = &object->un_pager.swp.swp_blocks[ix]; 31926f9a767SRodney W. Grimes ix = (offset % (SWB_NPAGES * PAGE_SIZE)) / PAGE_SIZE; 32026f9a767SRodney W. Grimes if (valid) 32126f9a767SRodney W. Grimes swb->swb_valid |= (1 << ix); 32226f9a767SRodney W. Grimes else 32326f9a767SRodney W. Grimes swb->swb_valid &= ~(1 << ix); 32426f9a767SRodney W. Grimes return; 32526f9a767SRodney W. Grimes } 32626f9a767SRodney W. Grimes 32726f9a767SRodney W. Grimes /* 32826f9a767SRodney W. Grimes * this routine allocates swap space with a fragmentation 32926f9a767SRodney W. Grimes * minimization policy. 33026f9a767SRodney W. Grimes */ 331f5a12711SPoul-Henning Kamp static int 3322a4895f4SDavid Greenman swap_pager_getswapspace(object, amount, rtval) 3332a4895f4SDavid Greenman vm_object_t object; 3342a4895f4SDavid Greenman unsigned int amount; 3352a4895f4SDavid Greenman unsigned int *rtval; 3360d94caffSDavid Greenman { 33724ea4a96SDavid Greenman vm_swap_size -= amount; 33824ea4a96SDavid Greenman if (!rlist_alloc(&swaplist, amount, rtval)) { 33924ea4a96SDavid Greenman vm_swap_size += amount; 34026f9a767SRodney W. Grimes return 0; 34124ea4a96SDavid Greenman } else { 34224ea4a96SDavid Greenman swapsizecheck(); 3432a4895f4SDavid Greenman object->un_pager.swp.swp_allocsize += amount; 34426f9a767SRodney W. Grimes return 1; 34526f9a767SRodney W. Grimes } 34626f9a767SRodney W. Grimes } 34726f9a767SRodney W. Grimes 34826f9a767SRodney W. Grimes /* 34926f9a767SRodney W. Grimes * this routine frees swap space with a fragmentation 35026f9a767SRodney W. Grimes * minimization policy. 35126f9a767SRodney W. Grimes */ 352f5a12711SPoul-Henning Kamp static void 3532a4895f4SDavid Greenman swap_pager_freeswapspace(object, from, to) 3542a4895f4SDavid Greenman vm_object_t object; 3552a4895f4SDavid Greenman unsigned int from; 3562a4895f4SDavid Greenman unsigned int to; 3570d94caffSDavid Greenman { 35835c10d22SDavid Greenman rlist_free(&swaplist, from, to); 35924ea4a96SDavid Greenman vm_swap_size += (to - from) + 1; 3602a4895f4SDavid Greenman object->un_pager.swp.swp_allocsize -= (to - from) + 1; 36124ea4a96SDavid Greenman swapsizecheck(); 36226f9a767SRodney W. Grimes } 36326f9a767SRodney W. Grimes /* 36426f9a767SRodney W. Grimes * this routine frees swap blocks from a specified pager 36526f9a767SRodney W. Grimes */ 36626f9a767SRodney W. Grimes void 36724a1cce3SDavid Greenman swap_pager_freespace(object, start, size) 36824a1cce3SDavid Greenman vm_object_t object; 36926f9a767SRodney W. Grimes vm_offset_t start; 37026f9a767SRodney W. Grimes vm_offset_t size; 37126f9a767SRodney W. Grimes { 37226f9a767SRodney W. Grimes vm_offset_t i; 37326f9a767SRodney W. Grimes int s; 37426f9a767SRodney W. Grimes 37526f9a767SRodney W. Grimes s = splbio(); 3766f7bc393SDavid Greenman for (i = start; i < round_page(start + size); i += PAGE_SIZE) { 37726f9a767SRodney W. Grimes int valid; 37824a1cce3SDavid Greenman int *addr = swap_pager_diskaddr(object, i, &valid); 3790d94caffSDavid Greenman 38026f9a767SRodney W. Grimes if (addr && *addr != SWB_EMPTY) { 3812a4895f4SDavid Greenman swap_pager_freeswapspace(object, *addr, *addr + btodb(PAGE_SIZE) - 1); 38226f9a767SRodney W. Grimes if (valid) { 3832a4895f4SDavid Greenman swap_pager_setvalid(object, i, 0); 38426f9a767SRodney W. Grimes } 38526f9a767SRodney W. Grimes *addr = SWB_EMPTY; 38626f9a767SRodney W. Grimes } 38726f9a767SRodney W. Grimes } 38826f9a767SRodney W. Grimes splx(s); 38926f9a767SRodney W. Grimes } 39026f9a767SRodney W. Grimes 391a1f6d91cSDavid Greenman static void 3922a4895f4SDavid Greenman swap_pager_free_swap(object) 3932a4895f4SDavid Greenman vm_object_t object; 394a1f6d91cSDavid Greenman { 395a1f6d91cSDavid Greenman register int i, j; 3962a4895f4SDavid Greenman register sw_blk_t swb; 397a1f6d91cSDavid Greenman int first_block=0, block_count=0; 398a1f6d91cSDavid Greenman int s; 399a1f6d91cSDavid Greenman /* 400a1f6d91cSDavid Greenman * Free left over swap blocks 401a1f6d91cSDavid Greenman */ 402a1f6d91cSDavid Greenman s = splbio(); 4032a4895f4SDavid Greenman for (i = 0, swb = object->un_pager.swp.swp_blocks; 4042a4895f4SDavid Greenman i < object->un_pager.swp.swp_nblocks; i++, swb++) { 405a1f6d91cSDavid Greenman for (j = 0; j < SWB_NPAGES; j++) { 4062a4895f4SDavid Greenman if (swb->swb_block[j] != SWB_EMPTY) { 407a1f6d91cSDavid Greenman /* 408a1f6d91cSDavid Greenman * initially the length of the run is zero 409a1f6d91cSDavid Greenman */ 410a1f6d91cSDavid Greenman if (block_count == 0) { 4112a4895f4SDavid Greenman first_block = swb->swb_block[j]; 412a1f6d91cSDavid Greenman block_count = btodb(PAGE_SIZE); 4132a4895f4SDavid Greenman swb->swb_block[j] = SWB_EMPTY; 414a1f6d91cSDavid Greenman /* 415a1f6d91cSDavid Greenman * if the new block can be included into the current run 416a1f6d91cSDavid Greenman */ 4172a4895f4SDavid Greenman } else if (swb->swb_block[j] == first_block + block_count) { 418a1f6d91cSDavid Greenman block_count += btodb(PAGE_SIZE); 4192a4895f4SDavid Greenman swb->swb_block[j] = SWB_EMPTY; 420a1f6d91cSDavid Greenman /* 421a1f6d91cSDavid Greenman * terminate the previous run, and start a new one 422a1f6d91cSDavid Greenman */ 423a1f6d91cSDavid Greenman } else { 4242a4895f4SDavid Greenman swap_pager_freeswapspace(object, first_block, 425a1f6d91cSDavid Greenman (unsigned) first_block + block_count - 1); 4262a4895f4SDavid Greenman first_block = swb->swb_block[j]; 427a1f6d91cSDavid Greenman block_count = btodb(PAGE_SIZE); 4282a4895f4SDavid Greenman swb->swb_block[j] = SWB_EMPTY; 429a1f6d91cSDavid Greenman } 430a1f6d91cSDavid Greenman } 431a1f6d91cSDavid Greenman } 432a1f6d91cSDavid Greenman } 433a1f6d91cSDavid Greenman 434a1f6d91cSDavid Greenman if (block_count) { 4352a4895f4SDavid Greenman swap_pager_freeswapspace(object, first_block, 436a1f6d91cSDavid Greenman (unsigned) first_block + block_count - 1); 437a1f6d91cSDavid Greenman } 438a1f6d91cSDavid Greenman splx(s); 439a1f6d91cSDavid Greenman } 440a1f6d91cSDavid Greenman 441a1f6d91cSDavid Greenman 44226f9a767SRodney W. Grimes /* 44326f9a767SRodney W. Grimes * swap_pager_reclaim frees up over-allocated space from all pagers 44426f9a767SRodney W. Grimes * this eliminates internal fragmentation due to allocation of space 44526f9a767SRodney W. Grimes * for segments that are never swapped to. It has been written so that 44626f9a767SRodney W. Grimes * it does not block until the rlist_free operation occurs; it keeps 44726f9a767SRodney W. Grimes * the queues consistant. 44826f9a767SRodney W. Grimes */ 44926f9a767SRodney W. Grimes 45026f9a767SRodney W. Grimes /* 45126f9a767SRodney W. Grimes * Maximum number of blocks (pages) to reclaim per pass 45226f9a767SRodney W. Grimes */ 453a1f6d91cSDavid Greenman #define MAXRECLAIM 128 45426f9a767SRodney W. Grimes 455f5a12711SPoul-Henning Kamp static void 45626f9a767SRodney W. Grimes swap_pager_reclaim() 45726f9a767SRodney W. Grimes { 45824a1cce3SDavid Greenman vm_object_t object; 45926f9a767SRodney W. Grimes int i, j, k; 46026f9a767SRodney W. Grimes int s; 46126f9a767SRodney W. Grimes int reclaimcount; 462a1f6d91cSDavid Greenman static struct { 463a1f6d91cSDavid Greenman int address; 4642a4895f4SDavid Greenman vm_object_t object; 465a1f6d91cSDavid Greenman } reclaims[MAXRECLAIM]; 46626f9a767SRodney W. Grimes static int in_reclaim; 46726f9a767SRodney W. Grimes 46826f9a767SRodney W. Grimes /* 46926f9a767SRodney W. Grimes * allow only one process to be in the swap_pager_reclaim subroutine 47026f9a767SRodney W. Grimes */ 47126f9a767SRodney W. Grimes s = splbio(); 47226f9a767SRodney W. Grimes if (in_reclaim) { 47324a1cce3SDavid Greenman tsleep(&in_reclaim, PSWP, "swrclm", 0); 47426f9a767SRodney W. Grimes splx(s); 47526f9a767SRodney W. Grimes return; 47626f9a767SRodney W. Grimes } 47726f9a767SRodney W. Grimes in_reclaim = 1; 47826f9a767SRodney W. Grimes reclaimcount = 0; 47926f9a767SRodney W. Grimes 48026f9a767SRodney W. Grimes /* for each pager queue */ 48126f9a767SRodney W. Grimes for (k = 0; swp_qs[k]; k++) { 48226f9a767SRodney W. Grimes 48324a1cce3SDavid Greenman object = swp_qs[k]->tqh_first; 48424a1cce3SDavid Greenman while (object && (reclaimcount < MAXRECLAIM)) { 48526f9a767SRodney W. Grimes 48626f9a767SRodney W. Grimes /* 48726f9a767SRodney W. Grimes * see if any blocks associated with a pager has been 48826f9a767SRodney W. Grimes * allocated but not used (written) 48926f9a767SRodney W. Grimes */ 4902a4895f4SDavid Greenman for (i = 0; i < object->un_pager.swp.swp_nblocks; i++) { 4912a4895f4SDavid Greenman sw_blk_t swb = &object->un_pager.swp.swp_blocks[i]; 4920d94caffSDavid Greenman 49326f9a767SRodney W. Grimes if (swb->swb_locked) 49426f9a767SRodney W. Grimes continue; 49526f9a767SRodney W. Grimes for (j = 0; j < SWB_NPAGES; j++) { 49626f9a767SRodney W. Grimes if (swb->swb_block[j] != SWB_EMPTY && 49726f9a767SRodney W. Grimes (swb->swb_valid & (1 << j)) == 0) { 498a1f6d91cSDavid Greenman reclaims[reclaimcount].address = swb->swb_block[j]; 4992a4895f4SDavid Greenman reclaims[reclaimcount++].object = object; 50026f9a767SRodney W. Grimes swb->swb_block[j] = SWB_EMPTY; 50126f9a767SRodney W. Grimes if (reclaimcount >= MAXRECLAIM) 50226f9a767SRodney W. Grimes goto rfinished; 50326f9a767SRodney W. Grimes } 50426f9a767SRodney W. Grimes } 50526f9a767SRodney W. Grimes } 50624a1cce3SDavid Greenman object = object->pager_object_list.tqe_next; 50726f9a767SRodney W. Grimes } 50826f9a767SRodney W. Grimes } 50926f9a767SRodney W. Grimes 51026f9a767SRodney W. Grimes rfinished: 51126f9a767SRodney W. Grimes 51226f9a767SRodney W. Grimes /* 51326f9a767SRodney W. Grimes * free the blocks that have been added to the reclaim list 51426f9a767SRodney W. Grimes */ 51526f9a767SRodney W. Grimes for (i = 0; i < reclaimcount; i++) { 5162a4895f4SDavid Greenman swap_pager_freeswapspace(reclaims[i].object, 5172a4895f4SDavid Greenman reclaims[i].address, reclaims[i].address + btodb(PAGE_SIZE) - 1); 51826f9a767SRodney W. Grimes } 51926f9a767SRodney W. Grimes splx(s); 52026f9a767SRodney W. Grimes in_reclaim = 0; 52124a1cce3SDavid Greenman wakeup(&in_reclaim); 52226f9a767SRodney W. Grimes } 52326f9a767SRodney W. Grimes 52426f9a767SRodney W. Grimes 52526f9a767SRodney W. Grimes /* 52626f9a767SRodney W. Grimes * swap_pager_copy copies blocks from one pager to another and 52726f9a767SRodney W. Grimes * destroys the source pager 52826f9a767SRodney W. Grimes */ 52926f9a767SRodney W. Grimes 53026f9a767SRodney W. Grimes void 53124a1cce3SDavid Greenman swap_pager_copy(srcobject, srcoffset, dstobject, dstoffset, offset) 53224a1cce3SDavid Greenman vm_object_t srcobject; 53326f9a767SRodney W. Grimes vm_offset_t srcoffset; 53424a1cce3SDavid Greenman vm_object_t dstobject; 53526f9a767SRodney W. Grimes vm_offset_t dstoffset; 53626f9a767SRodney W. Grimes vm_offset_t offset; 53726f9a767SRodney W. Grimes { 53826f9a767SRodney W. Grimes vm_offset_t i; 539a1f6d91cSDavid Greenman int origsize; 54026f9a767SRodney W. Grimes int s; 54126f9a767SRodney W. Grimes 54224ea4a96SDavid Greenman if (vm_swap_size) 54324ea4a96SDavid Greenman no_swap_space = 0; 54424ea4a96SDavid Greenman 5452a4895f4SDavid Greenman origsize = srcobject->un_pager.swp.swp_allocsize; 54626f9a767SRodney W. Grimes 54726f9a767SRodney W. Grimes /* 54824a1cce3SDavid Greenman * remove the source object from the swap_pager internal queue 54926f9a767SRodney W. Grimes */ 55024a1cce3SDavid Greenman if (srcobject->handle == NULL) { 55124a1cce3SDavid Greenman TAILQ_REMOVE(&swap_pager_un_object_list, srcobject, pager_object_list); 55226f9a767SRodney W. Grimes } else { 55324a1cce3SDavid Greenman TAILQ_REMOVE(&swap_pager_object_list, srcobject, pager_object_list); 55426f9a767SRodney W. Grimes } 55526f9a767SRodney W. Grimes 55624a1cce3SDavid Greenman s = splbio(); 5572a4895f4SDavid Greenman while (srcobject->un_pager.swp.swp_poip) { 5582a4895f4SDavid Greenman tsleep(srcobject, PVM, "spgout", 0); 55926f9a767SRodney W. Grimes } 56026f9a767SRodney W. Grimes splx(s); 56126f9a767SRodney W. Grimes 56226f9a767SRodney W. Grimes /* 56326f9a767SRodney W. Grimes * clean all of the pages that are currently active and finished 56426f9a767SRodney W. Grimes */ 56524a1cce3SDavid Greenman swap_pager_sync(); 56626f9a767SRodney W. Grimes 56726f9a767SRodney W. Grimes s = splbio(); 56826f9a767SRodney W. Grimes /* 56926f9a767SRodney W. Grimes * transfer source to destination 57026f9a767SRodney W. Grimes */ 57124a1cce3SDavid Greenman for (i = 0; i < dstobject->size; i += PAGE_SIZE) { 57226f9a767SRodney W. Grimes int srcvalid, dstvalid; 57324a1cce3SDavid Greenman int *srcaddrp = swap_pager_diskaddr(srcobject, i + offset + srcoffset, 57426f9a767SRodney W. Grimes &srcvalid); 57526f9a767SRodney W. Grimes int *dstaddrp; 5760d94caffSDavid Greenman 57726f9a767SRodney W. Grimes /* 57826f9a767SRodney W. Grimes * see if the source has space allocated 57926f9a767SRodney W. Grimes */ 58026f9a767SRodney W. Grimes if (srcaddrp && *srcaddrp != SWB_EMPTY) { 58126f9a767SRodney W. Grimes /* 5820d94caffSDavid Greenman * if the source is valid and the dest has no space, 5830d94caffSDavid Greenman * then copy the allocation from the srouce to the 5840d94caffSDavid Greenman * dest. 58526f9a767SRodney W. Grimes */ 58626f9a767SRodney W. Grimes if (srcvalid) { 58724a1cce3SDavid Greenman dstaddrp = swap_pager_diskaddr(dstobject, i + dstoffset, 588a1f6d91cSDavid Greenman &dstvalid); 58926f9a767SRodney W. Grimes /* 5900d94caffSDavid Greenman * if the dest already has a valid block, 5910d94caffSDavid Greenman * deallocate the source block without 5920d94caffSDavid Greenman * copying. 59326f9a767SRodney W. Grimes */ 59426f9a767SRodney W. Grimes if (!dstvalid && dstaddrp && *dstaddrp != SWB_EMPTY) { 5952a4895f4SDavid Greenman swap_pager_freeswapspace(dstobject, *dstaddrp, 596a1f6d91cSDavid Greenman *dstaddrp + btodb(PAGE_SIZE) - 1); 59726f9a767SRodney W. Grimes *dstaddrp = SWB_EMPTY; 59826f9a767SRodney W. Grimes } 59926f9a767SRodney W. Grimes if (dstaddrp && *dstaddrp == SWB_EMPTY) { 60026f9a767SRodney W. Grimes *dstaddrp = *srcaddrp; 60126f9a767SRodney W. Grimes *srcaddrp = SWB_EMPTY; 6022a4895f4SDavid Greenman dstobject->un_pager.swp.swp_allocsize += btodb(PAGE_SIZE); 6032a4895f4SDavid Greenman srcobject->un_pager.swp.swp_allocsize -= btodb(PAGE_SIZE); 6042a4895f4SDavid Greenman swap_pager_setvalid(dstobject, i + dstoffset, 1); 60526f9a767SRodney W. Grimes } 60626f9a767SRodney W. Grimes } 60726f9a767SRodney W. Grimes /* 6080d94caffSDavid Greenman * if the source is not empty at this point, then 6090d94caffSDavid Greenman * deallocate the space. 61026f9a767SRodney W. Grimes */ 61126f9a767SRodney W. Grimes if (*srcaddrp != SWB_EMPTY) { 6122a4895f4SDavid Greenman swap_pager_freeswapspace(srcobject, *srcaddrp, 613a1f6d91cSDavid Greenman *srcaddrp + btodb(PAGE_SIZE) - 1); 61426f9a767SRodney W. Grimes *srcaddrp = SWB_EMPTY; 61526f9a767SRodney W. Grimes } 61626f9a767SRodney W. Grimes } 61726f9a767SRodney W. Grimes } 61826f9a767SRodney W. Grimes splx(s); 61926f9a767SRodney W. Grimes 620a1f6d91cSDavid Greenman /* 621a1f6d91cSDavid Greenman * Free left over swap blocks 622a1f6d91cSDavid Greenman */ 6232a4895f4SDavid Greenman swap_pager_free_swap(srcobject); 624a1f6d91cSDavid Greenman 6252a4895f4SDavid Greenman if (srcobject->un_pager.swp.swp_allocsize) { 6262a4895f4SDavid Greenman printf("swap_pager_copy: *warning* pager with %d blocks (orig: %d)\n", 6272a4895f4SDavid Greenman srcobject->un_pager.swp.swp_allocsize, origsize); 6282a4895f4SDavid Greenman } 6292a4895f4SDavid Greenman 6302a4895f4SDavid Greenman free(srcobject->un_pager.swp.swp_blocks, M_VMPGDATA); 6312a4895f4SDavid Greenman srcobject->un_pager.swp.swp_blocks = NULL; 63226f9a767SRodney W. Grimes 63326f9a767SRodney W. Grimes return; 63426f9a767SRodney W. Grimes } 63526f9a767SRodney W. Grimes 636f5a12711SPoul-Henning Kamp static void 63724a1cce3SDavid Greenman swap_pager_dealloc(object) 63824a1cce3SDavid Greenman vm_object_t object; 639df8bae1dSRodney W. Grimes { 640df8bae1dSRodney W. Grimes int s; 641df8bae1dSRodney W. Grimes 642df8bae1dSRodney W. Grimes /* 6430d94caffSDavid Greenman * Remove from list right away so lookups will fail if we block for 6440d94caffSDavid Greenman * pageout completion. 645df8bae1dSRodney W. Grimes */ 64624a1cce3SDavid Greenman if (object->handle == NULL) { 64724a1cce3SDavid Greenman TAILQ_REMOVE(&swap_pager_un_object_list, object, pager_object_list); 64826f9a767SRodney W. Grimes } else { 64924a1cce3SDavid Greenman TAILQ_REMOVE(&swap_pager_object_list, object, pager_object_list); 650df8bae1dSRodney W. Grimes } 65124a1cce3SDavid Greenman 652df8bae1dSRodney W. Grimes /* 6530d94caffSDavid Greenman * Wait for all pageouts to finish and remove all entries from 6540d94caffSDavid Greenman * cleaning list. 655df8bae1dSRodney W. Grimes */ 65626f9a767SRodney W. Grimes 65724a1cce3SDavid Greenman s = splbio(); 6582a4895f4SDavid Greenman while (object->un_pager.swp.swp_poip) { 6592a4895f4SDavid Greenman tsleep(object, PVM, "swpout", 0); 660df8bae1dSRodney W. Grimes } 661df8bae1dSRodney W. Grimes splx(s); 66226f9a767SRodney W. Grimes 66326f9a767SRodney W. Grimes 66424a1cce3SDavid Greenman swap_pager_sync(); 665df8bae1dSRodney W. Grimes 666df8bae1dSRodney W. Grimes /* 667df8bae1dSRodney W. Grimes * Free left over swap blocks 668df8bae1dSRodney W. Grimes */ 6692a4895f4SDavid Greenman swap_pager_free_swap(object); 67026f9a767SRodney W. Grimes 6712a4895f4SDavid Greenman if (object->un_pager.swp.swp_allocsize) { 6722a4895f4SDavid Greenman printf("swap_pager_dealloc: *warning* freeing pager with %d blocks\n", 6732a4895f4SDavid Greenman object->un_pager.swp.swp_allocsize); 6742a4895f4SDavid Greenman } 675df8bae1dSRodney W. Grimes /* 676df8bae1dSRodney W. Grimes * Free swap management resources 677df8bae1dSRodney W. Grimes */ 6782a4895f4SDavid Greenman free(object->un_pager.swp.swp_blocks, M_VMPGDATA); 6792a4895f4SDavid Greenman object->un_pager.swp.swp_blocks = NULL; 68026f9a767SRodney W. Grimes } 68126f9a767SRodney W. Grimes 68226f9a767SRodney W. Grimes static inline int 6830d94caffSDavid Greenman const 6842a4895f4SDavid Greenman swap_pager_block_index(offset) 68526f9a767SRodney W. Grimes vm_offset_t offset; 68626f9a767SRodney W. Grimes { 68726f9a767SRodney W. Grimes return (offset / (SWB_NPAGES * PAGE_SIZE)); 68826f9a767SRodney W. Grimes } 68926f9a767SRodney W. Grimes 69026f9a767SRodney W. Grimes static inline int 6910d94caffSDavid Greenman const 6922a4895f4SDavid Greenman swap_pager_block_offset(offset) 69326f9a767SRodney W. Grimes vm_offset_t offset; 69426f9a767SRodney W. Grimes { 69526f9a767SRodney W. Grimes return ((offset % (PAGE_SIZE * SWB_NPAGES)) / PAGE_SIZE); 69626f9a767SRodney W. Grimes } 69726f9a767SRodney W. Grimes 69826f9a767SRodney W. Grimes /* 69924a1cce3SDavid Greenman * swap_pager_haspage returns TRUE if the pager has data that has 70026f9a767SRodney W. Grimes * been written out. 70126f9a767SRodney W. Grimes */ 702f5a12711SPoul-Henning Kamp static boolean_t 70324a1cce3SDavid Greenman swap_pager_haspage(object, offset, before, after) 70424a1cce3SDavid Greenman vm_object_t object; 70526f9a767SRodney W. Grimes vm_offset_t offset; 70624a1cce3SDavid Greenman int *before; 70724a1cce3SDavid Greenman int *after; 70826f9a767SRodney W. Grimes { 70926f9a767SRodney W. Grimes register sw_blk_t swb; 71026f9a767SRodney W. Grimes int ix; 711170db9c6SJohn Dyson int gix; 71226f9a767SRodney W. Grimes 71324a1cce3SDavid Greenman if (before != NULL) 71424a1cce3SDavid Greenman *before = 0; 71524a1cce3SDavid Greenman if (after != NULL) 71624a1cce3SDavid Greenman *after = 0; 71726f9a767SRodney W. Grimes ix = offset / (SWB_NPAGES * PAGE_SIZE); 7182a4895f4SDavid Greenman if (ix >= object->un_pager.swp.swp_nblocks) { 71926f9a767SRodney W. Grimes return (FALSE); 72026f9a767SRodney W. Grimes } 7212a4895f4SDavid Greenman swb = &object->un_pager.swp.swp_blocks[ix]; 722170db9c6SJohn Dyson gix = offset / PAGE_SIZE; 723170db9c6SJohn Dyson ix = gix % SWB_NPAGES; 724170db9c6SJohn Dyson 72526f9a767SRodney W. Grimes if (swb->swb_block[ix] != SWB_EMPTY) { 726170db9c6SJohn Dyson 727170db9c6SJohn Dyson if (swb->swb_valid & (1 << ix)) { 728170db9c6SJohn Dyson int tix; 729170db9c6SJohn Dyson if (before) { 730170db9c6SJohn Dyson for(tix = ix - 1; tix >= 0; --tix) { 7312f82e604SDavid Greenman if ((swb->swb_valid & (1 << tix)) == 0) 7322f82e604SDavid Greenman break; 733ca56715fSJohn Dyson if ((swb->swb_block[tix] + 734170db9c6SJohn Dyson (ix - tix) * (PAGE_SIZE/DEV_BSIZE)) != 735170db9c6SJohn Dyson swb->swb_block[ix]) 736170db9c6SJohn Dyson break; 737170db9c6SJohn Dyson (*before)++; 738170db9c6SJohn Dyson } 739170db9c6SJohn Dyson } 740170db9c6SJohn Dyson 741170db9c6SJohn Dyson if (after) { 742170db9c6SJohn Dyson for(tix = ix + 1; tix < SWB_NPAGES; tix++) { 7432f82e604SDavid Greenman if ((swb->swb_valid & (1 << tix)) == 0) 7442f82e604SDavid Greenman break; 745ca56715fSJohn Dyson if ((swb->swb_block[tix] - 746170db9c6SJohn Dyson (tix - ix) * (PAGE_SIZE/DEV_BSIZE)) != 747170db9c6SJohn Dyson swb->swb_block[ix]) 748170db9c6SJohn Dyson break; 749170db9c6SJohn Dyson (*after)++; 750170db9c6SJohn Dyson } 751170db9c6SJohn Dyson } 752170db9c6SJohn Dyson 75326f9a767SRodney W. Grimes return TRUE; 75426f9a767SRodney W. Grimes } 755170db9c6SJohn Dyson } 75626f9a767SRodney W. Grimes return (FALSE); 75726f9a767SRodney W. Grimes } 75826f9a767SRodney W. Grimes 75926f9a767SRodney W. Grimes /* 76026f9a767SRodney W. Grimes * swap_pager_freepage is a convienience routine that clears the busy 76126f9a767SRodney W. Grimes * bit and deallocates a page. 762df8bae1dSRodney W. Grimes */ 76326f9a767SRodney W. Grimes static void 76426f9a767SRodney W. Grimes swap_pager_freepage(m) 76526f9a767SRodney W. Grimes vm_page_t m; 76626f9a767SRodney W. Grimes { 76726f9a767SRodney W. Grimes PAGE_WAKEUP(m); 76826f9a767SRodney W. Grimes vm_page_free(m); 76926f9a767SRodney W. Grimes } 77026f9a767SRodney W. Grimes 77126f9a767SRodney W. Grimes /* 77226f9a767SRodney W. Grimes * swap_pager_ridpages is a convienience routine that deallocates all 77326f9a767SRodney W. Grimes * but the required page. this is usually used in error returns that 77426f9a767SRodney W. Grimes * need to invalidate the "extra" readahead pages. 77526f9a767SRodney W. Grimes */ 77626f9a767SRodney W. Grimes static void 77726f9a767SRodney W. Grimes swap_pager_ridpages(m, count, reqpage) 77826f9a767SRodney W. Grimes vm_page_t *m; 77926f9a767SRodney W. Grimes int count; 78026f9a767SRodney W. Grimes int reqpage; 78126f9a767SRodney W. Grimes { 78226f9a767SRodney W. Grimes int i; 7830d94caffSDavid Greenman 78426f9a767SRodney W. Grimes for (i = 0; i < count; i++) 78526f9a767SRodney W. Grimes if (i != reqpage) 78626f9a767SRodney W. Grimes swap_pager_freepage(m[i]); 78726f9a767SRodney W. Grimes } 78826f9a767SRodney W. Grimes 78926f9a767SRodney W. Grimes /* 79026f9a767SRodney W. Grimes * swap_pager_iodone1 is the completion routine for both reads and async writes 79126f9a767SRodney W. Grimes */ 792f5a12711SPoul-Henning Kamp static void 79326f9a767SRodney W. Grimes swap_pager_iodone1(bp) 79426f9a767SRodney W. Grimes struct buf *bp; 79526f9a767SRodney W. Grimes { 79626f9a767SRodney W. Grimes bp->b_flags |= B_DONE; 79726f9a767SRodney W. Grimes bp->b_flags &= ~B_ASYNC; 79824a1cce3SDavid Greenman wakeup(bp); 79926f9a767SRodney W. Grimes } 80026f9a767SRodney W. Grimes 80126f9a767SRodney W. Grimes int 80224a1cce3SDavid Greenman swap_pager_getpages(object, m, count, reqpage) 80324a1cce3SDavid Greenman vm_object_t object; 80426f9a767SRodney W. Grimes vm_page_t *m; 80526f9a767SRodney W. Grimes int count, reqpage; 806df8bae1dSRodney W. Grimes { 807df8bae1dSRodney W. Grimes register struct buf *bp; 80826f9a767SRodney W. Grimes sw_blk_t swb[count]; 809df8bae1dSRodney W. Grimes register int s; 81026f9a767SRodney W. Grimes int i; 811df8bae1dSRodney W. Grimes boolean_t rv; 81226f9a767SRodney W. Grimes vm_offset_t kva, off[count]; 813df8bae1dSRodney W. Grimes swp_clean_t spc; 81426f9a767SRodney W. Grimes vm_offset_t paging_offset; 81526f9a767SRodney W. Grimes int reqaddr[count]; 8166d40c3d3SDavid Greenman int sequential; 817df8bae1dSRodney W. Grimes 81826f9a767SRodney W. Grimes int first, last; 81926f9a767SRodney W. Grimes int failed; 82026f9a767SRodney W. Grimes int reqdskregion; 821df8bae1dSRodney W. Grimes 82226f9a767SRodney W. Grimes object = m[reqpage]->object; 82326f9a767SRodney W. Grimes paging_offset = object->paging_offset; 8246d40c3d3SDavid Greenman sequential = (m[reqpage]->offset == (object->last_read + PAGE_SIZE)); 8252a4895f4SDavid Greenman 82626f9a767SRodney W. Grimes for (i = 0; i < count; i++) { 82726f9a767SRodney W. Grimes vm_offset_t foff = m[i]->offset + paging_offset; 8282a4895f4SDavid Greenman int ix = swap_pager_block_index(foff); 8290d94caffSDavid Greenman 8302a4895f4SDavid Greenman if (ix >= object->un_pager.swp.swp_nblocks) { 83126f9a767SRodney W. Grimes int j; 8320d94caffSDavid Greenman 83326f9a767SRodney W. Grimes if (i <= reqpage) { 83426f9a767SRodney W. Grimes swap_pager_ridpages(m, count, reqpage); 835df8bae1dSRodney W. Grimes return (VM_PAGER_FAIL); 83626f9a767SRodney W. Grimes } 83726f9a767SRodney W. Grimes for (j = i; j < count; j++) { 83826f9a767SRodney W. Grimes swap_pager_freepage(m[j]); 83926f9a767SRodney W. Grimes } 84026f9a767SRodney W. Grimes count = i; 84126f9a767SRodney W. Grimes break; 84226f9a767SRodney W. Grimes } 8432a4895f4SDavid Greenman swb[i] = &object->un_pager.swp.swp_blocks[ix]; 8442a4895f4SDavid Greenman off[i] = swap_pager_block_offset(foff); 84526f9a767SRodney W. Grimes reqaddr[i] = swb[i]->swb_block[off[i]]; 84626f9a767SRodney W. Grimes } 84726f9a767SRodney W. Grimes 84826f9a767SRodney W. Grimes /* make sure that our required input request is existant */ 84926f9a767SRodney W. Grimes 85026f9a767SRodney W. Grimes if (reqaddr[reqpage] == SWB_EMPTY || 85126f9a767SRodney W. Grimes (swb[reqpage]->swb_valid & (1 << off[reqpage])) == 0) { 85226f9a767SRodney W. Grimes swap_pager_ridpages(m, count, reqpage); 85326f9a767SRodney W. Grimes return (VM_PAGER_FAIL); 85426f9a767SRodney W. Grimes } 85526f9a767SRodney W. Grimes reqdskregion = reqaddr[reqpage] / dmmax; 856df8bae1dSRodney W. Grimes 857df8bae1dSRodney W. Grimes /* 85826f9a767SRodney W. Grimes * search backwards for the first contiguous page to transfer 859df8bae1dSRodney W. Grimes */ 86026f9a767SRodney W. Grimes failed = 0; 86126f9a767SRodney W. Grimes first = 0; 86226f9a767SRodney W. Grimes for (i = reqpage - 1; i >= 0; --i) { 8636d40c3d3SDavid Greenman if (sequential || failed || (reqaddr[i] == SWB_EMPTY) || 86426f9a767SRodney W. Grimes (swb[i]->swb_valid & (1 << off[i])) == 0 || 86526f9a767SRodney W. Grimes (reqaddr[i] != (reqaddr[reqpage] + (i - reqpage) * btodb(PAGE_SIZE))) || 86626f9a767SRodney W. Grimes ((reqaddr[i] / dmmax) != reqdskregion)) { 86726f9a767SRodney W. Grimes failed = 1; 86826f9a767SRodney W. Grimes swap_pager_freepage(m[i]); 86926f9a767SRodney W. Grimes if (first == 0) 87026f9a767SRodney W. Grimes first = i + 1; 87126f9a767SRodney W. Grimes } 872df8bae1dSRodney W. Grimes } 873df8bae1dSRodney W. Grimes /* 87426f9a767SRodney W. Grimes * search forwards for the last contiguous page to transfer 875df8bae1dSRodney W. Grimes */ 87626f9a767SRodney W. Grimes failed = 0; 87726f9a767SRodney W. Grimes last = count; 87826f9a767SRodney W. Grimes for (i = reqpage + 1; i < count; i++) { 87926f9a767SRodney W. Grimes if (failed || (reqaddr[i] == SWB_EMPTY) || 88026f9a767SRodney W. Grimes (swb[i]->swb_valid & (1 << off[i])) == 0 || 88126f9a767SRodney W. Grimes (reqaddr[i] != (reqaddr[reqpage] + (i - reqpage) * btodb(PAGE_SIZE))) || 88226f9a767SRodney W. Grimes ((reqaddr[i] / dmmax) != reqdskregion)) { 88326f9a767SRodney W. Grimes failed = 1; 88426f9a767SRodney W. Grimes swap_pager_freepage(m[i]); 88526f9a767SRodney W. Grimes if (last == count) 88626f9a767SRodney W. Grimes last = i; 88726f9a767SRodney W. Grimes } 88826f9a767SRodney W. Grimes } 88926f9a767SRodney W. Grimes 89026f9a767SRodney W. Grimes count = last; 89126f9a767SRodney W. Grimes if (first != 0) { 89226f9a767SRodney W. Grimes for (i = first; i < count; i++) { 89326f9a767SRodney W. Grimes m[i - first] = m[i]; 89426f9a767SRodney W. Grimes reqaddr[i - first] = reqaddr[i]; 89526f9a767SRodney W. Grimes off[i - first] = off[i]; 89626f9a767SRodney W. Grimes } 89726f9a767SRodney W. Grimes count -= first; 89826f9a767SRodney W. Grimes reqpage -= first; 89926f9a767SRodney W. Grimes } 90026f9a767SRodney W. Grimes ++swb[reqpage]->swb_locked; 90126f9a767SRodney W. Grimes 90226f9a767SRodney W. Grimes /* 9030d94caffSDavid Greenman * at this point: "m" is a pointer to the array of vm_page_t for 9040d94caffSDavid Greenman * paging I/O "count" is the number of vm_page_t entries represented 9050d94caffSDavid Greenman * by "m" "object" is the vm_object_t for I/O "reqpage" is the index 9060d94caffSDavid Greenman * into "m" for the page actually faulted 90726f9a767SRodney W. Grimes */ 90826f9a767SRodney W. Grimes 90926f9a767SRodney W. Grimes spc = NULL; /* we might not use an spc data structure */ 91026f9a767SRodney W. Grimes 911a1f6d91cSDavid Greenman if ((count == 1) && (swap_pager_free.tqh_first != NULL)) { 91226f9a767SRodney W. Grimes /* 9130d94caffSDavid Greenman * if a kva has not been allocated, we can only do a one page 9140d94caffSDavid Greenman * transfer, so we free the other pages that might have been 9150d94caffSDavid Greenman * allocated by vm_fault. 91626f9a767SRodney W. Grimes */ 91726f9a767SRodney W. Grimes swap_pager_ridpages(m, count, reqpage); 91826f9a767SRodney W. Grimes m[0] = m[reqpage]; 91926f9a767SRodney W. Grimes reqaddr[0] = reqaddr[reqpage]; 92026f9a767SRodney W. Grimes 92126f9a767SRodney W. Grimes count = 1; 92226f9a767SRodney W. Grimes reqpage = 0; 92326f9a767SRodney W. Grimes /* 9240d94caffSDavid Greenman * get a swap pager clean data structure, block until we get 9250d94caffSDavid Greenman * it 92626f9a767SRodney W. Grimes */ 927df8bae1dSRodney W. Grimes if (swap_pager_free.tqh_first == NULL) { 928df8bae1dSRodney W. Grimes s = splbio(); 92926f9a767SRodney W. Grimes if (curproc == pageproc) 93024a1cce3SDavid Greenman swap_pager_sync(); 93126f9a767SRodney W. Grimes else 932f919ebdeSDavid Greenman pagedaemon_wakeup(); 93326f9a767SRodney W. Grimes while (swap_pager_free.tqh_first == NULL) { 93426f9a767SRodney W. Grimes swap_pager_needflags |= SWAP_FREE_NEEDED; 935a1f6d91cSDavid Greenman if (curproc == pageproc) 936a1f6d91cSDavid Greenman swap_pager_needflags |= SWAP_FREE_NEEDED_BY_PAGEOUT; 93724a1cce3SDavid Greenman tsleep(&swap_pager_free, 93826f9a767SRodney W. Grimes PVM, "swpfre", 0); 93926f9a767SRodney W. Grimes if (curproc == pageproc) 94024a1cce3SDavid Greenman swap_pager_sync(); 94126f9a767SRodney W. Grimes else 942f919ebdeSDavid Greenman pagedaemon_wakeup(); 943df8bae1dSRodney W. Grimes } 944df8bae1dSRodney W. Grimes splx(s); 94526f9a767SRodney W. Grimes } 94626f9a767SRodney W. Grimes spc = swap_pager_free.tqh_first; 94726f9a767SRodney W. Grimes TAILQ_REMOVE(&swap_pager_free, spc, spc_list); 94826f9a767SRodney W. Grimes kva = spc->spc_kva; 94926f9a767SRodney W. Grimes bp = spc->spc_bp; 95026f9a767SRodney W. Grimes bzero(bp, sizeof *bp); 95126f9a767SRodney W. Grimes bp->b_spc = spc; 9527609ab12SDavid Greenman bp->b_vnbufs.le_next = NOLIST; 95326f9a767SRodney W. Grimes } else { 95416f62314SDavid Greenman /* 95516f62314SDavid Greenman * Get a swap buffer header to perform the IO 95616f62314SDavid Greenman */ 95726f9a767SRodney W. Grimes bp = getpbuf(); 95816f62314SDavid Greenman kva = (vm_offset_t) bp->b_data; 95926f9a767SRodney W. Grimes } 96026f9a767SRodney W. Grimes 96116f62314SDavid Greenman /* 96216f62314SDavid Greenman * map our page(s) into kva for input 96316f62314SDavid Greenman */ 96416f62314SDavid Greenman pmap_qenter(kva, m, count); 96516f62314SDavid Greenman 966aba8f38eSDavid Greenman bp->b_flags = B_BUSY | B_READ | B_CALL | B_PAGING; 96726f9a767SRodney W. Grimes bp->b_iodone = swap_pager_iodone1; 968df8bae1dSRodney W. Grimes bp->b_proc = &proc0; /* XXX (but without B_PHYS set this is ok) */ 96926f9a767SRodney W. Grimes bp->b_rcred = bp->b_wcred = bp->b_proc->p_ucred; 97026f9a767SRodney W. Grimes crhold(bp->b_rcred); 97126f9a767SRodney W. Grimes crhold(bp->b_wcred); 97226f9a767SRodney W. Grimes bp->b_un.b_addr = (caddr_t) kva; 97326f9a767SRodney W. Grimes bp->b_blkno = reqaddr[0]; 97426f9a767SRodney W. Grimes bp->b_bcount = PAGE_SIZE * count; 97526f9a767SRodney W. Grimes bp->b_bufsize = PAGE_SIZE * count; 97626f9a767SRodney W. Grimes 9770d94caffSDavid Greenman pbgetvp(swapdev_vp, bp); 978df8bae1dSRodney W. Grimes 979976e77fcSDavid Greenman cnt.v_swapin++; 980976e77fcSDavid Greenman cnt.v_swappgsin += count; 981df8bae1dSRodney W. Grimes /* 98226f9a767SRodney W. Grimes * perform the I/O 983df8bae1dSRodney W. Grimes */ 984df8bae1dSRodney W. Grimes VOP_STRATEGY(bp); 98526f9a767SRodney W. Grimes 98626f9a767SRodney W. Grimes /* 98726f9a767SRodney W. Grimes * wait for the sync I/O to complete 98826f9a767SRodney W. Grimes */ 9897609ab12SDavid Greenman s = splbio(); 99026f9a767SRodney W. Grimes while ((bp->b_flags & B_DONE) == 0) { 99124a1cce3SDavid Greenman tsleep(bp, PVM, "swread", 0); 992df8bae1dSRodney W. Grimes } 9931b119d9dSDavid Greenman 9941b119d9dSDavid Greenman if (bp->b_flags & B_ERROR) { 9951b119d9dSDavid Greenman printf("swap_pager: I/O error - pagein failed; blkno %d, size %d, error %d\n", 9961b119d9dSDavid Greenman bp->b_blkno, bp->b_bcount, bp->b_error); 997a83c285cSDavid Greenman rv = VM_PAGER_ERROR; 9981b119d9dSDavid Greenman } else { 9991b119d9dSDavid Greenman rv = VM_PAGER_OK; 10001b119d9dSDavid Greenman } 100126f9a767SRodney W. Grimes 100226f9a767SRodney W. Grimes /* 10030d94caffSDavid Greenman * relpbuf does this, but we maintain our own buffer list also... 100426f9a767SRodney W. Grimes */ 1005df8bae1dSRodney W. Grimes if (bp->b_vp) 10060d94caffSDavid Greenman pbrelvp(bp); 100726f9a767SRodney W. Grimes 1008df8bae1dSRodney W. Grimes splx(s); 10092a4895f4SDavid Greenman swb[reqpage]->swb_locked--; 101026f9a767SRodney W. Grimes 101126f9a767SRodney W. Grimes /* 101226f9a767SRodney W. Grimes * remove the mapping for kernel virtual 101326f9a767SRodney W. Grimes */ 101416f62314SDavid Greenman pmap_qremove(kva, count); 101526f9a767SRodney W. Grimes 101626f9a767SRodney W. Grimes if (spc) { 10176d40c3d3SDavid Greenman m[reqpage]->object->last_read = m[reqpage]->offset; 10180d94caffSDavid Greenman if (bp->b_flags & B_WANTED) 101924a1cce3SDavid Greenman wakeup(bp); 102026f9a767SRodney W. Grimes /* 102126f9a767SRodney W. Grimes * if we have used an spc, we need to free it. 102226f9a767SRodney W. Grimes */ 102326f9a767SRodney W. Grimes if (bp->b_rcred != NOCRED) 102426f9a767SRodney W. Grimes crfree(bp->b_rcred); 102526f9a767SRodney W. Grimes if (bp->b_wcred != NOCRED) 102626f9a767SRodney W. Grimes crfree(bp->b_wcred); 102726f9a767SRodney W. Grimes TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 102826f9a767SRodney W. Grimes if (swap_pager_needflags & SWAP_FREE_NEEDED) { 102924a1cce3SDavid Greenman wakeup(&swap_pager_free); 103026f9a767SRodney W. Grimes } 1031a1f6d91cSDavid Greenman if (swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) 1032f919ebdeSDavid Greenman pagedaemon_wakeup(); 1033a1f6d91cSDavid Greenman swap_pager_needflags &= ~(SWAP_FREE_NEEDED|SWAP_FREE_NEEDED_BY_PAGEOUT); 103426f9a767SRodney W. Grimes } else { 103526f9a767SRodney W. Grimes /* 103626f9a767SRodney W. Grimes * release the physical I/O buffer 103726f9a767SRodney W. Grimes */ 103826f9a767SRodney W. Grimes relpbuf(bp); 103926f9a767SRodney W. Grimes /* 104026f9a767SRodney W. Grimes * finish up input if everything is ok 104126f9a767SRodney W. Grimes */ 104226f9a767SRodney W. Grimes if (rv == VM_PAGER_OK) { 104326f9a767SRodney W. Grimes for (i = 0; i < count; i++) { 104426f9a767SRodney W. Grimes pmap_clear_modify(VM_PAGE_TO_PHYS(m[i])); 10450d94caffSDavid Greenman m[i]->dirty = 0; 1046894048d7SJohn Dyson m[i]->flags &= ~PG_ZERO; 104726f9a767SRodney W. Grimes if (i != reqpage) { 104826f9a767SRodney W. Grimes /* 10490d94caffSDavid Greenman * whether or not to leave the page 10500d94caffSDavid Greenman * activated is up in the air, but we 10510d94caffSDavid Greenman * should put the page on a page queue 10520d94caffSDavid Greenman * somewhere. (it already is in the 10530d94caffSDavid Greenman * object). After some emperical 10540d94caffSDavid Greenman * results, it is best to deactivate 10550d94caffSDavid Greenman * the readahead pages. 105626f9a767SRodney W. Grimes */ 105726f9a767SRodney W. Grimes vm_page_deactivate(m[i]); 105826f9a767SRodney W. Grimes 105926f9a767SRodney W. Grimes /* 10600d94caffSDavid Greenman * just in case someone was asking for 10610d94caffSDavid Greenman * this page we now tell them that it 10620d94caffSDavid Greenman * is ok to use 106326f9a767SRodney W. Grimes */ 10640d94caffSDavid Greenman m[i]->valid = VM_PAGE_BITS_ALL; 106526f9a767SRodney W. Grimes PAGE_WAKEUP(m[i]); 106626f9a767SRodney W. Grimes } 106726f9a767SRodney W. Grimes } 10686d40c3d3SDavid Greenman 10696d40c3d3SDavid Greenman m[reqpage]->object->last_read = m[count-1]->offset; 10706d40c3d3SDavid Greenman 10712e1e24ddSDavid Greenman /* 10722e1e24ddSDavid Greenman * If we're out of swap space, then attempt to free 10732e1e24ddSDavid Greenman * some whenever pages are brought in. We must clear 10742e1e24ddSDavid Greenman * the clean flag so that the page contents will be 10752e1e24ddSDavid Greenman * preserved. 10762e1e24ddSDavid Greenman */ 107726f9a767SRodney W. Grimes if (swap_pager_full) { 10782e1e24ddSDavid Greenman for (i = 0; i < count; i++) { 10790d94caffSDavid Greenman m[i]->dirty = VM_PAGE_BITS_ALL; 10802e1e24ddSDavid Greenman } 108124a1cce3SDavid Greenman swap_pager_freespace(object, m[0]->offset + paging_offset, count * PAGE_SIZE); 108226f9a767SRodney W. Grimes } 108326f9a767SRodney W. Grimes } else { 108426f9a767SRodney W. Grimes swap_pager_ridpages(m, count, reqpage); 108526f9a767SRodney W. Grimes } 108626f9a767SRodney W. Grimes } 108723922ccaSDavid Greenman if (rv == VM_PAGER_OK) { 108823922ccaSDavid Greenman pmap_clear_modify(VM_PAGE_TO_PHYS(m[reqpage])); 108923922ccaSDavid Greenman m[reqpage]->valid = VM_PAGE_BITS_ALL; 109023922ccaSDavid Greenman m[reqpage]->dirty = 0; 109123922ccaSDavid Greenman } 1092df8bae1dSRodney W. Grimes return (rv); 1093df8bae1dSRodney W. Grimes } 1094df8bae1dSRodney W. Grimes 109526f9a767SRodney W. Grimes int 109624a1cce3SDavid Greenman swap_pager_putpages(object, m, count, sync, rtvals) 109724a1cce3SDavid Greenman vm_object_t object; 109826f9a767SRodney W. Grimes vm_page_t *m; 109926f9a767SRodney W. Grimes int count; 110024a1cce3SDavid Greenman boolean_t sync; 110126f9a767SRodney W. Grimes int *rtvals; 1102df8bae1dSRodney W. Grimes { 110326f9a767SRodney W. Grimes register struct buf *bp; 110426f9a767SRodney W. Grimes sw_blk_t swb[count]; 110526f9a767SRodney W. Grimes register int s; 110626f9a767SRodney W. Grimes int i, j, ix; 110726f9a767SRodney W. Grimes boolean_t rv; 110826f9a767SRodney W. Grimes vm_offset_t kva, off, foff; 110926f9a767SRodney W. Grimes swp_clean_t spc; 111026f9a767SRodney W. Grimes vm_offset_t paging_offset; 111126f9a767SRodney W. Grimes int reqaddr[count]; 111226f9a767SRodney W. Grimes int failed; 1113df8bae1dSRodney W. Grimes 111424ea4a96SDavid Greenman if (vm_swap_size) 111524ea4a96SDavid Greenman no_swap_space = 0; 111624ea4a96SDavid Greenman if (no_swap_space) { 11175663e6deSDavid Greenman for (i = 0; i < count; i++) 11185663e6deSDavid Greenman rtvals[i] = VM_PAGER_FAIL; 11195663e6deSDavid Greenman return VM_PAGER_FAIL; 11205663e6deSDavid Greenman } 112126f9a767SRodney W. Grimes spc = NULL; 112226f9a767SRodney W. Grimes 112326f9a767SRodney W. Grimes object = m[0]->object; 112426f9a767SRodney W. Grimes paging_offset = object->paging_offset; 112526f9a767SRodney W. Grimes 112626f9a767SRodney W. Grimes failed = 0; 112726f9a767SRodney W. Grimes for (j = 0; j < count; j++) { 112826f9a767SRodney W. Grimes foff = m[j]->offset + paging_offset; 11292a4895f4SDavid Greenman ix = swap_pager_block_index(foff); 113026f9a767SRodney W. Grimes swb[j] = 0; 11312a4895f4SDavid Greenman if (ix >= object->un_pager.swp.swp_nblocks) { 113226f9a767SRodney W. Grimes rtvals[j] = VM_PAGER_FAIL; 113326f9a767SRodney W. Grimes failed = 1; 113426f9a767SRodney W. Grimes continue; 113526f9a767SRodney W. Grimes } else { 113626f9a767SRodney W. Grimes rtvals[j] = VM_PAGER_OK; 113726f9a767SRodney W. Grimes } 11382a4895f4SDavid Greenman swb[j] = &object->un_pager.swp.swp_blocks[ix]; 11392a4895f4SDavid Greenman swb[j]->swb_locked++; 114026f9a767SRodney W. Grimes if (failed) { 114126f9a767SRodney W. Grimes rtvals[j] = VM_PAGER_FAIL; 114226f9a767SRodney W. Grimes continue; 114326f9a767SRodney W. Grimes } 11442a4895f4SDavid Greenman off = swap_pager_block_offset(foff); 114526f9a767SRodney W. Grimes reqaddr[j] = swb[j]->swb_block[off]; 114626f9a767SRodney W. Grimes if (reqaddr[j] == SWB_EMPTY) { 114726f9a767SRodney W. Grimes int blk; 114826f9a767SRodney W. Grimes int tries; 114926f9a767SRodney W. Grimes int ntoget; 11500d94caffSDavid Greenman 115126f9a767SRodney W. Grimes tries = 0; 1152df8bae1dSRodney W. Grimes s = splbio(); 115326f9a767SRodney W. Grimes 1154df8bae1dSRodney W. Grimes /* 11550d94caffSDavid Greenman * if any other pages have been allocated in this 11560d94caffSDavid Greenman * block, we only try to get one page. 1157df8bae1dSRodney W. Grimes */ 115826f9a767SRodney W. Grimes for (i = 0; i < SWB_NPAGES; i++) { 115926f9a767SRodney W. Grimes if (swb[j]->swb_block[i] != SWB_EMPTY) 1160df8bae1dSRodney W. Grimes break; 1161df8bae1dSRodney W. Grimes } 116226f9a767SRodney W. Grimes 116326f9a767SRodney W. Grimes ntoget = (i == SWB_NPAGES) ? SWB_NPAGES : 1; 116426f9a767SRodney W. Grimes /* 11650d94caffSDavid Greenman * this code is alittle conservative, but works (the 11660d94caffSDavid Greenman * intent of this code is to allocate small chunks for 11670d94caffSDavid Greenman * small objects) 116826f9a767SRodney W. Grimes */ 116911fda60bSJohn Dyson if ((foff == 0) && 117011fda60bSJohn Dyson ((ntoget * PAGE_SIZE) > object->size)) { 117111fda60bSJohn Dyson ntoget = (object->size + (PAGE_SIZE - 1)) / PAGE_SIZE; 117226f9a767SRodney W. Grimes } 117326f9a767SRodney W. Grimes retrygetspace: 117426f9a767SRodney W. Grimes if (!swap_pager_full && ntoget > 1 && 11752a4895f4SDavid Greenman swap_pager_getswapspace(object, ntoget * btodb(PAGE_SIZE), &blk)) { 117626f9a767SRodney W. Grimes 117726f9a767SRodney W. Grimes for (i = 0; i < ntoget; i++) { 117826f9a767SRodney W. Grimes swb[j]->swb_block[i] = blk + btodb(PAGE_SIZE) * i; 117926f9a767SRodney W. Grimes swb[j]->swb_valid = 0; 118026f9a767SRodney W. Grimes } 118126f9a767SRodney W. Grimes 118226f9a767SRodney W. Grimes reqaddr[j] = swb[j]->swb_block[off]; 11832a4895f4SDavid Greenman } else if (!swap_pager_getswapspace(object, btodb(PAGE_SIZE), 118426f9a767SRodney W. Grimes &swb[j]->swb_block[off])) { 118526f9a767SRodney W. Grimes /* 11860d94caffSDavid Greenman * if the allocation has failed, we try to 11870d94caffSDavid Greenman * reclaim space and retry. 118826f9a767SRodney W. Grimes */ 118926f9a767SRodney W. Grimes if (++tries == 1) { 119026f9a767SRodney W. Grimes swap_pager_reclaim(); 119126f9a767SRodney W. Grimes goto retrygetspace; 119226f9a767SRodney W. Grimes } 119326f9a767SRodney W. Grimes rtvals[j] = VM_PAGER_AGAIN; 119426f9a767SRodney W. Grimes failed = 1; 119524ea4a96SDavid Greenman swap_pager_full = 1; 119626f9a767SRodney W. Grimes } else { 119726f9a767SRodney W. Grimes reqaddr[j] = swb[j]->swb_block[off]; 119826f9a767SRodney W. Grimes swb[j]->swb_valid &= ~(1 << off); 1199df8bae1dSRodney W. Grimes } 1200df8bae1dSRodney W. Grimes splx(s); 120126f9a767SRodney W. Grimes } 120226f9a767SRodney W. Grimes } 120326f9a767SRodney W. Grimes 120426f9a767SRodney W. Grimes /* 120526f9a767SRodney W. Grimes * search forwards for the last contiguous page to transfer 120626f9a767SRodney W. Grimes */ 120726f9a767SRodney W. Grimes failed = 0; 120826f9a767SRodney W. Grimes for (i = 0; i < count; i++) { 120926f9a767SRodney W. Grimes if (failed || (reqaddr[i] != reqaddr[0] + i * btodb(PAGE_SIZE)) || 121026f9a767SRodney W. Grimes (reqaddr[i] / dmmax) != (reqaddr[0] / dmmax) || 121126f9a767SRodney W. Grimes (rtvals[i] != VM_PAGER_OK)) { 121226f9a767SRodney W. Grimes failed = 1; 121326f9a767SRodney W. Grimes if (rtvals[i] == VM_PAGER_OK) 121426f9a767SRodney W. Grimes rtvals[i] = VM_PAGER_AGAIN; 121526f9a767SRodney W. Grimes } 121626f9a767SRodney W. Grimes } 121726f9a767SRodney W. Grimes 121826f9a767SRodney W. Grimes for (i = 0; i < count; i++) { 121926f9a767SRodney W. Grimes if (rtvals[i] != VM_PAGER_OK) { 122026f9a767SRodney W. Grimes if (swb[i]) 122126f9a767SRodney W. Grimes --swb[i]->swb_locked; 122226f9a767SRodney W. Grimes } 122326f9a767SRodney W. Grimes } 122426f9a767SRodney W. Grimes 122526f9a767SRodney W. Grimes for (i = 0; i < count; i++) 122626f9a767SRodney W. Grimes if (rtvals[i] != VM_PAGER_OK) 122726f9a767SRodney W. Grimes break; 122826f9a767SRodney W. Grimes 122926f9a767SRodney W. Grimes if (i == 0) { 123026f9a767SRodney W. Grimes return VM_PAGER_AGAIN; 123126f9a767SRodney W. Grimes } 123226f9a767SRodney W. Grimes count = i; 123326f9a767SRodney W. Grimes for (i = 0; i < count; i++) { 123426f9a767SRodney W. Grimes if (reqaddr[i] == SWB_EMPTY) 123526f9a767SRodney W. Grimes printf("I/O to empty block????\n"); 123626f9a767SRodney W. Grimes } 123726f9a767SRodney W. Grimes 123826f9a767SRodney W. Grimes /* 12390d94caffSDavid Greenman * For synchronous writes, we clean up all completed async pageouts. 124026f9a767SRodney W. Grimes */ 124124a1cce3SDavid Greenman if (sync == TRUE) { 124224a1cce3SDavid Greenman swap_pager_sync(); 124326f9a767SRodney W. Grimes } 124426f9a767SRodney W. Grimes kva = 0; 124526f9a767SRodney W. Grimes 124626f9a767SRodney W. Grimes /* 124726f9a767SRodney W. Grimes * get a swap pager clean data structure, block until we get it 124826f9a767SRodney W. Grimes */ 1249a1f6d91cSDavid Greenman if (swap_pager_free.tqh_first == NULL || 1250a1f6d91cSDavid Greenman swap_pager_free.tqh_first->spc_list.tqe_next == NULL || 1251a1f6d91cSDavid Greenman swap_pager_free.tqh_first->spc_list.tqe_next->spc_list.tqe_next == NULL) { 125226f9a767SRodney W. Grimes s = splbio(); 12530d94caffSDavid Greenman if (curproc == pageproc) { 125424a1cce3SDavid Greenman swap_pager_sync(); 1255a1f6d91cSDavid Greenman #if 0 12560d94caffSDavid Greenman splx(s); 12570d94caffSDavid Greenman return VM_PAGER_AGAIN; 1258a1f6d91cSDavid Greenman #endif 12590d94caffSDavid Greenman } else 1260f919ebdeSDavid Greenman pagedaemon_wakeup(); 1261a1f6d91cSDavid Greenman while (swap_pager_free.tqh_first == NULL || 1262a1f6d91cSDavid Greenman swap_pager_free.tqh_first->spc_list.tqe_next == NULL || 1263a1f6d91cSDavid Greenman swap_pager_free.tqh_first->spc_list.tqe_next->spc_list.tqe_next == NULL) { 1264a1f6d91cSDavid Greenman if (curproc == pageproc) { 1265a1f6d91cSDavid Greenman swap_pager_needflags |= SWAP_FREE_NEEDED_BY_PAGEOUT; 1266a1f6d91cSDavid Greenman if((cnt.v_free_count + cnt.v_cache_count) > cnt.v_free_reserved) 126724a1cce3SDavid Greenman wakeup(&cnt.v_free_count); 1268a1f6d91cSDavid Greenman } 12690d94caffSDavid Greenman 127026f9a767SRodney W. Grimes swap_pager_needflags |= SWAP_FREE_NEEDED; 127124a1cce3SDavid Greenman tsleep(&swap_pager_free, PVM, "swpfre", 0); 127226f9a767SRodney W. Grimes if (curproc == pageproc) 127324a1cce3SDavid Greenman swap_pager_sync(); 127426f9a767SRodney W. Grimes else 1275f919ebdeSDavid Greenman pagedaemon_wakeup(); 127626f9a767SRodney W. Grimes } 127726f9a767SRodney W. Grimes splx(s); 127826f9a767SRodney W. Grimes } 127926f9a767SRodney W. Grimes spc = swap_pager_free.tqh_first; 128026f9a767SRodney W. Grimes TAILQ_REMOVE(&swap_pager_free, spc, spc_list); 1281fff93ab6SDavid Greenman 128226f9a767SRodney W. Grimes kva = spc->spc_kva; 128326f9a767SRodney W. Grimes 128426f9a767SRodney W. Grimes /* 128526f9a767SRodney W. Grimes * map our page(s) into kva for I/O 128626f9a767SRodney W. Grimes */ 128716f62314SDavid Greenman pmap_qenter(kva, m, count); 128826f9a767SRodney W. Grimes 128926f9a767SRodney W. Grimes /* 129026f9a767SRodney W. Grimes * get the base I/O offset into the swap file 129126f9a767SRodney W. Grimes */ 129226f9a767SRodney W. Grimes for (i = 0; i < count; i++) { 129326f9a767SRodney W. Grimes foff = m[i]->offset + paging_offset; 12942a4895f4SDavid Greenman off = swap_pager_block_offset(foff); 129526f9a767SRodney W. Grimes /* 129626f9a767SRodney W. Grimes * set the valid bit 129726f9a767SRodney W. Grimes */ 129826f9a767SRodney W. Grimes swb[i]->swb_valid |= (1 << off); 129926f9a767SRodney W. Grimes /* 130026f9a767SRodney W. Grimes * and unlock the data structure 130126f9a767SRodney W. Grimes */ 13022a4895f4SDavid Greenman swb[i]->swb_locked--; 130326f9a767SRodney W. Grimes } 130426f9a767SRodney W. Grimes 130526f9a767SRodney W. Grimes /* 130626f9a767SRodney W. Grimes * Get a swap buffer header and perform the IO 130726f9a767SRodney W. Grimes */ 130826f9a767SRodney W. Grimes bp = spc->spc_bp; 130926f9a767SRodney W. Grimes bzero(bp, sizeof *bp); 131026f9a767SRodney W. Grimes bp->b_spc = spc; 13117609ab12SDavid Greenman bp->b_vnbufs.le_next = NOLIST; 131226f9a767SRodney W. Grimes 1313aba8f38eSDavid Greenman bp->b_flags = B_BUSY | B_PAGING; 131426f9a767SRodney W. Grimes bp->b_proc = &proc0; /* XXX (but without B_PHYS set this is ok) */ 131526f9a767SRodney W. Grimes bp->b_rcred = bp->b_wcred = bp->b_proc->p_ucred; 1316a481f200SDavid Greenman if (bp->b_rcred != NOCRED) 131726f9a767SRodney W. Grimes crhold(bp->b_rcred); 1318a481f200SDavid Greenman if (bp->b_wcred != NOCRED) 131926f9a767SRodney W. Grimes crhold(bp->b_wcred); 1320a481f200SDavid Greenman bp->b_data = (caddr_t) kva; 132126f9a767SRodney W. Grimes bp->b_blkno = reqaddr[0]; 13220d94caffSDavid Greenman pbgetvp(swapdev_vp, bp); 132316f62314SDavid Greenman 132426f9a767SRodney W. Grimes bp->b_bcount = PAGE_SIZE * count; 132526f9a767SRodney W. Grimes bp->b_bufsize = PAGE_SIZE * count; 132626f9a767SRodney W. Grimes swapdev_vp->v_numoutput++; 132726f9a767SRodney W. Grimes 132826f9a767SRodney W. Grimes /* 13290d94caffSDavid Greenman * If this is an async write we set up additional buffer fields and 13300d94caffSDavid Greenman * place a "cleaning" entry on the inuse queue. 133126f9a767SRodney W. Grimes */ 13327609ab12SDavid Greenman s = splbio(); 133324a1cce3SDavid Greenman if (sync == FALSE) { 133426f9a767SRodney W. Grimes spc->spc_flags = 0; 13352a4895f4SDavid Greenman spc->spc_object = object; 133626f9a767SRodney W. Grimes for (i = 0; i < count; i++) 133726f9a767SRodney W. Grimes spc->spc_m[i] = m[i]; 133826f9a767SRodney W. Grimes spc->spc_count = count; 133926f9a767SRodney W. Grimes /* 134026f9a767SRodney W. Grimes * the completion routine for async writes 134126f9a767SRodney W. Grimes */ 134226f9a767SRodney W. Grimes bp->b_flags |= B_CALL; 134326f9a767SRodney W. Grimes bp->b_iodone = swap_pager_iodone; 134426f9a767SRodney W. Grimes bp->b_dirtyoff = 0; 134526f9a767SRodney W. Grimes bp->b_dirtyend = bp->b_bcount; 13462a4895f4SDavid Greenman object->un_pager.swp.swp_poip++; 134726f9a767SRodney W. Grimes TAILQ_INSERT_TAIL(&swap_pager_inuse, spc, spc_list); 134826f9a767SRodney W. Grimes } else { 13492a4895f4SDavid Greenman object->un_pager.swp.swp_poip++; 135026f9a767SRodney W. Grimes bp->b_flags |= B_CALL; 135126f9a767SRodney W. Grimes bp->b_iodone = swap_pager_iodone1; 135226f9a767SRodney W. Grimes } 1353976e77fcSDavid Greenman 1354976e77fcSDavid Greenman cnt.v_swapout++; 1355976e77fcSDavid Greenman cnt.v_swappgsout += count; 135626f9a767SRodney W. Grimes /* 135726f9a767SRodney W. Grimes * perform the I/O 135826f9a767SRodney W. Grimes */ 135926f9a767SRodney W. Grimes VOP_STRATEGY(bp); 136024a1cce3SDavid Greenman if (sync == FALSE) { 136126f9a767SRodney W. Grimes if ((bp->b_flags & B_DONE) == B_DONE) { 136224a1cce3SDavid Greenman swap_pager_sync(); 136326f9a767SRodney W. Grimes } 136426f9a767SRodney W. Grimes splx(s); 136526f9a767SRodney W. Grimes for (i = 0; i < count; i++) { 136626f9a767SRodney W. Grimes rtvals[i] = VM_PAGER_PEND; 136726f9a767SRodney W. Grimes } 136826f9a767SRodney W. Grimes return VM_PAGER_PEND; 136926f9a767SRodney W. Grimes } 137026f9a767SRodney W. Grimes /* 137126f9a767SRodney W. Grimes * wait for the sync I/O to complete 137226f9a767SRodney W. Grimes */ 137326f9a767SRodney W. Grimes while ((bp->b_flags & B_DONE) == 0) { 137424a1cce3SDavid Greenman tsleep(bp, PVM, "swwrt", 0); 137526f9a767SRodney W. Grimes } 13761b119d9dSDavid Greenman if (bp->b_flags & B_ERROR) { 13771b119d9dSDavid Greenman printf("swap_pager: I/O error - pageout failed; blkno %d, size %d, error %d\n", 13781b119d9dSDavid Greenman bp->b_blkno, bp->b_bcount, bp->b_error); 1379a83c285cSDavid Greenman rv = VM_PAGER_ERROR; 13801b119d9dSDavid Greenman } else { 13811b119d9dSDavid Greenman rv = VM_PAGER_OK; 13821b119d9dSDavid Greenman } 138326f9a767SRodney W. Grimes 13842a4895f4SDavid Greenman object->un_pager.swp.swp_poip--; 13852a4895f4SDavid Greenman if (object->un_pager.swp.swp_poip == 0) 13862a4895f4SDavid Greenman wakeup(object); 138726f9a767SRodney W. Grimes 138826f9a767SRodney W. Grimes if (bp->b_vp) 13890d94caffSDavid Greenman pbrelvp(bp); 13900d94caffSDavid Greenman if (bp->b_flags & B_WANTED) 139124a1cce3SDavid Greenman wakeup(bp); 139226f9a767SRodney W. Grimes 139326f9a767SRodney W. Grimes splx(s); 139426f9a767SRodney W. Grimes 139526f9a767SRodney W. Grimes /* 139626f9a767SRodney W. Grimes * remove the mapping for kernel virtual 139726f9a767SRodney W. Grimes */ 139816f62314SDavid Greenman pmap_qremove(kva, count); 139926f9a767SRodney W. Grimes 140026f9a767SRodney W. Grimes /* 14010d94caffSDavid Greenman * if we have written the page, then indicate that the page is clean. 140226f9a767SRodney W. Grimes */ 140326f9a767SRodney W. Grimes if (rv == VM_PAGER_OK) { 140426f9a767SRodney W. Grimes for (i = 0; i < count; i++) { 140526f9a767SRodney W. Grimes if (rtvals[i] == VM_PAGER_OK) { 140626f9a767SRodney W. Grimes pmap_clear_modify(VM_PAGE_TO_PHYS(m[i])); 14070d94caffSDavid Greenman m[i]->dirty = 0; 140826f9a767SRodney W. Grimes /* 14090d94caffSDavid Greenman * optimization, if a page has been read 14100d94caffSDavid Greenman * during the pageout process, we activate it. 141126f9a767SRodney W. Grimes */ 141226f9a767SRodney W. Grimes if ((m[i]->flags & PG_ACTIVE) == 0 && 14137fb0c17eSDavid Greenman ((m[i]->flags & (PG_WANTED|PG_REFERENCED)) || 14147fb0c17eSDavid Greenman pmap_is_referenced(VM_PAGE_TO_PHYS(m[i])))) { 141526f9a767SRodney W. Grimes vm_page_activate(m[i]); 141626f9a767SRodney W. Grimes } 141726f9a767SRodney W. Grimes } 14187fb0c17eSDavid Greenman } 141926f9a767SRodney W. Grimes } else { 142026f9a767SRodney W. Grimes for (i = 0; i < count; i++) { 142126f9a767SRodney W. Grimes rtvals[i] = rv; 142226f9a767SRodney W. Grimes } 142326f9a767SRodney W. Grimes } 142426f9a767SRodney W. Grimes 142526f9a767SRodney W. Grimes if (bp->b_rcred != NOCRED) 142626f9a767SRodney W. Grimes crfree(bp->b_rcred); 142726f9a767SRodney W. Grimes if (bp->b_wcred != NOCRED) 142826f9a767SRodney W. Grimes crfree(bp->b_wcred); 142926f9a767SRodney W. Grimes TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 143026f9a767SRodney W. Grimes if (swap_pager_needflags & SWAP_FREE_NEEDED) { 143124a1cce3SDavid Greenman wakeup(&swap_pager_free); 143226f9a767SRodney W. Grimes } 1433a1f6d91cSDavid Greenman if (swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) 1434f919ebdeSDavid Greenman pagedaemon_wakeup(); 1435a1f6d91cSDavid Greenman swap_pager_needflags &= ~(SWAP_FREE_NEEDED|SWAP_FREE_NEEDED_BY_PAGEOUT); 143626f9a767SRodney W. Grimes return (rv); 143726f9a767SRodney W. Grimes } 143826f9a767SRodney W. Grimes 143924a1cce3SDavid Greenman void 144024a1cce3SDavid Greenman swap_pager_sync() 144126f9a767SRodney W. Grimes { 144226f9a767SRodney W. Grimes register swp_clean_t spc, tspc; 144326f9a767SRodney W. Grimes register int s; 144426f9a767SRodney W. Grimes 144526f9a767SRodney W. Grimes tspc = NULL; 144626f9a767SRodney W. Grimes if (swap_pager_done.tqh_first == NULL) 144724a1cce3SDavid Greenman return; 144826f9a767SRodney W. Grimes for (;;) { 144926f9a767SRodney W. Grimes s = splbio(); 145026f9a767SRodney W. Grimes /* 14510d94caffSDavid Greenman * Look up and removal from done list must be done at splbio() 14520d94caffSDavid Greenman * to avoid conflicts with swap_pager_iodone. 145326f9a767SRodney W. Grimes */ 145405f0fdd2SPoul-Henning Kamp while ((spc = swap_pager_done.tqh_first) != 0) { 1455fff93ab6SDavid Greenman pmap_qremove(spc->spc_kva, spc->spc_count); 145626f9a767SRodney W. Grimes swap_pager_finish(spc); 145726f9a767SRodney W. Grimes TAILQ_REMOVE(&swap_pager_done, spc, spc_list); 145826f9a767SRodney W. Grimes goto doclean; 145926f9a767SRodney W. Grimes } 1460df8bae1dSRodney W. Grimes 1461df8bae1dSRodney W. Grimes /* 1462df8bae1dSRodney W. Grimes * No operations done, thats all we can do for now. 1463df8bae1dSRodney W. Grimes */ 146426f9a767SRodney W. Grimes 146526f9a767SRodney W. Grimes splx(s); 1466df8bae1dSRodney W. Grimes break; 1467df8bae1dSRodney W. Grimes 1468df8bae1dSRodney W. Grimes /* 14690d94caffSDavid Greenman * The desired page was found to be busy earlier in the scan 14700d94caffSDavid Greenman * but has since completed. 1471df8bae1dSRodney W. Grimes */ 147226f9a767SRodney W. Grimes doclean: 147326f9a767SRodney W. Grimes if (tspc && tspc == spc) { 147426f9a767SRodney W. Grimes tspc = NULL; 147526f9a767SRodney W. Grimes } 147626f9a767SRodney W. Grimes spc->spc_flags = 0; 147726f9a767SRodney W. Grimes TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 147826f9a767SRodney W. Grimes if (swap_pager_needflags & SWAP_FREE_NEEDED) { 147924a1cce3SDavid Greenman wakeup(&swap_pager_free); 148026f9a767SRodney W. Grimes } 1481a1f6d91cSDavid Greenman if( swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) 1482f919ebdeSDavid Greenman pagedaemon_wakeup(); 1483a1f6d91cSDavid Greenman swap_pager_needflags &= ~(SWAP_FREE_NEEDED|SWAP_FREE_NEEDED_BY_PAGEOUT); 148426f9a767SRodney W. Grimes splx(s); 148526f9a767SRodney W. Grimes } 148626f9a767SRodney W. Grimes 148724a1cce3SDavid Greenman return; 148826f9a767SRodney W. Grimes } 148926f9a767SRodney W. Grimes 149026f9a767SRodney W. Grimes void 149126f9a767SRodney W. Grimes swap_pager_finish(spc) 149226f9a767SRodney W. Grimes register swp_clean_t spc; 149326f9a767SRodney W. Grimes { 149426f9a767SRodney W. Grimes vm_object_t object = spc->spc_m[0]->object; 149526f9a767SRodney W. Grimes int i; 149626f9a767SRodney W. Grimes 1497c0503609SDavid Greenman object->paging_in_progress -= spc->spc_count; 1498c0503609SDavid Greenman if ((object->paging_in_progress == 0) && 1499c0503609SDavid Greenman (object->flags & OBJ_PIPWNT)) { 1500c0503609SDavid Greenman object->flags &= ~OBJ_PIPWNT; 150124a1cce3SDavid Greenman wakeup(object); 1502c0503609SDavid Greenman } 1503df8bae1dSRodney W. Grimes 1504df8bae1dSRodney W. Grimes /* 15055f55e841SDavid Greenman * If no error, mark as clean and inform the pmap system. If error, 15060d94caffSDavid Greenman * mark as dirty so we will try again. (XXX could get stuck doing 15070d94caffSDavid Greenman * this, should give up after awhile) 1508df8bae1dSRodney W. Grimes */ 1509df8bae1dSRodney W. Grimes if (spc->spc_flags & SPC_ERROR) { 151026f9a767SRodney W. Grimes for (i = 0; i < spc->spc_count; i++) { 1511a83c285cSDavid Greenman printf("swap_pager_finish: I/O error, clean of page %lx failed\n", 151205f0fdd2SPoul-Henning Kamp (u_long) VM_PAGE_TO_PHYS(spc->spc_m[i])); 151326f9a767SRodney W. Grimes } 1514df8bae1dSRodney W. Grimes } else { 151526f9a767SRodney W. Grimes for (i = 0; i < spc->spc_count; i++) { 151626f9a767SRodney W. Grimes pmap_clear_modify(VM_PAGE_TO_PHYS(spc->spc_m[i])); 15170d94caffSDavid Greenman spc->spc_m[i]->dirty = 0; 15180d94caffSDavid Greenman if ((spc->spc_m[i]->flags & PG_ACTIVE) == 0 && 15190d94caffSDavid Greenman ((spc->spc_m[i]->flags & PG_WANTED) || pmap_is_referenced(VM_PAGE_TO_PHYS(spc->spc_m[i])))) 15200d94caffSDavid Greenman vm_page_activate(spc->spc_m[i]); 1521df8bae1dSRodney W. Grimes } 1522df8bae1dSRodney W. Grimes } 1523df8bae1dSRodney W. Grimes 152426f9a767SRodney W. Grimes 152526f9a767SRodney W. Grimes for (i = 0; i < spc->spc_count; i++) { 1526df8bae1dSRodney W. Grimes /* 15270d94caffSDavid Greenman * we wakeup any processes that are waiting on these pages. 1528df8bae1dSRodney W. Grimes */ 152926f9a767SRodney W. Grimes PAGE_WAKEUP(spc->spc_m[i]); 1530df8bae1dSRodney W. Grimes } 153126f9a767SRodney W. Grimes nswiodone -= spc->spc_count; 1532df8bae1dSRodney W. Grimes 1533df8bae1dSRodney W. Grimes return; 153426f9a767SRodney W. Grimes } 1535df8bae1dSRodney W. Grimes 153626f9a767SRodney W. Grimes /* 153726f9a767SRodney W. Grimes * swap_pager_iodone 153826f9a767SRodney W. Grimes */ 1539f5a12711SPoul-Henning Kamp static void 1540df8bae1dSRodney W. Grimes swap_pager_iodone(bp) 1541df8bae1dSRodney W. Grimes register struct buf *bp; 1542df8bae1dSRodney W. Grimes { 1543df8bae1dSRodney W. Grimes register swp_clean_t spc; 1544df8bae1dSRodney W. Grimes int s; 1545df8bae1dSRodney W. Grimes 1546df8bae1dSRodney W. Grimes s = splbio(); 154726f9a767SRodney W. Grimes spc = (swp_clean_t) bp->b_spc; 154826f9a767SRodney W. Grimes TAILQ_REMOVE(&swap_pager_inuse, spc, spc_list); 154926f9a767SRodney W. Grimes TAILQ_INSERT_TAIL(&swap_pager_done, spc, spc_list); 155026f9a767SRodney W. Grimes if (bp->b_flags & B_ERROR) { 1551df8bae1dSRodney W. Grimes spc->spc_flags |= SPC_ERROR; 1552c3a1e425SDavid Greenman printf("swap_pager: I/O error - async %s failed; blkno %lu, size %ld, error %d\n", 15531b119d9dSDavid Greenman (bp->b_flags & B_READ) ? "pagein" : "pageout", 1554c3a1e425SDavid Greenman (u_long) bp->b_blkno, bp->b_bcount, bp->b_error); 1555df8bae1dSRodney W. Grimes } 155626f9a767SRodney W. Grimes 15570d94caffSDavid Greenman if (bp->b_vp) 15580d94caffSDavid Greenman pbrelvp(bp); 15590d94caffSDavid Greenman 15600d94caffSDavid Greenman if (bp->b_flags & B_WANTED) 156124a1cce3SDavid Greenman wakeup(bp); 15620d94caffSDavid Greenman 156326f9a767SRodney W. Grimes if (bp->b_rcred != NOCRED) 156426f9a767SRodney W. Grimes crfree(bp->b_rcred); 156526f9a767SRodney W. Grimes if (bp->b_wcred != NOCRED) 156626f9a767SRodney W. Grimes crfree(bp->b_wcred); 156726f9a767SRodney W. Grimes 156826f9a767SRodney W. Grimes nswiodone += spc->spc_count; 15692a4895f4SDavid Greenman if (--spc->spc_object->un_pager.swp.swp_poip == 0) { 15702a4895f4SDavid Greenman wakeup(spc->spc_object); 157126f9a767SRodney W. Grimes } 157226f9a767SRodney W. Grimes if ((swap_pager_needflags & SWAP_FREE_NEEDED) || 157326f9a767SRodney W. Grimes swap_pager_inuse.tqh_first == 0) { 157426f9a767SRodney W. Grimes swap_pager_needflags &= ~SWAP_FREE_NEEDED; 157524a1cce3SDavid Greenman wakeup(&swap_pager_free); 1576a1f6d91cSDavid Greenman } 1577a1f6d91cSDavid Greenman 1578a1f6d91cSDavid Greenman if( swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) { 1579a1f6d91cSDavid Greenman swap_pager_needflags &= ~SWAP_FREE_NEEDED_BY_PAGEOUT; 1580f919ebdeSDavid Greenman pagedaemon_wakeup(); 158126f9a767SRodney W. Grimes } 1582a1f6d91cSDavid Greenman 158326f9a767SRodney W. Grimes if (vm_pageout_pages_needed) { 158424a1cce3SDavid Greenman wakeup(&vm_pageout_pages_needed); 1585a1f6d91cSDavid Greenman vm_pageout_pages_needed = 0; 158626f9a767SRodney W. Grimes } 158726f9a767SRodney W. Grimes if ((swap_pager_inuse.tqh_first == NULL) || 15880d94caffSDavid Greenman ((cnt.v_free_count + cnt.v_cache_count) < cnt.v_free_min && 15890d94caffSDavid Greenman nswiodone + cnt.v_free_count + cnt.v_cache_count >= cnt.v_free_min)) { 1590f919ebdeSDavid Greenman pagedaemon_wakeup(); 159126f9a767SRodney W. Grimes } 159226f9a767SRodney W. Grimes splx(s); 159326f9a767SRodney W. Grimes } 159426f9a767SRodney W. Grimes 159526f9a767SRodney W. Grimes /* 159626f9a767SRodney W. Grimes * return true if any swap control structures can be allocated 159726f9a767SRodney W. Grimes */ 159826f9a767SRodney W. Grimes int 15990d94caffSDavid Greenman swap_pager_ready() 16000d94caffSDavid Greenman { 160126f9a767SRodney W. Grimes if (swap_pager_free.tqh_first) 160226f9a767SRodney W. Grimes return 1; 160326f9a767SRodney W. Grimes else 160426f9a767SRodney W. Grimes return 0; 160526f9a767SRodney W. Grimes } 1606