1df8bae1dSRodney W. Grimes /* 226f9a767SRodney W. Grimes * Copyright (c) 1994 John S. Dyson 3df8bae1dSRodney W. Grimes * Copyright (c) 1990 University of Utah. 4df8bae1dSRodney W. Grimes * Copyright (c) 1991, 1993 5df8bae1dSRodney W. Grimes * The Regents of the University of California. All rights reserved. 6df8bae1dSRodney W. Grimes * 7df8bae1dSRodney W. Grimes * This code is derived from software contributed to Berkeley by 8df8bae1dSRodney W. Grimes * the Systems Programming Group of the University of Utah Computer 9df8bae1dSRodney W. Grimes * Science Department. 10df8bae1dSRodney W. Grimes * 11df8bae1dSRodney W. Grimes * Redistribution and use in source and binary forms, with or without 12df8bae1dSRodney W. Grimes * modification, are permitted provided that the following conditions 13df8bae1dSRodney W. Grimes * are met: 14df8bae1dSRodney W. Grimes * 1. Redistributions of source code must retain the above copyright 15df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer. 16df8bae1dSRodney W. Grimes * 2. Redistributions in binary form must reproduce the above copyright 17df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer in the 18df8bae1dSRodney W. Grimes * documentation and/or other materials provided with the distribution. 19df8bae1dSRodney W. Grimes * 3. All advertising materials mentioning features or use of this software 20df8bae1dSRodney W. Grimes * must display the following acknowledgement: 21df8bae1dSRodney W. Grimes * This product includes software developed by the University of 22df8bae1dSRodney W. Grimes * California, Berkeley and its contributors. 23df8bae1dSRodney W. Grimes * 4. Neither the name of the University nor the names of its contributors 24df8bae1dSRodney W. Grimes * may be used to endorse or promote products derived from this software 25df8bae1dSRodney W. Grimes * without specific prior written permission. 26df8bae1dSRodney W. Grimes * 27df8bae1dSRodney W. Grimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 28df8bae1dSRodney W. Grimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29df8bae1dSRodney W. Grimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 30df8bae1dSRodney W. Grimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 31df8bae1dSRodney W. Grimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32df8bae1dSRodney W. Grimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 33df8bae1dSRodney W. Grimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 34df8bae1dSRodney W. Grimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 35df8bae1dSRodney W. Grimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 36df8bae1dSRodney W. Grimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37df8bae1dSRodney W. Grimes * SUCH DAMAGE. 38df8bae1dSRodney W. Grimes * 39df8bae1dSRodney W. Grimes * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$ 40df8bae1dSRodney W. Grimes * 41df8bae1dSRodney W. Grimes * @(#)swap_pager.c 8.9 (Berkeley) 3/21/94 4224a1cce3SDavid Greenman * $Id: swap_pager.c,v 1.41 1995/05/30 08:15:55 rgrimes Exp $ 43df8bae1dSRodney W. Grimes */ 44df8bae1dSRodney W. Grimes 45df8bae1dSRodney W. Grimes /* 46df8bae1dSRodney W. Grimes * Quick hack to page to dedicated partition(s). 47df8bae1dSRodney W. Grimes * TODO: 48df8bae1dSRodney W. Grimes * Add multiprocessor locks 49df8bae1dSRodney W. Grimes * Deal with async writes in a better fashion 50df8bae1dSRodney W. Grimes */ 51df8bae1dSRodney W. Grimes 52df8bae1dSRodney W. Grimes #include <sys/param.h> 53df8bae1dSRodney W. Grimes #include <sys/systm.h> 5464abb5a5SDavid Greenman #include <sys/kernel.h> 55df8bae1dSRodney W. Grimes #include <sys/proc.h> 56df8bae1dSRodney W. Grimes #include <sys/buf.h> 57df8bae1dSRodney W. Grimes #include <sys/vnode.h> 58df8bae1dSRodney W. Grimes #include <sys/malloc.h> 59df8bae1dSRodney W. Grimes 60df8bae1dSRodney W. Grimes #include <miscfs/specfs/specdev.h> 6126f9a767SRodney W. Grimes #include <sys/rlist.h> 62df8bae1dSRodney W. Grimes 63df8bae1dSRodney W. Grimes #include <vm/vm.h> 6426f9a767SRodney W. Grimes #include <vm/vm_pager.h> 65df8bae1dSRodney W. Grimes #include <vm/vm_page.h> 66df8bae1dSRodney W. Grimes #include <vm/vm_pageout.h> 67df8bae1dSRodney W. Grimes #include <vm/swap_pager.h> 6864abb5a5SDavid Greenman #include <vm/vm_kern.h> 69df8bae1dSRodney W. Grimes 70df8bae1dSRodney W. Grimes #ifndef NPENDINGIO 710d94caffSDavid Greenman #define NPENDINGIO 10 72df8bae1dSRodney W. Grimes #endif 73df8bae1dSRodney W. Grimes 7426f9a767SRodney W. Grimes int nswiodone; 7526f9a767SRodney W. Grimes int swap_pager_full; 7626f9a767SRodney W. Grimes extern int vm_swap_size; 7724ea4a96SDavid Greenman int no_swap_space = 1; 7835c10d22SDavid Greenman struct rlist *swaplist; 7935c10d22SDavid Greenman int nswaplist; 8026f9a767SRodney W. Grimes 8126f9a767SRodney W. Grimes #define MAX_PAGEOUT_CLUSTER 8 82df8bae1dSRodney W. Grimes 83df8bae1dSRodney W. Grimes TAILQ_HEAD(swpclean, swpagerclean); 84df8bae1dSRodney W. Grimes 8526f9a767SRodney W. Grimes typedef struct swpagerclean *swp_clean_t; 8626f9a767SRodney W. Grimes 87df8bae1dSRodney W. Grimes struct swpagerclean { 88df8bae1dSRodney W. Grimes TAILQ_ENTRY(swpagerclean) spc_list; 89df8bae1dSRodney W. Grimes int spc_flags; 90df8bae1dSRodney W. Grimes struct buf *spc_bp; 91df8bae1dSRodney W. Grimes sw_pager_t spc_swp; 92df8bae1dSRodney W. Grimes vm_offset_t spc_kva; 9326f9a767SRodney W. Grimes int spc_count; 9426f9a767SRodney W. Grimes vm_page_t spc_m[MAX_PAGEOUT_CLUSTER]; 95df8bae1dSRodney W. Grimes } swcleanlist[NPENDINGIO]; 9626f9a767SRodney W. Grimes 9726f9a767SRodney W. Grimes 98df8bae1dSRodney W. Grimes /* spc_flags values */ 9926f9a767SRodney W. Grimes #define SPC_ERROR 0x01 100df8bae1dSRodney W. Grimes 10126f9a767SRodney W. Grimes #define SWB_EMPTY (-1) 102df8bae1dSRodney W. Grimes 103a401ebbeSDavid Greenman struct swpclean swap_pager_done; /* list of completed page cleans */ 104df8bae1dSRodney W. Grimes struct swpclean swap_pager_inuse; /* list of pending page cleans */ 105df8bae1dSRodney W. Grimes struct swpclean swap_pager_free; /* list of free pager clean structs */ 10624a1cce3SDavid Greenman struct pagerlst swap_pager_object_list; /* list of "named" anon region objects */ 10724a1cce3SDavid Greenman struct pagerlst swap_pager_un_object_list; /* list of "unnamed" anon region objects */ 108df8bae1dSRodney W. Grimes 10926f9a767SRodney W. Grimes #define SWAP_FREE_NEEDED 0x1 /* need a swap block */ 110a1f6d91cSDavid Greenman #define SWAP_FREE_NEEDED_BY_PAGEOUT 0x2 11126f9a767SRodney W. Grimes int swap_pager_needflags; 11226f9a767SRodney W. Grimes 11326f9a767SRodney W. Grimes struct pagerlst *swp_qs[] = { 11424a1cce3SDavid Greenman &swap_pager_object_list, &swap_pager_un_object_list, (struct pagerlst *) 0 11526f9a767SRodney W. Grimes }; 11626f9a767SRodney W. Grimes 11724a1cce3SDavid Greenman /* 11824a1cce3SDavid Greenman * pagerops for OBJT_SWAP - "swap pager". 11924a1cce3SDavid Greenman */ 120df8bae1dSRodney W. Grimes struct pagerops swappagerops = { 121df8bae1dSRodney W. Grimes swap_pager_init, 122df8bae1dSRodney W. Grimes swap_pager_alloc, 123df8bae1dSRodney W. Grimes swap_pager_dealloc, 12424a1cce3SDavid Greenman swap_pager_getpages, 12524a1cce3SDavid Greenman swap_pager_putpages, 12624a1cce3SDavid Greenman swap_pager_haspage, 12724a1cce3SDavid Greenman swap_pager_sync 128df8bae1dSRodney W. Grimes }; 129df8bae1dSRodney W. Grimes 13026f9a767SRodney W. Grimes int npendingio = NPENDINGIO; 13126f9a767SRodney W. Grimes void swap_pager_finish(); 13226f9a767SRodney W. Grimes int dmmin, dmmax; 13326f9a767SRodney W. Grimes 13424a1cce3SDavid Greenman 1350d94caffSDavid Greenman static inline void 1360d94caffSDavid Greenman swapsizecheck() 1370d94caffSDavid Greenman { 13826f9a767SRodney W. Grimes if (vm_swap_size < 128 * btodb(PAGE_SIZE)) { 139a1f6d91cSDavid Greenman if (swap_pager_full == 0) 14026f9a767SRodney W. Grimes printf("swap_pager: out of space\n"); 14126f9a767SRodney W. Grimes swap_pager_full = 1; 14226f9a767SRodney W. Grimes } else if (vm_swap_size > 192 * btodb(PAGE_SIZE)) 14326f9a767SRodney W. Grimes swap_pager_full = 0; 14426f9a767SRodney W. Grimes } 14526f9a767SRodney W. Grimes 14626f9a767SRodney W. Grimes void 147df8bae1dSRodney W. Grimes swap_pager_init() 148df8bae1dSRodney W. Grimes { 14924a1cce3SDavid Greenman TAILQ_INIT(&swap_pager_object_list); 15024a1cce3SDavid Greenman TAILQ_INIT(&swap_pager_un_object_list); 151df8bae1dSRodney W. Grimes 152df8bae1dSRodney W. Grimes /* 153df8bae1dSRodney W. Grimes * Initialize clean lists 154df8bae1dSRodney W. Grimes */ 155df8bae1dSRodney W. Grimes TAILQ_INIT(&swap_pager_inuse); 15626f9a767SRodney W. Grimes TAILQ_INIT(&swap_pager_done); 157df8bae1dSRodney W. Grimes TAILQ_INIT(&swap_pager_free); 15826f9a767SRodney W. Grimes 159df8bae1dSRodney W. Grimes /* 160df8bae1dSRodney W. Grimes * Calculate the swap allocation constants. 161df8bae1dSRodney W. Grimes */ 162df8bae1dSRodney W. Grimes 16326f9a767SRodney W. Grimes dmmin = CLBYTES / DEV_BSIZE; 16426f9a767SRodney W. Grimes dmmax = btodb(SWB_NPAGES * PAGE_SIZE) * 2; 16526f9a767SRodney W. Grimes 166df8bae1dSRodney W. Grimes } 167df8bae1dSRodney W. Grimes 16824a1cce3SDavid Greenman void 16924a1cce3SDavid Greenman swap_pager_swap_init() 170df8bae1dSRodney W. Grimes { 17126f9a767SRodney W. Grimes swp_clean_t spc; 17226f9a767SRodney W. Grimes struct buf *bp; 17324a1cce3SDavid Greenman int i; 1740d94caffSDavid Greenman 17526f9a767SRodney W. Grimes /* 1760d94caffSDavid Greenman * kva's are allocated here so that we dont need to keep doing 1770d94caffSDavid Greenman * kmem_alloc pageables at runtime 17826f9a767SRodney W. Grimes */ 17926f9a767SRodney W. Grimes for (i = 0, spc = swcleanlist; i < npendingio; i++, spc++) { 180fff93ab6SDavid Greenman spc->spc_kva = kmem_alloc_pageable(pager_map, PAGE_SIZE * MAX_PAGEOUT_CLUSTER); 18126f9a767SRodney W. Grimes if (!spc->spc_kva) { 18226f9a767SRodney W. Grimes break; 18326f9a767SRodney W. Grimes } 184a1f6d91cSDavid Greenman spc->spc_bp = malloc(sizeof(*bp), M_TEMP, M_KERNEL); 18526f9a767SRodney W. Grimes if (!spc->spc_bp) { 18626f9a767SRodney W. Grimes kmem_free_wakeup(pager_map, spc->spc_kva, PAGE_SIZE); 18726f9a767SRodney W. Grimes break; 18826f9a767SRodney W. Grimes } 18926f9a767SRodney W. Grimes spc->spc_flags = 0; 19026f9a767SRodney W. Grimes TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 19126f9a767SRodney W. Grimes } 19226f9a767SRodney W. Grimes } 19324a1cce3SDavid Greenman 19424a1cce3SDavid Greenman int 19524a1cce3SDavid Greenman swap_pager_swp_alloc(object, wait) 19624a1cce3SDavid Greenman vm_object_t object; 19724a1cce3SDavid Greenman int wait; 19824a1cce3SDavid Greenman { 19924a1cce3SDavid Greenman register sw_pager_t swp; 20024a1cce3SDavid Greenman int i, j; 20124a1cce3SDavid Greenman 20224a1cce3SDavid Greenman if (object->pg_data != NULL) 20324a1cce3SDavid Greenman panic("swap_pager_swp_alloc: swp already allocated"); 20424a1cce3SDavid Greenman 20524a1cce3SDavid Greenman swp = (sw_pager_t) malloc(sizeof *swp, M_VMPGDATA, wait); 206df8bae1dSRodney W. Grimes if (swp == NULL) { 20724a1cce3SDavid Greenman return 1; 208df8bae1dSRodney W. Grimes } 20924a1cce3SDavid Greenman 21024a1cce3SDavid Greenman swp->sw_nblocks = (btodb(object->size) + btodb(SWB_NPAGES * PAGE_SIZE) - 1) / btodb(SWB_NPAGES * PAGE_SIZE); 21124a1cce3SDavid Greenman 21224a1cce3SDavid Greenman swp->sw_blocks = (sw_blk_t) malloc(swp->sw_nblocks * sizeof(*swp->sw_blocks), M_VMPGDATA, wait); 213df8bae1dSRodney W. Grimes if (swp->sw_blocks == NULL) { 214df8bae1dSRodney W. Grimes free((caddr_t) swp, M_VMPGDATA); 21524a1cce3SDavid Greenman return 1; 216df8bae1dSRodney W. Grimes } 21724a1cce3SDavid Greenman 21826f9a767SRodney W. Grimes for (i = 0; i < swp->sw_nblocks; i++) { 21926f9a767SRodney W. Grimes swp->sw_blocks[i].swb_valid = 0; 22026f9a767SRodney W. Grimes swp->sw_blocks[i].swb_locked = 0; 22126f9a767SRodney W. Grimes for (j = 0; j < SWB_NPAGES; j++) 22226f9a767SRodney W. Grimes swp->sw_blocks[i].swb_block[j] = SWB_EMPTY; 22326f9a767SRodney W. Grimes } 22426f9a767SRodney W. Grimes 225df8bae1dSRodney W. Grimes swp->sw_poip = 0; 226a1f6d91cSDavid Greenman swp->sw_allocsize = 0; 227df8bae1dSRodney W. Grimes 22824a1cce3SDavid Greenman object->pg_data = swp; 22924a1cce3SDavid Greenman 23024a1cce3SDavid Greenman if (object->handle != NULL) { 23124a1cce3SDavid Greenman TAILQ_INSERT_TAIL(&swap_pager_object_list, object, pager_object_list); 232df8bae1dSRodney W. Grimes } else { 23324a1cce3SDavid Greenman TAILQ_INSERT_TAIL(&swap_pager_un_object_list, object, pager_object_list); 234df8bae1dSRodney W. Grimes } 235df8bae1dSRodney W. Grimes 23624a1cce3SDavid Greenman return 0; 23724a1cce3SDavid Greenman } 23824a1cce3SDavid Greenman 23924a1cce3SDavid Greenman /* 24024a1cce3SDavid Greenman * Allocate a pager structure and associated resources. 24124a1cce3SDavid Greenman * Note that if we are called from the pageout daemon (handle == NULL) 24224a1cce3SDavid Greenman * we should not wait for memory as it could resulting in deadlock. 24324a1cce3SDavid Greenman */ 24424a1cce3SDavid Greenman vm_object_t 24524a1cce3SDavid Greenman swap_pager_alloc(handle, size, prot, offset) 24624a1cce3SDavid Greenman void *handle; 24724a1cce3SDavid Greenman register vm_size_t size; 24824a1cce3SDavid Greenman vm_prot_t prot; 24924a1cce3SDavid Greenman vm_offset_t offset; 25024a1cce3SDavid Greenman { 25124a1cce3SDavid Greenman vm_object_t object; 25224a1cce3SDavid Greenman int i; 25324a1cce3SDavid Greenman 25424a1cce3SDavid Greenman /* 25524a1cce3SDavid Greenman * If this is a "named" anonymous region, look it up and use the 25624a1cce3SDavid Greenman * object if it exists, otherwise allocate a new one. 25724a1cce3SDavid Greenman */ 25824a1cce3SDavid Greenman if (handle) { 25924a1cce3SDavid Greenman object = vm_pager_object_lookup(&swap_pager_object_list, handle); 26024a1cce3SDavid Greenman if (object != NULL) { 26124a1cce3SDavid Greenman vm_object_reference(object); 26224a1cce3SDavid Greenman } else { 26324a1cce3SDavid Greenman /* 26424a1cce3SDavid Greenman * XXX - there is a race condition here. Two processes 26524a1cce3SDavid Greenman * can request the same named object simultaneuously, 26624a1cce3SDavid Greenman * and if one blocks for memory, the result is a disaster. 26724a1cce3SDavid Greenman * Probably quite rare, but is yet another reason to just 26824a1cce3SDavid Greenman * rip support of "named anonymous regions" out altogether. 26924a1cce3SDavid Greenman */ 27024a1cce3SDavid Greenman object = vm_object_allocate(OBJT_SWAP, offset + size); 27124a1cce3SDavid Greenman object->handle = handle; 27224a1cce3SDavid Greenman (void) swap_pager_swp_alloc(object, M_WAITOK); 27324a1cce3SDavid Greenman } 27424a1cce3SDavid Greenman } else { 27524a1cce3SDavid Greenman object = vm_object_allocate(OBJT_SWAP, offset + size); 27624a1cce3SDavid Greenman (void) swap_pager_swp_alloc(object, M_WAITOK); 27724a1cce3SDavid Greenman } 27824a1cce3SDavid Greenman 27924a1cce3SDavid Greenman return (object); 280df8bae1dSRodney W. Grimes } 281df8bae1dSRodney W. Grimes 28226f9a767SRodney W. Grimes /* 28326f9a767SRodney W. Grimes * returns disk block associated with pager and offset 28426f9a767SRodney W. Grimes * additionally, as a side effect returns a flag indicating 28526f9a767SRodney W. Grimes * if the block has been written 28626f9a767SRodney W. Grimes */ 28726f9a767SRodney W. Grimes 288a1f6d91cSDavid Greenman inline static int * 28924a1cce3SDavid Greenman swap_pager_diskaddr(object, offset, valid) 29024a1cce3SDavid Greenman vm_object_t object; 29126f9a767SRodney W. Grimes vm_offset_t offset; 29226f9a767SRodney W. Grimes int *valid; 29326f9a767SRodney W. Grimes { 29424a1cce3SDavid Greenman sw_pager_t swp = object->pg_data; 29526f9a767SRodney W. Grimes register sw_blk_t swb; 29626f9a767SRodney W. Grimes int ix; 29726f9a767SRodney W. Grimes 29826f9a767SRodney W. Grimes if (valid) 29926f9a767SRodney W. Grimes *valid = 0; 30026f9a767SRodney W. Grimes ix = offset / (SWB_NPAGES * PAGE_SIZE); 301a1f6d91cSDavid Greenman if ((swp->sw_blocks == NULL) || (ix >= swp->sw_nblocks) || 30224a1cce3SDavid Greenman (offset >= object->size)) { 30326f9a767SRodney W. Grimes return (FALSE); 30426f9a767SRodney W. Grimes } 30526f9a767SRodney W. Grimes swb = &swp->sw_blocks[ix]; 30626f9a767SRodney W. Grimes ix = (offset % (SWB_NPAGES * PAGE_SIZE)) / PAGE_SIZE; 30726f9a767SRodney W. Grimes if (valid) 30826f9a767SRodney W. Grimes *valid = swb->swb_valid & (1 << ix); 30926f9a767SRodney W. Grimes return &swb->swb_block[ix]; 31026f9a767SRodney W. Grimes } 31126f9a767SRodney W. Grimes 31226f9a767SRodney W. Grimes /* 31326f9a767SRodney W. Grimes * Utility routine to set the valid (written) bit for 31426f9a767SRodney W. Grimes * a block associated with a pager and offset 31526f9a767SRodney W. Grimes */ 316df8bae1dSRodney W. Grimes static void 31726f9a767SRodney W. Grimes swap_pager_setvalid(swp, offset, valid) 31826f9a767SRodney W. Grimes sw_pager_t swp; 31926f9a767SRodney W. Grimes vm_offset_t offset; 32026f9a767SRodney W. Grimes int valid; 32126f9a767SRodney W. Grimes { 32226f9a767SRodney W. Grimes register sw_blk_t swb; 32326f9a767SRodney W. Grimes int ix; 32426f9a767SRodney W. Grimes 32526f9a767SRodney W. Grimes ix = offset / (SWB_NPAGES * PAGE_SIZE); 32626f9a767SRodney W. Grimes if (swp->sw_blocks == NULL || ix >= swp->sw_nblocks) 32726f9a767SRodney W. Grimes return; 32826f9a767SRodney W. Grimes 32926f9a767SRodney W. Grimes swb = &swp->sw_blocks[ix]; 33026f9a767SRodney W. Grimes ix = (offset % (SWB_NPAGES * PAGE_SIZE)) / PAGE_SIZE; 33126f9a767SRodney W. Grimes if (valid) 33226f9a767SRodney W. Grimes swb->swb_valid |= (1 << ix); 33326f9a767SRodney W. Grimes else 33426f9a767SRodney W. Grimes swb->swb_valid &= ~(1 << ix); 33526f9a767SRodney W. Grimes return; 33626f9a767SRodney W. Grimes } 33726f9a767SRodney W. Grimes 33826f9a767SRodney W. Grimes /* 33926f9a767SRodney W. Grimes * this routine allocates swap space with a fragmentation 34026f9a767SRodney W. Grimes * minimization policy. 34126f9a767SRodney W. Grimes */ 34226f9a767SRodney W. Grimes int 343a1f6d91cSDavid Greenman swap_pager_getswapspace(sw_pager_t swp, unsigned amount, unsigned *rtval) 3440d94caffSDavid Greenman { 34524ea4a96SDavid Greenman vm_swap_size -= amount; 34624ea4a96SDavid Greenman if (!rlist_alloc(&swaplist, amount, rtval)) { 34724ea4a96SDavid Greenman vm_swap_size += amount; 34826f9a767SRodney W. Grimes return 0; 34924ea4a96SDavid Greenman } else { 35024ea4a96SDavid Greenman swapsizecheck(); 351a1f6d91cSDavid Greenman swp->sw_allocsize += amount; 35226f9a767SRodney W. Grimes return 1; 35326f9a767SRodney W. Grimes } 35426f9a767SRodney W. Grimes } 35526f9a767SRodney W. Grimes 35626f9a767SRodney W. Grimes /* 35726f9a767SRodney W. Grimes * this routine frees swap space with a fragmentation 35826f9a767SRodney W. Grimes * minimization policy. 35926f9a767SRodney W. Grimes */ 36026f9a767SRodney W. Grimes void 361a1f6d91cSDavid Greenman swap_pager_freeswapspace(sw_pager_t swp, unsigned from, unsigned to) 3620d94caffSDavid Greenman { 36335c10d22SDavid Greenman rlist_free(&swaplist, from, to); 36424ea4a96SDavid Greenman vm_swap_size += (to - from) + 1; 365a1f6d91cSDavid Greenman swp->sw_allocsize -= (to - from) + 1; 36624ea4a96SDavid Greenman swapsizecheck(); 36726f9a767SRodney W. Grimes } 36826f9a767SRodney W. Grimes /* 36926f9a767SRodney W. Grimes * this routine frees swap blocks from a specified pager 37026f9a767SRodney W. Grimes */ 37126f9a767SRodney W. Grimes void 37224a1cce3SDavid Greenman swap_pager_freespace(object, start, size) 37324a1cce3SDavid Greenman vm_object_t object; 37426f9a767SRodney W. Grimes vm_offset_t start; 37526f9a767SRodney W. Grimes vm_offset_t size; 37626f9a767SRodney W. Grimes { 37724a1cce3SDavid Greenman sw_pager_t swp = object->pg_data; 37826f9a767SRodney W. Grimes vm_offset_t i; 37926f9a767SRodney W. Grimes int s; 38026f9a767SRodney W. Grimes 38126f9a767SRodney W. Grimes s = splbio(); 3826f7bc393SDavid Greenman for (i = start; i < round_page(start + size); i += PAGE_SIZE) { 38326f9a767SRodney W. Grimes int valid; 38424a1cce3SDavid Greenman int *addr = swap_pager_diskaddr(object, i, &valid); 3850d94caffSDavid Greenman 38626f9a767SRodney W. Grimes if (addr && *addr != SWB_EMPTY) { 387a1f6d91cSDavid Greenman swap_pager_freeswapspace(swp, *addr, *addr + btodb(PAGE_SIZE) - 1); 38826f9a767SRodney W. Grimes if (valid) { 38926f9a767SRodney W. Grimes swap_pager_setvalid(swp, i, 0); 39026f9a767SRodney W. Grimes } 39126f9a767SRodney W. Grimes *addr = SWB_EMPTY; 39226f9a767SRodney W. Grimes } 39326f9a767SRodney W. Grimes } 39426f9a767SRodney W. Grimes splx(s); 39526f9a767SRodney W. Grimes } 39626f9a767SRodney W. Grimes 397a1f6d91cSDavid Greenman static void 398a1f6d91cSDavid Greenman swap_pager_free_swap(swp) 399a1f6d91cSDavid Greenman sw_pager_t swp; 400a1f6d91cSDavid Greenman { 401a1f6d91cSDavid Greenman register int i, j; 402a1f6d91cSDavid Greenman register sw_blk_t bp; 403a1f6d91cSDavid Greenman int first_block=0, block_count=0; 404a1f6d91cSDavid Greenman int s; 405a1f6d91cSDavid Greenman /* 406a1f6d91cSDavid Greenman * Free left over swap blocks 407a1f6d91cSDavid Greenman */ 408a1f6d91cSDavid Greenman s = splbio(); 409a1f6d91cSDavid Greenman for (i = 0, bp = swp->sw_blocks; i < swp->sw_nblocks; i++, bp++) { 410a1f6d91cSDavid Greenman for (j = 0; j < SWB_NPAGES; j++) { 411a1f6d91cSDavid Greenman if (bp->swb_block[j] != SWB_EMPTY) { 412a1f6d91cSDavid Greenman /* 413a1f6d91cSDavid Greenman * initially the length of the run is zero 414a1f6d91cSDavid Greenman */ 415a1f6d91cSDavid Greenman if( block_count == 0) { 416a1f6d91cSDavid Greenman first_block = bp->swb_block[j]; 417a1f6d91cSDavid Greenman block_count = btodb(PAGE_SIZE); 418a1f6d91cSDavid Greenman bp->swb_block[j] = SWB_EMPTY; 419a1f6d91cSDavid Greenman /* 420a1f6d91cSDavid Greenman * if the new block can be included into the current run 421a1f6d91cSDavid Greenman */ 422a1f6d91cSDavid Greenman } else if( bp->swb_block[j] == first_block + block_count) { 423a1f6d91cSDavid Greenman block_count += btodb(PAGE_SIZE); 424a1f6d91cSDavid Greenman bp->swb_block[j] = SWB_EMPTY; 425a1f6d91cSDavid Greenman /* 426a1f6d91cSDavid Greenman * terminate the previous run, and start a new one 427a1f6d91cSDavid Greenman */ 428a1f6d91cSDavid Greenman } else { 429a1f6d91cSDavid Greenman swap_pager_freeswapspace(swp, first_block, 430a1f6d91cSDavid Greenman (unsigned) first_block + block_count - 1); 431a1f6d91cSDavid Greenman first_block = bp->swb_block[j]; 432a1f6d91cSDavid Greenman block_count = btodb(PAGE_SIZE); 433a1f6d91cSDavid Greenman bp->swb_block[j] = SWB_EMPTY; 434a1f6d91cSDavid Greenman } 435a1f6d91cSDavid Greenman } 436a1f6d91cSDavid Greenman } 437a1f6d91cSDavid Greenman } 438a1f6d91cSDavid Greenman 439a1f6d91cSDavid Greenman if( block_count) { 440a1f6d91cSDavid Greenman swap_pager_freeswapspace(swp, first_block, 441a1f6d91cSDavid Greenman (unsigned) first_block + block_count - 1); 442a1f6d91cSDavid Greenman } 443a1f6d91cSDavid Greenman splx(s); 444a1f6d91cSDavid Greenman } 445a1f6d91cSDavid Greenman 446a1f6d91cSDavid Greenman 44726f9a767SRodney W. Grimes /* 44826f9a767SRodney W. Grimes * swap_pager_reclaim frees up over-allocated space from all pagers 44926f9a767SRodney W. Grimes * this eliminates internal fragmentation due to allocation of space 45026f9a767SRodney W. Grimes * for segments that are never swapped to. It has been written so that 45126f9a767SRodney W. Grimes * it does not block until the rlist_free operation occurs; it keeps 45226f9a767SRodney W. Grimes * the queues consistant. 45326f9a767SRodney W. Grimes */ 45426f9a767SRodney W. Grimes 45526f9a767SRodney W. Grimes /* 45626f9a767SRodney W. Grimes * Maximum number of blocks (pages) to reclaim per pass 45726f9a767SRodney W. Grimes */ 458a1f6d91cSDavid Greenman #define MAXRECLAIM 128 45926f9a767SRodney W. Grimes 46026f9a767SRodney W. Grimes void 46126f9a767SRodney W. Grimes swap_pager_reclaim() 46226f9a767SRodney W. Grimes { 46324a1cce3SDavid Greenman vm_object_t object; 46426f9a767SRodney W. Grimes sw_pager_t swp; 46526f9a767SRodney W. Grimes int i, j, k; 46626f9a767SRodney W. Grimes int s; 46726f9a767SRodney W. Grimes int reclaimcount; 468a1f6d91cSDavid Greenman static struct { 469a1f6d91cSDavid Greenman int address; 470a1f6d91cSDavid Greenman sw_pager_t pager; 471a1f6d91cSDavid Greenman } reclaims[MAXRECLAIM]; 47226f9a767SRodney W. Grimes static int in_reclaim; 47326f9a767SRodney W. Grimes 47426f9a767SRodney W. Grimes /* 47526f9a767SRodney W. Grimes * allow only one process to be in the swap_pager_reclaim subroutine 47626f9a767SRodney W. Grimes */ 47726f9a767SRodney W. Grimes s = splbio(); 47826f9a767SRodney W. Grimes if (in_reclaim) { 47924a1cce3SDavid Greenman tsleep(&in_reclaim, PSWP, "swrclm", 0); 48026f9a767SRodney W. Grimes splx(s); 48126f9a767SRodney W. Grimes return; 48226f9a767SRodney W. Grimes } 48326f9a767SRodney W. Grimes in_reclaim = 1; 48426f9a767SRodney W. Grimes reclaimcount = 0; 48526f9a767SRodney W. Grimes 48626f9a767SRodney W. Grimes /* for each pager queue */ 48726f9a767SRodney W. Grimes for (k = 0; swp_qs[k]; k++) { 48826f9a767SRodney W. Grimes 48924a1cce3SDavid Greenman object = swp_qs[k]->tqh_first; 49024a1cce3SDavid Greenman while (object && (reclaimcount < MAXRECLAIM)) { 49126f9a767SRodney W. Grimes 49226f9a767SRodney W. Grimes /* 49326f9a767SRodney W. Grimes * see if any blocks associated with a pager has been 49426f9a767SRodney W. Grimes * allocated but not used (written) 49526f9a767SRodney W. Grimes */ 49624a1cce3SDavid Greenman swp = (sw_pager_t) object->pg_data; 49726f9a767SRodney W. Grimes for (i = 0; i < swp->sw_nblocks; i++) { 49826f9a767SRodney W. Grimes sw_blk_t swb = &swp->sw_blocks[i]; 4990d94caffSDavid Greenman 50026f9a767SRodney W. Grimes if (swb->swb_locked) 50126f9a767SRodney W. Grimes continue; 50226f9a767SRodney W. Grimes for (j = 0; j < SWB_NPAGES; j++) { 50326f9a767SRodney W. Grimes if (swb->swb_block[j] != SWB_EMPTY && 50426f9a767SRodney W. Grimes (swb->swb_valid & (1 << j)) == 0) { 505a1f6d91cSDavid Greenman reclaims[reclaimcount].address = swb->swb_block[j]; 506a1f6d91cSDavid Greenman reclaims[reclaimcount++].pager = swp; 50726f9a767SRodney W. Grimes swb->swb_block[j] = SWB_EMPTY; 50826f9a767SRodney W. Grimes if (reclaimcount >= MAXRECLAIM) 50926f9a767SRodney W. Grimes goto rfinished; 51026f9a767SRodney W. Grimes } 51126f9a767SRodney W. Grimes } 51226f9a767SRodney W. Grimes } 51324a1cce3SDavid Greenman object = object->pager_object_list.tqe_next; 51426f9a767SRodney W. Grimes } 51526f9a767SRodney W. Grimes } 51626f9a767SRodney W. Grimes 51726f9a767SRodney W. Grimes rfinished: 51826f9a767SRodney W. Grimes 51926f9a767SRodney W. Grimes /* 52026f9a767SRodney W. Grimes * free the blocks that have been added to the reclaim list 52126f9a767SRodney W. Grimes */ 52226f9a767SRodney W. Grimes for (i = 0; i < reclaimcount; i++) { 523a1f6d91cSDavid Greenman swap_pager_freeswapspace(reclaims[i].pager, reclaims[i].address, reclaims[i].address + btodb(PAGE_SIZE) - 1); 52426f9a767SRodney W. Grimes } 52526f9a767SRodney W. Grimes splx(s); 52626f9a767SRodney W. Grimes in_reclaim = 0; 52724a1cce3SDavid Greenman wakeup(&in_reclaim); 52826f9a767SRodney W. Grimes } 52926f9a767SRodney W. Grimes 53026f9a767SRodney W. Grimes 53126f9a767SRodney W. Grimes /* 53226f9a767SRodney W. Grimes * swap_pager_copy copies blocks from one pager to another and 53326f9a767SRodney W. Grimes * destroys the source pager 53426f9a767SRodney W. Grimes */ 53526f9a767SRodney W. Grimes 53626f9a767SRodney W. Grimes void 53724a1cce3SDavid Greenman swap_pager_copy(srcobject, srcoffset, dstobject, dstoffset, offset) 53824a1cce3SDavid Greenman vm_object_t srcobject; 53926f9a767SRodney W. Grimes vm_offset_t srcoffset; 54024a1cce3SDavid Greenman vm_object_t dstobject; 54126f9a767SRodney W. Grimes vm_offset_t dstoffset; 54226f9a767SRodney W. Grimes vm_offset_t offset; 54326f9a767SRodney W. Grimes { 54426f9a767SRodney W. Grimes sw_pager_t srcswp, dstswp; 54526f9a767SRodney W. Grimes vm_offset_t i; 546a1f6d91cSDavid Greenman int origsize; 54726f9a767SRodney W. Grimes int s; 54826f9a767SRodney W. Grimes 54924ea4a96SDavid Greenman if (vm_swap_size) 55024ea4a96SDavid Greenman no_swap_space = 0; 55124ea4a96SDavid Greenman 55224a1cce3SDavid Greenman srcswp = (sw_pager_t) srcobject->pg_data; 553a1f6d91cSDavid Greenman origsize = srcswp->sw_allocsize; 55424a1cce3SDavid Greenman dstswp = (sw_pager_t) dstobject->pg_data; 55526f9a767SRodney W. Grimes 55626f9a767SRodney W. Grimes /* 55724a1cce3SDavid Greenman * remove the source object from the swap_pager internal queue 55826f9a767SRodney W. Grimes */ 55924a1cce3SDavid Greenman if (srcobject->handle == NULL) { 56024a1cce3SDavid Greenman TAILQ_REMOVE(&swap_pager_un_object_list, srcobject, pager_object_list); 56126f9a767SRodney W. Grimes } else { 56224a1cce3SDavid Greenman TAILQ_REMOVE(&swap_pager_object_list, srcobject, pager_object_list); 56326f9a767SRodney W. Grimes } 56426f9a767SRodney W. Grimes 56524a1cce3SDavid Greenman s = splbio(); 56626f9a767SRodney W. Grimes while (srcswp->sw_poip) { 56724a1cce3SDavid Greenman tsleep(srcswp, PVM, "spgout", 0); 56826f9a767SRodney W. Grimes } 56926f9a767SRodney W. Grimes splx(s); 57026f9a767SRodney W. Grimes 57126f9a767SRodney W. Grimes /* 57226f9a767SRodney W. Grimes * clean all of the pages that are currently active and finished 57326f9a767SRodney W. Grimes */ 57424a1cce3SDavid Greenman swap_pager_sync(); 57526f9a767SRodney W. Grimes 57626f9a767SRodney W. Grimes s = splbio(); 57726f9a767SRodney W. Grimes /* 57826f9a767SRodney W. Grimes * transfer source to destination 57926f9a767SRodney W. Grimes */ 58024a1cce3SDavid Greenman for (i = 0; i < dstobject->size; i += PAGE_SIZE) { 58126f9a767SRodney W. Grimes int srcvalid, dstvalid; 58224a1cce3SDavid Greenman int *srcaddrp = swap_pager_diskaddr(srcobject, i + offset + srcoffset, 58326f9a767SRodney W. Grimes &srcvalid); 58426f9a767SRodney W. Grimes int *dstaddrp; 5850d94caffSDavid Greenman 58626f9a767SRodney W. Grimes /* 58726f9a767SRodney W. Grimes * see if the source has space allocated 58826f9a767SRodney W. Grimes */ 58926f9a767SRodney W. Grimes if (srcaddrp && *srcaddrp != SWB_EMPTY) { 59026f9a767SRodney W. Grimes /* 5910d94caffSDavid Greenman * if the source is valid and the dest has no space, 5920d94caffSDavid Greenman * then copy the allocation from the srouce to the 5930d94caffSDavid Greenman * dest. 59426f9a767SRodney W. Grimes */ 59526f9a767SRodney W. Grimes if (srcvalid) { 59624a1cce3SDavid Greenman dstaddrp = swap_pager_diskaddr(dstobject, i + dstoffset, 597a1f6d91cSDavid Greenman &dstvalid); 59826f9a767SRodney W. Grimes /* 5990d94caffSDavid Greenman * if the dest already has a valid block, 6000d94caffSDavid Greenman * deallocate the source block without 6010d94caffSDavid Greenman * copying. 60226f9a767SRodney W. Grimes */ 60326f9a767SRodney W. Grimes if (!dstvalid && dstaddrp && *dstaddrp != SWB_EMPTY) { 604a1f6d91cSDavid Greenman swap_pager_freeswapspace(dstswp, *dstaddrp, 605a1f6d91cSDavid Greenman *dstaddrp + btodb(PAGE_SIZE) - 1); 60626f9a767SRodney W. Grimes *dstaddrp = SWB_EMPTY; 60726f9a767SRodney W. Grimes } 60826f9a767SRodney W. Grimes if (dstaddrp && *dstaddrp == SWB_EMPTY) { 60926f9a767SRodney W. Grimes *dstaddrp = *srcaddrp; 61026f9a767SRodney W. Grimes *srcaddrp = SWB_EMPTY; 611a1f6d91cSDavid Greenman dstswp->sw_allocsize += btodb(PAGE_SIZE); 612a1f6d91cSDavid Greenman srcswp->sw_allocsize -= btodb(PAGE_SIZE); 61326f9a767SRodney W. Grimes swap_pager_setvalid(dstswp, i + dstoffset, 1); 61426f9a767SRodney W. Grimes } 61526f9a767SRodney W. Grimes } 61626f9a767SRodney W. Grimes /* 6170d94caffSDavid Greenman * if the source is not empty at this point, then 6180d94caffSDavid Greenman * deallocate the space. 61926f9a767SRodney W. Grimes */ 62026f9a767SRodney W. Grimes if (*srcaddrp != SWB_EMPTY) { 621a1f6d91cSDavid Greenman swap_pager_freeswapspace(srcswp, *srcaddrp, 622a1f6d91cSDavid Greenman *srcaddrp + btodb(PAGE_SIZE) - 1); 62326f9a767SRodney W. Grimes *srcaddrp = SWB_EMPTY; 62426f9a767SRodney W. Grimes } 62526f9a767SRodney W. Grimes } 62626f9a767SRodney W. Grimes } 62726f9a767SRodney W. Grimes splx(s); 62826f9a767SRodney W. Grimes 629a1f6d91cSDavid Greenman /* 630a1f6d91cSDavid Greenman * Free left over swap blocks 631a1f6d91cSDavid Greenman */ 632a1f6d91cSDavid Greenman swap_pager_free_swap(srcswp); 633a1f6d91cSDavid Greenman 634a1f6d91cSDavid Greenman if( srcswp->sw_allocsize) 635a1f6d91cSDavid Greenman printf("swap_pager_copy: *warning* pager with %d blocks (orig: %d)\n", srcswp->sw_allocsize, origsize); 63626f9a767SRodney W. Grimes free((caddr_t) srcswp->sw_blocks, M_VMPGDATA); 63726f9a767SRodney W. Grimes srcswp->sw_blocks = 0; 63826f9a767SRodney W. Grimes free((caddr_t) srcswp, M_VMPGDATA); 63924a1cce3SDavid Greenman srcobject->pg_data = NULL; 64026f9a767SRodney W. Grimes 64126f9a767SRodney W. Grimes return; 64226f9a767SRodney W. Grimes } 64326f9a767SRodney W. Grimes 64426f9a767SRodney W. Grimes void 64524a1cce3SDavid Greenman swap_pager_dealloc(object) 64624a1cce3SDavid Greenman vm_object_t object; 647df8bae1dSRodney W. Grimes { 648df8bae1dSRodney W. Grimes register sw_pager_t swp; 649df8bae1dSRodney W. Grimes int s; 650df8bae1dSRodney W. Grimes 65124a1cce3SDavid Greenman swp = (sw_pager_t) object->pg_data; 65224a1cce3SDavid Greenman 65324a1cce3SDavid Greenman /* "Can't" happen. */ 65424a1cce3SDavid Greenman if (swp == NULL) 65524a1cce3SDavid Greenman panic("swap_pager_dealloc: no swp data"); 65624a1cce3SDavid Greenman 657df8bae1dSRodney W. Grimes /* 6580d94caffSDavid Greenman * Remove from list right away so lookups will fail if we block for 6590d94caffSDavid Greenman * pageout completion. 660df8bae1dSRodney W. Grimes */ 66124a1cce3SDavid Greenman if (object->handle == NULL) { 66224a1cce3SDavid Greenman TAILQ_REMOVE(&swap_pager_un_object_list, object, pager_object_list); 66326f9a767SRodney W. Grimes } else { 66424a1cce3SDavid Greenman TAILQ_REMOVE(&swap_pager_object_list, object, pager_object_list); 665df8bae1dSRodney W. Grimes } 66624a1cce3SDavid Greenman 667df8bae1dSRodney W. Grimes /* 6680d94caffSDavid Greenman * Wait for all pageouts to finish and remove all entries from 6690d94caffSDavid Greenman * cleaning list. 670df8bae1dSRodney W. Grimes */ 67126f9a767SRodney W. Grimes 67224a1cce3SDavid Greenman s = splbio(); 673df8bae1dSRodney W. Grimes while (swp->sw_poip) { 67424a1cce3SDavid Greenman tsleep(swp, PVM, "swpout", 0); 675df8bae1dSRodney W. Grimes } 676df8bae1dSRodney W. Grimes splx(s); 67726f9a767SRodney W. Grimes 67826f9a767SRodney W. Grimes 67924a1cce3SDavid Greenman swap_pager_sync(); 680df8bae1dSRodney W. Grimes 681df8bae1dSRodney W. Grimes /* 682df8bae1dSRodney W. Grimes * Free left over swap blocks 683df8bae1dSRodney W. Grimes */ 684a1f6d91cSDavid Greenman swap_pager_free_swap(swp); 68526f9a767SRodney W. Grimes 686a1f6d91cSDavid Greenman if( swp->sw_allocsize) 687a1f6d91cSDavid Greenman printf("swap_pager_dealloc: *warning* freeing pager with %d blocks\n", swp->sw_allocsize); 688df8bae1dSRodney W. Grimes /* 689df8bae1dSRodney W. Grimes * Free swap management resources 690df8bae1dSRodney W. Grimes */ 691df8bae1dSRodney W. Grimes free((caddr_t) swp->sw_blocks, M_VMPGDATA); 69226f9a767SRodney W. Grimes swp->sw_blocks = 0; 693df8bae1dSRodney W. Grimes free((caddr_t) swp, M_VMPGDATA); 69424a1cce3SDavid Greenman object->pg_data = 0; 69526f9a767SRodney W. Grimes } 69626f9a767SRodney W. Grimes 69726f9a767SRodney W. Grimes static inline int 6980d94caffSDavid Greenman const 6990d94caffSDavid Greenman swap_pager_block_index(swp, offset) 70026f9a767SRodney W. Grimes sw_pager_t swp; 70126f9a767SRodney W. Grimes vm_offset_t offset; 70226f9a767SRodney W. Grimes { 70326f9a767SRodney W. Grimes return (offset / (SWB_NPAGES * PAGE_SIZE)); 70426f9a767SRodney W. Grimes } 70526f9a767SRodney W. Grimes 70626f9a767SRodney W. Grimes static inline int 7070d94caffSDavid Greenman const 7080d94caffSDavid Greenman swap_pager_block_offset(swp, offset) 70926f9a767SRodney W. Grimes sw_pager_t swp; 71026f9a767SRodney W. Grimes vm_offset_t offset; 71126f9a767SRodney W. Grimes { 71226f9a767SRodney W. Grimes return ((offset % (PAGE_SIZE * SWB_NPAGES)) / PAGE_SIZE); 71326f9a767SRodney W. Grimes } 71426f9a767SRodney W. Grimes 71526f9a767SRodney W. Grimes /* 71624a1cce3SDavid Greenman * swap_pager_haspage returns TRUE if the pager has data that has 71726f9a767SRodney W. Grimes * been written out. 71826f9a767SRodney W. Grimes */ 71924a1cce3SDavid Greenman boolean_t 72024a1cce3SDavid Greenman swap_pager_haspage(object, offset, before, after) 72124a1cce3SDavid Greenman vm_object_t object; 72226f9a767SRodney W. Grimes vm_offset_t offset; 72324a1cce3SDavid Greenman int *before; 72424a1cce3SDavid Greenman int *after; 72526f9a767SRodney W. Grimes { 72624a1cce3SDavid Greenman sw_pager_t swp = object->pg_data; 72726f9a767SRodney W. Grimes register sw_blk_t swb; 72826f9a767SRodney W. Grimes int ix; 72926f9a767SRodney W. Grimes 73024a1cce3SDavid Greenman if (before != NULL) 73124a1cce3SDavid Greenman *before = 0; 73224a1cce3SDavid Greenman if (after != NULL) 73324a1cce3SDavid Greenman *after = 0; 73426f9a767SRodney W. Grimes ix = offset / (SWB_NPAGES * PAGE_SIZE); 73526f9a767SRodney W. Grimes if (swp->sw_blocks == NULL || ix >= swp->sw_nblocks) { 73626f9a767SRodney W. Grimes return (FALSE); 73726f9a767SRodney W. Grimes } 73826f9a767SRodney W. Grimes swb = &swp->sw_blocks[ix]; 73926f9a767SRodney W. Grimes ix = (offset % (SWB_NPAGES * PAGE_SIZE)) / PAGE_SIZE; 74026f9a767SRodney W. Grimes if (swb->swb_block[ix] != SWB_EMPTY) { 74126f9a767SRodney W. Grimes if (swb->swb_valid & (1 << ix)) 74226f9a767SRodney W. Grimes return TRUE; 74326f9a767SRodney W. Grimes } 74426f9a767SRodney W. Grimes return (FALSE); 74526f9a767SRodney W. Grimes } 74626f9a767SRodney W. Grimes 74726f9a767SRodney W. Grimes /* 74826f9a767SRodney W. Grimes * swap_pager_freepage is a convienience routine that clears the busy 74926f9a767SRodney W. Grimes * bit and deallocates a page. 750df8bae1dSRodney W. Grimes */ 75126f9a767SRodney W. Grimes static void 75226f9a767SRodney W. Grimes swap_pager_freepage(m) 75326f9a767SRodney W. Grimes vm_page_t m; 75426f9a767SRodney W. Grimes { 75526f9a767SRodney W. Grimes PAGE_WAKEUP(m); 75626f9a767SRodney W. Grimes vm_page_free(m); 75726f9a767SRodney W. Grimes } 75826f9a767SRodney W. Grimes 75926f9a767SRodney W. Grimes /* 76026f9a767SRodney W. Grimes * swap_pager_ridpages is a convienience routine that deallocates all 76126f9a767SRodney W. Grimes * but the required page. this is usually used in error returns that 76226f9a767SRodney W. Grimes * need to invalidate the "extra" readahead pages. 76326f9a767SRodney W. Grimes */ 76426f9a767SRodney W. Grimes static void 76526f9a767SRodney W. Grimes swap_pager_ridpages(m, count, reqpage) 76626f9a767SRodney W. Grimes vm_page_t *m; 76726f9a767SRodney W. Grimes int count; 76826f9a767SRodney W. Grimes int reqpage; 76926f9a767SRodney W. Grimes { 77026f9a767SRodney W. Grimes int i; 7710d94caffSDavid Greenman 77226f9a767SRodney W. Grimes for (i = 0; i < count; i++) 77326f9a767SRodney W. Grimes if (i != reqpage) 77426f9a767SRodney W. Grimes swap_pager_freepage(m[i]); 77526f9a767SRodney W. Grimes } 77626f9a767SRodney W. Grimes 77726f9a767SRodney W. Grimes /* 77826f9a767SRodney W. Grimes * swap_pager_iodone1 is the completion routine for both reads and async writes 77926f9a767SRodney W. Grimes */ 78026f9a767SRodney W. Grimes void 78126f9a767SRodney W. Grimes swap_pager_iodone1(bp) 78226f9a767SRodney W. Grimes struct buf *bp; 78326f9a767SRodney W. Grimes { 78426f9a767SRodney W. Grimes bp->b_flags |= B_DONE; 78526f9a767SRodney W. Grimes bp->b_flags &= ~B_ASYNC; 78624a1cce3SDavid Greenman wakeup(bp); 78726f9a767SRodney W. Grimes } 78826f9a767SRodney W. Grimes 78926f9a767SRodney W. Grimes 79026f9a767SRodney W. Grimes int 79124a1cce3SDavid Greenman swap_pager_getpages(object, m, count, reqpage) 79224a1cce3SDavid Greenman vm_object_t object; 79326f9a767SRodney W. Grimes vm_page_t *m; 79426f9a767SRodney W. Grimes int count, reqpage; 795df8bae1dSRodney W. Grimes { 79624a1cce3SDavid Greenman register sw_pager_t swp = object->pg_data; 797df8bae1dSRodney W. Grimes register struct buf *bp; 79826f9a767SRodney W. Grimes sw_blk_t swb[count]; 799df8bae1dSRodney W. Grimes register int s; 80026f9a767SRodney W. Grimes int i; 801df8bae1dSRodney W. Grimes boolean_t rv; 80226f9a767SRodney W. Grimes vm_offset_t kva, off[count]; 803df8bae1dSRodney W. Grimes swp_clean_t spc; 80426f9a767SRodney W. Grimes vm_offset_t paging_offset; 80526f9a767SRodney W. Grimes int reqaddr[count]; 8066d40c3d3SDavid Greenman int sequential; 807df8bae1dSRodney W. Grimes 80826f9a767SRodney W. Grimes int first, last; 80926f9a767SRodney W. Grimes int failed; 81026f9a767SRodney W. Grimes int reqdskregion; 811df8bae1dSRodney W. Grimes 81226f9a767SRodney W. Grimes object = m[reqpage]->object; 81326f9a767SRodney W. Grimes paging_offset = object->paging_offset; 8146d40c3d3SDavid Greenman sequential = (m[reqpage]->offset == (object->last_read + PAGE_SIZE)); 815df8bae1dSRodney W. Grimes /* 8160d94caffSDavid Greenman * First determine if the page exists in the pager if this is a sync 8170d94caffSDavid Greenman * read. This quickly handles cases where we are following shadow 8180d94caffSDavid Greenman * chains looking for the top level object with the page. 819df8bae1dSRodney W. Grimes */ 82026f9a767SRodney W. Grimes if (swp->sw_blocks == NULL) { 82126f9a767SRodney W. Grimes swap_pager_ridpages(m, count, reqpage); 822df8bae1dSRodney W. Grimes return (VM_PAGER_FAIL); 823df8bae1dSRodney W. Grimes } 82426f9a767SRodney W. Grimes for (i = 0; i < count; i++) { 82526f9a767SRodney W. Grimes vm_offset_t foff = m[i]->offset + paging_offset; 82626f9a767SRodney W. Grimes int ix = swap_pager_block_index(swp, foff); 8270d94caffSDavid Greenman 82826f9a767SRodney W. Grimes if (ix >= swp->sw_nblocks) { 82926f9a767SRodney W. Grimes int j; 8300d94caffSDavid Greenman 83126f9a767SRodney W. Grimes if (i <= reqpage) { 83226f9a767SRodney W. Grimes swap_pager_ridpages(m, count, reqpage); 833df8bae1dSRodney W. Grimes return (VM_PAGER_FAIL); 83426f9a767SRodney W. Grimes } 83526f9a767SRodney W. Grimes for (j = i; j < count; j++) { 83626f9a767SRodney W. Grimes swap_pager_freepage(m[j]); 83726f9a767SRodney W. Grimes } 83826f9a767SRodney W. Grimes count = i; 83926f9a767SRodney W. Grimes break; 84026f9a767SRodney W. Grimes } 84126f9a767SRodney W. Grimes swb[i] = &swp->sw_blocks[ix]; 84226f9a767SRodney W. Grimes off[i] = swap_pager_block_offset(swp, foff); 84326f9a767SRodney W. Grimes reqaddr[i] = swb[i]->swb_block[off[i]]; 84426f9a767SRodney W. Grimes } 84526f9a767SRodney W. Grimes 84626f9a767SRodney W. Grimes /* make sure that our required input request is existant */ 84726f9a767SRodney W. Grimes 84826f9a767SRodney W. Grimes if (reqaddr[reqpage] == SWB_EMPTY || 84926f9a767SRodney W. Grimes (swb[reqpage]->swb_valid & (1 << off[reqpage])) == 0) { 85026f9a767SRodney W. Grimes swap_pager_ridpages(m, count, reqpage); 85126f9a767SRodney W. Grimes return (VM_PAGER_FAIL); 85226f9a767SRodney W. Grimes } 85326f9a767SRodney W. Grimes reqdskregion = reqaddr[reqpage] / dmmax; 854df8bae1dSRodney W. Grimes 855df8bae1dSRodney W. Grimes /* 85626f9a767SRodney W. Grimes * search backwards for the first contiguous page to transfer 857df8bae1dSRodney W. Grimes */ 85826f9a767SRodney W. Grimes failed = 0; 85926f9a767SRodney W. Grimes first = 0; 86026f9a767SRodney W. Grimes for (i = reqpage - 1; i >= 0; --i) { 8616d40c3d3SDavid Greenman if (sequential || failed || (reqaddr[i] == SWB_EMPTY) || 86226f9a767SRodney W. Grimes (swb[i]->swb_valid & (1 << off[i])) == 0 || 86326f9a767SRodney W. Grimes (reqaddr[i] != (reqaddr[reqpage] + (i - reqpage) * btodb(PAGE_SIZE))) || 86426f9a767SRodney W. Grimes ((reqaddr[i] / dmmax) != reqdskregion)) { 86526f9a767SRodney W. Grimes failed = 1; 86626f9a767SRodney W. Grimes swap_pager_freepage(m[i]); 86726f9a767SRodney W. Grimes if (first == 0) 86826f9a767SRodney W. Grimes first = i + 1; 86926f9a767SRodney W. Grimes } 870df8bae1dSRodney W. Grimes } 871df8bae1dSRodney W. Grimes /* 87226f9a767SRodney W. Grimes * search forwards for the last contiguous page to transfer 873df8bae1dSRodney W. Grimes */ 87426f9a767SRodney W. Grimes failed = 0; 87526f9a767SRodney W. Grimes last = count; 87626f9a767SRodney W. Grimes for (i = reqpage + 1; i < count; i++) { 87726f9a767SRodney W. Grimes if (failed || (reqaddr[i] == SWB_EMPTY) || 87826f9a767SRodney W. Grimes (swb[i]->swb_valid & (1 << off[i])) == 0 || 87926f9a767SRodney W. Grimes (reqaddr[i] != (reqaddr[reqpage] + (i - reqpage) * btodb(PAGE_SIZE))) || 88026f9a767SRodney W. Grimes ((reqaddr[i] / dmmax) != reqdskregion)) { 88126f9a767SRodney W. Grimes failed = 1; 88226f9a767SRodney W. Grimes swap_pager_freepage(m[i]); 88326f9a767SRodney W. Grimes if (last == count) 88426f9a767SRodney W. Grimes last = i; 88526f9a767SRodney W. Grimes } 88626f9a767SRodney W. Grimes } 88726f9a767SRodney W. Grimes 88826f9a767SRodney W. Grimes count = last; 88926f9a767SRodney W. Grimes if (first != 0) { 89026f9a767SRodney W. Grimes for (i = first; i < count; i++) { 89126f9a767SRodney W. Grimes m[i - first] = m[i]; 89226f9a767SRodney W. Grimes reqaddr[i - first] = reqaddr[i]; 89326f9a767SRodney W. Grimes off[i - first] = off[i]; 89426f9a767SRodney W. Grimes } 89526f9a767SRodney W. Grimes count -= first; 89626f9a767SRodney W. Grimes reqpage -= first; 89726f9a767SRodney W. Grimes } 89826f9a767SRodney W. Grimes ++swb[reqpage]->swb_locked; 89926f9a767SRodney W. Grimes 90026f9a767SRodney W. Grimes /* 9010d94caffSDavid Greenman * at this point: "m" is a pointer to the array of vm_page_t for 9020d94caffSDavid Greenman * paging I/O "count" is the number of vm_page_t entries represented 9030d94caffSDavid Greenman * by "m" "object" is the vm_object_t for I/O "reqpage" is the index 9040d94caffSDavid Greenman * into "m" for the page actually faulted 90526f9a767SRodney W. Grimes */ 90626f9a767SRodney W. Grimes 90726f9a767SRodney W. Grimes spc = NULL; /* we might not use an spc data structure */ 90826f9a767SRodney W. Grimes 909a1f6d91cSDavid Greenman if ((count == 1) && (swap_pager_free.tqh_first != NULL)) { 91026f9a767SRodney W. Grimes /* 9110d94caffSDavid Greenman * if a kva has not been allocated, we can only do a one page 9120d94caffSDavid Greenman * transfer, so we free the other pages that might have been 9130d94caffSDavid Greenman * allocated by vm_fault. 91426f9a767SRodney W. Grimes */ 91526f9a767SRodney W. Grimes swap_pager_ridpages(m, count, reqpage); 91626f9a767SRodney W. Grimes m[0] = m[reqpage]; 91726f9a767SRodney W. Grimes reqaddr[0] = reqaddr[reqpage]; 91826f9a767SRodney W. Grimes 91926f9a767SRodney W. Grimes count = 1; 92026f9a767SRodney W. Grimes reqpage = 0; 92126f9a767SRodney W. Grimes /* 9220d94caffSDavid Greenman * get a swap pager clean data structure, block until we get 9230d94caffSDavid Greenman * it 92426f9a767SRodney W. Grimes */ 925df8bae1dSRodney W. Grimes if (swap_pager_free.tqh_first == NULL) { 926df8bae1dSRodney W. Grimes s = splbio(); 92726f9a767SRodney W. Grimes if (curproc == pageproc) 92824a1cce3SDavid Greenman swap_pager_sync(); 92926f9a767SRodney W. Grimes else 930f919ebdeSDavid Greenman pagedaemon_wakeup(); 93126f9a767SRodney W. Grimes while (swap_pager_free.tqh_first == NULL) { 93226f9a767SRodney W. Grimes swap_pager_needflags |= SWAP_FREE_NEEDED; 933a1f6d91cSDavid Greenman if (curproc == pageproc) 934a1f6d91cSDavid Greenman swap_pager_needflags |= SWAP_FREE_NEEDED_BY_PAGEOUT; 93524a1cce3SDavid Greenman tsleep(&swap_pager_free, 93626f9a767SRodney W. Grimes PVM, "swpfre", 0); 93726f9a767SRodney W. Grimes if (curproc == pageproc) 93824a1cce3SDavid Greenman swap_pager_sync(); 93926f9a767SRodney W. Grimes else 940f919ebdeSDavid Greenman pagedaemon_wakeup(); 941df8bae1dSRodney W. Grimes } 942df8bae1dSRodney W. Grimes splx(s); 94326f9a767SRodney W. Grimes } 94426f9a767SRodney W. Grimes spc = swap_pager_free.tqh_first; 94526f9a767SRodney W. Grimes TAILQ_REMOVE(&swap_pager_free, spc, spc_list); 94626f9a767SRodney W. Grimes kva = spc->spc_kva; 94726f9a767SRodney W. Grimes bp = spc->spc_bp; 94826f9a767SRodney W. Grimes bzero(bp, sizeof *bp); 94926f9a767SRodney W. Grimes bp->b_spc = spc; 9507609ab12SDavid Greenman bp->b_vnbufs.le_next = NOLIST; 95126f9a767SRodney W. Grimes } else { 95216f62314SDavid Greenman /* 95316f62314SDavid Greenman * Get a swap buffer header to perform the IO 95416f62314SDavid Greenman */ 95526f9a767SRodney W. Grimes bp = getpbuf(); 95616f62314SDavid Greenman kva = (vm_offset_t) bp->b_data; 95726f9a767SRodney W. Grimes } 95826f9a767SRodney W. Grimes 95916f62314SDavid Greenman /* 96016f62314SDavid Greenman * map our page(s) into kva for input 96116f62314SDavid Greenman */ 96216f62314SDavid Greenman pmap_qenter(kva, m, count); 96316f62314SDavid Greenman 964aba8f38eSDavid Greenman bp->b_flags = B_BUSY | B_READ | B_CALL | B_PAGING; 96526f9a767SRodney W. Grimes bp->b_iodone = swap_pager_iodone1; 966df8bae1dSRodney W. Grimes bp->b_proc = &proc0; /* XXX (but without B_PHYS set this is ok) */ 96726f9a767SRodney W. Grimes bp->b_rcred = bp->b_wcred = bp->b_proc->p_ucred; 96826f9a767SRodney W. Grimes crhold(bp->b_rcred); 96926f9a767SRodney W. Grimes crhold(bp->b_wcred); 97026f9a767SRodney W. Grimes bp->b_un.b_addr = (caddr_t) kva; 97126f9a767SRodney W. Grimes bp->b_blkno = reqaddr[0]; 97226f9a767SRodney W. Grimes bp->b_bcount = PAGE_SIZE * count; 97326f9a767SRodney W. Grimes bp->b_bufsize = PAGE_SIZE * count; 97426f9a767SRodney W. Grimes 9750d94caffSDavid Greenman pbgetvp(swapdev_vp, bp); 97626f9a767SRodney W. Grimes swp->sw_piip++; 977df8bae1dSRodney W. Grimes 978976e77fcSDavid Greenman cnt.v_swapin++; 979976e77fcSDavid Greenman cnt.v_swappgsin += count; 980df8bae1dSRodney W. Grimes /* 98126f9a767SRodney W. Grimes * perform the I/O 982df8bae1dSRodney W. Grimes */ 983df8bae1dSRodney W. Grimes VOP_STRATEGY(bp); 98426f9a767SRodney W. Grimes 98526f9a767SRodney W. Grimes /* 98626f9a767SRodney W. Grimes * wait for the sync I/O to complete 98726f9a767SRodney W. Grimes */ 9887609ab12SDavid Greenman s = splbio(); 98926f9a767SRodney W. Grimes while ((bp->b_flags & B_DONE) == 0) { 99024a1cce3SDavid Greenman tsleep(bp, PVM, "swread", 0); 991df8bae1dSRodney W. Grimes } 9921b119d9dSDavid Greenman 9931b119d9dSDavid Greenman if (bp->b_flags & B_ERROR) { 9941b119d9dSDavid Greenman printf("swap_pager: I/O error - pagein failed; blkno %d, size %d, error %d\n", 9951b119d9dSDavid Greenman bp->b_blkno, bp->b_bcount, bp->b_error); 996a83c285cSDavid Greenman rv = VM_PAGER_ERROR; 9971b119d9dSDavid Greenman } else { 9981b119d9dSDavid Greenman rv = VM_PAGER_OK; 9991b119d9dSDavid Greenman } 100026f9a767SRodney W. Grimes 100126f9a767SRodney W. Grimes --swp->sw_piip; 100226f9a767SRodney W. Grimes if (swp->sw_piip == 0) 100324a1cce3SDavid Greenman wakeup(swp); 100426f9a767SRodney W. Grimes 10050d94caffSDavid Greenman 100626f9a767SRodney W. Grimes /* 10070d94caffSDavid Greenman * relpbuf does this, but we maintain our own buffer list also... 100826f9a767SRodney W. Grimes */ 1009df8bae1dSRodney W. Grimes if (bp->b_vp) 10100d94caffSDavid Greenman pbrelvp(bp); 101126f9a767SRodney W. Grimes 1012df8bae1dSRodney W. Grimes splx(s); 101326f9a767SRodney W. Grimes --swb[reqpage]->swb_locked; 101426f9a767SRodney W. Grimes 101526f9a767SRodney W. Grimes /* 101626f9a767SRodney W. Grimes * remove the mapping for kernel virtual 101726f9a767SRodney W. Grimes */ 101816f62314SDavid Greenman pmap_qremove(kva, count); 101926f9a767SRodney W. Grimes 102026f9a767SRodney W. Grimes if (spc) { 10216d40c3d3SDavid Greenman m[reqpage]->object->last_read = m[reqpage]->offset; 10220d94caffSDavid Greenman if (bp->b_flags & B_WANTED) 102324a1cce3SDavid Greenman wakeup(bp); 102426f9a767SRodney W. Grimes /* 102526f9a767SRodney W. Grimes * if we have used an spc, we need to free it. 102626f9a767SRodney W. Grimes */ 102726f9a767SRodney W. Grimes if (bp->b_rcred != NOCRED) 102826f9a767SRodney W. Grimes crfree(bp->b_rcred); 102926f9a767SRodney W. Grimes if (bp->b_wcred != NOCRED) 103026f9a767SRodney W. Grimes crfree(bp->b_wcred); 103126f9a767SRodney W. Grimes TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 103226f9a767SRodney W. Grimes if (swap_pager_needflags & SWAP_FREE_NEEDED) { 103324a1cce3SDavid Greenman wakeup(&swap_pager_free); 103426f9a767SRodney W. Grimes } 1035a1f6d91cSDavid Greenman if( swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) 1036f919ebdeSDavid Greenman pagedaemon_wakeup(); 1037a1f6d91cSDavid Greenman swap_pager_needflags &= ~(SWAP_FREE_NEEDED|SWAP_FREE_NEEDED_BY_PAGEOUT); 103826f9a767SRodney W. Grimes } else { 103926f9a767SRodney W. Grimes /* 104026f9a767SRodney W. Grimes * release the physical I/O buffer 104126f9a767SRodney W. Grimes */ 104226f9a767SRodney W. Grimes relpbuf(bp); 104326f9a767SRodney W. Grimes /* 104426f9a767SRodney W. Grimes * finish up input if everything is ok 104526f9a767SRodney W. Grimes */ 104626f9a767SRodney W. Grimes if (rv == VM_PAGER_OK) { 104726f9a767SRodney W. Grimes for (i = 0; i < count; i++) { 104826f9a767SRodney W. Grimes pmap_clear_modify(VM_PAGE_TO_PHYS(m[i])); 10490d94caffSDavid Greenman m[i]->dirty = 0; 105026f9a767SRodney W. Grimes if (i != reqpage) { 105126f9a767SRodney W. Grimes /* 10520d94caffSDavid Greenman * whether or not to leave the page 10530d94caffSDavid Greenman * activated is up in the air, but we 10540d94caffSDavid Greenman * should put the page on a page queue 10550d94caffSDavid Greenman * somewhere. (it already is in the 10560d94caffSDavid Greenman * object). After some emperical 10570d94caffSDavid Greenman * results, it is best to deactivate 10580d94caffSDavid Greenman * the readahead pages. 105926f9a767SRodney W. Grimes */ 106026f9a767SRodney W. Grimes vm_page_deactivate(m[i]); 106126f9a767SRodney W. Grimes 106226f9a767SRodney W. Grimes /* 10630d94caffSDavid Greenman * just in case someone was asking for 10640d94caffSDavid Greenman * this page we now tell them that it 10650d94caffSDavid Greenman * is ok to use 106626f9a767SRodney W. Grimes */ 10670d94caffSDavid Greenman m[i]->valid = VM_PAGE_BITS_ALL; 106826f9a767SRodney W. Grimes PAGE_WAKEUP(m[i]); 106926f9a767SRodney W. Grimes } 107026f9a767SRodney W. Grimes } 10716d40c3d3SDavid Greenman 10726d40c3d3SDavid Greenman m[reqpage]->object->last_read = m[count-1]->offset; 10736d40c3d3SDavid Greenman 10742e1e24ddSDavid Greenman /* 10752e1e24ddSDavid Greenman * If we're out of swap space, then attempt to free 10762e1e24ddSDavid Greenman * some whenever pages are brought in. We must clear 10772e1e24ddSDavid Greenman * the clean flag so that the page contents will be 10782e1e24ddSDavid Greenman * preserved. 10792e1e24ddSDavid Greenman */ 108026f9a767SRodney W. Grimes if (swap_pager_full) { 10812e1e24ddSDavid Greenman for (i = 0; i < count; i++) { 10820d94caffSDavid Greenman m[i]->dirty = VM_PAGE_BITS_ALL; 10832e1e24ddSDavid Greenman } 108424a1cce3SDavid Greenman swap_pager_freespace(object, m[0]->offset + paging_offset, count * PAGE_SIZE); 108526f9a767SRodney W. Grimes } 108626f9a767SRodney W. Grimes } else { 108726f9a767SRodney W. Grimes swap_pager_ridpages(m, count, reqpage); 108826f9a767SRodney W. Grimes } 108926f9a767SRodney W. Grimes } 1090df8bae1dSRodney W. Grimes return (rv); 1091df8bae1dSRodney W. Grimes } 1092df8bae1dSRodney W. Grimes 109326f9a767SRodney W. Grimes int 109424a1cce3SDavid Greenman swap_pager_putpages(object, m, count, sync, rtvals) 109524a1cce3SDavid Greenman vm_object_t object; 109626f9a767SRodney W. Grimes vm_page_t *m; 109726f9a767SRodney W. Grimes int count; 109824a1cce3SDavid Greenman boolean_t sync; 109926f9a767SRodney W. Grimes int *rtvals; 1100df8bae1dSRodney W. Grimes { 110124a1cce3SDavid Greenman register sw_pager_t swp = object->pg_data; 110226f9a767SRodney W. Grimes register struct buf *bp; 110326f9a767SRodney W. Grimes sw_blk_t swb[count]; 110426f9a767SRodney W. Grimes register int s; 110526f9a767SRodney W. Grimes int i, j, ix; 110626f9a767SRodney W. Grimes boolean_t rv; 110726f9a767SRodney W. Grimes vm_offset_t kva, off, foff; 110826f9a767SRodney W. Grimes swp_clean_t spc; 110926f9a767SRodney W. Grimes vm_offset_t paging_offset; 111026f9a767SRodney W. Grimes int reqaddr[count]; 111126f9a767SRodney W. Grimes int failed; 1112df8bae1dSRodney W. Grimes 111324ea4a96SDavid Greenman if (vm_swap_size) 111424ea4a96SDavid Greenman no_swap_space = 0; 111524ea4a96SDavid Greenman if (no_swap_space) { 11165663e6deSDavid Greenman for (i = 0; i < count; i++) 11175663e6deSDavid Greenman rtvals[i] = VM_PAGER_FAIL; 11185663e6deSDavid Greenman return VM_PAGER_FAIL; 11195663e6deSDavid Greenman } 112026f9a767SRodney W. Grimes spc = NULL; 112126f9a767SRodney W. Grimes 112226f9a767SRodney W. Grimes object = m[0]->object; 112326f9a767SRodney W. Grimes paging_offset = object->paging_offset; 112426f9a767SRodney W. Grimes 112526f9a767SRodney W. Grimes failed = 0; 112626f9a767SRodney W. Grimes for (j = 0; j < count; j++) { 112726f9a767SRodney W. Grimes foff = m[j]->offset + paging_offset; 112826f9a767SRodney W. Grimes ix = swap_pager_block_index(swp, foff); 112926f9a767SRodney W. Grimes swb[j] = 0; 113026f9a767SRodney W. Grimes if (swp->sw_blocks == NULL || ix >= swp->sw_nblocks) { 113126f9a767SRodney W. Grimes rtvals[j] = VM_PAGER_FAIL; 113226f9a767SRodney W. Grimes failed = 1; 113326f9a767SRodney W. Grimes continue; 113426f9a767SRodney W. Grimes } else { 113526f9a767SRodney W. Grimes rtvals[j] = VM_PAGER_OK; 113626f9a767SRodney W. Grimes } 113726f9a767SRodney W. Grimes swb[j] = &swp->sw_blocks[ix]; 113826f9a767SRodney W. Grimes ++swb[j]->swb_locked; 113926f9a767SRodney W. Grimes if (failed) { 114026f9a767SRodney W. Grimes rtvals[j] = VM_PAGER_FAIL; 114126f9a767SRodney W. Grimes continue; 114226f9a767SRodney W. Grimes } 114326f9a767SRodney W. Grimes off = swap_pager_block_offset(swp, foff); 114426f9a767SRodney W. Grimes reqaddr[j] = swb[j]->swb_block[off]; 114526f9a767SRodney W. Grimes if (reqaddr[j] == SWB_EMPTY) { 114626f9a767SRodney W. Grimes int blk; 114726f9a767SRodney W. Grimes int tries; 114826f9a767SRodney W. Grimes int ntoget; 11490d94caffSDavid Greenman 115026f9a767SRodney W. Grimes tries = 0; 1151df8bae1dSRodney W. Grimes s = splbio(); 115226f9a767SRodney W. Grimes 1153df8bae1dSRodney W. Grimes /* 11540d94caffSDavid Greenman * if any other pages have been allocated in this 11550d94caffSDavid Greenman * block, we only try to get one page. 1156df8bae1dSRodney W. Grimes */ 115726f9a767SRodney W. Grimes for (i = 0; i < SWB_NPAGES; i++) { 115826f9a767SRodney W. Grimes if (swb[j]->swb_block[i] != SWB_EMPTY) 1159df8bae1dSRodney W. Grimes break; 1160df8bae1dSRodney W. Grimes } 116126f9a767SRodney W. Grimes 116226f9a767SRodney W. Grimes ntoget = (i == SWB_NPAGES) ? SWB_NPAGES : 1; 116326f9a767SRodney W. Grimes /* 11640d94caffSDavid Greenman * this code is alittle conservative, but works (the 11650d94caffSDavid Greenman * intent of this code is to allocate small chunks for 11660d94caffSDavid Greenman * small objects) 116726f9a767SRodney W. Grimes */ 116811fda60bSJohn Dyson if ((foff == 0) && 116911fda60bSJohn Dyson ((ntoget * PAGE_SIZE) > object->size)) { 117011fda60bSJohn Dyson ntoget = (object->size + (PAGE_SIZE - 1)) / PAGE_SIZE; 117126f9a767SRodney W. Grimes } 117226f9a767SRodney W. Grimes retrygetspace: 117326f9a767SRodney W. Grimes if (!swap_pager_full && ntoget > 1 && 1174a1f6d91cSDavid Greenman swap_pager_getswapspace(swp, ntoget * btodb(PAGE_SIZE), &blk)) { 117526f9a767SRodney W. Grimes 117626f9a767SRodney W. Grimes for (i = 0; i < ntoget; i++) { 117726f9a767SRodney W. Grimes swb[j]->swb_block[i] = blk + btodb(PAGE_SIZE) * i; 117826f9a767SRodney W. Grimes swb[j]->swb_valid = 0; 117926f9a767SRodney W. Grimes } 118026f9a767SRodney W. Grimes 118126f9a767SRodney W. Grimes reqaddr[j] = swb[j]->swb_block[off]; 1182a1f6d91cSDavid Greenman } else if (!swap_pager_getswapspace(swp, btodb(PAGE_SIZE), 118326f9a767SRodney W. Grimes &swb[j]->swb_block[off])) { 118426f9a767SRodney W. Grimes /* 11850d94caffSDavid Greenman * if the allocation has failed, we try to 11860d94caffSDavid Greenman * reclaim space and retry. 118726f9a767SRodney W. Grimes */ 118826f9a767SRodney W. Grimes if (++tries == 1) { 118926f9a767SRodney W. Grimes swap_pager_reclaim(); 119026f9a767SRodney W. Grimes goto retrygetspace; 119126f9a767SRodney W. Grimes } 119226f9a767SRodney W. Grimes rtvals[j] = VM_PAGER_AGAIN; 119326f9a767SRodney W. Grimes failed = 1; 119424ea4a96SDavid Greenman swap_pager_full = 1; 119526f9a767SRodney W. Grimes } else { 119626f9a767SRodney W. Grimes reqaddr[j] = swb[j]->swb_block[off]; 119726f9a767SRodney W. Grimes swb[j]->swb_valid &= ~(1 << off); 1198df8bae1dSRodney W. Grimes } 1199df8bae1dSRodney W. Grimes splx(s); 120026f9a767SRodney W. Grimes } 120126f9a767SRodney W. Grimes } 120226f9a767SRodney W. Grimes 120326f9a767SRodney W. Grimes /* 120426f9a767SRodney W. Grimes * search forwards for the last contiguous page to transfer 120526f9a767SRodney W. Grimes */ 120626f9a767SRodney W. Grimes failed = 0; 120726f9a767SRodney W. Grimes for (i = 0; i < count; i++) { 120826f9a767SRodney W. Grimes if (failed || (reqaddr[i] != reqaddr[0] + i * btodb(PAGE_SIZE)) || 120926f9a767SRodney W. Grimes (reqaddr[i] / dmmax) != (reqaddr[0] / dmmax) || 121026f9a767SRodney W. Grimes (rtvals[i] != VM_PAGER_OK)) { 121126f9a767SRodney W. Grimes failed = 1; 121226f9a767SRodney W. Grimes if (rtvals[i] == VM_PAGER_OK) 121326f9a767SRodney W. Grimes rtvals[i] = VM_PAGER_AGAIN; 121426f9a767SRodney W. Grimes } 121526f9a767SRodney W. Grimes } 121626f9a767SRodney W. Grimes 121726f9a767SRodney W. Grimes for (i = 0; i < count; i++) { 121826f9a767SRodney W. Grimes if (rtvals[i] != VM_PAGER_OK) { 121926f9a767SRodney W. Grimes if (swb[i]) 122026f9a767SRodney W. Grimes --swb[i]->swb_locked; 122126f9a767SRodney W. Grimes } 122226f9a767SRodney W. Grimes } 122326f9a767SRodney W. Grimes 122426f9a767SRodney W. Grimes for (i = 0; i < count; i++) 122526f9a767SRodney W. Grimes if (rtvals[i] != VM_PAGER_OK) 122626f9a767SRodney W. Grimes break; 122726f9a767SRodney W. Grimes 122826f9a767SRodney W. Grimes if (i == 0) { 122926f9a767SRodney W. Grimes return VM_PAGER_AGAIN; 123026f9a767SRodney W. Grimes } 123126f9a767SRodney W. Grimes count = i; 123226f9a767SRodney W. Grimes for (i = 0; i < count; i++) { 123326f9a767SRodney W. Grimes if (reqaddr[i] == SWB_EMPTY) 123426f9a767SRodney W. Grimes printf("I/O to empty block????\n"); 123526f9a767SRodney W. Grimes } 123626f9a767SRodney W. Grimes 123726f9a767SRodney W. Grimes /* 12380d94caffSDavid Greenman * For synchronous writes, we clean up all completed async pageouts. 123926f9a767SRodney W. Grimes */ 124024a1cce3SDavid Greenman if (sync == TRUE) { 124124a1cce3SDavid Greenman swap_pager_sync(); 124226f9a767SRodney W. Grimes } 124326f9a767SRodney W. Grimes kva = 0; 124426f9a767SRodney W. Grimes 124526f9a767SRodney W. Grimes /* 124626f9a767SRodney W. Grimes * get a swap pager clean data structure, block until we get it 124726f9a767SRodney W. Grimes */ 1248a1f6d91cSDavid Greenman if (swap_pager_free.tqh_first == NULL || 1249a1f6d91cSDavid Greenman swap_pager_free.tqh_first->spc_list.tqe_next == NULL || 1250a1f6d91cSDavid Greenman swap_pager_free.tqh_first->spc_list.tqe_next->spc_list.tqe_next == NULL) { 125126f9a767SRodney W. Grimes s = splbio(); 12520d94caffSDavid Greenman if (curproc == pageproc) { 125324a1cce3SDavid Greenman swap_pager_sync(); 1254a1f6d91cSDavid Greenman #if 0 12550d94caffSDavid Greenman splx(s); 12560d94caffSDavid Greenman return VM_PAGER_AGAIN; 1257a1f6d91cSDavid Greenman #endif 12580d94caffSDavid Greenman } else 1259f919ebdeSDavid Greenman pagedaemon_wakeup(); 1260a1f6d91cSDavid Greenman while (swap_pager_free.tqh_first == NULL || 1261a1f6d91cSDavid Greenman swap_pager_free.tqh_first->spc_list.tqe_next == NULL || 1262a1f6d91cSDavid Greenman swap_pager_free.tqh_first->spc_list.tqe_next->spc_list.tqe_next == NULL) { 1263a1f6d91cSDavid Greenman if (curproc == pageproc) { 1264a1f6d91cSDavid Greenman swap_pager_needflags |= SWAP_FREE_NEEDED_BY_PAGEOUT; 1265a1f6d91cSDavid Greenman if((cnt.v_free_count + cnt.v_cache_count) > cnt.v_free_reserved) 126624a1cce3SDavid Greenman wakeup(&cnt.v_free_count); 1267a1f6d91cSDavid Greenman } 12680d94caffSDavid Greenman 126926f9a767SRodney W. Grimes swap_pager_needflags |= SWAP_FREE_NEEDED; 127024a1cce3SDavid Greenman tsleep(&swap_pager_free, PVM, "swpfre", 0); 127126f9a767SRodney W. Grimes if (curproc == pageproc) 127224a1cce3SDavid Greenman swap_pager_sync(); 127326f9a767SRodney W. Grimes else 1274f919ebdeSDavid Greenman pagedaemon_wakeup(); 127526f9a767SRodney W. Grimes } 127626f9a767SRodney W. Grimes splx(s); 127726f9a767SRodney W. Grimes } 127826f9a767SRodney W. Grimes spc = swap_pager_free.tqh_first; 127926f9a767SRodney W. Grimes TAILQ_REMOVE(&swap_pager_free, spc, spc_list); 1280fff93ab6SDavid Greenman 128126f9a767SRodney W. Grimes kva = spc->spc_kva; 128226f9a767SRodney W. Grimes 128326f9a767SRodney W. Grimes /* 128426f9a767SRodney W. Grimes * map our page(s) into kva for I/O 128526f9a767SRodney W. Grimes */ 128616f62314SDavid Greenman pmap_qenter(kva, m, count); 128726f9a767SRodney W. Grimes 128826f9a767SRodney W. Grimes /* 128926f9a767SRodney W. Grimes * get the base I/O offset into the swap file 129026f9a767SRodney W. Grimes */ 129126f9a767SRodney W. Grimes for (i = 0; i < count; i++) { 129226f9a767SRodney W. Grimes foff = m[i]->offset + paging_offset; 129326f9a767SRodney W. Grimes off = swap_pager_block_offset(swp, foff); 129426f9a767SRodney W. Grimes /* 129526f9a767SRodney W. Grimes * set the valid bit 129626f9a767SRodney W. Grimes */ 129726f9a767SRodney W. Grimes swb[i]->swb_valid |= (1 << off); 129826f9a767SRodney W. Grimes /* 129926f9a767SRodney W. Grimes * and unlock the data structure 130026f9a767SRodney W. Grimes */ 130126f9a767SRodney W. Grimes --swb[i]->swb_locked; 130226f9a767SRodney W. Grimes } 130326f9a767SRodney W. Grimes 130426f9a767SRodney W. Grimes /* 130526f9a767SRodney W. Grimes * Get a swap buffer header and perform the IO 130626f9a767SRodney W. Grimes */ 130726f9a767SRodney W. Grimes bp = spc->spc_bp; 130826f9a767SRodney W. Grimes bzero(bp, sizeof *bp); 130926f9a767SRodney W. Grimes bp->b_spc = spc; 13107609ab12SDavid Greenman bp->b_vnbufs.le_next = NOLIST; 131126f9a767SRodney W. Grimes 1312aba8f38eSDavid Greenman bp->b_flags = B_BUSY | B_PAGING; 131326f9a767SRodney W. Grimes bp->b_proc = &proc0; /* XXX (but without B_PHYS set this is ok) */ 131426f9a767SRodney W. Grimes bp->b_rcred = bp->b_wcred = bp->b_proc->p_ucred; 1315a481f200SDavid Greenman if (bp->b_rcred != NOCRED) 131626f9a767SRodney W. Grimes crhold(bp->b_rcred); 1317a481f200SDavid Greenman if (bp->b_wcred != NOCRED) 131826f9a767SRodney W. Grimes crhold(bp->b_wcred); 1319a481f200SDavid Greenman bp->b_data = (caddr_t) kva; 132026f9a767SRodney W. Grimes bp->b_blkno = reqaddr[0]; 13210d94caffSDavid Greenman pbgetvp(swapdev_vp, bp); 132216f62314SDavid Greenman 132326f9a767SRodney W. Grimes bp->b_bcount = PAGE_SIZE * count; 132426f9a767SRodney W. Grimes bp->b_bufsize = PAGE_SIZE * count; 132526f9a767SRodney W. Grimes swapdev_vp->v_numoutput++; 132626f9a767SRodney W. Grimes 132726f9a767SRodney W. Grimes /* 13280d94caffSDavid Greenman * If this is an async write we set up additional buffer fields and 13290d94caffSDavid Greenman * place a "cleaning" entry on the inuse queue. 133026f9a767SRodney W. Grimes */ 13317609ab12SDavid Greenman s = splbio(); 133224a1cce3SDavid Greenman if (sync == FALSE) { 133326f9a767SRodney W. Grimes spc->spc_flags = 0; 133426f9a767SRodney W. Grimes spc->spc_swp = swp; 133526f9a767SRodney W. Grimes for (i = 0; i < count; i++) 133626f9a767SRodney W. Grimes spc->spc_m[i] = m[i]; 133726f9a767SRodney W. Grimes spc->spc_count = count; 133826f9a767SRodney W. Grimes /* 133926f9a767SRodney W. Grimes * the completion routine for async writes 134026f9a767SRodney W. Grimes */ 134126f9a767SRodney W. Grimes bp->b_flags |= B_CALL; 134226f9a767SRodney W. Grimes bp->b_iodone = swap_pager_iodone; 134326f9a767SRodney W. Grimes bp->b_dirtyoff = 0; 134426f9a767SRodney W. Grimes bp->b_dirtyend = bp->b_bcount; 134526f9a767SRodney W. Grimes swp->sw_poip++; 134626f9a767SRodney W. Grimes TAILQ_INSERT_TAIL(&swap_pager_inuse, spc, spc_list); 134726f9a767SRodney W. Grimes } else { 134826f9a767SRodney W. Grimes swp->sw_poip++; 134926f9a767SRodney W. Grimes bp->b_flags |= B_CALL; 135026f9a767SRodney W. Grimes bp->b_iodone = swap_pager_iodone1; 135126f9a767SRodney W. Grimes } 1352976e77fcSDavid Greenman 1353976e77fcSDavid Greenman cnt.v_swapout++; 1354976e77fcSDavid Greenman cnt.v_swappgsout += count; 135526f9a767SRodney W. Grimes /* 135626f9a767SRodney W. Grimes * perform the I/O 135726f9a767SRodney W. Grimes */ 135826f9a767SRodney W. Grimes VOP_STRATEGY(bp); 135924a1cce3SDavid Greenman if (sync == FALSE) { 136026f9a767SRodney W. Grimes if ((bp->b_flags & B_DONE) == B_DONE) { 136124a1cce3SDavid Greenman swap_pager_sync(); 136226f9a767SRodney W. Grimes } 136326f9a767SRodney W. Grimes splx(s); 136426f9a767SRodney W. Grimes for (i = 0; i < count; i++) { 136526f9a767SRodney W. Grimes rtvals[i] = VM_PAGER_PEND; 136626f9a767SRodney W. Grimes } 136726f9a767SRodney W. Grimes return VM_PAGER_PEND; 136826f9a767SRodney W. Grimes } 136926f9a767SRodney W. Grimes /* 137026f9a767SRodney W. Grimes * wait for the sync I/O to complete 137126f9a767SRodney W. Grimes */ 137226f9a767SRodney W. Grimes while ((bp->b_flags & B_DONE) == 0) { 137324a1cce3SDavid Greenman tsleep(bp, PVM, "swwrt", 0); 137426f9a767SRodney W. Grimes } 13751b119d9dSDavid Greenman if (bp->b_flags & B_ERROR) { 13761b119d9dSDavid Greenman printf("swap_pager: I/O error - pageout failed; blkno %d, size %d, error %d\n", 13771b119d9dSDavid Greenman bp->b_blkno, bp->b_bcount, bp->b_error); 1378a83c285cSDavid Greenman rv = VM_PAGER_ERROR; 13791b119d9dSDavid Greenman } else { 13801b119d9dSDavid Greenman rv = VM_PAGER_OK; 13811b119d9dSDavid Greenman } 138226f9a767SRodney W. Grimes 138326f9a767SRodney W. Grimes --swp->sw_poip; 138426f9a767SRodney W. Grimes if (swp->sw_poip == 0) 138524a1cce3SDavid Greenman wakeup(swp); 138626f9a767SRodney W. Grimes 138726f9a767SRodney W. Grimes if (bp->b_vp) 13880d94caffSDavid Greenman pbrelvp(bp); 13890d94caffSDavid Greenman if (bp->b_flags & B_WANTED) 139024a1cce3SDavid Greenman wakeup(bp); 139126f9a767SRodney W. Grimes 139226f9a767SRodney W. Grimes splx(s); 139326f9a767SRodney W. Grimes 139426f9a767SRodney W. Grimes /* 139526f9a767SRodney W. Grimes * remove the mapping for kernel virtual 139626f9a767SRodney W. Grimes */ 139716f62314SDavid Greenman pmap_qremove(kva, count); 139826f9a767SRodney W. Grimes 139926f9a767SRodney W. Grimes /* 14000d94caffSDavid Greenman * if we have written the page, then indicate that the page is clean. 140126f9a767SRodney W. Grimes */ 140226f9a767SRodney W. Grimes if (rv == VM_PAGER_OK) { 140326f9a767SRodney W. Grimes for (i = 0; i < count; i++) { 140426f9a767SRodney W. Grimes if (rtvals[i] == VM_PAGER_OK) { 140526f9a767SRodney W. Grimes pmap_clear_modify(VM_PAGE_TO_PHYS(m[i])); 14060d94caffSDavid Greenman m[i]->dirty = 0; 140726f9a767SRodney W. Grimes /* 14080d94caffSDavid Greenman * optimization, if a page has been read 14090d94caffSDavid Greenman * during the pageout process, we activate it. 141026f9a767SRodney W. Grimes */ 141126f9a767SRodney W. Grimes if ((m[i]->flags & PG_ACTIVE) == 0 && 14127fb0c17eSDavid Greenman ((m[i]->flags & (PG_WANTED|PG_REFERENCED)) || 14137fb0c17eSDavid Greenman pmap_is_referenced(VM_PAGE_TO_PHYS(m[i])))) { 141426f9a767SRodney W. Grimes vm_page_activate(m[i]); 141526f9a767SRodney W. Grimes } 141626f9a767SRodney W. Grimes } 14177fb0c17eSDavid Greenman } 141826f9a767SRodney W. Grimes } else { 141926f9a767SRodney W. Grimes for (i = 0; i < count; i++) { 142026f9a767SRodney W. Grimes rtvals[i] = rv; 142126f9a767SRodney W. Grimes } 142226f9a767SRodney W. Grimes } 142326f9a767SRodney W. Grimes 142426f9a767SRodney W. Grimes if (bp->b_rcred != NOCRED) 142526f9a767SRodney W. Grimes crfree(bp->b_rcred); 142626f9a767SRodney W. Grimes if (bp->b_wcred != NOCRED) 142726f9a767SRodney W. Grimes crfree(bp->b_wcred); 142826f9a767SRodney W. Grimes TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 142926f9a767SRodney W. Grimes if (swap_pager_needflags & SWAP_FREE_NEEDED) { 143024a1cce3SDavid Greenman wakeup(&swap_pager_free); 143126f9a767SRodney W. Grimes } 1432a1f6d91cSDavid Greenman if( swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) 1433f919ebdeSDavid Greenman pagedaemon_wakeup(); 1434a1f6d91cSDavid Greenman swap_pager_needflags &= ~(SWAP_FREE_NEEDED|SWAP_FREE_NEEDED_BY_PAGEOUT); 143526f9a767SRodney W. Grimes return (rv); 143626f9a767SRodney W. Grimes } 143726f9a767SRodney W. Grimes 143824a1cce3SDavid Greenman void 143924a1cce3SDavid Greenman swap_pager_sync() 144026f9a767SRodney W. Grimes { 144126f9a767SRodney W. Grimes register swp_clean_t spc, tspc; 144226f9a767SRodney W. Grimes register int s; 144326f9a767SRodney W. Grimes 144426f9a767SRodney W. Grimes tspc = NULL; 144526f9a767SRodney W. Grimes if (swap_pager_done.tqh_first == NULL) 144624a1cce3SDavid Greenman return; 144726f9a767SRodney W. Grimes for (;;) { 144826f9a767SRodney W. Grimes s = splbio(); 144926f9a767SRodney W. Grimes /* 14500d94caffSDavid Greenman * Look up and removal from done list must be done at splbio() 14510d94caffSDavid Greenman * to avoid conflicts with swap_pager_iodone. 145226f9a767SRodney W. Grimes */ 145305f0fdd2SPoul-Henning Kamp while ((spc = swap_pager_done.tqh_first) != 0) { 1454fff93ab6SDavid Greenman pmap_qremove(spc->spc_kva, spc->spc_count); 145526f9a767SRodney W. Grimes swap_pager_finish(spc); 145626f9a767SRodney W. Grimes TAILQ_REMOVE(&swap_pager_done, spc, spc_list); 145726f9a767SRodney W. Grimes goto doclean; 145826f9a767SRodney W. Grimes } 1459df8bae1dSRodney W. Grimes 1460df8bae1dSRodney W. Grimes /* 1461df8bae1dSRodney W. Grimes * No operations done, thats all we can do for now. 1462df8bae1dSRodney W. Grimes */ 146326f9a767SRodney W. Grimes 146426f9a767SRodney W. Grimes splx(s); 1465df8bae1dSRodney W. Grimes break; 1466df8bae1dSRodney W. Grimes 1467df8bae1dSRodney W. Grimes /* 14680d94caffSDavid Greenman * The desired page was found to be busy earlier in the scan 14690d94caffSDavid Greenman * but has since completed. 1470df8bae1dSRodney W. Grimes */ 147126f9a767SRodney W. Grimes doclean: 147226f9a767SRodney W. Grimes if (tspc && tspc == spc) { 147326f9a767SRodney W. Grimes tspc = NULL; 147426f9a767SRodney W. Grimes } 147526f9a767SRodney W. Grimes spc->spc_flags = 0; 147626f9a767SRodney W. Grimes TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 147726f9a767SRodney W. Grimes if (swap_pager_needflags & SWAP_FREE_NEEDED) { 147824a1cce3SDavid Greenman wakeup(&swap_pager_free); 147926f9a767SRodney W. Grimes } 1480a1f6d91cSDavid Greenman if( swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) 1481f919ebdeSDavid Greenman pagedaemon_wakeup(); 1482a1f6d91cSDavid Greenman swap_pager_needflags &= ~(SWAP_FREE_NEEDED|SWAP_FREE_NEEDED_BY_PAGEOUT); 148326f9a767SRodney W. Grimes splx(s); 148426f9a767SRodney W. Grimes } 148526f9a767SRodney W. Grimes 148624a1cce3SDavid Greenman return; 148726f9a767SRodney W. Grimes } 148826f9a767SRodney W. Grimes 148926f9a767SRodney W. Grimes void 149026f9a767SRodney W. Grimes swap_pager_finish(spc) 149126f9a767SRodney W. Grimes register swp_clean_t spc; 149226f9a767SRodney W. Grimes { 149326f9a767SRodney W. Grimes vm_object_t object = spc->spc_m[0]->object; 149426f9a767SRodney W. Grimes int i; 149526f9a767SRodney W. Grimes 1496c0503609SDavid Greenman object->paging_in_progress -= spc->spc_count; 1497c0503609SDavid Greenman if ((object->paging_in_progress == 0) && 1498c0503609SDavid Greenman (object->flags & OBJ_PIPWNT)) { 1499c0503609SDavid Greenman object->flags &= ~OBJ_PIPWNT; 150024a1cce3SDavid Greenman wakeup(object); 1501c0503609SDavid Greenman } 1502df8bae1dSRodney W. Grimes 1503df8bae1dSRodney W. Grimes /* 15045f55e841SDavid Greenman * If no error, mark as clean and inform the pmap system. If error, 15050d94caffSDavid Greenman * mark as dirty so we will try again. (XXX could get stuck doing 15060d94caffSDavid Greenman * this, should give up after awhile) 1507df8bae1dSRodney W. Grimes */ 1508df8bae1dSRodney W. Grimes if (spc->spc_flags & SPC_ERROR) { 150926f9a767SRodney W. Grimes for (i = 0; i < spc->spc_count; i++) { 1510a83c285cSDavid Greenman printf("swap_pager_finish: I/O error, clean of page %lx failed\n", 151105f0fdd2SPoul-Henning Kamp (u_long) VM_PAGE_TO_PHYS(spc->spc_m[i])); 151226f9a767SRodney W. Grimes } 1513df8bae1dSRodney W. Grimes } else { 151426f9a767SRodney W. Grimes for (i = 0; i < spc->spc_count; i++) { 151526f9a767SRodney W. Grimes pmap_clear_modify(VM_PAGE_TO_PHYS(spc->spc_m[i])); 15160d94caffSDavid Greenman spc->spc_m[i]->dirty = 0; 15170d94caffSDavid Greenman if ((spc->spc_m[i]->flags & PG_ACTIVE) == 0 && 15180d94caffSDavid Greenman ((spc->spc_m[i]->flags & PG_WANTED) || pmap_is_referenced(VM_PAGE_TO_PHYS(spc->spc_m[i])))) 15190d94caffSDavid Greenman vm_page_activate(spc->spc_m[i]); 1520df8bae1dSRodney W. Grimes } 1521df8bae1dSRodney W. Grimes } 1522df8bae1dSRodney W. Grimes 152326f9a767SRodney W. Grimes 152426f9a767SRodney W. Grimes for (i = 0; i < spc->spc_count; i++) { 1525df8bae1dSRodney W. Grimes /* 15260d94caffSDavid Greenman * we wakeup any processes that are waiting on these pages. 1527df8bae1dSRodney W. Grimes */ 152826f9a767SRodney W. Grimes PAGE_WAKEUP(spc->spc_m[i]); 1529df8bae1dSRodney W. Grimes } 153026f9a767SRodney W. Grimes nswiodone -= spc->spc_count; 1531df8bae1dSRodney W. Grimes 1532df8bae1dSRodney W. Grimes return; 153326f9a767SRodney W. Grimes } 1534df8bae1dSRodney W. Grimes 153526f9a767SRodney W. Grimes /* 153626f9a767SRodney W. Grimes * swap_pager_iodone 153726f9a767SRodney W. Grimes */ 153826f9a767SRodney W. Grimes void 1539df8bae1dSRodney W. Grimes swap_pager_iodone(bp) 1540df8bae1dSRodney W. Grimes register struct buf *bp; 1541df8bae1dSRodney W. Grimes { 1542df8bae1dSRodney W. Grimes register swp_clean_t spc; 1543df8bae1dSRodney W. Grimes int s; 1544df8bae1dSRodney W. Grimes 1545df8bae1dSRodney W. Grimes s = splbio(); 154626f9a767SRodney W. Grimes spc = (swp_clean_t) bp->b_spc; 154726f9a767SRodney W. Grimes TAILQ_REMOVE(&swap_pager_inuse, spc, spc_list); 154826f9a767SRodney W. Grimes TAILQ_INSERT_TAIL(&swap_pager_done, spc, spc_list); 154926f9a767SRodney W. Grimes if (bp->b_flags & B_ERROR) { 1550df8bae1dSRodney W. Grimes spc->spc_flags |= SPC_ERROR; 1551c3a1e425SDavid Greenman printf("swap_pager: I/O error - async %s failed; blkno %lu, size %ld, error %d\n", 15521b119d9dSDavid Greenman (bp->b_flags & B_READ) ? "pagein" : "pageout", 1553c3a1e425SDavid Greenman (u_long) bp->b_blkno, bp->b_bcount, bp->b_error); 1554df8bae1dSRodney W. Grimes } 155526f9a767SRodney W. Grimes 15560d94caffSDavid Greenman if (bp->b_vp) 15570d94caffSDavid Greenman pbrelvp(bp); 15580d94caffSDavid Greenman 15590d94caffSDavid Greenman if (bp->b_flags & B_WANTED) 156024a1cce3SDavid Greenman wakeup(bp); 15610d94caffSDavid Greenman 156226f9a767SRodney W. Grimes if (bp->b_rcred != NOCRED) 156326f9a767SRodney W. Grimes crfree(bp->b_rcred); 156426f9a767SRodney W. Grimes if (bp->b_wcred != NOCRED) 156526f9a767SRodney W. Grimes crfree(bp->b_wcred); 156626f9a767SRodney W. Grimes 156726f9a767SRodney W. Grimes nswiodone += spc->spc_count; 156826f9a767SRodney W. Grimes if (--spc->spc_swp->sw_poip == 0) { 156924a1cce3SDavid Greenman wakeup(spc->spc_swp); 157026f9a767SRodney W. Grimes } 157126f9a767SRodney W. Grimes if ((swap_pager_needflags & SWAP_FREE_NEEDED) || 157226f9a767SRodney W. Grimes swap_pager_inuse.tqh_first == 0) { 157326f9a767SRodney W. Grimes swap_pager_needflags &= ~SWAP_FREE_NEEDED; 157424a1cce3SDavid Greenman wakeup(&swap_pager_free); 1575a1f6d91cSDavid Greenman } 1576a1f6d91cSDavid Greenman 1577a1f6d91cSDavid Greenman if( swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) { 1578a1f6d91cSDavid Greenman swap_pager_needflags &= ~SWAP_FREE_NEEDED_BY_PAGEOUT; 1579f919ebdeSDavid Greenman pagedaemon_wakeup(); 158026f9a767SRodney W. Grimes } 1581a1f6d91cSDavid Greenman 158226f9a767SRodney W. Grimes if (vm_pageout_pages_needed) { 158324a1cce3SDavid Greenman wakeup(&vm_pageout_pages_needed); 1584a1f6d91cSDavid Greenman vm_pageout_pages_needed = 0; 158526f9a767SRodney W. Grimes } 158626f9a767SRodney W. Grimes if ((swap_pager_inuse.tqh_first == NULL) || 15870d94caffSDavid Greenman ((cnt.v_free_count + cnt.v_cache_count) < cnt.v_free_min && 15880d94caffSDavid Greenman nswiodone + cnt.v_free_count + cnt.v_cache_count >= cnt.v_free_min)) { 1589f919ebdeSDavid Greenman pagedaemon_wakeup(); 159026f9a767SRodney W. Grimes } 159126f9a767SRodney W. Grimes splx(s); 159226f9a767SRodney W. Grimes } 159326f9a767SRodney W. Grimes 159426f9a767SRodney W. Grimes /* 159526f9a767SRodney W. Grimes * return true if any swap control structures can be allocated 159626f9a767SRodney W. Grimes */ 159726f9a767SRodney W. Grimes int 15980d94caffSDavid Greenman swap_pager_ready() 15990d94caffSDavid Greenman { 160026f9a767SRodney W. Grimes if (swap_pager_free.tqh_first) 160126f9a767SRodney W. Grimes return 1; 160226f9a767SRodney W. Grimes else 160326f9a767SRodney W. Grimes return 0; 160426f9a767SRodney W. Grimes } 1605