xref: /freebsd/sys/vm/swap_pager.c (revision f5a127111043fead9d4b117b5112666ea4adcbcc)
1df8bae1dSRodney W. Grimes /*
226f9a767SRodney W. Grimes  * Copyright (c) 1994 John S. Dyson
3df8bae1dSRodney W. Grimes  * Copyright (c) 1990 University of Utah.
4df8bae1dSRodney W. Grimes  * Copyright (c) 1991, 1993
5df8bae1dSRodney W. Grimes  *	The Regents of the University of California.  All rights reserved.
6df8bae1dSRodney W. Grimes  *
7df8bae1dSRodney W. Grimes  * This code is derived from software contributed to Berkeley by
8df8bae1dSRodney W. Grimes  * the Systems Programming Group of the University of Utah Computer
9df8bae1dSRodney W. Grimes  * Science Department.
10df8bae1dSRodney W. Grimes  *
11df8bae1dSRodney W. Grimes  * Redistribution and use in source and binary forms, with or without
12df8bae1dSRodney W. Grimes  * modification, are permitted provided that the following conditions
13df8bae1dSRodney W. Grimes  * are met:
14df8bae1dSRodney W. Grimes  * 1. Redistributions of source code must retain the above copyright
15df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer.
16df8bae1dSRodney W. Grimes  * 2. Redistributions in binary form must reproduce the above copyright
17df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer in the
18df8bae1dSRodney W. Grimes  *    documentation and/or other materials provided with the distribution.
19df8bae1dSRodney W. Grimes  * 3. All advertising materials mentioning features or use of this software
20df8bae1dSRodney W. Grimes  *    must display the following acknowledgement:
21df8bae1dSRodney W. Grimes  *	This product includes software developed by the University of
22df8bae1dSRodney W. Grimes  *	California, Berkeley and its contributors.
23df8bae1dSRodney W. Grimes  * 4. Neither the name of the University nor the names of its contributors
24df8bae1dSRodney W. Grimes  *    may be used to endorse or promote products derived from this software
25df8bae1dSRodney W. Grimes  *    without specific prior written permission.
26df8bae1dSRodney W. Grimes  *
27df8bae1dSRodney W. Grimes  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
28df8bae1dSRodney W. Grimes  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29df8bae1dSRodney W. Grimes  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
30df8bae1dSRodney W. Grimes  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
31df8bae1dSRodney W. Grimes  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
32df8bae1dSRodney W. Grimes  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
33df8bae1dSRodney W. Grimes  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
34df8bae1dSRodney W. Grimes  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
35df8bae1dSRodney W. Grimes  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
36df8bae1dSRodney W. Grimes  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37df8bae1dSRodney W. Grimes  * SUCH DAMAGE.
38df8bae1dSRodney W. Grimes  *
39df8bae1dSRodney W. Grimes  * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
40df8bae1dSRodney W. Grimes  *
41df8bae1dSRodney W. Grimes  *	@(#)swap_pager.c	8.9 (Berkeley) 3/21/94
42f5a12711SPoul-Henning Kamp  * $Id: swap_pager.c,v 1.48 1995/11/02 06:42:45 davidg Exp $
43df8bae1dSRodney W. Grimes  */
44df8bae1dSRodney W. Grimes 
45df8bae1dSRodney W. Grimes /*
46df8bae1dSRodney W. Grimes  * Quick hack to page to dedicated partition(s).
47df8bae1dSRodney W. Grimes  * TODO:
48df8bae1dSRodney W. Grimes  *	Add multiprocessor locks
49df8bae1dSRodney W. Grimes  *	Deal with async writes in a better fashion
50df8bae1dSRodney W. Grimes  */
51df8bae1dSRodney W. Grimes 
52df8bae1dSRodney W. Grimes #include <sys/param.h>
53df8bae1dSRodney W. Grimes #include <sys/systm.h>
5464abb5a5SDavid Greenman #include <sys/kernel.h>
55df8bae1dSRodney W. Grimes #include <sys/proc.h>
56df8bae1dSRodney W. Grimes #include <sys/buf.h>
57df8bae1dSRodney W. Grimes #include <sys/vnode.h>
58df8bae1dSRodney W. Grimes #include <sys/malloc.h>
59df8bae1dSRodney W. Grimes 
60df8bae1dSRodney W. Grimes #include <miscfs/specfs/specdev.h>
6126f9a767SRodney W. Grimes #include <sys/rlist.h>
62df8bae1dSRodney W. Grimes 
63df8bae1dSRodney W. Grimes #include <vm/vm.h>
6426f9a767SRodney W. Grimes #include <vm/vm_pager.h>
65df8bae1dSRodney W. Grimes #include <vm/vm_page.h>
66df8bae1dSRodney W. Grimes #include <vm/vm_pageout.h>
67df8bae1dSRodney W. Grimes #include <vm/swap_pager.h>
6864abb5a5SDavid Greenman #include <vm/vm_kern.h>
69df8bae1dSRodney W. Grimes 
70df8bae1dSRodney W. Grimes #ifndef NPENDINGIO
710d94caffSDavid Greenman #define NPENDINGIO	10
72df8bae1dSRodney W. Grimes #endif
73df8bae1dSRodney W. Grimes 
7426f9a767SRodney W. Grimes int nswiodone;
7526f9a767SRodney W. Grimes int swap_pager_full;
7626f9a767SRodney W. Grimes extern int vm_swap_size;
77f5a12711SPoul-Henning Kamp static int no_swap_space = 1;
7835c10d22SDavid Greenman struct rlist *swaplist;
7935c10d22SDavid Greenman int nswaplist;
8026f9a767SRodney W. Grimes 
8126f9a767SRodney W. Grimes #define MAX_PAGEOUT_CLUSTER 8
82df8bae1dSRodney W. Grimes 
83df8bae1dSRodney W. Grimes TAILQ_HEAD(swpclean, swpagerclean);
84df8bae1dSRodney W. Grimes 
8526f9a767SRodney W. Grimes typedef struct swpagerclean *swp_clean_t;
8626f9a767SRodney W. Grimes 
87df8bae1dSRodney W. Grimes struct swpagerclean {
88df8bae1dSRodney W. Grimes 	TAILQ_ENTRY(swpagerclean) spc_list;
89df8bae1dSRodney W. Grimes 	int spc_flags;
90df8bae1dSRodney W. Grimes 	struct buf *spc_bp;
912a4895f4SDavid Greenman 	vm_object_t spc_object;
92df8bae1dSRodney W. Grimes 	vm_offset_t spc_kva;
9326f9a767SRodney W. Grimes 	int spc_count;
9426f9a767SRodney W. Grimes 	vm_page_t spc_m[MAX_PAGEOUT_CLUSTER];
95df8bae1dSRodney W. Grimes } swcleanlist[NPENDINGIO];
9626f9a767SRodney W. Grimes 
9726f9a767SRodney W. Grimes 
98df8bae1dSRodney W. Grimes /* spc_flags values */
9926f9a767SRodney W. Grimes #define SPC_ERROR	0x01
100df8bae1dSRodney W. Grimes 
10126f9a767SRodney W. Grimes #define SWB_EMPTY (-1)
102df8bae1dSRodney W. Grimes 
103a401ebbeSDavid Greenman struct swpclean swap_pager_done;	/* list of completed page cleans */
104df8bae1dSRodney W. Grimes struct swpclean swap_pager_inuse;	/* list of pending page cleans */
105df8bae1dSRodney W. Grimes struct swpclean swap_pager_free;	/* list of free pager clean structs */
10624a1cce3SDavid Greenman struct pagerlst swap_pager_object_list;	/* list of "named" anon region objects */
10724a1cce3SDavid Greenman struct pagerlst swap_pager_un_object_list; /* list of "unnamed" anon region objects */
108df8bae1dSRodney W. Grimes 
10926f9a767SRodney W. Grimes #define	SWAP_FREE_NEEDED	0x1	/* need a swap block */
110a1f6d91cSDavid Greenman #define SWAP_FREE_NEEDED_BY_PAGEOUT 0x2
11126f9a767SRodney W. Grimes int swap_pager_needflags;
11226f9a767SRodney W. Grimes 
113f5a12711SPoul-Henning Kamp static struct pagerlst *swp_qs[] = {
11424a1cce3SDavid Greenman 	&swap_pager_object_list, &swap_pager_un_object_list, (struct pagerlst *) 0
11526f9a767SRodney W. Grimes };
11626f9a767SRodney W. Grimes 
11724a1cce3SDavid Greenman /*
11824a1cce3SDavid Greenman  * pagerops for OBJT_SWAP - "swap pager".
11924a1cce3SDavid Greenman  */
120df8bae1dSRodney W. Grimes struct pagerops swappagerops = {
121df8bae1dSRodney W. Grimes 	swap_pager_init,
122df8bae1dSRodney W. Grimes 	swap_pager_alloc,
123df8bae1dSRodney W. Grimes 	swap_pager_dealloc,
12424a1cce3SDavid Greenman 	swap_pager_getpages,
12524a1cce3SDavid Greenman 	swap_pager_putpages,
12624a1cce3SDavid Greenman 	swap_pager_haspage,
12724a1cce3SDavid Greenman 	swap_pager_sync
128df8bae1dSRodney W. Grimes };
129df8bae1dSRodney W. Grimes 
130f5a12711SPoul-Henning Kamp static int npendingio = NPENDINGIO;
131f5a12711SPoul-Henning Kamp static void swap_pager_finish();
13226f9a767SRodney W. Grimes int dmmin, dmmax;
13326f9a767SRodney W. Grimes 
13424a1cce3SDavid Greenman 
1350d94caffSDavid Greenman static inline void
1360d94caffSDavid Greenman swapsizecheck()
1370d94caffSDavid Greenman {
13826f9a767SRodney W. Grimes 	if (vm_swap_size < 128 * btodb(PAGE_SIZE)) {
139a1f6d91cSDavid Greenman 		if (swap_pager_full == 0)
14026f9a767SRodney W. Grimes 			printf("swap_pager: out of space\n");
14126f9a767SRodney W. Grimes 		swap_pager_full = 1;
14226f9a767SRodney W. Grimes 	} else if (vm_swap_size > 192 * btodb(PAGE_SIZE))
14326f9a767SRodney W. Grimes 		swap_pager_full = 0;
14426f9a767SRodney W. Grimes }
14526f9a767SRodney W. Grimes 
146f5a12711SPoul-Henning Kamp static void
147df8bae1dSRodney W. Grimes swap_pager_init()
148df8bae1dSRodney W. Grimes {
14924a1cce3SDavid Greenman 	TAILQ_INIT(&swap_pager_object_list);
15024a1cce3SDavid Greenman 	TAILQ_INIT(&swap_pager_un_object_list);
151df8bae1dSRodney W. Grimes 
152df8bae1dSRodney W. Grimes 	/*
153df8bae1dSRodney W. Grimes 	 * Initialize clean lists
154df8bae1dSRodney W. Grimes 	 */
155df8bae1dSRodney W. Grimes 	TAILQ_INIT(&swap_pager_inuse);
15626f9a767SRodney W. Grimes 	TAILQ_INIT(&swap_pager_done);
157df8bae1dSRodney W. Grimes 	TAILQ_INIT(&swap_pager_free);
15826f9a767SRodney W. Grimes 
159df8bae1dSRodney W. Grimes 	/*
160df8bae1dSRodney W. Grimes 	 * Calculate the swap allocation constants.
161df8bae1dSRodney W. Grimes 	 */
16226f9a767SRodney W. Grimes 	dmmin = CLBYTES / DEV_BSIZE;
16326f9a767SRodney W. Grimes 	dmmax = btodb(SWB_NPAGES * PAGE_SIZE) * 2;
164df8bae1dSRodney W. Grimes }
165df8bae1dSRodney W. Grimes 
16624a1cce3SDavid Greenman void
16724a1cce3SDavid Greenman swap_pager_swap_init()
168df8bae1dSRodney W. Grimes {
16926f9a767SRodney W. Grimes 	swp_clean_t spc;
17026f9a767SRodney W. Grimes 	struct buf *bp;
17124a1cce3SDavid Greenman 	int i;
1720d94caffSDavid Greenman 
17326f9a767SRodney W. Grimes 	/*
1740d94caffSDavid Greenman 	 * kva's are allocated here so that we dont need to keep doing
1750d94caffSDavid Greenman 	 * kmem_alloc pageables at runtime
17626f9a767SRodney W. Grimes 	 */
17726f9a767SRodney W. Grimes 	for (i = 0, spc = swcleanlist; i < npendingio; i++, spc++) {
178fff93ab6SDavid Greenman 		spc->spc_kva = kmem_alloc_pageable(pager_map, PAGE_SIZE * MAX_PAGEOUT_CLUSTER);
17926f9a767SRodney W. Grimes 		if (!spc->spc_kva) {
18026f9a767SRodney W. Grimes 			break;
18126f9a767SRodney W. Grimes 		}
182a1f6d91cSDavid Greenman 		spc->spc_bp = malloc(sizeof(*bp), M_TEMP, M_KERNEL);
18326f9a767SRodney W. Grimes 		if (!spc->spc_bp) {
18426f9a767SRodney W. Grimes 			kmem_free_wakeup(pager_map, spc->spc_kva, PAGE_SIZE);
18526f9a767SRodney W. Grimes 			break;
18626f9a767SRodney W. Grimes 		}
18726f9a767SRodney W. Grimes 		spc->spc_flags = 0;
18826f9a767SRodney W. Grimes 		TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list);
18926f9a767SRodney W. Grimes 	}
19026f9a767SRodney W. Grimes }
19124a1cce3SDavid Greenman 
19224a1cce3SDavid Greenman int
19324a1cce3SDavid Greenman swap_pager_swp_alloc(object, wait)
19424a1cce3SDavid Greenman 	vm_object_t object;
19524a1cce3SDavid Greenman 	int wait;
19624a1cce3SDavid Greenman {
1972a4895f4SDavid Greenman 	sw_blk_t swb;
1982a4895f4SDavid Greenman 	int nblocks;
19924a1cce3SDavid Greenman 	int i, j;
20024a1cce3SDavid Greenman 
2012a4895f4SDavid Greenman 	nblocks = (btodb(object->size) + btodb(SWB_NPAGES * PAGE_SIZE) - 1) /
2022a4895f4SDavid Greenman 	    btodb(SWB_NPAGES * PAGE_SIZE);
20324a1cce3SDavid Greenman 
2042a4895f4SDavid Greenman 	swb = malloc(nblocks * sizeof(*swb), M_VMPGDATA, wait);
2052a4895f4SDavid Greenman 	if (swb == NULL)
20624a1cce3SDavid Greenman 		return 1;
20724a1cce3SDavid Greenman 
2082a4895f4SDavid Greenman 	for (i = 0; i < nblocks; i++) {
2092a4895f4SDavid Greenman 		swb[i].swb_valid = 0;
2102a4895f4SDavid Greenman 		swb[i].swb_locked = 0;
21126f9a767SRodney W. Grimes 		for (j = 0; j < SWB_NPAGES; j++)
2122a4895f4SDavid Greenman 			swb[i].swb_block[j] = SWB_EMPTY;
21326f9a767SRodney W. Grimes 	}
21426f9a767SRodney W. Grimes 
2152a4895f4SDavid Greenman 	object->un_pager.swp.swp_nblocks = nblocks;
2162a4895f4SDavid Greenman 	object->un_pager.swp.swp_allocsize = 0;
2172a4895f4SDavid Greenman 	object->un_pager.swp.swp_blocks = swb;
2182a4895f4SDavid Greenman 	object->un_pager.swp.swp_poip = 0;
21924a1cce3SDavid Greenman 
22024a1cce3SDavid Greenman 	if (object->handle != NULL) {
22124a1cce3SDavid Greenman 		TAILQ_INSERT_TAIL(&swap_pager_object_list, object, pager_object_list);
222df8bae1dSRodney W. Grimes 	} else {
22324a1cce3SDavid Greenman 		TAILQ_INSERT_TAIL(&swap_pager_un_object_list, object, pager_object_list);
224df8bae1dSRodney W. Grimes 	}
225df8bae1dSRodney W. Grimes 
22624a1cce3SDavid Greenman 	return 0;
22724a1cce3SDavid Greenman }
22824a1cce3SDavid Greenman 
22924a1cce3SDavid Greenman /*
2302a4895f4SDavid Greenman  * Allocate an object and associated resources.
23124a1cce3SDavid Greenman  * Note that if we are called from the pageout daemon (handle == NULL)
23224a1cce3SDavid Greenman  * we should not wait for memory as it could resulting in deadlock.
23324a1cce3SDavid Greenman  */
234f5a12711SPoul-Henning Kamp static vm_object_t
23524a1cce3SDavid Greenman swap_pager_alloc(handle, size, prot, offset)
23624a1cce3SDavid Greenman 	void *handle;
23724a1cce3SDavid Greenman 	register vm_size_t size;
23824a1cce3SDavid Greenman 	vm_prot_t prot;
23924a1cce3SDavid Greenman 	vm_offset_t offset;
24024a1cce3SDavid Greenman {
24124a1cce3SDavid Greenman 	vm_object_t object;
24224a1cce3SDavid Greenman 	int i;
24324a1cce3SDavid Greenman 
24424a1cce3SDavid Greenman 	/*
24524a1cce3SDavid Greenman 	 * If this is a "named" anonymous region, look it up and use the
24624a1cce3SDavid Greenman 	 * object if it exists, otherwise allocate a new one.
24724a1cce3SDavid Greenman 	 */
24824a1cce3SDavid Greenman 	if (handle) {
24924a1cce3SDavid Greenman 		object = vm_pager_object_lookup(&swap_pager_object_list, handle);
25024a1cce3SDavid Greenman 		if (object != NULL) {
25124a1cce3SDavid Greenman 			vm_object_reference(object);
25224a1cce3SDavid Greenman 		} else {
25324a1cce3SDavid Greenman 			/*
25424a1cce3SDavid Greenman 			 * XXX - there is a race condition here. Two processes
25524a1cce3SDavid Greenman 			 * can request the same named object simultaneuously,
25624a1cce3SDavid Greenman 			 * and if one blocks for memory, the result is a disaster.
25724a1cce3SDavid Greenman 			 * Probably quite rare, but is yet another reason to just
25824a1cce3SDavid Greenman 			 * rip support of "named anonymous regions" out altogether.
25924a1cce3SDavid Greenman 			 */
26024a1cce3SDavid Greenman 			object = vm_object_allocate(OBJT_SWAP, offset + size);
26124a1cce3SDavid Greenman 			object->handle = handle;
26224a1cce3SDavid Greenman 			(void) swap_pager_swp_alloc(object, M_WAITOK);
26324a1cce3SDavid Greenman 		}
26424a1cce3SDavid Greenman 	} else {
26524a1cce3SDavid Greenman 		object = vm_object_allocate(OBJT_SWAP, offset + size);
26624a1cce3SDavid Greenman 		(void) swap_pager_swp_alloc(object, M_WAITOK);
26724a1cce3SDavid Greenman 	}
26824a1cce3SDavid Greenman 
26924a1cce3SDavid Greenman 	return (object);
270df8bae1dSRodney W. Grimes }
271df8bae1dSRodney W. Grimes 
27226f9a767SRodney W. Grimes /*
27326f9a767SRodney W. Grimes  * returns disk block associated with pager and offset
27426f9a767SRodney W. Grimes  * additionally, as a side effect returns a flag indicating
27526f9a767SRodney W. Grimes  * if the block has been written
27626f9a767SRodney W. Grimes  */
27726f9a767SRodney W. Grimes 
278a1f6d91cSDavid Greenman inline static int *
27924a1cce3SDavid Greenman swap_pager_diskaddr(object, offset, valid)
28024a1cce3SDavid Greenman 	vm_object_t object;
28126f9a767SRodney W. Grimes 	vm_offset_t offset;
28226f9a767SRodney W. Grimes 	int *valid;
28326f9a767SRodney W. Grimes {
28426f9a767SRodney W. Grimes 	register sw_blk_t swb;
28526f9a767SRodney W. Grimes 	int ix;
28626f9a767SRodney W. Grimes 
28726f9a767SRodney W. Grimes 	if (valid)
28826f9a767SRodney W. Grimes 		*valid = 0;
28926f9a767SRodney W. Grimes 	ix = offset / (SWB_NPAGES * PAGE_SIZE);
2902a4895f4SDavid Greenman 	if ((ix >= object->un_pager.swp.swp_nblocks) ||
29124a1cce3SDavid Greenman 	    (offset >= object->size)) {
29226f9a767SRodney W. Grimes 		return (FALSE);
29326f9a767SRodney W. Grimes 	}
2942a4895f4SDavid Greenman 	swb = &object->un_pager.swp.swp_blocks[ix];
29526f9a767SRodney W. Grimes 	ix = (offset % (SWB_NPAGES * PAGE_SIZE)) / PAGE_SIZE;
29626f9a767SRodney W. Grimes 	if (valid)
29726f9a767SRodney W. Grimes 		*valid = swb->swb_valid & (1 << ix);
29826f9a767SRodney W. Grimes 	return &swb->swb_block[ix];
29926f9a767SRodney W. Grimes }
30026f9a767SRodney W. Grimes 
30126f9a767SRodney W. Grimes /*
30226f9a767SRodney W. Grimes  * Utility routine to set the valid (written) bit for
30326f9a767SRodney W. Grimes  * a block associated with a pager and offset
30426f9a767SRodney W. Grimes  */
305df8bae1dSRodney W. Grimes static void
3062a4895f4SDavid Greenman swap_pager_setvalid(object, offset, valid)
3072a4895f4SDavid Greenman 	vm_object_t object;
30826f9a767SRodney W. Grimes 	vm_offset_t offset;
30926f9a767SRodney W. Grimes 	int valid;
31026f9a767SRodney W. Grimes {
31126f9a767SRodney W. Grimes 	register sw_blk_t swb;
31226f9a767SRodney W. Grimes 	int ix;
31326f9a767SRodney W. Grimes 
31426f9a767SRodney W. Grimes 	ix = offset / (SWB_NPAGES * PAGE_SIZE);
3152a4895f4SDavid Greenman 	if (ix >= object->un_pager.swp.swp_nblocks)
31626f9a767SRodney W. Grimes 		return;
31726f9a767SRodney W. Grimes 
3182a4895f4SDavid Greenman 	swb = &object->un_pager.swp.swp_blocks[ix];
31926f9a767SRodney W. Grimes 	ix = (offset % (SWB_NPAGES * PAGE_SIZE)) / PAGE_SIZE;
32026f9a767SRodney W. Grimes 	if (valid)
32126f9a767SRodney W. Grimes 		swb->swb_valid |= (1 << ix);
32226f9a767SRodney W. Grimes 	else
32326f9a767SRodney W. Grimes 		swb->swb_valid &= ~(1 << ix);
32426f9a767SRodney W. Grimes 	return;
32526f9a767SRodney W. Grimes }
32626f9a767SRodney W. Grimes 
32726f9a767SRodney W. Grimes /*
32826f9a767SRodney W. Grimes  * this routine allocates swap space with a fragmentation
32926f9a767SRodney W. Grimes  * minimization policy.
33026f9a767SRodney W. Grimes  */
331f5a12711SPoul-Henning Kamp static int
3322a4895f4SDavid Greenman swap_pager_getswapspace(object, amount, rtval)
3332a4895f4SDavid Greenman 	vm_object_t object;
3342a4895f4SDavid Greenman 	unsigned int amount;
3352a4895f4SDavid Greenman 	unsigned int *rtval;
3360d94caffSDavid Greenman {
33724ea4a96SDavid Greenman 	vm_swap_size -= amount;
33824ea4a96SDavid Greenman 	if (!rlist_alloc(&swaplist, amount, rtval)) {
33924ea4a96SDavid Greenman 		vm_swap_size += amount;
34026f9a767SRodney W. Grimes 		return 0;
34124ea4a96SDavid Greenman 	} else {
34224ea4a96SDavid Greenman 		swapsizecheck();
3432a4895f4SDavid Greenman 		object->un_pager.swp.swp_allocsize += amount;
34426f9a767SRodney W. Grimes 		return 1;
34526f9a767SRodney W. Grimes 	}
34626f9a767SRodney W. Grimes }
34726f9a767SRodney W. Grimes 
34826f9a767SRodney W. Grimes /*
34926f9a767SRodney W. Grimes  * this routine frees swap space with a fragmentation
35026f9a767SRodney W. Grimes  * minimization policy.
35126f9a767SRodney W. Grimes  */
352f5a12711SPoul-Henning Kamp static void
3532a4895f4SDavid Greenman swap_pager_freeswapspace(object, from, to)
3542a4895f4SDavid Greenman 	vm_object_t object;
3552a4895f4SDavid Greenman 	unsigned int from;
3562a4895f4SDavid Greenman 	unsigned int to;
3570d94caffSDavid Greenman {
35835c10d22SDavid Greenman 	rlist_free(&swaplist, from, to);
35924ea4a96SDavid Greenman 	vm_swap_size += (to - from) + 1;
3602a4895f4SDavid Greenman 	object->un_pager.swp.swp_allocsize -= (to - from) + 1;
36124ea4a96SDavid Greenman 	swapsizecheck();
36226f9a767SRodney W. Grimes }
36326f9a767SRodney W. Grimes /*
36426f9a767SRodney W. Grimes  * this routine frees swap blocks from a specified pager
36526f9a767SRodney W. Grimes  */
36626f9a767SRodney W. Grimes void
36724a1cce3SDavid Greenman swap_pager_freespace(object, start, size)
36824a1cce3SDavid Greenman 	vm_object_t object;
36926f9a767SRodney W. Grimes 	vm_offset_t start;
37026f9a767SRodney W. Grimes 	vm_offset_t size;
37126f9a767SRodney W. Grimes {
37226f9a767SRodney W. Grimes 	vm_offset_t i;
37326f9a767SRodney W. Grimes 	int s;
37426f9a767SRodney W. Grimes 
37526f9a767SRodney W. Grimes 	s = splbio();
3766f7bc393SDavid Greenman 	for (i = start; i < round_page(start + size); i += PAGE_SIZE) {
37726f9a767SRodney W. Grimes 		int valid;
37824a1cce3SDavid Greenman 		int *addr = swap_pager_diskaddr(object, i, &valid);
3790d94caffSDavid Greenman 
38026f9a767SRodney W. Grimes 		if (addr && *addr != SWB_EMPTY) {
3812a4895f4SDavid Greenman 			swap_pager_freeswapspace(object, *addr, *addr + btodb(PAGE_SIZE) - 1);
38226f9a767SRodney W. Grimes 			if (valid) {
3832a4895f4SDavid Greenman 				swap_pager_setvalid(object, i, 0);
38426f9a767SRodney W. Grimes 			}
38526f9a767SRodney W. Grimes 			*addr = SWB_EMPTY;
38626f9a767SRodney W. Grimes 		}
38726f9a767SRodney W. Grimes 	}
38826f9a767SRodney W. Grimes 	splx(s);
38926f9a767SRodney W. Grimes }
39026f9a767SRodney W. Grimes 
391a1f6d91cSDavid Greenman static void
3922a4895f4SDavid Greenman swap_pager_free_swap(object)
3932a4895f4SDavid Greenman 	vm_object_t object;
394a1f6d91cSDavid Greenman {
395a1f6d91cSDavid Greenman 	register int i, j;
3962a4895f4SDavid Greenman 	register sw_blk_t swb;
397a1f6d91cSDavid Greenman 	int first_block=0, block_count=0;
398a1f6d91cSDavid Greenman 	int s;
399a1f6d91cSDavid Greenman 	/*
400a1f6d91cSDavid Greenman 	 * Free left over swap blocks
401a1f6d91cSDavid Greenman 	 */
402a1f6d91cSDavid Greenman 	s = splbio();
4032a4895f4SDavid Greenman 	for (i = 0, swb = object->un_pager.swp.swp_blocks;
4042a4895f4SDavid Greenman 	    i < object->un_pager.swp.swp_nblocks; i++, swb++) {
405a1f6d91cSDavid Greenman 		for (j = 0; j < SWB_NPAGES; j++) {
4062a4895f4SDavid Greenman 			if (swb->swb_block[j] != SWB_EMPTY) {
407a1f6d91cSDavid Greenman 				/*
408a1f6d91cSDavid Greenman 				 * initially the length of the run is zero
409a1f6d91cSDavid Greenman 				 */
410a1f6d91cSDavid Greenman 				if (block_count == 0) {
4112a4895f4SDavid Greenman 					first_block = swb->swb_block[j];
412a1f6d91cSDavid Greenman 					block_count = btodb(PAGE_SIZE);
4132a4895f4SDavid Greenman 					swb->swb_block[j] = SWB_EMPTY;
414a1f6d91cSDavid Greenman 				/*
415a1f6d91cSDavid Greenman 				 * if the new block can be included into the current run
416a1f6d91cSDavid Greenman 				 */
4172a4895f4SDavid Greenman 				} else if (swb->swb_block[j] == first_block + block_count) {
418a1f6d91cSDavid Greenman 					block_count += btodb(PAGE_SIZE);
4192a4895f4SDavid Greenman 					swb->swb_block[j] = SWB_EMPTY;
420a1f6d91cSDavid Greenman 				/*
421a1f6d91cSDavid Greenman 				 * terminate the previous run, and start a new one
422a1f6d91cSDavid Greenman 				 */
423a1f6d91cSDavid Greenman 				} else {
4242a4895f4SDavid Greenman 					swap_pager_freeswapspace(object, first_block,
425a1f6d91cSDavid Greenman 				   	 (unsigned) first_block + block_count - 1);
4262a4895f4SDavid Greenman 					first_block = swb->swb_block[j];
427a1f6d91cSDavid Greenman 					block_count = btodb(PAGE_SIZE);
4282a4895f4SDavid Greenman 					swb->swb_block[j] = SWB_EMPTY;
429a1f6d91cSDavid Greenman 				}
430a1f6d91cSDavid Greenman 			}
431a1f6d91cSDavid Greenman 		}
432a1f6d91cSDavid Greenman 	}
433a1f6d91cSDavid Greenman 
434a1f6d91cSDavid Greenman 	if (block_count) {
4352a4895f4SDavid Greenman 		swap_pager_freeswapspace(object, first_block,
436a1f6d91cSDavid Greenman 		   	 (unsigned) first_block + block_count - 1);
437a1f6d91cSDavid Greenman 	}
438a1f6d91cSDavid Greenman 	splx(s);
439a1f6d91cSDavid Greenman }
440a1f6d91cSDavid Greenman 
441a1f6d91cSDavid Greenman 
44226f9a767SRodney W. Grimes /*
44326f9a767SRodney W. Grimes  * swap_pager_reclaim frees up over-allocated space from all pagers
44426f9a767SRodney W. Grimes  * this eliminates internal fragmentation due to allocation of space
44526f9a767SRodney W. Grimes  * for segments that are never swapped to. It has been written so that
44626f9a767SRodney W. Grimes  * it does not block until the rlist_free operation occurs; it keeps
44726f9a767SRodney W. Grimes  * the queues consistant.
44826f9a767SRodney W. Grimes  */
44926f9a767SRodney W. Grimes 
45026f9a767SRodney W. Grimes /*
45126f9a767SRodney W. Grimes  * Maximum number of blocks (pages) to reclaim per pass
45226f9a767SRodney W. Grimes  */
453a1f6d91cSDavid Greenman #define MAXRECLAIM 128
45426f9a767SRodney W. Grimes 
455f5a12711SPoul-Henning Kamp static void
45626f9a767SRodney W. Grimes swap_pager_reclaim()
45726f9a767SRodney W. Grimes {
45824a1cce3SDavid Greenman 	vm_object_t object;
45926f9a767SRodney W. Grimes 	int i, j, k;
46026f9a767SRodney W. Grimes 	int s;
46126f9a767SRodney W. Grimes 	int reclaimcount;
462a1f6d91cSDavid Greenman 	static struct {
463a1f6d91cSDavid Greenman 		int address;
4642a4895f4SDavid Greenman 		vm_object_t object;
465a1f6d91cSDavid Greenman 	} reclaims[MAXRECLAIM];
46626f9a767SRodney W. Grimes 	static int in_reclaim;
46726f9a767SRodney W. Grimes 
46826f9a767SRodney W. Grimes 	/*
46926f9a767SRodney W. Grimes 	 * allow only one process to be in the swap_pager_reclaim subroutine
47026f9a767SRodney W. Grimes 	 */
47126f9a767SRodney W. Grimes 	s = splbio();
47226f9a767SRodney W. Grimes 	if (in_reclaim) {
47324a1cce3SDavid Greenman 		tsleep(&in_reclaim, PSWP, "swrclm", 0);
47426f9a767SRodney W. Grimes 		splx(s);
47526f9a767SRodney W. Grimes 		return;
47626f9a767SRodney W. Grimes 	}
47726f9a767SRodney W. Grimes 	in_reclaim = 1;
47826f9a767SRodney W. Grimes 	reclaimcount = 0;
47926f9a767SRodney W. Grimes 
48026f9a767SRodney W. Grimes 	/* for each pager queue */
48126f9a767SRodney W. Grimes 	for (k = 0; swp_qs[k]; k++) {
48226f9a767SRodney W. Grimes 
48324a1cce3SDavid Greenman 		object = swp_qs[k]->tqh_first;
48424a1cce3SDavid Greenman 		while (object && (reclaimcount < MAXRECLAIM)) {
48526f9a767SRodney W. Grimes 
48626f9a767SRodney W. Grimes 			/*
48726f9a767SRodney W. Grimes 			 * see if any blocks associated with a pager has been
48826f9a767SRodney W. Grimes 			 * allocated but not used (written)
48926f9a767SRodney W. Grimes 			 */
4902a4895f4SDavid Greenman 			for (i = 0; i < object->un_pager.swp.swp_nblocks; i++) {
4912a4895f4SDavid Greenman 				sw_blk_t swb = &object->un_pager.swp.swp_blocks[i];
4920d94caffSDavid Greenman 
49326f9a767SRodney W. Grimes 				if (swb->swb_locked)
49426f9a767SRodney W. Grimes 					continue;
49526f9a767SRodney W. Grimes 				for (j = 0; j < SWB_NPAGES; j++) {
49626f9a767SRodney W. Grimes 					if (swb->swb_block[j] != SWB_EMPTY &&
49726f9a767SRodney W. Grimes 					    (swb->swb_valid & (1 << j)) == 0) {
498a1f6d91cSDavid Greenman 						reclaims[reclaimcount].address = swb->swb_block[j];
4992a4895f4SDavid Greenman 						reclaims[reclaimcount++].object = object;
50026f9a767SRodney W. Grimes 						swb->swb_block[j] = SWB_EMPTY;
50126f9a767SRodney W. Grimes 						if (reclaimcount >= MAXRECLAIM)
50226f9a767SRodney W. Grimes 							goto rfinished;
50326f9a767SRodney W. Grimes 					}
50426f9a767SRodney W. Grimes 				}
50526f9a767SRodney W. Grimes 			}
50624a1cce3SDavid Greenman 			object = object->pager_object_list.tqe_next;
50726f9a767SRodney W. Grimes 		}
50826f9a767SRodney W. Grimes 	}
50926f9a767SRodney W. Grimes 
51026f9a767SRodney W. Grimes rfinished:
51126f9a767SRodney W. Grimes 
51226f9a767SRodney W. Grimes 	/*
51326f9a767SRodney W. Grimes 	 * free the blocks that have been added to the reclaim list
51426f9a767SRodney W. Grimes 	 */
51526f9a767SRodney W. Grimes 	for (i = 0; i < reclaimcount; i++) {
5162a4895f4SDavid Greenman 		swap_pager_freeswapspace(reclaims[i].object,
5172a4895f4SDavid Greenman 		    reclaims[i].address, reclaims[i].address + btodb(PAGE_SIZE) - 1);
51826f9a767SRodney W. Grimes 	}
51926f9a767SRodney W. Grimes 	splx(s);
52026f9a767SRodney W. Grimes 	in_reclaim = 0;
52124a1cce3SDavid Greenman 	wakeup(&in_reclaim);
52226f9a767SRodney W. Grimes }
52326f9a767SRodney W. Grimes 
52426f9a767SRodney W. Grimes 
52526f9a767SRodney W. Grimes /*
52626f9a767SRodney W. Grimes  * swap_pager_copy copies blocks from one pager to another and
52726f9a767SRodney W. Grimes  * destroys the source pager
52826f9a767SRodney W. Grimes  */
52926f9a767SRodney W. Grimes 
53026f9a767SRodney W. Grimes void
53124a1cce3SDavid Greenman swap_pager_copy(srcobject, srcoffset, dstobject, dstoffset, offset)
53224a1cce3SDavid Greenman 	vm_object_t srcobject;
53326f9a767SRodney W. Grimes 	vm_offset_t srcoffset;
53424a1cce3SDavid Greenman 	vm_object_t dstobject;
53526f9a767SRodney W. Grimes 	vm_offset_t dstoffset;
53626f9a767SRodney W. Grimes 	vm_offset_t offset;
53726f9a767SRodney W. Grimes {
53826f9a767SRodney W. Grimes 	vm_offset_t i;
539a1f6d91cSDavid Greenman 	int origsize;
54026f9a767SRodney W. Grimes 	int s;
54126f9a767SRodney W. Grimes 
54224ea4a96SDavid Greenman 	if (vm_swap_size)
54324ea4a96SDavid Greenman 		no_swap_space = 0;
54424ea4a96SDavid Greenman 
5452a4895f4SDavid Greenman 	origsize = srcobject->un_pager.swp.swp_allocsize;
54626f9a767SRodney W. Grimes 
54726f9a767SRodney W. Grimes 	/*
54824a1cce3SDavid Greenman 	 * remove the source object from the swap_pager internal queue
54926f9a767SRodney W. Grimes 	 */
55024a1cce3SDavid Greenman 	if (srcobject->handle == NULL) {
55124a1cce3SDavid Greenman 		TAILQ_REMOVE(&swap_pager_un_object_list, srcobject, pager_object_list);
55226f9a767SRodney W. Grimes 	} else {
55324a1cce3SDavid Greenman 		TAILQ_REMOVE(&swap_pager_object_list, srcobject, pager_object_list);
55426f9a767SRodney W. Grimes 	}
55526f9a767SRodney W. Grimes 
55624a1cce3SDavid Greenman 	s = splbio();
5572a4895f4SDavid Greenman 	while (srcobject->un_pager.swp.swp_poip) {
5582a4895f4SDavid Greenman 		tsleep(srcobject, PVM, "spgout", 0);
55926f9a767SRodney W. Grimes 	}
56026f9a767SRodney W. Grimes 	splx(s);
56126f9a767SRodney W. Grimes 
56226f9a767SRodney W. Grimes 	/*
56326f9a767SRodney W. Grimes 	 * clean all of the pages that are currently active and finished
56426f9a767SRodney W. Grimes 	 */
56524a1cce3SDavid Greenman 	swap_pager_sync();
56626f9a767SRodney W. Grimes 
56726f9a767SRodney W. Grimes 	s = splbio();
56826f9a767SRodney W. Grimes 	/*
56926f9a767SRodney W. Grimes 	 * transfer source to destination
57026f9a767SRodney W. Grimes 	 */
57124a1cce3SDavid Greenman 	for (i = 0; i < dstobject->size; i += PAGE_SIZE) {
57226f9a767SRodney W. Grimes 		int srcvalid, dstvalid;
57324a1cce3SDavid Greenman 		int *srcaddrp = swap_pager_diskaddr(srcobject, i + offset + srcoffset,
57426f9a767SRodney W. Grimes 						    &srcvalid);
57526f9a767SRodney W. Grimes 		int *dstaddrp;
5760d94caffSDavid Greenman 
57726f9a767SRodney W. Grimes 		/*
57826f9a767SRodney W. Grimes 		 * see if the source has space allocated
57926f9a767SRodney W. Grimes 		 */
58026f9a767SRodney W. Grimes 		if (srcaddrp && *srcaddrp != SWB_EMPTY) {
58126f9a767SRodney W. Grimes 			/*
5820d94caffSDavid Greenman 			 * if the source is valid and the dest has no space,
5830d94caffSDavid Greenman 			 * then copy the allocation from the srouce to the
5840d94caffSDavid Greenman 			 * dest.
58526f9a767SRodney W. Grimes 			 */
58626f9a767SRodney W. Grimes 			if (srcvalid) {
58724a1cce3SDavid Greenman 				dstaddrp = swap_pager_diskaddr(dstobject, i + dstoffset,
588a1f6d91cSDavid Greenman 							&dstvalid);
58926f9a767SRodney W. Grimes 				/*
5900d94caffSDavid Greenman 				 * if the dest already has a valid block,
5910d94caffSDavid Greenman 				 * deallocate the source block without
5920d94caffSDavid Greenman 				 * copying.
59326f9a767SRodney W. Grimes 				 */
59426f9a767SRodney W. Grimes 				if (!dstvalid && dstaddrp && *dstaddrp != SWB_EMPTY) {
5952a4895f4SDavid Greenman 					swap_pager_freeswapspace(dstobject, *dstaddrp,
596a1f6d91cSDavid Greenman 						*dstaddrp + btodb(PAGE_SIZE) - 1);
59726f9a767SRodney W. Grimes 					*dstaddrp = SWB_EMPTY;
59826f9a767SRodney W. Grimes 				}
59926f9a767SRodney W. Grimes 				if (dstaddrp && *dstaddrp == SWB_EMPTY) {
60026f9a767SRodney W. Grimes 					*dstaddrp = *srcaddrp;
60126f9a767SRodney W. Grimes 					*srcaddrp = SWB_EMPTY;
6022a4895f4SDavid Greenman 					dstobject->un_pager.swp.swp_allocsize += btodb(PAGE_SIZE);
6032a4895f4SDavid Greenman 					srcobject->un_pager.swp.swp_allocsize -= btodb(PAGE_SIZE);
6042a4895f4SDavid Greenman 					swap_pager_setvalid(dstobject, i + dstoffset, 1);
60526f9a767SRodney W. Grimes 				}
60626f9a767SRodney W. Grimes 			}
60726f9a767SRodney W. Grimes 			/*
6080d94caffSDavid Greenman 			 * if the source is not empty at this point, then
6090d94caffSDavid Greenman 			 * deallocate the space.
61026f9a767SRodney W. Grimes 			 */
61126f9a767SRodney W. Grimes 			if (*srcaddrp != SWB_EMPTY) {
6122a4895f4SDavid Greenman 				swap_pager_freeswapspace(srcobject, *srcaddrp,
613a1f6d91cSDavid Greenman 					*srcaddrp + btodb(PAGE_SIZE) - 1);
61426f9a767SRodney W. Grimes 				*srcaddrp = SWB_EMPTY;
61526f9a767SRodney W. Grimes 			}
61626f9a767SRodney W. Grimes 		}
61726f9a767SRodney W. Grimes 	}
61826f9a767SRodney W. Grimes 	splx(s);
61926f9a767SRodney W. Grimes 
620a1f6d91cSDavid Greenman 	/*
621a1f6d91cSDavid Greenman 	 * Free left over swap blocks
622a1f6d91cSDavid Greenman 	 */
6232a4895f4SDavid Greenman 	swap_pager_free_swap(srcobject);
624a1f6d91cSDavid Greenman 
6252a4895f4SDavid Greenman 	if (srcobject->un_pager.swp.swp_allocsize) {
6262a4895f4SDavid Greenman 		printf("swap_pager_copy: *warning* pager with %d blocks (orig: %d)\n",
6272a4895f4SDavid Greenman 		    srcobject->un_pager.swp.swp_allocsize, origsize);
6282a4895f4SDavid Greenman 	}
6292a4895f4SDavid Greenman 
6302a4895f4SDavid Greenman 	free(srcobject->un_pager.swp.swp_blocks, M_VMPGDATA);
6312a4895f4SDavid Greenman 	srcobject->un_pager.swp.swp_blocks = NULL;
63226f9a767SRodney W. Grimes 
63326f9a767SRodney W. Grimes 	return;
63426f9a767SRodney W. Grimes }
63526f9a767SRodney W. Grimes 
636f5a12711SPoul-Henning Kamp static void
63724a1cce3SDavid Greenman swap_pager_dealloc(object)
63824a1cce3SDavid Greenman 	vm_object_t object;
639df8bae1dSRodney W. Grimes {
640df8bae1dSRodney W. Grimes 	int s;
641df8bae1dSRodney W. Grimes 
642df8bae1dSRodney W. Grimes 	/*
6430d94caffSDavid Greenman 	 * Remove from list right away so lookups will fail if we block for
6440d94caffSDavid Greenman 	 * pageout completion.
645df8bae1dSRodney W. Grimes 	 */
64624a1cce3SDavid Greenman 	if (object->handle == NULL) {
64724a1cce3SDavid Greenman 		TAILQ_REMOVE(&swap_pager_un_object_list, object, pager_object_list);
64826f9a767SRodney W. Grimes 	} else {
64924a1cce3SDavid Greenman 		TAILQ_REMOVE(&swap_pager_object_list, object, pager_object_list);
650df8bae1dSRodney W. Grimes 	}
65124a1cce3SDavid Greenman 
652df8bae1dSRodney W. Grimes 	/*
6530d94caffSDavid Greenman 	 * Wait for all pageouts to finish and remove all entries from
6540d94caffSDavid Greenman 	 * cleaning list.
655df8bae1dSRodney W. Grimes 	 */
65626f9a767SRodney W. Grimes 
65724a1cce3SDavid Greenman 	s = splbio();
6582a4895f4SDavid Greenman 	while (object->un_pager.swp.swp_poip) {
6592a4895f4SDavid Greenman 		tsleep(object, PVM, "swpout", 0);
660df8bae1dSRodney W. Grimes 	}
661df8bae1dSRodney W. Grimes 	splx(s);
66226f9a767SRodney W. Grimes 
66326f9a767SRodney W. Grimes 
66424a1cce3SDavid Greenman 	swap_pager_sync();
665df8bae1dSRodney W. Grimes 
666df8bae1dSRodney W. Grimes 	/*
667df8bae1dSRodney W. Grimes 	 * Free left over swap blocks
668df8bae1dSRodney W. Grimes 	 */
6692a4895f4SDavid Greenman 	swap_pager_free_swap(object);
67026f9a767SRodney W. Grimes 
6712a4895f4SDavid Greenman 	if (object->un_pager.swp.swp_allocsize) {
6722a4895f4SDavid Greenman 		printf("swap_pager_dealloc: *warning* freeing pager with %d blocks\n",
6732a4895f4SDavid Greenman 		    object->un_pager.swp.swp_allocsize);
6742a4895f4SDavid Greenman 	}
675df8bae1dSRodney W. Grimes 	/*
676df8bae1dSRodney W. Grimes 	 * Free swap management resources
677df8bae1dSRodney W. Grimes 	 */
6782a4895f4SDavid Greenman 	free(object->un_pager.swp.swp_blocks, M_VMPGDATA);
6792a4895f4SDavid Greenman 	object->un_pager.swp.swp_blocks = NULL;
68026f9a767SRodney W. Grimes }
68126f9a767SRodney W. Grimes 
68226f9a767SRodney W. Grimes static inline int
6830d94caffSDavid Greenman const
6842a4895f4SDavid Greenman swap_pager_block_index(offset)
68526f9a767SRodney W. Grimes 	vm_offset_t offset;
68626f9a767SRodney W. Grimes {
68726f9a767SRodney W. Grimes 	return (offset / (SWB_NPAGES * PAGE_SIZE));
68826f9a767SRodney W. Grimes }
68926f9a767SRodney W. Grimes 
69026f9a767SRodney W. Grimes static inline int
6910d94caffSDavid Greenman const
6922a4895f4SDavid Greenman swap_pager_block_offset(offset)
69326f9a767SRodney W. Grimes 	vm_offset_t offset;
69426f9a767SRodney W. Grimes {
69526f9a767SRodney W. Grimes 	return ((offset % (PAGE_SIZE * SWB_NPAGES)) / PAGE_SIZE);
69626f9a767SRodney W. Grimes }
69726f9a767SRodney W. Grimes 
69826f9a767SRodney W. Grimes /*
69924a1cce3SDavid Greenman  * swap_pager_haspage returns TRUE if the pager has data that has
70026f9a767SRodney W. Grimes  * been written out.
70126f9a767SRodney W. Grimes  */
702f5a12711SPoul-Henning Kamp static boolean_t
70324a1cce3SDavid Greenman swap_pager_haspage(object, offset, before, after)
70424a1cce3SDavid Greenman 	vm_object_t object;
70526f9a767SRodney W. Grimes 	vm_offset_t offset;
70624a1cce3SDavid Greenman 	int *before;
70724a1cce3SDavid Greenman 	int *after;
70826f9a767SRodney W. Grimes {
70926f9a767SRodney W. Grimes 	register sw_blk_t swb;
71026f9a767SRodney W. Grimes 	int ix;
711170db9c6SJohn Dyson 	int gix;
71226f9a767SRodney W. Grimes 
71324a1cce3SDavid Greenman 	if (before != NULL)
71424a1cce3SDavid Greenman 		*before = 0;
71524a1cce3SDavid Greenman 	if (after != NULL)
71624a1cce3SDavid Greenman 		*after = 0;
71726f9a767SRodney W. Grimes 	ix = offset / (SWB_NPAGES * PAGE_SIZE);
7182a4895f4SDavid Greenman 	if (ix >= object->un_pager.swp.swp_nblocks) {
71926f9a767SRodney W. Grimes 		return (FALSE);
72026f9a767SRodney W. Grimes 	}
7212a4895f4SDavid Greenman 	swb = &object->un_pager.swp.swp_blocks[ix];
722170db9c6SJohn Dyson 	gix = offset / PAGE_SIZE;
723170db9c6SJohn Dyson 	ix = gix % SWB_NPAGES;
724170db9c6SJohn Dyson 
72526f9a767SRodney W. Grimes 	if (swb->swb_block[ix] != SWB_EMPTY) {
726170db9c6SJohn Dyson 
727170db9c6SJohn Dyson 		if (swb->swb_valid & (1 << ix)) {
728170db9c6SJohn Dyson 			int tix;
729170db9c6SJohn Dyson 			if (before) {
730170db9c6SJohn Dyson 				for(tix = ix - 1; tix >= 0; --tix) {
7312f82e604SDavid Greenman 					if ((swb->swb_valid & (1 << tix)) == 0)
7322f82e604SDavid Greenman 						break;
733ca56715fSJohn Dyson 					if ((swb->swb_block[tix] +
734170db9c6SJohn Dyson 						(ix - tix) * (PAGE_SIZE/DEV_BSIZE)) !=
735170db9c6SJohn Dyson 						swb->swb_block[ix])
736170db9c6SJohn Dyson 						break;
737170db9c6SJohn Dyson 					(*before)++;
738170db9c6SJohn Dyson 				}
739170db9c6SJohn Dyson 			}
740170db9c6SJohn Dyson 
741170db9c6SJohn Dyson 			if (after) {
742170db9c6SJohn Dyson 				for(tix = ix + 1; tix < SWB_NPAGES; tix++) {
7432f82e604SDavid Greenman 					if ((swb->swb_valid & (1 << tix)) == 0)
7442f82e604SDavid Greenman 						break;
745ca56715fSJohn Dyson 					if ((swb->swb_block[tix] -
746170db9c6SJohn Dyson 						(tix - ix) * (PAGE_SIZE/DEV_BSIZE)) !=
747170db9c6SJohn Dyson 						swb->swb_block[ix])
748170db9c6SJohn Dyson 						break;
749170db9c6SJohn Dyson 					(*after)++;
750170db9c6SJohn Dyson 				}
751170db9c6SJohn Dyson 			}
752170db9c6SJohn Dyson 
75326f9a767SRodney W. Grimes 			return TRUE;
75426f9a767SRodney W. Grimes 		}
755170db9c6SJohn Dyson 	}
75626f9a767SRodney W. Grimes 	return (FALSE);
75726f9a767SRodney W. Grimes }
75826f9a767SRodney W. Grimes 
75926f9a767SRodney W. Grimes /*
76026f9a767SRodney W. Grimes  * swap_pager_freepage is a convienience routine that clears the busy
76126f9a767SRodney W. Grimes  * bit and deallocates a page.
762df8bae1dSRodney W. Grimes  */
76326f9a767SRodney W. Grimes static void
76426f9a767SRodney W. Grimes swap_pager_freepage(m)
76526f9a767SRodney W. Grimes 	vm_page_t m;
76626f9a767SRodney W. Grimes {
76726f9a767SRodney W. Grimes 	PAGE_WAKEUP(m);
76826f9a767SRodney W. Grimes 	vm_page_free(m);
76926f9a767SRodney W. Grimes }
77026f9a767SRodney W. Grimes 
77126f9a767SRodney W. Grimes /*
77226f9a767SRodney W. Grimes  * swap_pager_ridpages is a convienience routine that deallocates all
77326f9a767SRodney W. Grimes  * but the required page.  this is usually used in error returns that
77426f9a767SRodney W. Grimes  * need to invalidate the "extra" readahead pages.
77526f9a767SRodney W. Grimes  */
77626f9a767SRodney W. Grimes static void
77726f9a767SRodney W. Grimes swap_pager_ridpages(m, count, reqpage)
77826f9a767SRodney W. Grimes 	vm_page_t *m;
77926f9a767SRodney W. Grimes 	int count;
78026f9a767SRodney W. Grimes 	int reqpage;
78126f9a767SRodney W. Grimes {
78226f9a767SRodney W. Grimes 	int i;
7830d94caffSDavid Greenman 
78426f9a767SRodney W. Grimes 	for (i = 0; i < count; i++)
78526f9a767SRodney W. Grimes 		if (i != reqpage)
78626f9a767SRodney W. Grimes 			swap_pager_freepage(m[i]);
78726f9a767SRodney W. Grimes }
78826f9a767SRodney W. Grimes 
78926f9a767SRodney W. Grimes /*
79026f9a767SRodney W. Grimes  * swap_pager_iodone1 is the completion routine for both reads and async writes
79126f9a767SRodney W. Grimes  */
792f5a12711SPoul-Henning Kamp static void
79326f9a767SRodney W. Grimes swap_pager_iodone1(bp)
79426f9a767SRodney W. Grimes 	struct buf *bp;
79526f9a767SRodney W. Grimes {
79626f9a767SRodney W. Grimes 	bp->b_flags |= B_DONE;
79726f9a767SRodney W. Grimes 	bp->b_flags &= ~B_ASYNC;
79824a1cce3SDavid Greenman 	wakeup(bp);
79926f9a767SRodney W. Grimes }
80026f9a767SRodney W. Grimes 
80126f9a767SRodney W. Grimes int
80224a1cce3SDavid Greenman swap_pager_getpages(object, m, count, reqpage)
80324a1cce3SDavid Greenman 	vm_object_t object;
80426f9a767SRodney W. Grimes 	vm_page_t *m;
80526f9a767SRodney W. Grimes 	int count, reqpage;
806df8bae1dSRodney W. Grimes {
807df8bae1dSRodney W. Grimes 	register struct buf *bp;
80826f9a767SRodney W. Grimes 	sw_blk_t swb[count];
809df8bae1dSRodney W. Grimes 	register int s;
81026f9a767SRodney W. Grimes 	int i;
811df8bae1dSRodney W. Grimes 	boolean_t rv;
81226f9a767SRodney W. Grimes 	vm_offset_t kva, off[count];
813df8bae1dSRodney W. Grimes 	swp_clean_t spc;
81426f9a767SRodney W. Grimes 	vm_offset_t paging_offset;
81526f9a767SRodney W. Grimes 	int reqaddr[count];
8166d40c3d3SDavid Greenman 	int sequential;
817df8bae1dSRodney W. Grimes 
81826f9a767SRodney W. Grimes 	int first, last;
81926f9a767SRodney W. Grimes 	int failed;
82026f9a767SRodney W. Grimes 	int reqdskregion;
821df8bae1dSRodney W. Grimes 
82226f9a767SRodney W. Grimes 	object = m[reqpage]->object;
82326f9a767SRodney W. Grimes 	paging_offset = object->paging_offset;
8246d40c3d3SDavid Greenman 	sequential = (m[reqpage]->offset == (object->last_read + PAGE_SIZE));
8252a4895f4SDavid Greenman 
82626f9a767SRodney W. Grimes 	for (i = 0; i < count; i++) {
82726f9a767SRodney W. Grimes 		vm_offset_t foff = m[i]->offset + paging_offset;
8282a4895f4SDavid Greenman 		int ix = swap_pager_block_index(foff);
8290d94caffSDavid Greenman 
8302a4895f4SDavid Greenman 		if (ix >= object->un_pager.swp.swp_nblocks) {
83126f9a767SRodney W. Grimes 			int j;
8320d94caffSDavid Greenman 
83326f9a767SRodney W. Grimes 			if (i <= reqpage) {
83426f9a767SRodney W. Grimes 				swap_pager_ridpages(m, count, reqpage);
835df8bae1dSRodney W. Grimes 				return (VM_PAGER_FAIL);
83626f9a767SRodney W. Grimes 			}
83726f9a767SRodney W. Grimes 			for (j = i; j < count; j++) {
83826f9a767SRodney W. Grimes 				swap_pager_freepage(m[j]);
83926f9a767SRodney W. Grimes 			}
84026f9a767SRodney W. Grimes 			count = i;
84126f9a767SRodney W. Grimes 			break;
84226f9a767SRodney W. Grimes 		}
8432a4895f4SDavid Greenman 		swb[i] = &object->un_pager.swp.swp_blocks[ix];
8442a4895f4SDavid Greenman 		off[i] = swap_pager_block_offset(foff);
84526f9a767SRodney W. Grimes 		reqaddr[i] = swb[i]->swb_block[off[i]];
84626f9a767SRodney W. Grimes 	}
84726f9a767SRodney W. Grimes 
84826f9a767SRodney W. Grimes 	/* make sure that our required input request is existant */
84926f9a767SRodney W. Grimes 
85026f9a767SRodney W. Grimes 	if (reqaddr[reqpage] == SWB_EMPTY ||
85126f9a767SRodney W. Grimes 	    (swb[reqpage]->swb_valid & (1 << off[reqpage])) == 0) {
85226f9a767SRodney W. Grimes 		swap_pager_ridpages(m, count, reqpage);
85326f9a767SRodney W. Grimes 		return (VM_PAGER_FAIL);
85426f9a767SRodney W. Grimes 	}
85526f9a767SRodney W. Grimes 	reqdskregion = reqaddr[reqpage] / dmmax;
856df8bae1dSRodney W. Grimes 
857df8bae1dSRodney W. Grimes 	/*
85826f9a767SRodney W. Grimes 	 * search backwards for the first contiguous page to transfer
859df8bae1dSRodney W. Grimes 	 */
86026f9a767SRodney W. Grimes 	failed = 0;
86126f9a767SRodney W. Grimes 	first = 0;
86226f9a767SRodney W. Grimes 	for (i = reqpage - 1; i >= 0; --i) {
8636d40c3d3SDavid Greenman 		if (sequential || failed || (reqaddr[i] == SWB_EMPTY) ||
86426f9a767SRodney W. Grimes 		    (swb[i]->swb_valid & (1 << off[i])) == 0 ||
86526f9a767SRodney W. Grimes 		    (reqaddr[i] != (reqaddr[reqpage] + (i - reqpage) * btodb(PAGE_SIZE))) ||
86626f9a767SRodney W. Grimes 		    ((reqaddr[i] / dmmax) != reqdskregion)) {
86726f9a767SRodney W. Grimes 			failed = 1;
86826f9a767SRodney W. Grimes 			swap_pager_freepage(m[i]);
86926f9a767SRodney W. Grimes 			if (first == 0)
87026f9a767SRodney W. Grimes 				first = i + 1;
87126f9a767SRodney W. Grimes 		}
872df8bae1dSRodney W. Grimes 	}
873df8bae1dSRodney W. Grimes 	/*
87426f9a767SRodney W. Grimes 	 * search forwards for the last contiguous page to transfer
875df8bae1dSRodney W. Grimes 	 */
87626f9a767SRodney W. Grimes 	failed = 0;
87726f9a767SRodney W. Grimes 	last = count;
87826f9a767SRodney W. Grimes 	for (i = reqpage + 1; i < count; i++) {
87926f9a767SRodney W. Grimes 		if (failed || (reqaddr[i] == SWB_EMPTY) ||
88026f9a767SRodney W. Grimes 		    (swb[i]->swb_valid & (1 << off[i])) == 0 ||
88126f9a767SRodney W. Grimes 		    (reqaddr[i] != (reqaddr[reqpage] + (i - reqpage) * btodb(PAGE_SIZE))) ||
88226f9a767SRodney W. Grimes 		    ((reqaddr[i] / dmmax) != reqdskregion)) {
88326f9a767SRodney W. Grimes 			failed = 1;
88426f9a767SRodney W. Grimes 			swap_pager_freepage(m[i]);
88526f9a767SRodney W. Grimes 			if (last == count)
88626f9a767SRodney W. Grimes 				last = i;
88726f9a767SRodney W. Grimes 		}
88826f9a767SRodney W. Grimes 	}
88926f9a767SRodney W. Grimes 
89026f9a767SRodney W. Grimes 	count = last;
89126f9a767SRodney W. Grimes 	if (first != 0) {
89226f9a767SRodney W. Grimes 		for (i = first; i < count; i++) {
89326f9a767SRodney W. Grimes 			m[i - first] = m[i];
89426f9a767SRodney W. Grimes 			reqaddr[i - first] = reqaddr[i];
89526f9a767SRodney W. Grimes 			off[i - first] = off[i];
89626f9a767SRodney W. Grimes 		}
89726f9a767SRodney W. Grimes 		count -= first;
89826f9a767SRodney W. Grimes 		reqpage -= first;
89926f9a767SRodney W. Grimes 	}
90026f9a767SRodney W. Grimes 	++swb[reqpage]->swb_locked;
90126f9a767SRodney W. Grimes 
90226f9a767SRodney W. Grimes 	/*
9030d94caffSDavid Greenman 	 * at this point: "m" is a pointer to the array of vm_page_t for
9040d94caffSDavid Greenman 	 * paging I/O "count" is the number of vm_page_t entries represented
9050d94caffSDavid Greenman 	 * by "m" "object" is the vm_object_t for I/O "reqpage" is the index
9060d94caffSDavid Greenman 	 * into "m" for the page actually faulted
90726f9a767SRodney W. Grimes 	 */
90826f9a767SRodney W. Grimes 
90926f9a767SRodney W. Grimes 	spc = NULL;	/* we might not use an spc data structure */
91026f9a767SRodney W. Grimes 
911a1f6d91cSDavid Greenman 	if ((count == 1) && (swap_pager_free.tqh_first != NULL)) {
91226f9a767SRodney W. Grimes 		/*
9130d94caffSDavid Greenman 		 * if a kva has not been allocated, we can only do a one page
9140d94caffSDavid Greenman 		 * transfer, so we free the other pages that might have been
9150d94caffSDavid Greenman 		 * allocated by vm_fault.
91626f9a767SRodney W. Grimes 		 */
91726f9a767SRodney W. Grimes 		swap_pager_ridpages(m, count, reqpage);
91826f9a767SRodney W. Grimes 		m[0] = m[reqpage];
91926f9a767SRodney W. Grimes 		reqaddr[0] = reqaddr[reqpage];
92026f9a767SRodney W. Grimes 
92126f9a767SRodney W. Grimes 		count = 1;
92226f9a767SRodney W. Grimes 		reqpage = 0;
92326f9a767SRodney W. Grimes 		/*
9240d94caffSDavid Greenman 		 * get a swap pager clean data structure, block until we get
9250d94caffSDavid Greenman 		 * it
92626f9a767SRodney W. Grimes 		 */
927df8bae1dSRodney W. Grimes 		if (swap_pager_free.tqh_first == NULL) {
928df8bae1dSRodney W. Grimes 			s = splbio();
92926f9a767SRodney W. Grimes 			if (curproc == pageproc)
93024a1cce3SDavid Greenman 				swap_pager_sync();
93126f9a767SRodney W. Grimes 			else
932f919ebdeSDavid Greenman 				pagedaemon_wakeup();
93326f9a767SRodney W. Grimes 			while (swap_pager_free.tqh_first == NULL) {
93426f9a767SRodney W. Grimes 				swap_pager_needflags |= SWAP_FREE_NEEDED;
935a1f6d91cSDavid Greenman 				if (curproc == pageproc)
936a1f6d91cSDavid Greenman 					swap_pager_needflags |= SWAP_FREE_NEEDED_BY_PAGEOUT;
93724a1cce3SDavid Greenman 				tsleep(&swap_pager_free,
93826f9a767SRodney W. Grimes 				    PVM, "swpfre", 0);
93926f9a767SRodney W. Grimes 				if (curproc == pageproc)
94024a1cce3SDavid Greenman 					swap_pager_sync();
94126f9a767SRodney W. Grimes 				else
942f919ebdeSDavid Greenman 					pagedaemon_wakeup();
943df8bae1dSRodney W. Grimes 			}
944df8bae1dSRodney W. Grimes 			splx(s);
94526f9a767SRodney W. Grimes 		}
94626f9a767SRodney W. Grimes 		spc = swap_pager_free.tqh_first;
94726f9a767SRodney W. Grimes 		TAILQ_REMOVE(&swap_pager_free, spc, spc_list);
94826f9a767SRodney W. Grimes 		kva = spc->spc_kva;
94926f9a767SRodney W. Grimes 		bp = spc->spc_bp;
95026f9a767SRodney W. Grimes 		bzero(bp, sizeof *bp);
95126f9a767SRodney W. Grimes 		bp->b_spc = spc;
9527609ab12SDavid Greenman 		bp->b_vnbufs.le_next = NOLIST;
95326f9a767SRodney W. Grimes 	} else {
95416f62314SDavid Greenman 		/*
95516f62314SDavid Greenman 		 * Get a swap buffer header to perform the IO
95616f62314SDavid Greenman 		 */
95726f9a767SRodney W. Grimes 		bp = getpbuf();
95816f62314SDavid Greenman 		kva = (vm_offset_t) bp->b_data;
95926f9a767SRodney W. Grimes 	}
96026f9a767SRodney W. Grimes 
96116f62314SDavid Greenman 	/*
96216f62314SDavid Greenman 	 * map our page(s) into kva for input
96316f62314SDavid Greenman 	 */
96416f62314SDavid Greenman 	pmap_qenter(kva, m, count);
96516f62314SDavid Greenman 
966aba8f38eSDavid Greenman 	bp->b_flags = B_BUSY | B_READ | B_CALL | B_PAGING;
96726f9a767SRodney W. Grimes 	bp->b_iodone = swap_pager_iodone1;
968df8bae1dSRodney W. Grimes 	bp->b_proc = &proc0;	/* XXX (but without B_PHYS set this is ok) */
96926f9a767SRodney W. Grimes 	bp->b_rcred = bp->b_wcred = bp->b_proc->p_ucred;
97026f9a767SRodney W. Grimes 	crhold(bp->b_rcred);
97126f9a767SRodney W. Grimes 	crhold(bp->b_wcred);
97226f9a767SRodney W. Grimes 	bp->b_un.b_addr = (caddr_t) kva;
97326f9a767SRodney W. Grimes 	bp->b_blkno = reqaddr[0];
97426f9a767SRodney W. Grimes 	bp->b_bcount = PAGE_SIZE * count;
97526f9a767SRodney W. Grimes 	bp->b_bufsize = PAGE_SIZE * count;
97626f9a767SRodney W. Grimes 
9770d94caffSDavid Greenman 	pbgetvp(swapdev_vp, bp);
978df8bae1dSRodney W. Grimes 
979976e77fcSDavid Greenman 	cnt.v_swapin++;
980976e77fcSDavid Greenman 	cnt.v_swappgsin += count;
981df8bae1dSRodney W. Grimes 	/*
98226f9a767SRodney W. Grimes 	 * perform the I/O
983df8bae1dSRodney W. Grimes 	 */
984df8bae1dSRodney W. Grimes 	VOP_STRATEGY(bp);
98526f9a767SRodney W. Grimes 
98626f9a767SRodney W. Grimes 	/*
98726f9a767SRodney W. Grimes 	 * wait for the sync I/O to complete
98826f9a767SRodney W. Grimes 	 */
9897609ab12SDavid Greenman 	s = splbio();
99026f9a767SRodney W. Grimes 	while ((bp->b_flags & B_DONE) == 0) {
99124a1cce3SDavid Greenman 		tsleep(bp, PVM, "swread", 0);
992df8bae1dSRodney W. Grimes 	}
9931b119d9dSDavid Greenman 
9941b119d9dSDavid Greenman 	if (bp->b_flags & B_ERROR) {
9951b119d9dSDavid Greenman 		printf("swap_pager: I/O error - pagein failed; blkno %d, size %d, error %d\n",
9961b119d9dSDavid Greenman 		    bp->b_blkno, bp->b_bcount, bp->b_error);
997a83c285cSDavid Greenman 		rv = VM_PAGER_ERROR;
9981b119d9dSDavid Greenman 	} else {
9991b119d9dSDavid Greenman 		rv = VM_PAGER_OK;
10001b119d9dSDavid Greenman 	}
100126f9a767SRodney W. Grimes 
100226f9a767SRodney W. Grimes 	/*
10030d94caffSDavid Greenman 	 * relpbuf does this, but we maintain our own buffer list also...
100426f9a767SRodney W. Grimes 	 */
1005df8bae1dSRodney W. Grimes 	if (bp->b_vp)
10060d94caffSDavid Greenman 		pbrelvp(bp);
100726f9a767SRodney W. Grimes 
1008df8bae1dSRodney W. Grimes 	splx(s);
10092a4895f4SDavid Greenman 	swb[reqpage]->swb_locked--;
101026f9a767SRodney W. Grimes 
101126f9a767SRodney W. Grimes 	/*
101226f9a767SRodney W. Grimes 	 * remove the mapping for kernel virtual
101326f9a767SRodney W. Grimes 	 */
101416f62314SDavid Greenman 	pmap_qremove(kva, count);
101526f9a767SRodney W. Grimes 
101626f9a767SRodney W. Grimes 	if (spc) {
10176d40c3d3SDavid Greenman 		m[reqpage]->object->last_read = m[reqpage]->offset;
10180d94caffSDavid Greenman 		if (bp->b_flags & B_WANTED)
101924a1cce3SDavid Greenman 			wakeup(bp);
102026f9a767SRodney W. Grimes 		/*
102126f9a767SRodney W. Grimes 		 * if we have used an spc, we need to free it.
102226f9a767SRodney W. Grimes 		 */
102326f9a767SRodney W. Grimes 		if (bp->b_rcred != NOCRED)
102426f9a767SRodney W. Grimes 			crfree(bp->b_rcred);
102526f9a767SRodney W. Grimes 		if (bp->b_wcred != NOCRED)
102626f9a767SRodney W. Grimes 			crfree(bp->b_wcred);
102726f9a767SRodney W. Grimes 		TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list);
102826f9a767SRodney W. Grimes 		if (swap_pager_needflags & SWAP_FREE_NEEDED) {
102924a1cce3SDavid Greenman 			wakeup(&swap_pager_free);
103026f9a767SRodney W. Grimes 		}
1031a1f6d91cSDavid Greenman 		if (swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT)
1032f919ebdeSDavid Greenman 			pagedaemon_wakeup();
1033a1f6d91cSDavid Greenman 		swap_pager_needflags &= ~(SWAP_FREE_NEEDED|SWAP_FREE_NEEDED_BY_PAGEOUT);
103426f9a767SRodney W. Grimes 	} else {
103526f9a767SRodney W. Grimes 		/*
103626f9a767SRodney W. Grimes 		 * release the physical I/O buffer
103726f9a767SRodney W. Grimes 		 */
103826f9a767SRodney W. Grimes 		relpbuf(bp);
103926f9a767SRodney W. Grimes 		/*
104026f9a767SRodney W. Grimes 		 * finish up input if everything is ok
104126f9a767SRodney W. Grimes 		 */
104226f9a767SRodney W. Grimes 		if (rv == VM_PAGER_OK) {
104326f9a767SRodney W. Grimes 			for (i = 0; i < count; i++) {
104426f9a767SRodney W. Grimes 				pmap_clear_modify(VM_PAGE_TO_PHYS(m[i]));
10450d94caffSDavid Greenman 				m[i]->dirty = 0;
1046894048d7SJohn Dyson 				m[i]->flags &= ~PG_ZERO;
104726f9a767SRodney W. Grimes 				if (i != reqpage) {
104826f9a767SRodney W. Grimes 					/*
10490d94caffSDavid Greenman 					 * whether or not to leave the page
10500d94caffSDavid Greenman 					 * activated is up in the air, but we
10510d94caffSDavid Greenman 					 * should put the page on a page queue
10520d94caffSDavid Greenman 					 * somewhere. (it already is in the
10530d94caffSDavid Greenman 					 * object). After some emperical
10540d94caffSDavid Greenman 					 * results, it is best to deactivate
10550d94caffSDavid Greenman 					 * the readahead pages.
105626f9a767SRodney W. Grimes 					 */
105726f9a767SRodney W. Grimes 					vm_page_deactivate(m[i]);
105826f9a767SRodney W. Grimes 
105926f9a767SRodney W. Grimes 					/*
10600d94caffSDavid Greenman 					 * just in case someone was asking for
10610d94caffSDavid Greenman 					 * this page we now tell them that it
10620d94caffSDavid Greenman 					 * is ok to use
106326f9a767SRodney W. Grimes 					 */
10640d94caffSDavid Greenman 					m[i]->valid = VM_PAGE_BITS_ALL;
106526f9a767SRodney W. Grimes 					PAGE_WAKEUP(m[i]);
106626f9a767SRodney W. Grimes 				}
106726f9a767SRodney W. Grimes 			}
10686d40c3d3SDavid Greenman 
10696d40c3d3SDavid Greenman 			m[reqpage]->object->last_read = m[count-1]->offset;
10706d40c3d3SDavid Greenman 
10712e1e24ddSDavid Greenman 			/*
10722e1e24ddSDavid Greenman 			 * If we're out of swap space, then attempt to free
10732e1e24ddSDavid Greenman 			 * some whenever pages are brought in. We must clear
10742e1e24ddSDavid Greenman 			 * the clean flag so that the page contents will be
10752e1e24ddSDavid Greenman 			 * preserved.
10762e1e24ddSDavid Greenman 			 */
107726f9a767SRodney W. Grimes 			if (swap_pager_full) {
10782e1e24ddSDavid Greenman 				for (i = 0; i < count; i++) {
10790d94caffSDavid Greenman 					m[i]->dirty = VM_PAGE_BITS_ALL;
10802e1e24ddSDavid Greenman 				}
108124a1cce3SDavid Greenman 				swap_pager_freespace(object, m[0]->offset + paging_offset, count * PAGE_SIZE);
108226f9a767SRodney W. Grimes 			}
108326f9a767SRodney W. Grimes 		} else {
108426f9a767SRodney W. Grimes 			swap_pager_ridpages(m, count, reqpage);
108526f9a767SRodney W. Grimes 		}
108626f9a767SRodney W. Grimes 	}
108723922ccaSDavid Greenman 	if (rv == VM_PAGER_OK) {
108823922ccaSDavid Greenman 		pmap_clear_modify(VM_PAGE_TO_PHYS(m[reqpage]));
108923922ccaSDavid Greenman 		m[reqpage]->valid = VM_PAGE_BITS_ALL;
109023922ccaSDavid Greenman 		m[reqpage]->dirty = 0;
109123922ccaSDavid Greenman 	}
1092df8bae1dSRodney W. Grimes 	return (rv);
1093df8bae1dSRodney W. Grimes }
1094df8bae1dSRodney W. Grimes 
109526f9a767SRodney W. Grimes int
109624a1cce3SDavid Greenman swap_pager_putpages(object, m, count, sync, rtvals)
109724a1cce3SDavid Greenman 	vm_object_t object;
109826f9a767SRodney W. Grimes 	vm_page_t *m;
109926f9a767SRodney W. Grimes 	int count;
110024a1cce3SDavid Greenman 	boolean_t sync;
110126f9a767SRodney W. Grimes 	int *rtvals;
1102df8bae1dSRodney W. Grimes {
110326f9a767SRodney W. Grimes 	register struct buf *bp;
110426f9a767SRodney W. Grimes 	sw_blk_t swb[count];
110526f9a767SRodney W. Grimes 	register int s;
110626f9a767SRodney W. Grimes 	int i, j, ix;
110726f9a767SRodney W. Grimes 	boolean_t rv;
110826f9a767SRodney W. Grimes 	vm_offset_t kva, off, foff;
110926f9a767SRodney W. Grimes 	swp_clean_t spc;
111026f9a767SRodney W. Grimes 	vm_offset_t paging_offset;
111126f9a767SRodney W. Grimes 	int reqaddr[count];
111226f9a767SRodney W. Grimes 	int failed;
1113df8bae1dSRodney W. Grimes 
111424ea4a96SDavid Greenman 	if (vm_swap_size)
111524ea4a96SDavid Greenman 		no_swap_space = 0;
111624ea4a96SDavid Greenman 	if (no_swap_space) {
11175663e6deSDavid Greenman 		for (i = 0; i < count; i++)
11185663e6deSDavid Greenman 			rtvals[i] = VM_PAGER_FAIL;
11195663e6deSDavid Greenman 		return VM_PAGER_FAIL;
11205663e6deSDavid Greenman 	}
112126f9a767SRodney W. Grimes 	spc = NULL;
112226f9a767SRodney W. Grimes 
112326f9a767SRodney W. Grimes 	object = m[0]->object;
112426f9a767SRodney W. Grimes 	paging_offset = object->paging_offset;
112526f9a767SRodney W. Grimes 
112626f9a767SRodney W. Grimes 	failed = 0;
112726f9a767SRodney W. Grimes 	for (j = 0; j < count; j++) {
112826f9a767SRodney W. Grimes 		foff = m[j]->offset + paging_offset;
11292a4895f4SDavid Greenman 		ix = swap_pager_block_index(foff);
113026f9a767SRodney W. Grimes 		swb[j] = 0;
11312a4895f4SDavid Greenman 		if (ix >= object->un_pager.swp.swp_nblocks) {
113226f9a767SRodney W. Grimes 			rtvals[j] = VM_PAGER_FAIL;
113326f9a767SRodney W. Grimes 			failed = 1;
113426f9a767SRodney W. Grimes 			continue;
113526f9a767SRodney W. Grimes 		} else {
113626f9a767SRodney W. Grimes 			rtvals[j] = VM_PAGER_OK;
113726f9a767SRodney W. Grimes 		}
11382a4895f4SDavid Greenman 		swb[j] = &object->un_pager.swp.swp_blocks[ix];
11392a4895f4SDavid Greenman 		swb[j]->swb_locked++;
114026f9a767SRodney W. Grimes 		if (failed) {
114126f9a767SRodney W. Grimes 			rtvals[j] = VM_PAGER_FAIL;
114226f9a767SRodney W. Grimes 			continue;
114326f9a767SRodney W. Grimes 		}
11442a4895f4SDavid Greenman 		off = swap_pager_block_offset(foff);
114526f9a767SRodney W. Grimes 		reqaddr[j] = swb[j]->swb_block[off];
114626f9a767SRodney W. Grimes 		if (reqaddr[j] == SWB_EMPTY) {
114726f9a767SRodney W. Grimes 			int blk;
114826f9a767SRodney W. Grimes 			int tries;
114926f9a767SRodney W. Grimes 			int ntoget;
11500d94caffSDavid Greenman 
115126f9a767SRodney W. Grimes 			tries = 0;
1152df8bae1dSRodney W. Grimes 			s = splbio();
115326f9a767SRodney W. Grimes 
1154df8bae1dSRodney W. Grimes 			/*
11550d94caffSDavid Greenman 			 * if any other pages have been allocated in this
11560d94caffSDavid Greenman 			 * block, we only try to get one page.
1157df8bae1dSRodney W. Grimes 			 */
115826f9a767SRodney W. Grimes 			for (i = 0; i < SWB_NPAGES; i++) {
115926f9a767SRodney W. Grimes 				if (swb[j]->swb_block[i] != SWB_EMPTY)
1160df8bae1dSRodney W. Grimes 					break;
1161df8bae1dSRodney W. Grimes 			}
116226f9a767SRodney W. Grimes 
116326f9a767SRodney W. Grimes 			ntoget = (i == SWB_NPAGES) ? SWB_NPAGES : 1;
116426f9a767SRodney W. Grimes 			/*
11650d94caffSDavid Greenman 			 * this code is alittle conservative, but works (the
11660d94caffSDavid Greenman 			 * intent of this code is to allocate small chunks for
11670d94caffSDavid Greenman 			 * small objects)
116826f9a767SRodney W. Grimes 			 */
116911fda60bSJohn Dyson 			if ((foff == 0) &&
117011fda60bSJohn Dyson 				((ntoget * PAGE_SIZE) > object->size)) {
117111fda60bSJohn Dyson 				ntoget = (object->size + (PAGE_SIZE - 1)) / PAGE_SIZE;
117226f9a767SRodney W. Grimes 			}
117326f9a767SRodney W. Grimes 	retrygetspace:
117426f9a767SRodney W. Grimes 			if (!swap_pager_full && ntoget > 1 &&
11752a4895f4SDavid Greenman 			    swap_pager_getswapspace(object, ntoget * btodb(PAGE_SIZE), &blk)) {
117626f9a767SRodney W. Grimes 
117726f9a767SRodney W. Grimes 				for (i = 0; i < ntoget; i++) {
117826f9a767SRodney W. Grimes 					swb[j]->swb_block[i] = blk + btodb(PAGE_SIZE) * i;
117926f9a767SRodney W. Grimes 					swb[j]->swb_valid = 0;
118026f9a767SRodney W. Grimes 				}
118126f9a767SRodney W. Grimes 
118226f9a767SRodney W. Grimes 				reqaddr[j] = swb[j]->swb_block[off];
11832a4895f4SDavid Greenman 			} else if (!swap_pager_getswapspace(object, btodb(PAGE_SIZE),
118426f9a767SRodney W. Grimes 				&swb[j]->swb_block[off])) {
118526f9a767SRodney W. Grimes 				/*
11860d94caffSDavid Greenman 				 * if the allocation has failed, we try to
11870d94caffSDavid Greenman 				 * reclaim space and retry.
118826f9a767SRodney W. Grimes 				 */
118926f9a767SRodney W. Grimes 				if (++tries == 1) {
119026f9a767SRodney W. Grimes 					swap_pager_reclaim();
119126f9a767SRodney W. Grimes 					goto retrygetspace;
119226f9a767SRodney W. Grimes 				}
119326f9a767SRodney W. Grimes 				rtvals[j] = VM_PAGER_AGAIN;
119426f9a767SRodney W. Grimes 				failed = 1;
119524ea4a96SDavid Greenman 				swap_pager_full = 1;
119626f9a767SRodney W. Grimes 			} else {
119726f9a767SRodney W. Grimes 				reqaddr[j] = swb[j]->swb_block[off];
119826f9a767SRodney W. Grimes 				swb[j]->swb_valid &= ~(1 << off);
1199df8bae1dSRodney W. Grimes 			}
1200df8bae1dSRodney W. Grimes 			splx(s);
120126f9a767SRodney W. Grimes 		}
120226f9a767SRodney W. Grimes 	}
120326f9a767SRodney W. Grimes 
120426f9a767SRodney W. Grimes 	/*
120526f9a767SRodney W. Grimes 	 * search forwards for the last contiguous page to transfer
120626f9a767SRodney W. Grimes 	 */
120726f9a767SRodney W. Grimes 	failed = 0;
120826f9a767SRodney W. Grimes 	for (i = 0; i < count; i++) {
120926f9a767SRodney W. Grimes 		if (failed || (reqaddr[i] != reqaddr[0] + i * btodb(PAGE_SIZE)) ||
121026f9a767SRodney W. Grimes 		    (reqaddr[i] / dmmax) != (reqaddr[0] / dmmax) ||
121126f9a767SRodney W. Grimes 		    (rtvals[i] != VM_PAGER_OK)) {
121226f9a767SRodney W. Grimes 			failed = 1;
121326f9a767SRodney W. Grimes 			if (rtvals[i] == VM_PAGER_OK)
121426f9a767SRodney W. Grimes 				rtvals[i] = VM_PAGER_AGAIN;
121526f9a767SRodney W. Grimes 		}
121626f9a767SRodney W. Grimes 	}
121726f9a767SRodney W. Grimes 
121826f9a767SRodney W. Grimes 	for (i = 0; i < count; i++) {
121926f9a767SRodney W. Grimes 		if (rtvals[i] != VM_PAGER_OK) {
122026f9a767SRodney W. Grimes 			if (swb[i])
122126f9a767SRodney W. Grimes 				--swb[i]->swb_locked;
122226f9a767SRodney W. Grimes 		}
122326f9a767SRodney W. Grimes 	}
122426f9a767SRodney W. Grimes 
122526f9a767SRodney W. Grimes 	for (i = 0; i < count; i++)
122626f9a767SRodney W. Grimes 		if (rtvals[i] != VM_PAGER_OK)
122726f9a767SRodney W. Grimes 			break;
122826f9a767SRodney W. Grimes 
122926f9a767SRodney W. Grimes 	if (i == 0) {
123026f9a767SRodney W. Grimes 		return VM_PAGER_AGAIN;
123126f9a767SRodney W. Grimes 	}
123226f9a767SRodney W. Grimes 	count = i;
123326f9a767SRodney W. Grimes 	for (i = 0; i < count; i++) {
123426f9a767SRodney W. Grimes 		if (reqaddr[i] == SWB_EMPTY)
123526f9a767SRodney W. Grimes 			printf("I/O to empty block????\n");
123626f9a767SRodney W. Grimes 	}
123726f9a767SRodney W. Grimes 
123826f9a767SRodney W. Grimes 	/*
12390d94caffSDavid Greenman 	 * For synchronous writes, we clean up all completed async pageouts.
124026f9a767SRodney W. Grimes 	 */
124124a1cce3SDavid Greenman 	if (sync == TRUE) {
124224a1cce3SDavid Greenman 		swap_pager_sync();
124326f9a767SRodney W. Grimes 	}
124426f9a767SRodney W. Grimes 	kva = 0;
124526f9a767SRodney W. Grimes 
124626f9a767SRodney W. Grimes 	/*
124726f9a767SRodney W. Grimes 	 * get a swap pager clean data structure, block until we get it
124826f9a767SRodney W. Grimes 	 */
1249a1f6d91cSDavid Greenman 	if (swap_pager_free.tqh_first == NULL ||
1250a1f6d91cSDavid Greenman 		swap_pager_free.tqh_first->spc_list.tqe_next == NULL ||
1251a1f6d91cSDavid Greenman 		swap_pager_free.tqh_first->spc_list.tqe_next->spc_list.tqe_next == NULL) {
125226f9a767SRodney W. Grimes 		s = splbio();
12530d94caffSDavid Greenman 		if (curproc == pageproc) {
125424a1cce3SDavid Greenman 			swap_pager_sync();
1255a1f6d91cSDavid Greenman #if 0
12560d94caffSDavid Greenman 			splx(s);
12570d94caffSDavid Greenman 			return VM_PAGER_AGAIN;
1258a1f6d91cSDavid Greenman #endif
12590d94caffSDavid Greenman 		} else
1260f919ebdeSDavid Greenman 			pagedaemon_wakeup();
1261a1f6d91cSDavid Greenman 		while (swap_pager_free.tqh_first == NULL ||
1262a1f6d91cSDavid Greenman 			swap_pager_free.tqh_first->spc_list.tqe_next == NULL ||
1263a1f6d91cSDavid Greenman 			swap_pager_free.tqh_first->spc_list.tqe_next->spc_list.tqe_next == NULL) {
1264a1f6d91cSDavid Greenman 			if (curproc == pageproc) {
1265a1f6d91cSDavid Greenman 				swap_pager_needflags |= SWAP_FREE_NEEDED_BY_PAGEOUT;
1266a1f6d91cSDavid Greenman 			    if((cnt.v_free_count + cnt.v_cache_count) > cnt.v_free_reserved)
126724a1cce3SDavid Greenman 					wakeup(&cnt.v_free_count);
1268a1f6d91cSDavid Greenman 			}
12690d94caffSDavid Greenman 
127026f9a767SRodney W. Grimes 			swap_pager_needflags |= SWAP_FREE_NEEDED;
127124a1cce3SDavid Greenman 			tsleep(&swap_pager_free, PVM, "swpfre", 0);
127226f9a767SRodney W. Grimes 			if (curproc == pageproc)
127324a1cce3SDavid Greenman 				swap_pager_sync();
127426f9a767SRodney W. Grimes 			else
1275f919ebdeSDavid Greenman 				pagedaemon_wakeup();
127626f9a767SRodney W. Grimes 		}
127726f9a767SRodney W. Grimes 		splx(s);
127826f9a767SRodney W. Grimes 	}
127926f9a767SRodney W. Grimes 	spc = swap_pager_free.tqh_first;
128026f9a767SRodney W. Grimes 	TAILQ_REMOVE(&swap_pager_free, spc, spc_list);
1281fff93ab6SDavid Greenman 
128226f9a767SRodney W. Grimes 	kva = spc->spc_kva;
128326f9a767SRodney W. Grimes 
128426f9a767SRodney W. Grimes 	/*
128526f9a767SRodney W. Grimes 	 * map our page(s) into kva for I/O
128626f9a767SRodney W. Grimes 	 */
128716f62314SDavid Greenman 	pmap_qenter(kva, m, count);
128826f9a767SRodney W. Grimes 
128926f9a767SRodney W. Grimes 	/*
129026f9a767SRodney W. Grimes 	 * get the base I/O offset into the swap file
129126f9a767SRodney W. Grimes 	 */
129226f9a767SRodney W. Grimes 	for (i = 0; i < count; i++) {
129326f9a767SRodney W. Grimes 		foff = m[i]->offset + paging_offset;
12942a4895f4SDavid Greenman 		off = swap_pager_block_offset(foff);
129526f9a767SRodney W. Grimes 		/*
129626f9a767SRodney W. Grimes 		 * set the valid bit
129726f9a767SRodney W. Grimes 		 */
129826f9a767SRodney W. Grimes 		swb[i]->swb_valid |= (1 << off);
129926f9a767SRodney W. Grimes 		/*
130026f9a767SRodney W. Grimes 		 * and unlock the data structure
130126f9a767SRodney W. Grimes 		 */
13022a4895f4SDavid Greenman 		swb[i]->swb_locked--;
130326f9a767SRodney W. Grimes 	}
130426f9a767SRodney W. Grimes 
130526f9a767SRodney W. Grimes 	/*
130626f9a767SRodney W. Grimes 	 * Get a swap buffer header and perform the IO
130726f9a767SRodney W. Grimes 	 */
130826f9a767SRodney W. Grimes 	bp = spc->spc_bp;
130926f9a767SRodney W. Grimes 	bzero(bp, sizeof *bp);
131026f9a767SRodney W. Grimes 	bp->b_spc = spc;
13117609ab12SDavid Greenman 	bp->b_vnbufs.le_next = NOLIST;
131226f9a767SRodney W. Grimes 
1313aba8f38eSDavid Greenman 	bp->b_flags = B_BUSY | B_PAGING;
131426f9a767SRodney W. Grimes 	bp->b_proc = &proc0;	/* XXX (but without B_PHYS set this is ok) */
131526f9a767SRodney W. Grimes 	bp->b_rcred = bp->b_wcred = bp->b_proc->p_ucred;
1316a481f200SDavid Greenman 	if (bp->b_rcred != NOCRED)
131726f9a767SRodney W. Grimes 		crhold(bp->b_rcred);
1318a481f200SDavid Greenman 	if (bp->b_wcred != NOCRED)
131926f9a767SRodney W. Grimes 		crhold(bp->b_wcred);
1320a481f200SDavid Greenman 	bp->b_data = (caddr_t) kva;
132126f9a767SRodney W. Grimes 	bp->b_blkno = reqaddr[0];
13220d94caffSDavid Greenman 	pbgetvp(swapdev_vp, bp);
132316f62314SDavid Greenman 
132426f9a767SRodney W. Grimes 	bp->b_bcount = PAGE_SIZE * count;
132526f9a767SRodney W. Grimes 	bp->b_bufsize = PAGE_SIZE * count;
132626f9a767SRodney W. Grimes 	swapdev_vp->v_numoutput++;
132726f9a767SRodney W. Grimes 
132826f9a767SRodney W. Grimes 	/*
13290d94caffSDavid Greenman 	 * If this is an async write we set up additional buffer fields and
13300d94caffSDavid Greenman 	 * place a "cleaning" entry on the inuse queue.
133126f9a767SRodney W. Grimes 	 */
13327609ab12SDavid Greenman 	s = splbio();
133324a1cce3SDavid Greenman 	if (sync == FALSE) {
133426f9a767SRodney W. Grimes 		spc->spc_flags = 0;
13352a4895f4SDavid Greenman 		spc->spc_object = object;
133626f9a767SRodney W. Grimes 		for (i = 0; i < count; i++)
133726f9a767SRodney W. Grimes 			spc->spc_m[i] = m[i];
133826f9a767SRodney W. Grimes 		spc->spc_count = count;
133926f9a767SRodney W. Grimes 		/*
134026f9a767SRodney W. Grimes 		 * the completion routine for async writes
134126f9a767SRodney W. Grimes 		 */
134226f9a767SRodney W. Grimes 		bp->b_flags |= B_CALL;
134326f9a767SRodney W. Grimes 		bp->b_iodone = swap_pager_iodone;
134426f9a767SRodney W. Grimes 		bp->b_dirtyoff = 0;
134526f9a767SRodney W. Grimes 		bp->b_dirtyend = bp->b_bcount;
13462a4895f4SDavid Greenman 		object->un_pager.swp.swp_poip++;
134726f9a767SRodney W. Grimes 		TAILQ_INSERT_TAIL(&swap_pager_inuse, spc, spc_list);
134826f9a767SRodney W. Grimes 	} else {
13492a4895f4SDavid Greenman 		object->un_pager.swp.swp_poip++;
135026f9a767SRodney W. Grimes 		bp->b_flags |= B_CALL;
135126f9a767SRodney W. Grimes 		bp->b_iodone = swap_pager_iodone1;
135226f9a767SRodney W. Grimes 	}
1353976e77fcSDavid Greenman 
1354976e77fcSDavid Greenman 	cnt.v_swapout++;
1355976e77fcSDavid Greenman 	cnt.v_swappgsout += count;
135626f9a767SRodney W. Grimes 	/*
135726f9a767SRodney W. Grimes 	 * perform the I/O
135826f9a767SRodney W. Grimes 	 */
135926f9a767SRodney W. Grimes 	VOP_STRATEGY(bp);
136024a1cce3SDavid Greenman 	if (sync == FALSE) {
136126f9a767SRodney W. Grimes 		if ((bp->b_flags & B_DONE) == B_DONE) {
136224a1cce3SDavid Greenman 			swap_pager_sync();
136326f9a767SRodney W. Grimes 		}
136426f9a767SRodney W. Grimes 		splx(s);
136526f9a767SRodney W. Grimes 		for (i = 0; i < count; i++) {
136626f9a767SRodney W. Grimes 			rtvals[i] = VM_PAGER_PEND;
136726f9a767SRodney W. Grimes 		}
136826f9a767SRodney W. Grimes 		return VM_PAGER_PEND;
136926f9a767SRodney W. Grimes 	}
137026f9a767SRodney W. Grimes 	/*
137126f9a767SRodney W. Grimes 	 * wait for the sync I/O to complete
137226f9a767SRodney W. Grimes 	 */
137326f9a767SRodney W. Grimes 	while ((bp->b_flags & B_DONE) == 0) {
137424a1cce3SDavid Greenman 		tsleep(bp, PVM, "swwrt", 0);
137526f9a767SRodney W. Grimes 	}
13761b119d9dSDavid Greenman 	if (bp->b_flags & B_ERROR) {
13771b119d9dSDavid Greenman 		printf("swap_pager: I/O error - pageout failed; blkno %d, size %d, error %d\n",
13781b119d9dSDavid Greenman 		    bp->b_blkno, bp->b_bcount, bp->b_error);
1379a83c285cSDavid Greenman 		rv = VM_PAGER_ERROR;
13801b119d9dSDavid Greenman 	} else {
13811b119d9dSDavid Greenman 		rv = VM_PAGER_OK;
13821b119d9dSDavid Greenman 	}
138326f9a767SRodney W. Grimes 
13842a4895f4SDavid Greenman 	object->un_pager.swp.swp_poip--;
13852a4895f4SDavid Greenman 	if (object->un_pager.swp.swp_poip == 0)
13862a4895f4SDavid Greenman 		wakeup(object);
138726f9a767SRodney W. Grimes 
138826f9a767SRodney W. Grimes 	if (bp->b_vp)
13890d94caffSDavid Greenman 		pbrelvp(bp);
13900d94caffSDavid Greenman 	if (bp->b_flags & B_WANTED)
139124a1cce3SDavid Greenman 		wakeup(bp);
139226f9a767SRodney W. Grimes 
139326f9a767SRodney W. Grimes 	splx(s);
139426f9a767SRodney W. Grimes 
139526f9a767SRodney W. Grimes 	/*
139626f9a767SRodney W. Grimes 	 * remove the mapping for kernel virtual
139726f9a767SRodney W. Grimes 	 */
139816f62314SDavid Greenman 	pmap_qremove(kva, count);
139926f9a767SRodney W. Grimes 
140026f9a767SRodney W. Grimes 	/*
14010d94caffSDavid Greenman 	 * if we have written the page, then indicate that the page is clean.
140226f9a767SRodney W. Grimes 	 */
140326f9a767SRodney W. Grimes 	if (rv == VM_PAGER_OK) {
140426f9a767SRodney W. Grimes 		for (i = 0; i < count; i++) {
140526f9a767SRodney W. Grimes 			if (rtvals[i] == VM_PAGER_OK) {
140626f9a767SRodney W. Grimes 				pmap_clear_modify(VM_PAGE_TO_PHYS(m[i]));
14070d94caffSDavid Greenman 				m[i]->dirty = 0;
140826f9a767SRodney W. Grimes 				/*
14090d94caffSDavid Greenman 				 * optimization, if a page has been read
14100d94caffSDavid Greenman 				 * during the pageout process, we activate it.
141126f9a767SRodney W. Grimes 				 */
141226f9a767SRodney W. Grimes 				if ((m[i]->flags & PG_ACTIVE) == 0 &&
14137fb0c17eSDavid Greenman 				    ((m[i]->flags & (PG_WANTED|PG_REFERENCED)) ||
14147fb0c17eSDavid Greenman 				    pmap_is_referenced(VM_PAGE_TO_PHYS(m[i])))) {
141526f9a767SRodney W. Grimes 					vm_page_activate(m[i]);
141626f9a767SRodney W. Grimes 				}
141726f9a767SRodney W. Grimes 			}
14187fb0c17eSDavid Greenman 		}
141926f9a767SRodney W. Grimes 	} else {
142026f9a767SRodney W. Grimes 		for (i = 0; i < count; i++) {
142126f9a767SRodney W. Grimes 			rtvals[i] = rv;
142226f9a767SRodney W. Grimes 		}
142326f9a767SRodney W. Grimes 	}
142426f9a767SRodney W. Grimes 
142526f9a767SRodney W. Grimes 	if (bp->b_rcred != NOCRED)
142626f9a767SRodney W. Grimes 		crfree(bp->b_rcred);
142726f9a767SRodney W. Grimes 	if (bp->b_wcred != NOCRED)
142826f9a767SRodney W. Grimes 		crfree(bp->b_wcred);
142926f9a767SRodney W. Grimes 	TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list);
143026f9a767SRodney W. Grimes 	if (swap_pager_needflags & SWAP_FREE_NEEDED) {
143124a1cce3SDavid Greenman 		wakeup(&swap_pager_free);
143226f9a767SRodney W. Grimes 	}
1433a1f6d91cSDavid Greenman 	if (swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT)
1434f919ebdeSDavid Greenman 		pagedaemon_wakeup();
1435a1f6d91cSDavid Greenman 	swap_pager_needflags &= ~(SWAP_FREE_NEEDED|SWAP_FREE_NEEDED_BY_PAGEOUT);
143626f9a767SRodney W. Grimes 	return (rv);
143726f9a767SRodney W. Grimes }
143826f9a767SRodney W. Grimes 
143924a1cce3SDavid Greenman void
144024a1cce3SDavid Greenman swap_pager_sync()
144126f9a767SRodney W. Grimes {
144226f9a767SRodney W. Grimes 	register swp_clean_t spc, tspc;
144326f9a767SRodney W. Grimes 	register int s;
144426f9a767SRodney W. Grimes 
144526f9a767SRodney W. Grimes 	tspc = NULL;
144626f9a767SRodney W. Grimes 	if (swap_pager_done.tqh_first == NULL)
144724a1cce3SDavid Greenman 		return;
144826f9a767SRodney W. Grimes 	for (;;) {
144926f9a767SRodney W. Grimes 		s = splbio();
145026f9a767SRodney W. Grimes 		/*
14510d94caffSDavid Greenman 		 * Look up and removal from done list must be done at splbio()
14520d94caffSDavid Greenman 		 * to avoid conflicts with swap_pager_iodone.
145326f9a767SRodney W. Grimes 		 */
145405f0fdd2SPoul-Henning Kamp 		while ((spc = swap_pager_done.tqh_first) != 0) {
1455fff93ab6SDavid Greenman 			pmap_qremove(spc->spc_kva, spc->spc_count);
145626f9a767SRodney W. Grimes 			swap_pager_finish(spc);
145726f9a767SRodney W. Grimes 			TAILQ_REMOVE(&swap_pager_done, spc, spc_list);
145826f9a767SRodney W. Grimes 			goto doclean;
145926f9a767SRodney W. Grimes 		}
1460df8bae1dSRodney W. Grimes 
1461df8bae1dSRodney W. Grimes 		/*
1462df8bae1dSRodney W. Grimes 		 * No operations done, thats all we can do for now.
1463df8bae1dSRodney W. Grimes 		 */
146426f9a767SRodney W. Grimes 
146526f9a767SRodney W. Grimes 		splx(s);
1466df8bae1dSRodney W. Grimes 		break;
1467df8bae1dSRodney W. Grimes 
1468df8bae1dSRodney W. Grimes 		/*
14690d94caffSDavid Greenman 		 * The desired page was found to be busy earlier in the scan
14700d94caffSDavid Greenman 		 * but has since completed.
1471df8bae1dSRodney W. Grimes 		 */
147226f9a767SRodney W. Grimes doclean:
147326f9a767SRodney W. Grimes 		if (tspc && tspc == spc) {
147426f9a767SRodney W. Grimes 			tspc = NULL;
147526f9a767SRodney W. Grimes 		}
147626f9a767SRodney W. Grimes 		spc->spc_flags = 0;
147726f9a767SRodney W. Grimes 		TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list);
147826f9a767SRodney W. Grimes 		if (swap_pager_needflags & SWAP_FREE_NEEDED) {
147924a1cce3SDavid Greenman 			wakeup(&swap_pager_free);
148026f9a767SRodney W. Grimes 		}
1481a1f6d91cSDavid Greenman 		if( swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT)
1482f919ebdeSDavid Greenman 			pagedaemon_wakeup();
1483a1f6d91cSDavid Greenman 		swap_pager_needflags &= ~(SWAP_FREE_NEEDED|SWAP_FREE_NEEDED_BY_PAGEOUT);
148426f9a767SRodney W. Grimes 		splx(s);
148526f9a767SRodney W. Grimes 	}
148626f9a767SRodney W. Grimes 
148724a1cce3SDavid Greenman 	return;
148826f9a767SRodney W. Grimes }
148926f9a767SRodney W. Grimes 
149026f9a767SRodney W. Grimes void
149126f9a767SRodney W. Grimes swap_pager_finish(spc)
149226f9a767SRodney W. Grimes 	register swp_clean_t spc;
149326f9a767SRodney W. Grimes {
149426f9a767SRodney W. Grimes 	vm_object_t object = spc->spc_m[0]->object;
149526f9a767SRodney W. Grimes 	int i;
149626f9a767SRodney W. Grimes 
1497c0503609SDavid Greenman 	object->paging_in_progress -= spc->spc_count;
1498c0503609SDavid Greenman 	if ((object->paging_in_progress == 0) &&
1499c0503609SDavid Greenman 	    (object->flags & OBJ_PIPWNT)) {
1500c0503609SDavid Greenman 		object->flags &= ~OBJ_PIPWNT;
150124a1cce3SDavid Greenman 		wakeup(object);
1502c0503609SDavid Greenman 	}
1503df8bae1dSRodney W. Grimes 
1504df8bae1dSRodney W. Grimes 	/*
15055f55e841SDavid Greenman 	 * If no error, mark as clean and inform the pmap system. If error,
15060d94caffSDavid Greenman 	 * mark as dirty so we will try again. (XXX could get stuck doing
15070d94caffSDavid Greenman 	 * this, should give up after awhile)
1508df8bae1dSRodney W. Grimes 	 */
1509df8bae1dSRodney W. Grimes 	if (spc->spc_flags & SPC_ERROR) {
151026f9a767SRodney W. Grimes 		for (i = 0; i < spc->spc_count; i++) {
1511a83c285cSDavid Greenman 			printf("swap_pager_finish: I/O error, clean of page %lx failed\n",
151205f0fdd2SPoul-Henning Kamp 			    (u_long) VM_PAGE_TO_PHYS(spc->spc_m[i]));
151326f9a767SRodney W. Grimes 		}
1514df8bae1dSRodney W. Grimes 	} else {
151526f9a767SRodney W. Grimes 		for (i = 0; i < spc->spc_count; i++) {
151626f9a767SRodney W. Grimes 			pmap_clear_modify(VM_PAGE_TO_PHYS(spc->spc_m[i]));
15170d94caffSDavid Greenman 			spc->spc_m[i]->dirty = 0;
15180d94caffSDavid Greenman 			if ((spc->spc_m[i]->flags & PG_ACTIVE) == 0 &&
15190d94caffSDavid Greenman 			    ((spc->spc_m[i]->flags & PG_WANTED) || pmap_is_referenced(VM_PAGE_TO_PHYS(spc->spc_m[i]))))
15200d94caffSDavid Greenman 				vm_page_activate(spc->spc_m[i]);
1521df8bae1dSRodney W. Grimes 		}
1522df8bae1dSRodney W. Grimes 	}
1523df8bae1dSRodney W. Grimes 
152426f9a767SRodney W. Grimes 
152526f9a767SRodney W. Grimes 	for (i = 0; i < spc->spc_count; i++) {
1526df8bae1dSRodney W. Grimes 		/*
15270d94caffSDavid Greenman 		 * we wakeup any processes that are waiting on these pages.
1528df8bae1dSRodney W. Grimes 		 */
152926f9a767SRodney W. Grimes 		PAGE_WAKEUP(spc->spc_m[i]);
1530df8bae1dSRodney W. Grimes 	}
153126f9a767SRodney W. Grimes 	nswiodone -= spc->spc_count;
1532df8bae1dSRodney W. Grimes 
1533df8bae1dSRodney W. Grimes 	return;
153426f9a767SRodney W. Grimes }
1535df8bae1dSRodney W. Grimes 
153626f9a767SRodney W. Grimes /*
153726f9a767SRodney W. Grimes  * swap_pager_iodone
153826f9a767SRodney W. Grimes  */
1539f5a12711SPoul-Henning Kamp static void
1540df8bae1dSRodney W. Grimes swap_pager_iodone(bp)
1541df8bae1dSRodney W. Grimes 	register struct buf *bp;
1542df8bae1dSRodney W. Grimes {
1543df8bae1dSRodney W. Grimes 	register swp_clean_t spc;
1544df8bae1dSRodney W. Grimes 	int s;
1545df8bae1dSRodney W. Grimes 
1546df8bae1dSRodney W. Grimes 	s = splbio();
154726f9a767SRodney W. Grimes 	spc = (swp_clean_t) bp->b_spc;
154826f9a767SRodney W. Grimes 	TAILQ_REMOVE(&swap_pager_inuse, spc, spc_list);
154926f9a767SRodney W. Grimes 	TAILQ_INSERT_TAIL(&swap_pager_done, spc, spc_list);
155026f9a767SRodney W. Grimes 	if (bp->b_flags & B_ERROR) {
1551df8bae1dSRodney W. Grimes 		spc->spc_flags |= SPC_ERROR;
1552c3a1e425SDavid Greenman 		printf("swap_pager: I/O error - async %s failed; blkno %lu, size %ld, error %d\n",
15531b119d9dSDavid Greenman 		    (bp->b_flags & B_READ) ? "pagein" : "pageout",
1554c3a1e425SDavid Greenman 		    (u_long) bp->b_blkno, bp->b_bcount, bp->b_error);
1555df8bae1dSRodney W. Grimes 	}
155626f9a767SRodney W. Grimes 
15570d94caffSDavid Greenman 	if (bp->b_vp)
15580d94caffSDavid Greenman 		pbrelvp(bp);
15590d94caffSDavid Greenman 
15600d94caffSDavid Greenman 	if (bp->b_flags & B_WANTED)
156124a1cce3SDavid Greenman 		wakeup(bp);
15620d94caffSDavid Greenman 
156326f9a767SRodney W. Grimes 	if (bp->b_rcred != NOCRED)
156426f9a767SRodney W. Grimes 		crfree(bp->b_rcred);
156526f9a767SRodney W. Grimes 	if (bp->b_wcred != NOCRED)
156626f9a767SRodney W. Grimes 		crfree(bp->b_wcred);
156726f9a767SRodney W. Grimes 
156826f9a767SRodney W. Grimes 	nswiodone += spc->spc_count;
15692a4895f4SDavid Greenman 	if (--spc->spc_object->un_pager.swp.swp_poip == 0) {
15702a4895f4SDavid Greenman 		wakeup(spc->spc_object);
157126f9a767SRodney W. Grimes 	}
157226f9a767SRodney W. Grimes 	if ((swap_pager_needflags & SWAP_FREE_NEEDED) ||
157326f9a767SRodney W. Grimes 	    swap_pager_inuse.tqh_first == 0) {
157426f9a767SRodney W. Grimes 		swap_pager_needflags &= ~SWAP_FREE_NEEDED;
157524a1cce3SDavid Greenman 		wakeup(&swap_pager_free);
1576a1f6d91cSDavid Greenman 	}
1577a1f6d91cSDavid Greenman 
1578a1f6d91cSDavid Greenman 	if( swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) {
1579a1f6d91cSDavid Greenman 		swap_pager_needflags &= ~SWAP_FREE_NEEDED_BY_PAGEOUT;
1580f919ebdeSDavid Greenman 		pagedaemon_wakeup();
158126f9a767SRodney W. Grimes 	}
1582a1f6d91cSDavid Greenman 
158326f9a767SRodney W. Grimes 	if (vm_pageout_pages_needed) {
158424a1cce3SDavid Greenman 		wakeup(&vm_pageout_pages_needed);
1585a1f6d91cSDavid Greenman 		vm_pageout_pages_needed = 0;
158626f9a767SRodney W. Grimes 	}
158726f9a767SRodney W. Grimes 	if ((swap_pager_inuse.tqh_first == NULL) ||
15880d94caffSDavid Greenman 	    ((cnt.v_free_count + cnt.v_cache_count) < cnt.v_free_min &&
15890d94caffSDavid Greenman 	    nswiodone + cnt.v_free_count + cnt.v_cache_count >= cnt.v_free_min)) {
1590f919ebdeSDavid Greenman 		pagedaemon_wakeup();
159126f9a767SRodney W. Grimes 	}
159226f9a767SRodney W. Grimes 	splx(s);
159326f9a767SRodney W. Grimes }
159426f9a767SRodney W. Grimes 
159526f9a767SRodney W. Grimes /*
159626f9a767SRodney W. Grimes  * return true if any swap control structures can be allocated
159726f9a767SRodney W. Grimes  */
159826f9a767SRodney W. Grimes int
15990d94caffSDavid Greenman swap_pager_ready()
16000d94caffSDavid Greenman {
160126f9a767SRodney W. Grimes 	if (swap_pager_free.tqh_first)
160226f9a767SRodney W. Grimes 		return 1;
160326f9a767SRodney W. Grimes 	else
160426f9a767SRodney W. Grimes 		return 0;
160526f9a767SRodney W. Grimes }
1606