xref: /freebsd/sys/vm/swap_pager.c (revision 5721c9c76a4ab8ff2d7f9d6a9811d8f647b7d3b6)
1df8bae1dSRodney W. Grimes /*
21c7c3c6aSMatthew Dillon  * Copyright (c) 1998 Matthew Dillon,
326f9a767SRodney W. Grimes  * Copyright (c) 1994 John S. Dyson
4df8bae1dSRodney W. Grimes  * Copyright (c) 1990 University of Utah.
5e9c0cc15SPoul-Henning Kamp  * Copyright (c) 1982, 1986, 1989, 1993
6df8bae1dSRodney W. Grimes  *	The Regents of the University of California.  All rights reserved.
7df8bae1dSRodney W. Grimes  *
8df8bae1dSRodney W. Grimes  * This code is derived from software contributed to Berkeley by
9df8bae1dSRodney W. Grimes  * the Systems Programming Group of the University of Utah Computer
10df8bae1dSRodney W. Grimes  * Science Department.
11df8bae1dSRodney W. Grimes  *
12df8bae1dSRodney W. Grimes  * Redistribution and use in source and binary forms, with or without
13df8bae1dSRodney W. Grimes  * modification, are permitted provided that the following conditions
14df8bae1dSRodney W. Grimes  * are met:
15df8bae1dSRodney W. Grimes  * 1. Redistributions of source code must retain the above copyright
16df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer.
17df8bae1dSRodney W. Grimes  * 2. Redistributions in binary form must reproduce the above copyright
18df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer in the
19df8bae1dSRodney W. Grimes  *    documentation and/or other materials provided with the distribution.
20df8bae1dSRodney W. Grimes  * 3. All advertising materials mentioning features or use of this software
215929bcfaSPhilippe Charnier  *    must display the following acknowledgement:
22df8bae1dSRodney W. Grimes  *	This product includes software developed by the University of
23df8bae1dSRodney W. Grimes  *	California, Berkeley and its contributors.
24df8bae1dSRodney W. Grimes  * 4. Neither the name of the University nor the names of its contributors
25df8bae1dSRodney W. Grimes  *    may be used to endorse or promote products derived from this software
26df8bae1dSRodney W. Grimes  *    without specific prior written permission.
27df8bae1dSRodney W. Grimes  *
28df8bae1dSRodney W. Grimes  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
29df8bae1dSRodney W. Grimes  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30df8bae1dSRodney W. Grimes  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31df8bae1dSRodney W. Grimes  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
32df8bae1dSRodney W. Grimes  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33df8bae1dSRodney W. Grimes  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34df8bae1dSRodney W. Grimes  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35df8bae1dSRodney W. Grimes  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36df8bae1dSRodney W. Grimes  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37df8bae1dSRodney W. Grimes  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38df8bae1dSRodney W. Grimes  * SUCH DAMAGE.
39df8bae1dSRodney W. Grimes  *
401c7c3c6aSMatthew Dillon  *				New Swap System
411c7c3c6aSMatthew Dillon  *				Matthew Dillon
421c7c3c6aSMatthew Dillon  *
431c7c3c6aSMatthew Dillon  * Radix Bitmap 'blists'.
441c7c3c6aSMatthew Dillon  *
451c7c3c6aSMatthew Dillon  *	- The new swapper uses the new radix bitmap code.  This should scale
461c7c3c6aSMatthew Dillon  *	  to arbitrarily small or arbitrarily large swap spaces and an almost
471c7c3c6aSMatthew Dillon  *	  arbitrary degree of fragmentation.
481c7c3c6aSMatthew Dillon  *
491c7c3c6aSMatthew Dillon  * Features:
501c7c3c6aSMatthew Dillon  *
511c7c3c6aSMatthew Dillon  *	- on the fly reallocation of swap during putpages.  The new system
521c7c3c6aSMatthew Dillon  *	  does not try to keep previously allocated swap blocks for dirty
531c7c3c6aSMatthew Dillon  *	  pages.
541c7c3c6aSMatthew Dillon  *
551c7c3c6aSMatthew Dillon  *	- on the fly deallocation of swap
561c7c3c6aSMatthew Dillon  *
571c7c3c6aSMatthew Dillon  *	- No more garbage collection required.  Unnecessarily allocated swap
581c7c3c6aSMatthew Dillon  *	  blocks only exist for dirty vm_page_t's now and these are already
591c7c3c6aSMatthew Dillon  *	  cycled (in a high-load system) by the pager.  We also do on-the-fly
601c7c3c6aSMatthew Dillon  *	  removal of invalidated swap blocks when a page is destroyed
611c7c3c6aSMatthew Dillon  *	  or renamed.
621c7c3c6aSMatthew Dillon  *
63df8bae1dSRodney W. Grimes  * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
64df8bae1dSRodney W. Grimes  *
65df8bae1dSRodney W. Grimes  *	@(#)swap_pager.c	8.9 (Berkeley) 3/21/94
66e9c0cc15SPoul-Henning Kamp  *	@(#)vm_swap.c	8.5 (Berkeley) 2/17/94
67df8bae1dSRodney W. Grimes  */
68df8bae1dSRodney W. Grimes 
69874651b1SDavid E. O'Brien #include <sys/cdefs.h>
70874651b1SDavid E. O'Brien __FBSDID("$FreeBSD$");
71874651b1SDavid E. O'Brien 
72e9c0cc15SPoul-Henning Kamp #include "opt_mac.h"
73e9c0cc15SPoul-Henning Kamp #include "opt_swap.h"
74e9c0cc15SPoul-Henning Kamp #include "opt_vm.h"
75e9c0cc15SPoul-Henning Kamp 
76df8bae1dSRodney W. Grimes #include <sys/param.h>
77df8bae1dSRodney W. Grimes #include <sys/systm.h>
78af647ddeSBruce Evans #include <sys/conf.h>
7964abb5a5SDavid Greenman #include <sys/kernel.h>
80df8bae1dSRodney W. Grimes #include <sys/proc.h>
819626b608SPoul-Henning Kamp #include <sys/bio.h>
82df8bae1dSRodney W. Grimes #include <sys/buf.h>
83e9c0cc15SPoul-Henning Kamp #include <sys/disk.h>
84e9c0cc15SPoul-Henning Kamp #include <sys/fcntl.h>
85e9c0cc15SPoul-Henning Kamp #include <sys/mount.h>
86e9c0cc15SPoul-Henning Kamp #include <sys/namei.h>
87df8bae1dSRodney W. Grimes #include <sys/vnode.h>
88e9c0cc15SPoul-Henning Kamp #include <sys/mac.h>
89df8bae1dSRodney W. Grimes #include <sys/malloc.h>
90327f4e83SMatthew Dillon #include <sys/sysctl.h>
91e9c0cc15SPoul-Henning Kamp #include <sys/sysproto.h>
921c7c3c6aSMatthew Dillon #include <sys/blist.h>
931c7c3c6aSMatthew Dillon #include <sys/lock.h>
940cddd8f0SMatthew Dillon #include <sys/sx.h>
95936524aaSMatthew Dillon #include <sys/vmmeter.h>
96df8bae1dSRodney W. Grimes 
97df8bae1dSRodney W. Grimes #include <vm/vm.h>
9821cd6e62SSeigo Tanimura #include <vm/pmap.h>
9921cd6e62SSeigo Tanimura #include <vm/vm_map.h>
10021cd6e62SSeigo Tanimura #include <vm/vm_kern.h>
101efeaf95aSDavid Greenman #include <vm/vm_object.h>
102df8bae1dSRodney W. Grimes #include <vm/vm_page.h>
103efeaf95aSDavid Greenman #include <vm/vm_pager.h>
104df8bae1dSRodney W. Grimes #include <vm/vm_pageout.h>
105e9c0cc15SPoul-Henning Kamp #include <vm/vm_param.h>
106df8bae1dSRodney W. Grimes #include <vm/swap_pager.h>
107efeaf95aSDavid Greenman #include <vm/vm_extern.h>
108670d17b5SJeff Roberson #include <vm/uma.h>
109df8bae1dSRodney W. Grimes 
110dee34ca4SPoul-Henning Kamp #include <geom/geom.h>
111dee34ca4SPoul-Henning Kamp 
112ec38b344SPoul-Henning Kamp /*
113ec38b344SPoul-Henning Kamp  * SWB_NPAGES must be a power of 2.  It may be set to 1, 2, 4, 8, or 16
114ec38b344SPoul-Henning Kamp  * pages per allocation.  We recommend you stick with the default of 8.
115ec38b344SPoul-Henning Kamp  * The 16-page limit is due to the radix code (kern/subr_blist.c).
116ec38b344SPoul-Henning Kamp  */
117ec38b344SPoul-Henning Kamp #ifndef MAX_PAGEOUT_CLUSTER
118ec38b344SPoul-Henning Kamp #define MAX_PAGEOUT_CLUSTER 16
119ec38b344SPoul-Henning Kamp #endif
120ec38b344SPoul-Henning Kamp 
121ec38b344SPoul-Henning Kamp #if !defined(SWB_NPAGES)
122ec38b344SPoul-Henning Kamp #define SWB_NPAGES	MAX_PAGEOUT_CLUSTER
123ec38b344SPoul-Henning Kamp #endif
124ec38b344SPoul-Henning Kamp 
125ec38b344SPoul-Henning Kamp /*
126ec38b344SPoul-Henning Kamp  * Piecemeal swap metadata structure.  Swap is stored in a radix tree.
127ec38b344SPoul-Henning Kamp  *
128ec38b344SPoul-Henning Kamp  * If SWB_NPAGES is 8 and sizeof(char *) == sizeof(daddr_t), our radix
129ec38b344SPoul-Henning Kamp  * is basically 8.  Assuming PAGE_SIZE == 4096, one tree level represents
130ec38b344SPoul-Henning Kamp  * 32K worth of data, two levels represent 256K, three levels represent
131ec38b344SPoul-Henning Kamp  * 2 MBytes.   This is acceptable.
132ec38b344SPoul-Henning Kamp  *
133ec38b344SPoul-Henning Kamp  * Overall memory utilization is about the same as the old swap structure.
134ec38b344SPoul-Henning Kamp  */
135ec38b344SPoul-Henning Kamp #define SWCORRECT(n) (sizeof(void *) * (n) / sizeof(daddr_t))
136ec38b344SPoul-Henning Kamp #define SWAP_META_PAGES		(SWB_NPAGES * 2)
137ec38b344SPoul-Henning Kamp #define SWAP_META_MASK		(SWAP_META_PAGES - 1)
138ec38b344SPoul-Henning Kamp 
1398f60c087SPoul-Henning Kamp typedef	int32_t	swblk_t;	/*
1408f60c087SPoul-Henning Kamp 				 * swap offset.  This is the type used to
1418f60c087SPoul-Henning Kamp 				 * address the "virtual swap device" and
1428f60c087SPoul-Henning Kamp 				 * therefore the maximum swap space is
1438f60c087SPoul-Henning Kamp 				 * 2^32 pages.
1448f60c087SPoul-Henning Kamp 				 */
145e9c0cc15SPoul-Henning Kamp 
1464b03903aSPoul-Henning Kamp struct swdevt;
1474b03903aSPoul-Henning Kamp typedef void sw_strategy_t(struct buf *bp, struct swdevt *sw);
148dee34ca4SPoul-Henning Kamp typedef void sw_close_t(struct thread *td, struct swdevt *sw);
1494b03903aSPoul-Henning Kamp 
1508d677ef9SPoul-Henning Kamp /*
1518d677ef9SPoul-Henning Kamp  * Swap device table
1528d677ef9SPoul-Henning Kamp  */
1538d677ef9SPoul-Henning Kamp struct swdevt {
1548d677ef9SPoul-Henning Kamp 	int	sw_flags;
1558d677ef9SPoul-Henning Kamp 	int	sw_nblks;
1568d677ef9SPoul-Henning Kamp 	int     sw_used;
157f3732fd1SPoul-Henning Kamp 	dev_t	sw_dev;
158dee34ca4SPoul-Henning Kamp 	struct vnode *sw_vp;
15959efee01SPoul-Henning Kamp 	void	*sw_id;
1608f60c087SPoul-Henning Kamp 	swblk_t	sw_first;
1618f60c087SPoul-Henning Kamp 	swblk_t	sw_end;
1628f60c087SPoul-Henning Kamp 	struct blist *sw_blist;
1638f60c087SPoul-Henning Kamp 	TAILQ_ENTRY(swdevt)	sw_list;
1644b03903aSPoul-Henning Kamp 	sw_strategy_t		*sw_strategy;
165dee34ca4SPoul-Henning Kamp 	sw_close_t		*sw_close;
1668d677ef9SPoul-Henning Kamp };
1678d677ef9SPoul-Henning Kamp 
1688d677ef9SPoul-Henning Kamp #define	SW_CLOSING	0x04
1698d677ef9SPoul-Henning Kamp 
170e9c0cc15SPoul-Henning Kamp struct swblock {
171e9c0cc15SPoul-Henning Kamp 	struct swblock	*swb_hnext;
172e9c0cc15SPoul-Henning Kamp 	vm_object_t	swb_object;
173e9c0cc15SPoul-Henning Kamp 	vm_pindex_t	swb_index;
174e9c0cc15SPoul-Henning Kamp 	int		swb_count;
175e9c0cc15SPoul-Henning Kamp 	daddr_t		swb_pages[SWAP_META_PAGES];
176e9c0cc15SPoul-Henning Kamp };
177e9c0cc15SPoul-Henning Kamp 
17820da9c2eSPoul-Henning Kamp static struct mtx sw_dev_mtx;
1798f60c087SPoul-Henning Kamp static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq);
1808f60c087SPoul-Henning Kamp static struct swdevt *swdevhd;	/* Allocate from here next */
1818f60c087SPoul-Henning Kamp static int nswapdev;		/* Number of swap devices */
1828f60c087SPoul-Henning Kamp int swap_pager_avail;
183e9c0cc15SPoul-Henning Kamp static int swdev_syscall_active = 0; /* serialize swap(on|off) */
184e9c0cc15SPoul-Henning Kamp 
1854b03903aSPoul-Henning Kamp static void swapdev_strategy(struct buf *, struct swdevt *sw);
186e9c0cc15SPoul-Henning Kamp 
1871c7c3c6aSMatthew Dillon #define SWM_FREE	0x02	/* free, period			*/
1881c7c3c6aSMatthew Dillon #define SWM_POP		0x04	/* pop out			*/
18926f9a767SRodney W. Grimes 
1907dea2c2eSAlan Cox int swap_pager_full = 2;	/* swap space exhaustion (task killing) */
1917dea2c2eSAlan Cox static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/
1921c7c3c6aSMatthew Dillon static int nsw_rcount;		/* free read buffers			*/
193327f4e83SMatthew Dillon static int nsw_wcount_sync;	/* limit write buffers / synchronous	*/
194327f4e83SMatthew Dillon static int nsw_wcount_async;	/* limit write buffers / asynchronous	*/
195327f4e83SMatthew Dillon static int nsw_wcount_async_max;/* assigned maximum			*/
196327f4e83SMatthew Dillon static int nsw_cluster_max;	/* maximum VOP I/O allowed		*/
1971c7c3c6aSMatthew Dillon 
1981c7c3c6aSMatthew Dillon static struct swblock **swhash;
1991c7c3c6aSMatthew Dillon static int swhash_mask;
2007827d9b0SAlan Cox static struct mtx swhash_mtx;
2017827d9b0SAlan Cox 
202327f4e83SMatthew Dillon static int swap_async_max = 4;	/* maximum in-progress async I/O's	*/
2030cddd8f0SMatthew Dillon static struct sx sw_alloc_sx;
204327f4e83SMatthew Dillon 
20524e7ab7cSPoul-Henning Kamp 
206327f4e83SMatthew Dillon SYSCTL_INT(_vm, OID_AUTO, swap_async_max,
207327f4e83SMatthew Dillon         CTLFLAG_RW, &swap_async_max, 0, "Maximum running async swap ops");
2081c7c3c6aSMatthew Dillon 
2091c7c3c6aSMatthew Dillon /*
2101c7c3c6aSMatthew Dillon  * "named" and "unnamed" anon region objects.  Try to reduce the overhead
2111c7c3c6aSMatthew Dillon  * of searching a named list by hashing it just a little.
2121c7c3c6aSMatthew Dillon  */
2131c7c3c6aSMatthew Dillon 
2141c7c3c6aSMatthew Dillon #define NOBJLISTS		8
2151c7c3c6aSMatthew Dillon 
2161c7c3c6aSMatthew Dillon #define NOBJLIST(handle)	\
217af647ddeSBruce Evans 	(&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)])
2181c7c3c6aSMatthew Dillon 
219a9fa2c05SAlfred Perlstein static struct mtx sw_alloc_mtx;	/* protect list manipulation */
2201c7c3c6aSMatthew Dillon static struct pagerlst	swap_pager_object_list[NOBJLISTS];
221e9c0cc15SPoul-Henning Kamp static uma_zone_t	swap_zone;
2225285558aSAlan Cox static struct vm_object	swap_zone_obj;
2231c7c3c6aSMatthew Dillon 
2241c7c3c6aSMatthew Dillon /*
2251c7c3c6aSMatthew Dillon  * pagerops for OBJT_SWAP - "swap pager".  Some ops are also global procedure
2261c7c3c6aSMatthew Dillon  * calls hooked from other parts of the VM system and do not appear here.
2271c7c3c6aSMatthew Dillon  * (see vm/swap_pager.h).
2281c7c3c6aSMatthew Dillon  */
229ff98689dSBruce Evans static vm_object_t
23011caded3SAlfred Perlstein 		swap_pager_alloc(void *handle, vm_ooffset_t size,
23111caded3SAlfred Perlstein 				      vm_prot_t prot, vm_ooffset_t offset);
23211caded3SAlfred Perlstein static void	swap_pager_dealloc(vm_object_t object);
23311caded3SAlfred Perlstein static int	swap_pager_getpages(vm_object_t, vm_page_t *, int, int);
234751221fdSPoul-Henning Kamp static void	swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *);
2355ea4972cSAlan Cox static boolean_t
2365ea4972cSAlan Cox 		swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
23711caded3SAlfred Perlstein static void	swap_pager_init(void);
23811caded3SAlfred Perlstein static void	swap_pager_unswapped(vm_page_t);
2398f60c087SPoul-Henning Kamp static void	swap_pager_swapoff(struct swdevt *sp, int *sw_used);
240f708ef1bSPoul-Henning Kamp 
241df8bae1dSRodney W. Grimes struct pagerops swappagerops = {
242e04e4bacSPoul-Henning Kamp 	.pgo_init =	swap_pager_init,	/* early system initialization of pager	*/
243e04e4bacSPoul-Henning Kamp 	.pgo_alloc =	swap_pager_alloc,	/* allocate an OBJT_SWAP object		*/
244e04e4bacSPoul-Henning Kamp 	.pgo_dealloc =	swap_pager_dealloc,	/* deallocate an OBJT_SWAP object	*/
245e04e4bacSPoul-Henning Kamp 	.pgo_getpages =	swap_pager_getpages,	/* pagein				*/
246e04e4bacSPoul-Henning Kamp 	.pgo_putpages =	swap_pager_putpages,	/* pageout				*/
247e04e4bacSPoul-Henning Kamp 	.pgo_haspage =	swap_pager_haspage,	/* get backing store status for page	*/
248e04e4bacSPoul-Henning Kamp 	.pgo_pageunswapped = swap_pager_unswapped,	/* remove swap related to page		*/
249df8bae1dSRodney W. Grimes };
250df8bae1dSRodney W. Grimes 
2511c7c3c6aSMatthew Dillon /*
2521c7c3c6aSMatthew Dillon  * dmmax is in page-sized chunks with the new swap system.  It was
25364bcb9c8SMatthew Dillon  * dev-bsized chunks in the old.  dmmax is always a power of 2.
2541c7c3c6aSMatthew Dillon  *
2551c7c3c6aSMatthew Dillon  * swap_*() routines are externally accessible.  swp_*() routines are
2561c7c3c6aSMatthew Dillon  * internal.
2571c7c3c6aSMatthew Dillon  */
258751221fdSPoul-Henning Kamp static int dmmax;
259e9c0cc15SPoul-Henning Kamp static int nswap_lowat = 128;	/* in pages, swap_pager_almost_full warn */
260e9c0cc15SPoul-Henning Kamp static int nswap_hiwat = 512;	/* in pages, swap_pager_almost_full warn */
26126f9a767SRodney W. Grimes 
262cee313c4SRobert Watson SYSCTL_INT(_vm, OID_AUTO, dmmax,
263cee313c4SRobert Watson 	CTLFLAG_RD, &dmmax, 0, "Maximum size of a swap block");
264cee313c4SRobert Watson 
265a5edd34aSPoul-Henning Kamp static void	swp_sizecheck(void);
26611caded3SAlfred Perlstein static void	swp_pager_async_iodone(struct buf *bp);
267dee34ca4SPoul-Henning Kamp static int	swapongeom(struct thread *, struct vnode *);
26859efee01SPoul-Henning Kamp static int	swaponvp(struct thread *, struct vnode *, u_long);
26924a1cce3SDavid Greenman 
2701c7c3c6aSMatthew Dillon /*
2711c7c3c6aSMatthew Dillon  * Swap bitmap functions
2721c7c3c6aSMatthew Dillon  */
273a5edd34aSPoul-Henning Kamp static void	swp_pager_freeswapspace(daddr_t blk, int npages);
274a5edd34aSPoul-Henning Kamp static daddr_t	swp_pager_getswapspace(int npages);
2751c7c3c6aSMatthew Dillon 
2761c7c3c6aSMatthew Dillon /*
2771c7c3c6aSMatthew Dillon  * Metadata functions
2781c7c3c6aSMatthew Dillon  */
279a5edd34aSPoul-Henning Kamp static struct swblock **swp_pager_hash(vm_object_t object, vm_pindex_t index);
28011caded3SAlfred Perlstein static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
28111caded3SAlfred Perlstein static void swp_pager_meta_free(vm_object_t, vm_pindex_t, daddr_t);
28211caded3SAlfred Perlstein static void swp_pager_meta_free_all(vm_object_t);
28311caded3SAlfred Perlstein static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);
2841c7c3c6aSMatthew Dillon 
2851c7c3c6aSMatthew Dillon /*
2861c7c3c6aSMatthew Dillon  * SWP_SIZECHECK() -	update swap_pager_full indication
2871c7c3c6aSMatthew Dillon  *
28820d3034fSMatthew Dillon  *	update the swap_pager_almost_full indication and warn when we are
28920d3034fSMatthew Dillon  *	about to run out of swap space, using lowat/hiwat hysteresis.
29020d3034fSMatthew Dillon  *
29120d3034fSMatthew Dillon  *	Clear swap_pager_full ( task killing ) indication when lowat is met.
2921c7c3c6aSMatthew Dillon  *
2931c7c3c6aSMatthew Dillon  *	No restrictions on call
2941c7c3c6aSMatthew Dillon  *	This routine may not block.
2951c7c3c6aSMatthew Dillon  *	This routine must be called at splvm()
2961c7c3c6aSMatthew Dillon  */
297a5edd34aSPoul-Henning Kamp static void
2982f249180SPoul-Henning Kamp swp_sizecheck(void)
2990d94caffSDavid Greenman {
30023955314SAlfred Perlstein 
3018f60c087SPoul-Henning Kamp 	if (swap_pager_avail < nswap_lowat) {
30220d3034fSMatthew Dillon 		if (swap_pager_almost_full == 0) {
3031af87c92SDavid Greenman 			printf("swap_pager: out of swap space\n");
30420d3034fSMatthew Dillon 			swap_pager_almost_full = 1;
3052b0d37a4SMatthew Dillon 		}
30620d3034fSMatthew Dillon 	} else {
30726f9a767SRodney W. Grimes 		swap_pager_full = 0;
3088f60c087SPoul-Henning Kamp 		if (swap_pager_avail > nswap_hiwat)
30920d3034fSMatthew Dillon 			swap_pager_almost_full = 0;
31026f9a767SRodney W. Grimes 	}
3111c7c3c6aSMatthew Dillon }
3121c7c3c6aSMatthew Dillon 
3131c7c3c6aSMatthew Dillon /*
314da5fd145SPeter Wemm  * SWP_PAGER_HASH() -	hash swap meta data
315da5fd145SPeter Wemm  *
316a5edd34aSPoul-Henning Kamp  *	This is an helper function which hashes the swapblk given
317da5fd145SPeter Wemm  *	the object and page index.  It returns a pointer to a pointer
318da5fd145SPeter Wemm  *	to the object, or a pointer to a NULL pointer if it could not
319da5fd145SPeter Wemm  *	find a swapblk.
320da5fd145SPeter Wemm  *
321da5fd145SPeter Wemm  *	This routine must be called at splvm().
322da5fd145SPeter Wemm  */
323a5edd34aSPoul-Henning Kamp static struct swblock **
324da5fd145SPeter Wemm swp_pager_hash(vm_object_t object, vm_pindex_t index)
325da5fd145SPeter Wemm {
326da5fd145SPeter Wemm 	struct swblock **pswap;
327da5fd145SPeter Wemm 	struct swblock *swap;
328da5fd145SPeter Wemm 
329da5fd145SPeter Wemm 	index &= ~(vm_pindex_t)SWAP_META_MASK;
330da5fd145SPeter Wemm 	pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask];
331da5fd145SPeter Wemm 	while ((swap = *pswap) != NULL) {
332da5fd145SPeter Wemm 		if (swap->swb_object == object &&
333da5fd145SPeter Wemm 		    swap->swb_index == index
334da5fd145SPeter Wemm 		) {
335da5fd145SPeter Wemm 			break;
336da5fd145SPeter Wemm 		}
337da5fd145SPeter Wemm 		pswap = &swap->swb_hnext;
338da5fd145SPeter Wemm 	}
339da5fd145SPeter Wemm 	return (pswap);
340da5fd145SPeter Wemm }
341da5fd145SPeter Wemm 
342da5fd145SPeter Wemm /*
3431c7c3c6aSMatthew Dillon  * SWAP_PAGER_INIT() -	initialize the swap pager!
3441c7c3c6aSMatthew Dillon  *
3451c7c3c6aSMatthew Dillon  *	Expected to be started from system init.  NOTE:  This code is run
3461c7c3c6aSMatthew Dillon  *	before much else so be careful what you depend on.  Most of the VM
3471c7c3c6aSMatthew Dillon  *	system has yet to be initialized at this point.
3481c7c3c6aSMatthew Dillon  */
349f5a12711SPoul-Henning Kamp static void
3502f249180SPoul-Henning Kamp swap_pager_init(void)
351df8bae1dSRodney W. Grimes {
3521c7c3c6aSMatthew Dillon 	/*
3531c7c3c6aSMatthew Dillon 	 * Initialize object lists
3541c7c3c6aSMatthew Dillon 	 */
3551c7c3c6aSMatthew Dillon 	int i;
3561c7c3c6aSMatthew Dillon 
3571c7c3c6aSMatthew Dillon 	for (i = 0; i < NOBJLISTS; ++i)
3581c7c3c6aSMatthew Dillon 		TAILQ_INIT(&swap_pager_object_list[i]);
3596008862bSJohn Baldwin 	mtx_init(&sw_alloc_mtx, "swap_pager list", NULL, MTX_DEF);
36020da9c2eSPoul-Henning Kamp 	mtx_init(&sw_dev_mtx, "swapdev", NULL, MTX_DEF);
361df8bae1dSRodney W. Grimes 
362df8bae1dSRodney W. Grimes 	/*
3631c7c3c6aSMatthew Dillon 	 * Device Stripe, in PAGE_SIZE'd blocks
364df8bae1dSRodney W. Grimes 	 */
3651c7c3c6aSMatthew Dillon 	dmmax = SWB_NPAGES * 2;
3661c7c3c6aSMatthew Dillon }
36726f9a767SRodney W. Grimes 
368df8bae1dSRodney W. Grimes /*
3691c7c3c6aSMatthew Dillon  * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process
3701c7c3c6aSMatthew Dillon  *
3711c7c3c6aSMatthew Dillon  *	Expected to be started from pageout process once, prior to entering
3721c7c3c6aSMatthew Dillon  *	its main loop.
373df8bae1dSRodney W. Grimes  */
37424a1cce3SDavid Greenman void
3752f249180SPoul-Henning Kamp swap_pager_swap_init(void)
376df8bae1dSRodney W. Grimes {
37721cd6e62SSeigo Tanimura 	int n, n2;
3780d94caffSDavid Greenman 
37926f9a767SRodney W. Grimes 	/*
3801c7c3c6aSMatthew Dillon 	 * Number of in-transit swap bp operations.  Don't
3811c7c3c6aSMatthew Dillon 	 * exhaust the pbufs completely.  Make sure we
3821c7c3c6aSMatthew Dillon 	 * initialize workable values (0 will work for hysteresis
3831c7c3c6aSMatthew Dillon 	 * but it isn't very efficient).
3841c7c3c6aSMatthew Dillon 	 *
385327f4e83SMatthew Dillon 	 * The nsw_cluster_max is constrained by the bp->b_pages[]
3861c7c3c6aSMatthew Dillon 	 * array (MAXPHYS/PAGE_SIZE) and our locally defined
3871c7c3c6aSMatthew Dillon 	 * MAX_PAGEOUT_CLUSTER.   Also be aware that swap ops are
3881c7c3c6aSMatthew Dillon 	 * constrained by the swap device interleave stripe size.
389327f4e83SMatthew Dillon 	 *
390327f4e83SMatthew Dillon 	 * Currently we hardwire nsw_wcount_async to 4.  This limit is
391327f4e83SMatthew Dillon 	 * designed to prevent other I/O from having high latencies due to
392327f4e83SMatthew Dillon 	 * our pageout I/O.  The value 4 works well for one or two active swap
393327f4e83SMatthew Dillon 	 * devices but is probably a little low if you have more.  Even so,
394327f4e83SMatthew Dillon 	 * a higher value would probably generate only a limited improvement
395327f4e83SMatthew Dillon 	 * with three or four active swap devices since the system does not
396327f4e83SMatthew Dillon 	 * typically have to pageout at extreme bandwidths.   We will want
397327f4e83SMatthew Dillon 	 * at least 2 per swap devices, and 4 is a pretty good value if you
398327f4e83SMatthew Dillon 	 * have one NFS swap device due to the command/ack latency over NFS.
399327f4e83SMatthew Dillon 	 * So it all works out pretty well.
40026f9a767SRodney W. Grimes 	 */
401ad3cce20SMatthew Dillon 	nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER);
402327f4e83SMatthew Dillon 
4036d541bf1SJohn Baldwin 	mtx_lock(&pbuf_mtx);
4041c7c3c6aSMatthew Dillon 	nsw_rcount = (nswbuf + 1) / 2;
405327f4e83SMatthew Dillon 	nsw_wcount_sync = (nswbuf + 3) / 4;
406327f4e83SMatthew Dillon 	nsw_wcount_async = 4;
407327f4e83SMatthew Dillon 	nsw_wcount_async_max = nsw_wcount_async;
4086d541bf1SJohn Baldwin 	mtx_unlock(&pbuf_mtx);
40924a1cce3SDavid Greenman 
4101c7c3c6aSMatthew Dillon 	/*
4111c7c3c6aSMatthew Dillon 	 * Initialize our zone.  Right now I'm just guessing on the number
4121c7c3c6aSMatthew Dillon 	 * we need based on the number of pages in the system.  Each swblock
4132f9e4e80SMatthew Dillon 	 * can hold 16 pages, so this is probably overkill.  This reservation
414ec61f55dSMatthew Dillon 	 * is typically limited to around 32MB by default.
4151c7c3c6aSMatthew Dillon 	 */
416ec61f55dSMatthew Dillon 	n = cnt.v_page_count / 2;
4172f9e4e80SMatthew Dillon 	if (maxswzone && n > maxswzone / sizeof(struct swblock))
4182f9e4e80SMatthew Dillon 		n = maxswzone / sizeof(struct swblock);
41921cd6e62SSeigo Tanimura 	n2 = n;
420670d17b5SJeff Roberson 	swap_zone = uma_zcreate("SWAPMETA", sizeof(struct swblock), NULL, NULL,
4217827d9b0SAlan Cox 	    NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM);
4228355f576SJeff Roberson 	do {
4235285558aSAlan Cox 		if (uma_zone_set_obj(swap_zone, &swap_zone_obj, n))
42461ce6eeeSAlfred Perlstein 			break;
42561ce6eeeSAlfred Perlstein 		/*
42661ce6eeeSAlfred Perlstein 		 * if the allocation failed, try a zone two thirds the
42761ce6eeeSAlfred Perlstein 		 * size of the previous attempt.
42861ce6eeeSAlfred Perlstein 		 */
42961ce6eeeSAlfred Perlstein 		n -= ((n + 2) / 3);
43061ce6eeeSAlfred Perlstein 	} while (n > 0);
43121cd6e62SSeigo Tanimura 	if (swap_zone == NULL)
432670d17b5SJeff Roberson 		panic("failed to create swap_zone.");
43321cd6e62SSeigo Tanimura 	if (n2 != n)
43461ce6eeeSAlfred Perlstein 		printf("Swap zone entries reduced from %d to %d.\n", n2, n);
43521cd6e62SSeigo Tanimura 	n2 = n;
43624a1cce3SDavid Greenman 
4371c7c3c6aSMatthew Dillon 	/*
4381c7c3c6aSMatthew Dillon 	 * Initialize our meta-data hash table.  The swapper does not need to
4391c7c3c6aSMatthew Dillon 	 * be quite as efficient as the VM system, so we do not use an
4401c7c3c6aSMatthew Dillon 	 * oversized hash table.
4411c7c3c6aSMatthew Dillon 	 *
4421c7c3c6aSMatthew Dillon 	 * 	n: 		size of hash table, must be power of 2
4431c7c3c6aSMatthew Dillon 	 *	swhash_mask:	hash table index mask
4441c7c3c6aSMatthew Dillon 	 */
44561ce6eeeSAlfred Perlstein 	for (n = 1; n < n2 / 8; n *= 2)
4461c7c3c6aSMatthew Dillon 		;
447a163d034SWarner Losh 	swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO);
4481c7c3c6aSMatthew Dillon 	swhash_mask = n - 1;
4497827d9b0SAlan Cox 	mtx_init(&swhash_mtx, "swap_pager swhash", NULL, MTX_DEF);
45024a1cce3SDavid Greenman }
45124a1cce3SDavid Greenman 
45224a1cce3SDavid Greenman /*
4531c7c3c6aSMatthew Dillon  * SWAP_PAGER_ALLOC() -	allocate a new OBJT_SWAP VM object and instantiate
4541c7c3c6aSMatthew Dillon  *			its metadata structures.
4551c7c3c6aSMatthew Dillon  *
4561c7c3c6aSMatthew Dillon  *	This routine is called from the mmap and fork code to create a new
4571c7c3c6aSMatthew Dillon  *	OBJT_SWAP object.  We do this by creating an OBJT_DEFAULT object
4581c7c3c6aSMatthew Dillon  *	and then converting it with swp_pager_meta_build().
4591c7c3c6aSMatthew Dillon  *
4601c7c3c6aSMatthew Dillon  *	This routine may block in vm_object_allocate() and create a named
4611c7c3c6aSMatthew Dillon  *	object lookup race, so we must interlock.   We must also run at
4621c7c3c6aSMatthew Dillon  *	splvm() for the object lookup to handle races with interrupts, but
4631c7c3c6aSMatthew Dillon  *	we do not have to maintain splvm() in between the lookup and the
4641c7c3c6aSMatthew Dillon  *	add because (I believe) it is not possible to attempt to create
4651c7c3c6aSMatthew Dillon  *	a new swap object w/handle when a default object with that handle
4661c7c3c6aSMatthew Dillon  *	already exists.
46724c46d03SAlan Cox  *
46824c46d03SAlan Cox  * MPSAFE
46924a1cce3SDavid Greenman  */
470f5a12711SPoul-Henning Kamp static vm_object_t
4716cde7a16SDavid Greenman swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
472b9dcd593SBruce Evans 		 vm_ooffset_t offset)
47324a1cce3SDavid Greenman {
47424a1cce3SDavid Greenman 	vm_object_t object;
4752f7af3dbSAlan Cox 	vm_pindex_t pindex;
4762f7af3dbSAlan Cox 
4772f7af3dbSAlan Cox 	pindex = OFF_TO_IDX(offset + PAGE_MASK + size);
47824a1cce3SDavid Greenman 
47924a1cce3SDavid Greenman 	if (handle) {
480e793b779SAlan Cox 		mtx_lock(&Giant);
4811c7c3c6aSMatthew Dillon 		/*
4821c7c3c6aSMatthew Dillon 		 * Reference existing named region or allocate new one.  There
4831c7c3c6aSMatthew Dillon 		 * should not be a race here against swp_pager_meta_build()
4841c7c3c6aSMatthew Dillon 		 * as called from vm_page_remove() in regards to the lookup
4851c7c3c6aSMatthew Dillon 		 * of the handle.
4861c7c3c6aSMatthew Dillon 		 */
4870cddd8f0SMatthew Dillon 		sx_xlock(&sw_alloc_sx);
4881c7c3c6aSMatthew Dillon 		object = vm_pager_object_lookup(NOBJLIST(handle), handle);
4891c7c3c6aSMatthew Dillon 
49024a1cce3SDavid Greenman 		if (object != NULL) {
49124a1cce3SDavid Greenman 			vm_object_reference(object);
49224a1cce3SDavid Greenman 		} else {
4932f7af3dbSAlan Cox 			object = vm_object_allocate(OBJT_DEFAULT, pindex);
49424a1cce3SDavid Greenman 			object->handle = handle;
4951c7c3c6aSMatthew Dillon 
496ee3dc7d7SAlan Cox 			VM_OBJECT_LOCK(object);
4974dcc5c2dSMatthew Dillon 			swp_pager_meta_build(object, 0, SWAPBLK_NONE);
498ee3dc7d7SAlan Cox 			VM_OBJECT_UNLOCK(object);
49924a1cce3SDavid Greenman 		}
5000cddd8f0SMatthew Dillon 		sx_xunlock(&sw_alloc_sx);
501e793b779SAlan Cox 		mtx_unlock(&Giant);
50224a1cce3SDavid Greenman 	} else {
5032f7af3dbSAlan Cox 		object = vm_object_allocate(OBJT_DEFAULT, pindex);
5041c7c3c6aSMatthew Dillon 
505ee3dc7d7SAlan Cox 		VM_OBJECT_LOCK(object);
5064dcc5c2dSMatthew Dillon 		swp_pager_meta_build(object, 0, SWAPBLK_NONE);
507ee3dc7d7SAlan Cox 		VM_OBJECT_UNLOCK(object);
50824a1cce3SDavid Greenman 	}
50924a1cce3SDavid Greenman 	return (object);
510df8bae1dSRodney W. Grimes }
511df8bae1dSRodney W. Grimes 
51226f9a767SRodney W. Grimes /*
5131c7c3c6aSMatthew Dillon  * SWAP_PAGER_DEALLOC() -	remove swap metadata from object
5141c7c3c6aSMatthew Dillon  *
5151c7c3c6aSMatthew Dillon  *	The swap backing for the object is destroyed.  The code is
5161c7c3c6aSMatthew Dillon  *	designed such that we can reinstantiate it later, but this
5171c7c3c6aSMatthew Dillon  *	routine is typically called only when the entire object is
5181c7c3c6aSMatthew Dillon  *	about to be destroyed.
5191c7c3c6aSMatthew Dillon  *
5201c7c3c6aSMatthew Dillon  *	This routine may block, but no longer does.
5211c7c3c6aSMatthew Dillon  *
5221c7c3c6aSMatthew Dillon  *	The object must be locked or unreferenceable.
52326f9a767SRodney W. Grimes  */
524df8bae1dSRodney W. Grimes static void
5252f249180SPoul-Henning Kamp swap_pager_dealloc(vm_object_t object)
52626f9a767SRodney W. Grimes {
5274dcc5c2dSMatthew Dillon 	int s;
5284dcc5c2dSMatthew Dillon 
52926f9a767SRodney W. Grimes 	/*
5301c7c3c6aSMatthew Dillon 	 * Remove from list right away so lookups will fail if we block for
5311c7c3c6aSMatthew Dillon 	 * pageout completion.
53226f9a767SRodney W. Grimes 	 */
533bd228075SAlan Cox 	if (object->handle != NULL) {
534a9fa2c05SAlfred Perlstein 		mtx_lock(&sw_alloc_mtx);
5351c7c3c6aSMatthew Dillon 		TAILQ_REMOVE(NOBJLIST(object->handle), object, pager_object_list);
536a9fa2c05SAlfred Perlstein 		mtx_unlock(&sw_alloc_mtx);
537bd228075SAlan Cox 	}
5381c7c3c6aSMatthew Dillon 
539658ad5ffSAlan Cox 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
5401c7c3c6aSMatthew Dillon 	vm_object_pip_wait(object, "swpdea");
5411c7c3c6aSMatthew Dillon 
5421c7c3c6aSMatthew Dillon 	/*
5431c7c3c6aSMatthew Dillon 	 * Free all remaining metadata.  We only bother to free it from
5441c7c3c6aSMatthew Dillon 	 * the swap meta data.  We do not attempt to free swapblk's still
5451c7c3c6aSMatthew Dillon 	 * associated with vm_page_t's for this object.  We do not care
5461c7c3c6aSMatthew Dillon 	 * if paging is still in progress on some objects.
5471c7c3c6aSMatthew Dillon 	 */
5484dcc5c2dSMatthew Dillon 	s = splvm();
5491c7c3c6aSMatthew Dillon 	swp_pager_meta_free_all(object);
5504dcc5c2dSMatthew Dillon 	splx(s);
5511c7c3c6aSMatthew Dillon }
5521c7c3c6aSMatthew Dillon 
5531c7c3c6aSMatthew Dillon /************************************************************************
5541c7c3c6aSMatthew Dillon  *			SWAP PAGER BITMAP ROUTINES			*
5551c7c3c6aSMatthew Dillon  ************************************************************************/
5561c7c3c6aSMatthew Dillon 
5571c7c3c6aSMatthew Dillon /*
5581c7c3c6aSMatthew Dillon  * SWP_PAGER_GETSWAPSPACE() -	allocate raw swap space
5591c7c3c6aSMatthew Dillon  *
5601c7c3c6aSMatthew Dillon  *	Allocate swap for the requested number of pages.  The starting
5611c7c3c6aSMatthew Dillon  *	swap block number (a page index) is returned or SWAPBLK_NONE
5621c7c3c6aSMatthew Dillon  *	if the allocation failed.
5631c7c3c6aSMatthew Dillon  *
5641c7c3c6aSMatthew Dillon  *	Also has the side effect of advising that somebody made a mistake
5651c7c3c6aSMatthew Dillon  *	when they configured swap and didn't configure enough.
5661c7c3c6aSMatthew Dillon  *
5671c7c3c6aSMatthew Dillon  *	Must be called at splvm() to avoid races with bitmap frees from
5681c7c3c6aSMatthew Dillon  *	vm_page_remove() aka swap_pager_page_removed().
5691c7c3c6aSMatthew Dillon  *
5701c7c3c6aSMatthew Dillon  *	This routine may not block
5711c7c3c6aSMatthew Dillon  *	This routine must be called at splvm().
5728f60c087SPoul-Henning Kamp  *
5738f60c087SPoul-Henning Kamp  *	We allocate in round-robin fashion from the configured devices.
5741c7c3c6aSMatthew Dillon  */
575a5edd34aSPoul-Henning Kamp static daddr_t
5762f249180SPoul-Henning Kamp swp_pager_getswapspace(int npages)
5771c7c3c6aSMatthew Dillon {
5781c7c3c6aSMatthew Dillon 	daddr_t blk;
5798f60c087SPoul-Henning Kamp 	struct swdevt *sp;
5808f60c087SPoul-Henning Kamp 	int i;
5811c7c3c6aSMatthew Dillon 
5828f60c087SPoul-Henning Kamp 	blk = SWAPBLK_NONE;
58320da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
5848f60c087SPoul-Henning Kamp 	sp = swdevhd;
5858f60c087SPoul-Henning Kamp 	for (i = 0; i < nswapdev; i++) {
5868f60c087SPoul-Henning Kamp 		if (sp == NULL)
5878f60c087SPoul-Henning Kamp 			sp = TAILQ_FIRST(&swtailq);
5888f60c087SPoul-Henning Kamp 		if (!(sp->sw_flags & SW_CLOSING)) {
5898f60c087SPoul-Henning Kamp 			blk = blist_alloc(sp->sw_blist, npages);
5908f60c087SPoul-Henning Kamp 			if (blk != SWAPBLK_NONE) {
5918f60c087SPoul-Henning Kamp 				blk += sp->sw_first;
5928f60c087SPoul-Henning Kamp 				sp->sw_used += npages;
593d05bc129SAlan Cox 				swap_pager_avail -= npages;
5948f60c087SPoul-Henning Kamp 				swp_sizecheck();
5958f60c087SPoul-Henning Kamp 				swdevhd = TAILQ_NEXT(sp, sw_list);
596d05bc129SAlan Cox 				goto done;
5978f60c087SPoul-Henning Kamp 			}
5988f60c087SPoul-Henning Kamp 		}
5998f60c087SPoul-Henning Kamp 		sp = TAILQ_NEXT(sp, sw_list);
6008f60c087SPoul-Henning Kamp 	}
6012b0d37a4SMatthew Dillon 	if (swap_pager_full != 2) {
60220da9c2eSPoul-Henning Kamp 		printf("swap_pager_getswapspace(%d): failed\n", npages);
6032b0d37a4SMatthew Dillon 		swap_pager_full = 2;
60420d3034fSMatthew Dillon 		swap_pager_almost_full = 1;
6052b0d37a4SMatthew Dillon 	}
6068f60c087SPoul-Henning Kamp 	swdevhd = NULL;
607d05bc129SAlan Cox done:
608d05bc129SAlan Cox 	mtx_unlock(&sw_dev_mtx);
6091c7c3c6aSMatthew Dillon 	return (blk);
61026f9a767SRodney W. Grimes }
61126f9a767SRodney W. Grimes 
6128f60c087SPoul-Henning Kamp static struct swdevt *
6134b03903aSPoul-Henning Kamp swp_pager_find_dev(daddr_t blk)
6148f60c087SPoul-Henning Kamp {
6158f60c087SPoul-Henning Kamp 	struct swdevt *sp;
6168f60c087SPoul-Henning Kamp 
61720da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
6188f60c087SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
61920da9c2eSPoul-Henning Kamp 		if (blk >= sp->sw_first && blk < sp->sw_end) {
62020da9c2eSPoul-Henning Kamp 			mtx_unlock(&sw_dev_mtx);
6218f60c087SPoul-Henning Kamp 			return (sp);
6228f60c087SPoul-Henning Kamp 		}
62320da9c2eSPoul-Henning Kamp 	}
62420da9c2eSPoul-Henning Kamp 	panic("Swapdev not found");
6258f60c087SPoul-Henning Kamp }
6268f60c087SPoul-Henning Kamp 
6274b03903aSPoul-Henning Kamp static void
6284b03903aSPoul-Henning Kamp swp_pager_strategy(struct buf *bp)
6294b03903aSPoul-Henning Kamp {
6304b03903aSPoul-Henning Kamp 	struct swdevt *sp;
6314b03903aSPoul-Henning Kamp 
63220da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
6334b03903aSPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
6344b03903aSPoul-Henning Kamp 		if (bp->b_blkno >= sp->sw_first && bp->b_blkno < sp->sw_end) {
63520da9c2eSPoul-Henning Kamp 			mtx_unlock(&sw_dev_mtx);
6364b03903aSPoul-Henning Kamp 			sp->sw_strategy(bp, sp);
6374b03903aSPoul-Henning Kamp 			return;
6384b03903aSPoul-Henning Kamp 		}
6394b03903aSPoul-Henning Kamp 	}
64020da9c2eSPoul-Henning Kamp 	panic("Swapdev not found");
6414b03903aSPoul-Henning Kamp }
6424b03903aSPoul-Henning Kamp 
6438f60c087SPoul-Henning Kamp 
64426f9a767SRodney W. Grimes /*
6451c7c3c6aSMatthew Dillon  * SWP_PAGER_FREESWAPSPACE() -	free raw swap space
6461c7c3c6aSMatthew Dillon  *
6471c7c3c6aSMatthew Dillon  *	This routine returns the specified swap blocks back to the bitmap.
6481c7c3c6aSMatthew Dillon  *
6491c7c3c6aSMatthew Dillon  *	Note:  This routine may not block (it could in the old swap code),
6501c7c3c6aSMatthew Dillon  *	and through the use of the new blist routines it does not block.
6511c7c3c6aSMatthew Dillon  *
6521c7c3c6aSMatthew Dillon  *	We must be called at splvm() to avoid races with bitmap frees from
6531c7c3c6aSMatthew Dillon  *	vm_page_remove() aka swap_pager_page_removed().
6541c7c3c6aSMatthew Dillon  *
6551c7c3c6aSMatthew Dillon  *	This routine may not block
6561c7c3c6aSMatthew Dillon  *	This routine must be called at splvm().
65726f9a767SRodney W. Grimes  */
658a5edd34aSPoul-Henning Kamp static void
6598f60c087SPoul-Henning Kamp swp_pager_freeswapspace(daddr_t blk, int npages)
6600d94caffSDavid Greenman {
6618f60c087SPoul-Henning Kamp 	struct swdevt *sp;
66292da00bbSMatthew Dillon 
6637645e885SAlan Cox 	mtx_lock(&sw_dev_mtx);
6647645e885SAlan Cox 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
6657645e885SAlan Cox 		if (blk >= sp->sw_first && blk < sp->sw_end) {
66692da00bbSMatthew Dillon 			sp->sw_used -= npages;
66792da00bbSMatthew Dillon 			/*
6687645e885SAlan Cox 			 * If we are attempting to stop swapping on
6697645e885SAlan Cox 			 * this device, we don't want to mark any
6707645e885SAlan Cox 			 * blocks free lest they be reused.
67192da00bbSMatthew Dillon 			 */
6727645e885SAlan Cox 			if ((sp->sw_flags & SW_CLOSING) == 0) {
6737645e885SAlan Cox 				blist_free(sp->sw_blist, blk - sp->sw_first,
6747645e885SAlan Cox 				    npages);
6758f60c087SPoul-Henning Kamp 				swap_pager_avail += npages;
6761c7c3c6aSMatthew Dillon 				swp_sizecheck();
67726f9a767SRodney W. Grimes 			}
6787645e885SAlan Cox 			mtx_unlock(&sw_dev_mtx);
6797645e885SAlan Cox 			return;
6807645e885SAlan Cox 		}
6817645e885SAlan Cox 	}
6827645e885SAlan Cox 	panic("Swapdev not found");
6837645e885SAlan Cox }
6841c7c3c6aSMatthew Dillon 
68526f9a767SRodney W. Grimes /*
6861c7c3c6aSMatthew Dillon  * SWAP_PAGER_FREESPACE() -	frees swap blocks associated with a page
6871c7c3c6aSMatthew Dillon  *				range within an object.
6881c7c3c6aSMatthew Dillon  *
6891c7c3c6aSMatthew Dillon  *	This is a globally accessible routine.
6901c7c3c6aSMatthew Dillon  *
6911c7c3c6aSMatthew Dillon  *	This routine removes swapblk assignments from swap metadata.
6921c7c3c6aSMatthew Dillon  *
6931c7c3c6aSMatthew Dillon  *	The external callers of this routine typically have already destroyed
6941c7c3c6aSMatthew Dillon  *	or renamed vm_page_t's associated with this range in the object so
6951c7c3c6aSMatthew Dillon  *	we should be ok.
6964dcc5c2dSMatthew Dillon  *
6974dcc5c2dSMatthew Dillon  *	This routine may be called at any spl.  We up our spl to splvm temporarily
6984dcc5c2dSMatthew Dillon  *	in order to perform the metadata removal.
69926f9a767SRodney W. Grimes  */
70026f9a767SRodney W. Grimes void
7012f249180SPoul-Henning Kamp swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size)
70226f9a767SRodney W. Grimes {
7034dcc5c2dSMatthew Dillon 	int s = splvm();
70423955314SAlfred Perlstein 
70519ba4c8eSAlan Cox 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
7061c7c3c6aSMatthew Dillon 	swp_pager_meta_free(object, start, size);
7074dcc5c2dSMatthew Dillon 	splx(s);
7084dcc5c2dSMatthew Dillon }
7094dcc5c2dSMatthew Dillon 
7104dcc5c2dSMatthew Dillon /*
7114dcc5c2dSMatthew Dillon  * SWAP_PAGER_RESERVE() - reserve swap blocks in object
7124dcc5c2dSMatthew Dillon  *
7134dcc5c2dSMatthew Dillon  *	Assigns swap blocks to the specified range within the object.  The
7144dcc5c2dSMatthew Dillon  *	swap blocks are not zerod.  Any previous swap assignment is destroyed.
7154dcc5c2dSMatthew Dillon  *
7164dcc5c2dSMatthew Dillon  *	Returns 0 on success, -1 on failure.
7174dcc5c2dSMatthew Dillon  */
7184dcc5c2dSMatthew Dillon int
7194dcc5c2dSMatthew Dillon swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
7204dcc5c2dSMatthew Dillon {
7214dcc5c2dSMatthew Dillon 	int s;
7224dcc5c2dSMatthew Dillon 	int n = 0;
7234dcc5c2dSMatthew Dillon 	daddr_t blk = SWAPBLK_NONE;
7244dcc5c2dSMatthew Dillon 	vm_pindex_t beg = start;	/* save start index */
7254dcc5c2dSMatthew Dillon 
7264dcc5c2dSMatthew Dillon 	s = splvm();
727ee3dc7d7SAlan Cox 	VM_OBJECT_LOCK(object);
7284dcc5c2dSMatthew Dillon 	while (size) {
7294dcc5c2dSMatthew Dillon 		if (n == 0) {
7304dcc5c2dSMatthew Dillon 			n = BLIST_MAX_ALLOC;
7314dcc5c2dSMatthew Dillon 			while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
7324dcc5c2dSMatthew Dillon 				n >>= 1;
7334dcc5c2dSMatthew Dillon 				if (n == 0) {
7344dcc5c2dSMatthew Dillon 					swp_pager_meta_free(object, beg, start - beg);
735ee3dc7d7SAlan Cox 					VM_OBJECT_UNLOCK(object);
7364dcc5c2dSMatthew Dillon 					splx(s);
7374dcc5c2dSMatthew Dillon 					return (-1);
7384dcc5c2dSMatthew Dillon 				}
7394dcc5c2dSMatthew Dillon 			}
7404dcc5c2dSMatthew Dillon 		}
7414dcc5c2dSMatthew Dillon 		swp_pager_meta_build(object, start, blk);
7424dcc5c2dSMatthew Dillon 		--size;
7434dcc5c2dSMatthew Dillon 		++start;
7444dcc5c2dSMatthew Dillon 		++blk;
7454dcc5c2dSMatthew Dillon 		--n;
7464dcc5c2dSMatthew Dillon 	}
7474dcc5c2dSMatthew Dillon 	swp_pager_meta_free(object, start, n);
748ee3dc7d7SAlan Cox 	VM_OBJECT_UNLOCK(object);
7494dcc5c2dSMatthew Dillon 	splx(s);
7504dcc5c2dSMatthew Dillon 	return (0);
75126f9a767SRodney W. Grimes }
75226f9a767SRodney W. Grimes 
7530a47b48bSJohn Dyson /*
7541c7c3c6aSMatthew Dillon  * SWAP_PAGER_COPY() -  copy blocks from source pager to destination pager
7551c7c3c6aSMatthew Dillon  *			and destroy the source.
7561c7c3c6aSMatthew Dillon  *
7571c7c3c6aSMatthew Dillon  *	Copy any valid swapblks from the source to the destination.  In
7581c7c3c6aSMatthew Dillon  *	cases where both the source and destination have a valid swapblk,
7591c7c3c6aSMatthew Dillon  *	we keep the destination's.
7601c7c3c6aSMatthew Dillon  *
7611c7c3c6aSMatthew Dillon  *	This routine is allowed to block.  It may block allocating metadata
7621c7c3c6aSMatthew Dillon  *	indirectly through swp_pager_meta_build() or if paging is still in
7631c7c3c6aSMatthew Dillon  *	progress on the source.
7641c7c3c6aSMatthew Dillon  *
7654dcc5c2dSMatthew Dillon  *	This routine can be called at any spl
7664dcc5c2dSMatthew Dillon  *
7671c7c3c6aSMatthew Dillon  *	XXX vm_page_collapse() kinda expects us not to block because we
7681c7c3c6aSMatthew Dillon  *	supposedly do not need to allocate memory, but for the moment we
7691c7c3c6aSMatthew Dillon  *	*may* have to get a little memory from the zone allocator, but
7701c7c3c6aSMatthew Dillon  *	it is taken from the interrupt memory.  We should be ok.
7711c7c3c6aSMatthew Dillon  *
7721c7c3c6aSMatthew Dillon  *	The source object contains no vm_page_t's (which is just as well)
7731c7c3c6aSMatthew Dillon  *
7741c7c3c6aSMatthew Dillon  *	The source object is of type OBJT_SWAP.
7751c7c3c6aSMatthew Dillon  *
7764dcc5c2dSMatthew Dillon  *	The source and destination objects must be locked or
7774dcc5c2dSMatthew Dillon  *	inaccessible (XXX are they ?)
77826f9a767SRodney W. Grimes  */
77926f9a767SRodney W. Grimes void
7802f249180SPoul-Henning Kamp swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
7812f249180SPoul-Henning Kamp     vm_pindex_t offset, int destroysource)
78226f9a767SRodney W. Grimes {
783a316d390SJohn Dyson 	vm_pindex_t i;
7844dcc5c2dSMatthew Dillon 	int s;
7854dcc5c2dSMatthew Dillon 
786c7c8dd7eSAlan Cox 	VM_OBJECT_LOCK_ASSERT(srcobject, MA_OWNED);
787c7c8dd7eSAlan Cox 	VM_OBJECT_LOCK_ASSERT(dstobject, MA_OWNED);
7880cddd8f0SMatthew Dillon 
7894dcc5c2dSMatthew Dillon 	s = splvm();
79026f9a767SRodney W. Grimes 	/*
7911c7c3c6aSMatthew Dillon 	 * If destroysource is set, we remove the source object from the
7921c7c3c6aSMatthew Dillon 	 * swap_pager internal queue now.
79326f9a767SRodney W. Grimes 	 */
794cbd8ec09SJohn Dyson 	if (destroysource) {
795bd228075SAlan Cox 		if (srcobject->handle != NULL) {
796a9fa2c05SAlfred Perlstein 			mtx_lock(&sw_alloc_mtx);
7971c7c3c6aSMatthew Dillon 			TAILQ_REMOVE(
7981c7c3c6aSMatthew Dillon 			    NOBJLIST(srcobject->handle),
7991c7c3c6aSMatthew Dillon 			    srcobject,
8001c7c3c6aSMatthew Dillon 			    pager_object_list
8011c7c3c6aSMatthew Dillon 			);
802a9fa2c05SAlfred Perlstein 			mtx_unlock(&sw_alloc_mtx);
803cbd8ec09SJohn Dyson 		}
804bd228075SAlan Cox 	}
80526f9a767SRodney W. Grimes 
8061c7c3c6aSMatthew Dillon 	/*
8071c7c3c6aSMatthew Dillon 	 * transfer source to destination.
8081c7c3c6aSMatthew Dillon 	 */
8091c7c3c6aSMatthew Dillon 	for (i = 0; i < dstobject->size; ++i) {
8101c7c3c6aSMatthew Dillon 		daddr_t dstaddr;
8111c7c3c6aSMatthew Dillon 
8121c7c3c6aSMatthew Dillon 		/*
8131c7c3c6aSMatthew Dillon 		 * Locate (without changing) the swapblk on the destination,
8141c7c3c6aSMatthew Dillon 		 * unless it is invalid in which case free it silently, or
8151c7c3c6aSMatthew Dillon 		 * if the destination is a resident page, in which case the
8161c7c3c6aSMatthew Dillon 		 * source is thrown away.
8171c7c3c6aSMatthew Dillon 		 */
8181c7c3c6aSMatthew Dillon 		dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
8191c7c3c6aSMatthew Dillon 
8201c7c3c6aSMatthew Dillon 		if (dstaddr == SWAPBLK_NONE) {
8211c7c3c6aSMatthew Dillon 			/*
8221c7c3c6aSMatthew Dillon 			 * Destination has no swapblk and is not resident,
8231c7c3c6aSMatthew Dillon 			 * copy source.
8241c7c3c6aSMatthew Dillon 			 */
8251c7c3c6aSMatthew Dillon 			daddr_t srcaddr;
8261c7c3c6aSMatthew Dillon 
8271c7c3c6aSMatthew Dillon 			srcaddr = swp_pager_meta_ctl(
8281c7c3c6aSMatthew Dillon 			    srcobject,
8291c7c3c6aSMatthew Dillon 			    i + offset,
8301c7c3c6aSMatthew Dillon 			    SWM_POP
8311c7c3c6aSMatthew Dillon 			);
8321c7c3c6aSMatthew Dillon 
833ee3dc7d7SAlan Cox 			if (srcaddr != SWAPBLK_NONE) {
834c7c8dd7eSAlan Cox 				/*
835c7c8dd7eSAlan Cox 				 * swp_pager_meta_build() can sleep.
836c7c8dd7eSAlan Cox 				 */
837c7c8dd7eSAlan Cox 				vm_object_pip_add(srcobject, 1);
838c7c8dd7eSAlan Cox 				VM_OBJECT_UNLOCK(srcobject);
839c7c8dd7eSAlan Cox 				vm_object_pip_add(dstobject, 1);
8404dcc5c2dSMatthew Dillon 				swp_pager_meta_build(dstobject, i, srcaddr);
841c7c8dd7eSAlan Cox 				vm_object_pip_wakeup(dstobject);
842c7c8dd7eSAlan Cox 				VM_OBJECT_LOCK(srcobject);
843c7c8dd7eSAlan Cox 				vm_object_pip_wakeup(srcobject);
844ee3dc7d7SAlan Cox 			}
8451c7c3c6aSMatthew Dillon 		} else {
8461c7c3c6aSMatthew Dillon 			/*
8471c7c3c6aSMatthew Dillon 			 * Destination has valid swapblk or it is represented
8481c7c3c6aSMatthew Dillon 			 * by a resident page.  We destroy the sourceblock.
8491c7c3c6aSMatthew Dillon 			 */
8501c7c3c6aSMatthew Dillon 
8511c7c3c6aSMatthew Dillon 			swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE);
8521c7c3c6aSMatthew Dillon 		}
85326f9a767SRodney W. Grimes 	}
85426f9a767SRodney W. Grimes 
85526f9a767SRodney W. Grimes 	/*
8561c7c3c6aSMatthew Dillon 	 * Free left over swap blocks in source.
8571c7c3c6aSMatthew Dillon 	 *
8581c7c3c6aSMatthew Dillon 	 * We have to revert the type to OBJT_DEFAULT so we do not accidently
8591c7c3c6aSMatthew Dillon 	 * double-remove the object from the swap queues.
86026f9a767SRodney W. Grimes 	 */
861c0877f10SJohn Dyson 	if (destroysource) {
8621c7c3c6aSMatthew Dillon 		swp_pager_meta_free_all(srcobject);
8631c7c3c6aSMatthew Dillon 		/*
8641c7c3c6aSMatthew Dillon 		 * Reverting the type is not necessary, the caller is going
8651c7c3c6aSMatthew Dillon 		 * to destroy srcobject directly, but I'm doing it here
866956f3135SPhilippe Charnier 		 * for consistency since we've removed the object from its
8671c7c3c6aSMatthew Dillon 		 * queues.
8681c7c3c6aSMatthew Dillon 		 */
8691c7c3c6aSMatthew Dillon 		srcobject->type = OBJT_DEFAULT;
870c0877f10SJohn Dyson 	}
8714dcc5c2dSMatthew Dillon 	splx(s);
87226f9a767SRodney W. Grimes }
87326f9a767SRodney W. Grimes 
874df8bae1dSRodney W. Grimes /*
8751c7c3c6aSMatthew Dillon  * SWAP_PAGER_HASPAGE() -	determine if we have good backing store for
8761c7c3c6aSMatthew Dillon  *				the requested page.
8771c7c3c6aSMatthew Dillon  *
8781c7c3c6aSMatthew Dillon  *	We determine whether good backing store exists for the requested
8791c7c3c6aSMatthew Dillon  *	page and return TRUE if it does, FALSE if it doesn't.
8801c7c3c6aSMatthew Dillon  *
8811c7c3c6aSMatthew Dillon  *	If TRUE, we also try to determine how much valid, contiguous backing
8821c7c3c6aSMatthew Dillon  *	store exists before and after the requested page within a reasonable
8831c7c3c6aSMatthew Dillon  *	distance.  We do not try to restrict it to the swap device stripe
8841c7c3c6aSMatthew Dillon  *	(that is handled in getpages/putpages).  It probably isn't worth
8851c7c3c6aSMatthew Dillon  *	doing here.
886df8bae1dSRodney W. Grimes  */
8875ea4972cSAlan Cox static boolean_t
8882f249180SPoul-Henning Kamp swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after)
88926f9a767SRodney W. Grimes {
8901c7c3c6aSMatthew Dillon 	daddr_t blk0;
89125db2c54SMatthew Dillon 	int s;
89226f9a767SRodney W. Grimes 
893ee3dc7d7SAlan Cox 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
8941c7c3c6aSMatthew Dillon 	/*
8951c7c3c6aSMatthew Dillon 	 * do we have good backing store at the requested index ?
8961c7c3c6aSMatthew Dillon 	 */
89725db2c54SMatthew Dillon 	s = splvm();
8981c7c3c6aSMatthew Dillon 	blk0 = swp_pager_meta_ctl(object, pindex, 0);
8991c7c3c6aSMatthew Dillon 
9004dcc5c2dSMatthew Dillon 	if (blk0 == SWAPBLK_NONE) {
90125db2c54SMatthew Dillon 		splx(s);
9021c7c3c6aSMatthew Dillon 		if (before)
90324a1cce3SDavid Greenman 			*before = 0;
9041c7c3c6aSMatthew Dillon 		if (after)
90524a1cce3SDavid Greenman 			*after = 0;
90626f9a767SRodney W. Grimes 		return (FALSE);
90726f9a767SRodney W. Grimes 	}
90826f9a767SRodney W. Grimes 
90926f9a767SRodney W. Grimes 	/*
9101c7c3c6aSMatthew Dillon 	 * find backwards-looking contiguous good backing store
911e47ed70bSJohn Dyson 	 */
9121c7c3c6aSMatthew Dillon 	if (before != NULL) {
91326f9a767SRodney W. Grimes 		int i;
9140d94caffSDavid Greenman 
9151c7c3c6aSMatthew Dillon 		for (i = 1; i < (SWB_NPAGES/2); ++i) {
9161c7c3c6aSMatthew Dillon 			daddr_t blk;
9171c7c3c6aSMatthew Dillon 
9181c7c3c6aSMatthew Dillon 			if (i > pindex)
9191c7c3c6aSMatthew Dillon 				break;
9201c7c3c6aSMatthew Dillon 			blk = swp_pager_meta_ctl(object, pindex - i, 0);
9211c7c3c6aSMatthew Dillon 			if (blk != blk0 - i)
9221c7c3c6aSMatthew Dillon 				break;
923ffc82b0aSJohn Dyson 		}
9241c7c3c6aSMatthew Dillon 		*before = (i - 1);
92526f9a767SRodney W. Grimes 	}
92626f9a767SRodney W. Grimes 
92726f9a767SRodney W. Grimes 	/*
9281c7c3c6aSMatthew Dillon 	 * find forward-looking contiguous good backing store
92926f9a767SRodney W. Grimes 	 */
9301c7c3c6aSMatthew Dillon 	if (after != NULL) {
9311c7c3c6aSMatthew Dillon 		int i;
9321c7c3c6aSMatthew Dillon 
9331c7c3c6aSMatthew Dillon 		for (i = 1; i < (SWB_NPAGES/2); ++i) {
9341c7c3c6aSMatthew Dillon 			daddr_t blk;
9351c7c3c6aSMatthew Dillon 
9361c7c3c6aSMatthew Dillon 			blk = swp_pager_meta_ctl(object, pindex + i, 0);
9371c7c3c6aSMatthew Dillon 			if (blk != blk0 + i)
9381c7c3c6aSMatthew Dillon 				break;
93926f9a767SRodney W. Grimes 		}
9401c7c3c6aSMatthew Dillon 		*after = (i - 1);
9411c7c3c6aSMatthew Dillon 	}
94225db2c54SMatthew Dillon 	splx(s);
9431c7c3c6aSMatthew Dillon 	return (TRUE);
9441c7c3c6aSMatthew Dillon }
9451c7c3c6aSMatthew Dillon 
9461c7c3c6aSMatthew Dillon /*
9471c7c3c6aSMatthew Dillon  * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page
9481c7c3c6aSMatthew Dillon  *
9491c7c3c6aSMatthew Dillon  *	This removes any associated swap backing store, whether valid or
9501c7c3c6aSMatthew Dillon  *	not, from the page.
9511c7c3c6aSMatthew Dillon  *
9521c7c3c6aSMatthew Dillon  *	This routine is typically called when a page is made dirty, at
9531c7c3c6aSMatthew Dillon  *	which point any associated swap can be freed.  MADV_FREE also
9541c7c3c6aSMatthew Dillon  *	calls us in a special-case situation
9551c7c3c6aSMatthew Dillon  *
9561c7c3c6aSMatthew Dillon  *	NOTE!!!  If the page is clean and the swap was valid, the caller
9571c7c3c6aSMatthew Dillon  *	should make the page dirty before calling this routine.  This routine
9581c7c3c6aSMatthew Dillon  *	does NOT change the m->dirty status of the page.  Also: MADV_FREE
9591c7c3c6aSMatthew Dillon  *	depends on it.
9601c7c3c6aSMatthew Dillon  *
9611c7c3c6aSMatthew Dillon  *	This routine may not block
9624dcc5c2dSMatthew Dillon  *	This routine must be called at splvm()
9631c7c3c6aSMatthew Dillon  */
9641c7c3c6aSMatthew Dillon static void
9652f249180SPoul-Henning Kamp swap_pager_unswapped(vm_page_t m)
9661c7c3c6aSMatthew Dillon {
9672f249180SPoul-Henning Kamp 
968ee3dc7d7SAlan Cox 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
9691c7c3c6aSMatthew Dillon 	swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE);
9701c7c3c6aSMatthew Dillon }
9711c7c3c6aSMatthew Dillon 
9721c7c3c6aSMatthew Dillon /*
9731c7c3c6aSMatthew Dillon  * SWAP_PAGER_GETPAGES() - bring pages in from swap
9741c7c3c6aSMatthew Dillon  *
9751c7c3c6aSMatthew Dillon  *	Attempt to retrieve (m, count) pages from backing store, but make
9761c7c3c6aSMatthew Dillon  *	sure we retrieve at least m[reqpage].  We try to load in as large
9771c7c3c6aSMatthew Dillon  *	a chunk surrounding m[reqpage] as is contiguous in swap and which
9781c7c3c6aSMatthew Dillon  *	belongs to the same object.
9791c7c3c6aSMatthew Dillon  *
9801c7c3c6aSMatthew Dillon  *	The code is designed for asynchronous operation and
9811c7c3c6aSMatthew Dillon  *	immediate-notification of 'reqpage' but tends not to be
9821c7c3c6aSMatthew Dillon  *	used that way.  Please do not optimize-out this algorithmic
9831c7c3c6aSMatthew Dillon  *	feature, I intend to improve on it in the future.
9841c7c3c6aSMatthew Dillon  *
9851c7c3c6aSMatthew Dillon  *	The parent has a single vm_object_pip_add() reference prior to
9861c7c3c6aSMatthew Dillon  *	calling us and we should return with the same.
9871c7c3c6aSMatthew Dillon  *
9881c7c3c6aSMatthew Dillon  *	The parent has BUSY'd the pages.  We should return with 'm'
9891c7c3c6aSMatthew Dillon  *	left busy, but the others adjusted.
9901c7c3c6aSMatthew Dillon  */
991f708ef1bSPoul-Henning Kamp static int
9922f249180SPoul-Henning Kamp swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage)
993df8bae1dSRodney W. Grimes {
9941c7c3c6aSMatthew Dillon 	struct buf *bp;
9951c7c3c6aSMatthew Dillon 	vm_page_t mreq;
9961c7c3c6aSMatthew Dillon 	int s;
99726f9a767SRodney W. Grimes 	int i;
99826f9a767SRodney W. Grimes 	int j;
9991c7c3c6aSMatthew Dillon 	daddr_t blk;
10000d94caffSDavid Greenman 
10011c7c3c6aSMatthew Dillon 	mreq = m[reqpage];
10021c7c3c6aSMatthew Dillon 
1003e04e4bacSPoul-Henning Kamp 	KASSERT(mreq->object == object,
1004e04e4bacSPoul-Henning Kamp 	    ("swap_pager_getpages: object mismatch %p/%p",
1005e04e4bacSPoul-Henning Kamp 	    object, mreq->object));
1006e04e4bacSPoul-Henning Kamp 
10071c7c3c6aSMatthew Dillon 	/*
10081c7c3c6aSMatthew Dillon 	 * Calculate range to retrieve.  The pages have already been assigned
1009751221fdSPoul-Henning Kamp 	 * their swapblks.  We require a *contiguous* range but we know it to
1010751221fdSPoul-Henning Kamp 	 * not span devices.   If we do not supply it, bad things
10114dcc5c2dSMatthew Dillon 	 * happen.  Note that blk, iblk & jblk can be SWAPBLK_NONE, but the
10124dcc5c2dSMatthew Dillon 	 * loops are set up such that the case(s) are handled implicitly.
10134dcc5c2dSMatthew Dillon 	 *
10144dcc5c2dSMatthew Dillon 	 * The swp_*() calls must be made at splvm().  vm_page_free() does
10154dcc5c2dSMatthew Dillon 	 * not need to be, but it will go a little faster if it is.
10161c7c3c6aSMatthew Dillon 	 */
10174dcc5c2dSMatthew Dillon 	s = splvm();
10181c7c3c6aSMatthew Dillon 	blk = swp_pager_meta_ctl(mreq->object, mreq->pindex, 0);
10191c7c3c6aSMatthew Dillon 
10201c7c3c6aSMatthew Dillon 	for (i = reqpage - 1; i >= 0; --i) {
10211c7c3c6aSMatthew Dillon 		daddr_t iblk;
10221c7c3c6aSMatthew Dillon 
10231c7c3c6aSMatthew Dillon 		iblk = swp_pager_meta_ctl(m[i]->object, m[i]->pindex, 0);
10241c7c3c6aSMatthew Dillon 		if (blk != iblk + (reqpage - i))
102526f9a767SRodney W. Grimes 			break;
102626f9a767SRodney W. Grimes 	}
10271c7c3c6aSMatthew Dillon 	++i;
10281c7c3c6aSMatthew Dillon 
10291c7c3c6aSMatthew Dillon 	for (j = reqpage + 1; j < count; ++j) {
10301c7c3c6aSMatthew Dillon 		daddr_t jblk;
10311c7c3c6aSMatthew Dillon 
10321c7c3c6aSMatthew Dillon 		jblk = swp_pager_meta_ctl(m[j]->object, m[j]->pindex, 0);
10331c7c3c6aSMatthew Dillon 		if (blk != jblk - (j - reqpage))
10341c7c3c6aSMatthew Dillon 			break;
10351c7c3c6aSMatthew Dillon 	}
10361c7c3c6aSMatthew Dillon 
10371c7c3c6aSMatthew Dillon 	/*
10381c7c3c6aSMatthew Dillon 	 * free pages outside our collection range.   Note: we never free
10391c7c3c6aSMatthew Dillon 	 * mreq, it must remain busy throughout.
10401c7c3c6aSMatthew Dillon 	 */
1041ab9abe5dSAlan Cox 	vm_page_lock_queues();
10421c7c3c6aSMatthew Dillon 	{
10431c7c3c6aSMatthew Dillon 		int k;
10441c7c3c6aSMatthew Dillon 
10454dcc5c2dSMatthew Dillon 		for (k = 0; k < i; ++k)
10464dcc5c2dSMatthew Dillon 			vm_page_free(m[k]);
10474dcc5c2dSMatthew Dillon 		for (k = j; k < count; ++k)
10481c7c3c6aSMatthew Dillon 			vm_page_free(m[k]);
10491c7c3c6aSMatthew Dillon 	}
1050ab9abe5dSAlan Cox 	vm_page_unlock_queues();
10514dcc5c2dSMatthew Dillon 	splx(s);
10524dcc5c2dSMatthew Dillon 
10531c7c3c6aSMatthew Dillon 
10541c7c3c6aSMatthew Dillon 	/*
10554dcc5c2dSMatthew Dillon 	 * Return VM_PAGER_FAIL if we have nothing to do.  Return mreq
10564dcc5c2dSMatthew Dillon 	 * still busy, but the others unbusied.
10571c7c3c6aSMatthew Dillon 	 */
10584dcc5c2dSMatthew Dillon 	if (blk == SWAPBLK_NONE)
105926f9a767SRodney W. Grimes 		return (VM_PAGER_FAIL);
1060df8bae1dSRodney W. Grimes 
106116f62314SDavid Greenman 	/*
10628630c117SAlan Cox 	 * Getpbuf() can sleep.
10638630c117SAlan Cox 	 */
10648630c117SAlan Cox 	VM_OBJECT_UNLOCK(object);
10658630c117SAlan Cox 	/*
106616f62314SDavid Greenman 	 * Get a swap buffer header to perform the IO
106716f62314SDavid Greenman 	 */
10681c7c3c6aSMatthew Dillon 	bp = getpbuf(&nsw_rcount);
10695e04322aSPoul-Henning Kamp 	bp->b_flags |= B_PAGING;
107026f9a767SRodney W. Grimes 
107116f62314SDavid Greenman 	/*
107216f62314SDavid Greenman 	 * map our page(s) into kva for input
107316f62314SDavid Greenman 	 */
1074d68d828bSAlan Cox 	pmap_qenter((vm_offset_t)bp->b_data, m + i, j - i);
10751c7c3c6aSMatthew Dillon 
107621144e3bSPoul-Henning Kamp 	bp->b_iocmd = BIO_READ;
10771c7c3c6aSMatthew Dillon 	bp->b_iodone = swp_pager_async_iodone;
1078fdcc1cc0SJohn Baldwin 	bp->b_rcred = crhold(thread0.td_ucred);
1079fdcc1cc0SJohn Baldwin 	bp->b_wcred = crhold(thread0.td_ucred);
10801c7c3c6aSMatthew Dillon 	bp->b_blkno = blk - (reqpage - i);
10811c7c3c6aSMatthew Dillon 	bp->b_bcount = PAGE_SIZE * (j - i);
10821c7c3c6aSMatthew Dillon 	bp->b_bufsize = PAGE_SIZE * (j - i);
10831c7c3c6aSMatthew Dillon 	bp->b_pager.pg_reqpage = reqpage - i;
10841c7c3c6aSMatthew Dillon 
10858630c117SAlan Cox 	VM_OBJECT_LOCK(object);
1086b365ea9eSAlan Cox 	vm_page_lock_queues();
10871c7c3c6aSMatthew Dillon 	{
10881c7c3c6aSMatthew Dillon 		int k;
10891c7c3c6aSMatthew Dillon 
10901c7c3c6aSMatthew Dillon 		for (k = i; k < j; ++k) {
10911c7c3c6aSMatthew Dillon 			bp->b_pages[k - i] = m[k];
10921c7c3c6aSMatthew Dillon 			vm_page_flag_set(m[k], PG_SWAPINPROG);
10931c7c3c6aSMatthew Dillon 		}
10941c7c3c6aSMatthew Dillon 	}
1095b365ea9eSAlan Cox 	vm_page_unlock_queues();
10968630c117SAlan Cox 	VM_OBJECT_UNLOCK(object);
10971c7c3c6aSMatthew Dillon 	bp->b_npages = j - i;
109826f9a767SRodney W. Grimes 
1099976e77fcSDavid Greenman 	cnt.v_swapin++;
11001c7c3c6aSMatthew Dillon 	cnt.v_swappgsin += bp->b_npages;
11011c7c3c6aSMatthew Dillon 
1102df8bae1dSRodney W. Grimes 	/*
11031c7c3c6aSMatthew Dillon 	 * We still hold the lock on mreq, and our automatic completion routine
11041c7c3c6aSMatthew Dillon 	 * does not remove it.
1105df8bae1dSRodney W. Grimes 	 */
1106d68d828bSAlan Cox 	VM_OBJECT_LOCK(mreq->object);
11071c7c3c6aSMatthew Dillon 	vm_object_pip_add(mreq->object, bp->b_npages);
1108d68d828bSAlan Cox 	VM_OBJECT_UNLOCK(mreq->object);
11091c7c3c6aSMatthew Dillon 
11101c7c3c6aSMatthew Dillon 	/*
11111c7c3c6aSMatthew Dillon 	 * perform the I/O.  NOTE!!!  bp cannot be considered valid after
11121c7c3c6aSMatthew Dillon 	 * this point because we automatically release it on completion.
11131c7c3c6aSMatthew Dillon 	 * Instead, we look at the one page we are interested in which we
11141c7c3c6aSMatthew Dillon 	 * still hold a lock on even through the I/O completion.
11151c7c3c6aSMatthew Dillon 	 *
11161c7c3c6aSMatthew Dillon 	 * The other pages in our m[] array are also released on completion,
11171c7c3c6aSMatthew Dillon 	 * so we cannot assume they are valid anymore either.
11181c7c3c6aSMatthew Dillon 	 *
1119c37a77eeSPoul-Henning Kamp 	 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
11201c7c3c6aSMatthew Dillon 	 */
1121b890cb2cSPeter Wemm 	BUF_KERNPROC(bp);
11224b03903aSPoul-Henning Kamp 	swp_pager_strategy(bp);
112326f9a767SRodney W. Grimes 
112426f9a767SRodney W. Grimes 	/*
11251c7c3c6aSMatthew Dillon 	 * wait for the page we want to complete.  PG_SWAPINPROG is always
11261c7c3c6aSMatthew Dillon 	 * cleared on completion.  If an I/O error occurs, SWAPBLK_NONE
11271c7c3c6aSMatthew Dillon 	 * is set in the meta-data.
112826f9a767SRodney W. Grimes 	 */
11291c7c3c6aSMatthew Dillon 	s = splvm();
1130b365ea9eSAlan Cox 	vm_page_lock_queues();
11311c7c3c6aSMatthew Dillon 	while ((mreq->flags & PG_SWAPINPROG) != 0) {
11321c7c3c6aSMatthew Dillon 		vm_page_flag_set(mreq, PG_WANTED | PG_REFERENCED);
11331c7c3c6aSMatthew Dillon 		cnt.v_intrans++;
1134b365ea9eSAlan Cox 		if (msleep(mreq, &vm_page_queue_mtx, PSWP, "swread", hz*20)) {
11359bd86a98SBruce M Simpson 			printf(
11369bd86a98SBruce M Simpson "swap_pager: indefinite wait buffer: device: %s, blkno: %jd, size: %ld\n",
11370e3fe6e3SBruce M Simpson 			    bp->b_dev == NULL ? "[NULL]" : devtoname(bp->b_dev),
11380e3fe6e3SBruce M Simpson 			    (intmax_t)bp->b_blkno, bp->b_bcount);
11391c7c3c6aSMatthew Dillon 		}
11401b119d9dSDavid Greenman 	}
1141b365ea9eSAlan Cox 	vm_page_unlock_queues();
1142df8bae1dSRodney W. Grimes 	splx(s);
114326f9a767SRodney W. Grimes 
11448630c117SAlan Cox 	VM_OBJECT_LOCK(mreq->object);
114526f9a767SRodney W. Grimes 	/*
1146a1287949SEivind Eklund 	 * mreq is left busied after completion, but all the other pages
11471c7c3c6aSMatthew Dillon 	 * are freed.  If we had an unrecoverable read error the page will
11481c7c3c6aSMatthew Dillon 	 * not be valid.
114926f9a767SRodney W. Grimes 	 */
11501c7c3c6aSMatthew Dillon 	if (mreq->valid != VM_PAGE_BITS_ALL) {
11511c7c3c6aSMatthew Dillon 		return (VM_PAGER_ERROR);
115226f9a767SRodney W. Grimes 	} else {
11531c7c3c6aSMatthew Dillon 		return (VM_PAGER_OK);
115426f9a767SRodney W. Grimes 	}
11551c7c3c6aSMatthew Dillon 
11561c7c3c6aSMatthew Dillon 	/*
11571c7c3c6aSMatthew Dillon 	 * A final note: in a low swap situation, we cannot deallocate swap
11581c7c3c6aSMatthew Dillon 	 * and mark a page dirty here because the caller is likely to mark
11591c7c3c6aSMatthew Dillon 	 * the page clean when we return, causing the page to possibly revert
11601c7c3c6aSMatthew Dillon 	 * to all-zero's later.
11611c7c3c6aSMatthew Dillon 	 */
1162df8bae1dSRodney W. Grimes }
1163df8bae1dSRodney W. Grimes 
11641c7c3c6aSMatthew Dillon /*
11651c7c3c6aSMatthew Dillon  *	swap_pager_putpages:
11661c7c3c6aSMatthew Dillon  *
11671c7c3c6aSMatthew Dillon  *	Assign swap (if necessary) and initiate I/O on the specified pages.
11681c7c3c6aSMatthew Dillon  *
11691c7c3c6aSMatthew Dillon  *	We support both OBJT_DEFAULT and OBJT_SWAP objects.  DEFAULT objects
11701c7c3c6aSMatthew Dillon  *	are automatically converted to SWAP objects.
11711c7c3c6aSMatthew Dillon  *
1172ea3aecf5SPeter Wemm  *	In a low memory situation we may block in VOP_STRATEGY(), but the new
11731c7c3c6aSMatthew Dillon  *	vm_page reservation system coupled with properly written VFS devices
11741c7c3c6aSMatthew Dillon  *	should ensure that no low-memory deadlock occurs.  This is an area
11751c7c3c6aSMatthew Dillon  *	which needs work.
11761c7c3c6aSMatthew Dillon  *
11771c7c3c6aSMatthew Dillon  *	The parent has N vm_object_pip_add() references prior to
11781c7c3c6aSMatthew Dillon  *	calling us and will remove references for rtvals[] that are
11791c7c3c6aSMatthew Dillon  *	not set to VM_PAGER_PEND.  We need to remove the rest on I/O
11801c7c3c6aSMatthew Dillon  *	completion.
11811c7c3c6aSMatthew Dillon  *
11821c7c3c6aSMatthew Dillon  *	The parent has soft-busy'd the pages it passes us and will unbusy
11831c7c3c6aSMatthew Dillon  *	those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
11841c7c3c6aSMatthew Dillon  *	We need to unbusy the rest on I/O completion.
11851c7c3c6aSMatthew Dillon  */
1186e4542174SMatthew Dillon void
11872f249180SPoul-Henning Kamp swap_pager_putpages(vm_object_t object, vm_page_t *m, int count,
11882f249180SPoul-Henning Kamp     boolean_t sync, int *rtvals)
1189df8bae1dSRodney W. Grimes {
11901c7c3c6aSMatthew Dillon 	int i;
11911c7c3c6aSMatthew Dillon 	int n = 0;
1192df8bae1dSRodney W. Grimes 
11930cddd8f0SMatthew Dillon 	GIANT_REQUIRED;
11941c7c3c6aSMatthew Dillon 	if (count && m[0]->object != object) {
11951c7c3c6aSMatthew Dillon 		panic("swap_pager_getpages: object mismatch %p/%p",
11961c7c3c6aSMatthew Dillon 		    object,
11971c7c3c6aSMatthew Dillon 		    m[0]->object
11981c7c3c6aSMatthew Dillon 		);
11991c7c3c6aSMatthew Dillon 	}
1200ee3dc7d7SAlan Cox 
12011c7c3c6aSMatthew Dillon 	/*
12021c7c3c6aSMatthew Dillon 	 * Step 1
12031c7c3c6aSMatthew Dillon 	 *
12041c7c3c6aSMatthew Dillon 	 * Turn object into OBJT_SWAP
12051c7c3c6aSMatthew Dillon 	 * check for bogus sysops
12061c7c3c6aSMatthew Dillon 	 * force sync if not pageout process
12071c7c3c6aSMatthew Dillon 	 */
12084dcc5c2dSMatthew Dillon 	if (object->type != OBJT_SWAP)
12094dcc5c2dSMatthew Dillon 		swp_pager_meta_build(object, 0, SWAPBLK_NONE);
1210ee3dc7d7SAlan Cox 	VM_OBJECT_UNLOCK(object);
1211e47ed70bSJohn Dyson 
1212e47ed70bSJohn Dyson 	if (curproc != pageproc)
1213e47ed70bSJohn Dyson 		sync = TRUE;
121426f9a767SRodney W. Grimes 
12151c7c3c6aSMatthew Dillon 	/*
12161c7c3c6aSMatthew Dillon 	 * Step 2
12171c7c3c6aSMatthew Dillon 	 *
1218ad3cce20SMatthew Dillon 	 * Update nsw parameters from swap_async_max sysctl values.
1219ad3cce20SMatthew Dillon 	 * Do not let the sysop crash the machine with bogus numbers.
1220327f4e83SMatthew Dillon 	 */
12216d541bf1SJohn Baldwin 	mtx_lock(&pbuf_mtx);
1222327f4e83SMatthew Dillon 	if (swap_async_max != nsw_wcount_async_max) {
1223327f4e83SMatthew Dillon 		int n;
1224327f4e83SMatthew Dillon 		int s;
1225327f4e83SMatthew Dillon 
1226327f4e83SMatthew Dillon 		/*
1227327f4e83SMatthew Dillon 		 * limit range
1228327f4e83SMatthew Dillon 		 */
1229327f4e83SMatthew Dillon 		if ((n = swap_async_max) > nswbuf / 2)
1230327f4e83SMatthew Dillon 			n = nswbuf / 2;
1231327f4e83SMatthew Dillon 		if (n < 1)
1232327f4e83SMatthew Dillon 			n = 1;
1233327f4e83SMatthew Dillon 		swap_async_max = n;
1234327f4e83SMatthew Dillon 
1235327f4e83SMatthew Dillon 		/*
1236327f4e83SMatthew Dillon 		 * Adjust difference ( if possible ).  If the current async
1237327f4e83SMatthew Dillon 		 * count is too low, we may not be able to make the adjustment
1238327f4e83SMatthew Dillon 		 * at this time.
1239327f4e83SMatthew Dillon 		 */
1240327f4e83SMatthew Dillon 		s = splvm();
1241327f4e83SMatthew Dillon 		n -= nsw_wcount_async_max;
1242327f4e83SMatthew Dillon 		if (nsw_wcount_async + n >= 0) {
1243327f4e83SMatthew Dillon 			nsw_wcount_async += n;
1244327f4e83SMatthew Dillon 			nsw_wcount_async_max += n;
1245327f4e83SMatthew Dillon 			wakeup(&nsw_wcount_async);
1246327f4e83SMatthew Dillon 		}
1247327f4e83SMatthew Dillon 		splx(s);
1248327f4e83SMatthew Dillon 	}
12496d541bf1SJohn Baldwin 	mtx_unlock(&pbuf_mtx);
1250327f4e83SMatthew Dillon 
1251327f4e83SMatthew Dillon 	/*
1252327f4e83SMatthew Dillon 	 * Step 3
1253327f4e83SMatthew Dillon 	 *
12541c7c3c6aSMatthew Dillon 	 * Assign swap blocks and issue I/O.  We reallocate swap on the fly.
12551c7c3c6aSMatthew Dillon 	 * The page is left dirty until the pageout operation completes
12561c7c3c6aSMatthew Dillon 	 * successfully.
12571c7c3c6aSMatthew Dillon 	 */
12581c7c3c6aSMatthew Dillon 	for (i = 0; i < count; i += n) {
12591c7c3c6aSMatthew Dillon 		int s;
12601c7c3c6aSMatthew Dillon 		int j;
12611c7c3c6aSMatthew Dillon 		struct buf *bp;
1262a316d390SJohn Dyson 		daddr_t blk;
126326f9a767SRodney W. Grimes 
1264df8bae1dSRodney W. Grimes 		/*
12651c7c3c6aSMatthew Dillon 		 * Maximum I/O size is limited by a number of factors.
1266df8bae1dSRodney W. Grimes 		 */
12671c7c3c6aSMatthew Dillon 		n = min(BLIST_MAX_ALLOC, count - i);
1268327f4e83SMatthew Dillon 		n = min(n, nsw_cluster_max);
12691c7c3c6aSMatthew Dillon 
12704dcc5c2dSMatthew Dillon 		s = splvm();
12714dcc5c2dSMatthew Dillon 
127226f9a767SRodney W. Grimes 		/*
12731c7c3c6aSMatthew Dillon 		 * Get biggest block of swap we can.  If we fail, fall
12741c7c3c6aSMatthew Dillon 		 * back and try to allocate a smaller block.  Don't go
12751c7c3c6aSMatthew Dillon 		 * overboard trying to allocate space if it would overly
12761c7c3c6aSMatthew Dillon 		 * fragment swap.
127726f9a767SRodney W. Grimes 		 */
12781c7c3c6aSMatthew Dillon 		while (
12791c7c3c6aSMatthew Dillon 		    (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE &&
12801c7c3c6aSMatthew Dillon 		    n > 4
12811c7c3c6aSMatthew Dillon 		) {
12821c7c3c6aSMatthew Dillon 			n >>= 1;
128326f9a767SRodney W. Grimes 		}
12841c7c3c6aSMatthew Dillon 		if (blk == SWAPBLK_NONE) {
12854dcc5c2dSMatthew Dillon 			for (j = 0; j < n; ++j)
12861c7c3c6aSMatthew Dillon 				rtvals[i+j] = VM_PAGER_FAIL;
12874dcc5c2dSMatthew Dillon 			splx(s);
12881c7c3c6aSMatthew Dillon 			continue;
128926f9a767SRodney W. Grimes 		}
129026f9a767SRodney W. Grimes 
129126f9a767SRodney W. Grimes 		/*
12921c7c3c6aSMatthew Dillon 		 * All I/O parameters have been satisfied, build the I/O
12931c7c3c6aSMatthew Dillon 		 * request and assign the swap space.
129426f9a767SRodney W. Grimes 		 */
1295327f4e83SMatthew Dillon 		if (sync == TRUE) {
1296327f4e83SMatthew Dillon 			bp = getpbuf(&nsw_wcount_sync);
1297327f4e83SMatthew Dillon 		} else {
1298327f4e83SMatthew Dillon 			bp = getpbuf(&nsw_wcount_async);
129921144e3bSPoul-Henning Kamp 			bp->b_flags = B_ASYNC;
1300327f4e83SMatthew Dillon 		}
13015e04322aSPoul-Henning Kamp 		bp->b_flags |= B_PAGING;
1302912e4ae9SPoul-Henning Kamp 		bp->b_iocmd = BIO_WRITE;
130326f9a767SRodney W. Grimes 
13041c7c3c6aSMatthew Dillon 		pmap_qenter((vm_offset_t)bp->b_data, &m[i], n);
13051c7c3c6aSMatthew Dillon 
1306fdcc1cc0SJohn Baldwin 		bp->b_rcred = crhold(thread0.td_ucred);
1307fdcc1cc0SJohn Baldwin 		bp->b_wcred = crhold(thread0.td_ucred);
13081c7c3c6aSMatthew Dillon 		bp->b_bcount = PAGE_SIZE * n;
13091c7c3c6aSMatthew Dillon 		bp->b_bufsize = PAGE_SIZE * n;
13101c7c3c6aSMatthew Dillon 		bp->b_blkno = blk;
1311e47ed70bSJohn Dyson 
1312ee3dc7d7SAlan Cox 		VM_OBJECT_LOCK(object);
13131c7c3c6aSMatthew Dillon 		for (j = 0; j < n; ++j) {
13141c7c3c6aSMatthew Dillon 			vm_page_t mreq = m[i+j];
13151c7c3c6aSMatthew Dillon 
13161c7c3c6aSMatthew Dillon 			swp_pager_meta_build(
13171c7c3c6aSMatthew Dillon 			    mreq->object,
13181c7c3c6aSMatthew Dillon 			    mreq->pindex,
13194dcc5c2dSMatthew Dillon 			    blk + j
13201c7c3c6aSMatthew Dillon 			);
13217dbf82dcSMatthew Dillon 			vm_page_dirty(mreq);
13221c7c3c6aSMatthew Dillon 			rtvals[i+j] = VM_PAGER_OK;
13231c7c3c6aSMatthew Dillon 
1324b365ea9eSAlan Cox 			vm_page_lock_queues();
13251c7c3c6aSMatthew Dillon 			vm_page_flag_set(mreq, PG_SWAPINPROG);
1326b365ea9eSAlan Cox 			vm_page_unlock_queues();
13271c7c3c6aSMatthew Dillon 			bp->b_pages[j] = mreq;
13281c7c3c6aSMatthew Dillon 		}
1329ee3dc7d7SAlan Cox 		VM_OBJECT_UNLOCK(object);
13301c7c3c6aSMatthew Dillon 		bp->b_npages = n;
1331a5296b05SJulian Elischer 		/*
1332a5296b05SJulian Elischer 		 * Must set dirty range for NFS to work.
1333a5296b05SJulian Elischer 		 */
1334a5296b05SJulian Elischer 		bp->b_dirtyoff = 0;
1335a5296b05SJulian Elischer 		bp->b_dirtyend = bp->b_bcount;
13361c7c3c6aSMatthew Dillon 
13371c7c3c6aSMatthew Dillon 		cnt.v_swapout++;
13381c7c3c6aSMatthew Dillon 		cnt.v_swappgsout += bp->b_npages;
133926f9a767SRodney W. Grimes 
13404dcc5c2dSMatthew Dillon 		splx(s);
13414dcc5c2dSMatthew Dillon 
134226f9a767SRodney W. Grimes 		/*
13431c7c3c6aSMatthew Dillon 		 * asynchronous
13441c7c3c6aSMatthew Dillon 		 *
1345c37a77eeSPoul-Henning Kamp 		 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
134626f9a767SRodney W. Grimes 		 */
13471c7c3c6aSMatthew Dillon 		if (sync == FALSE) {
13481c7c3c6aSMatthew Dillon 			bp->b_iodone = swp_pager_async_iodone;
134967812eacSKirk McKusick 			BUF_KERNPROC(bp);
13504b03903aSPoul-Henning Kamp 			swp_pager_strategy(bp);
13511c7c3c6aSMatthew Dillon 
13521c7c3c6aSMatthew Dillon 			for (j = 0; j < n; ++j)
13531c7c3c6aSMatthew Dillon 				rtvals[i+j] = VM_PAGER_PEND;
135423955314SAlfred Perlstein 			/* restart outter loop */
13551c7c3c6aSMatthew Dillon 			continue;
135626f9a767SRodney W. Grimes 		}
1357e47ed70bSJohn Dyson 
135826f9a767SRodney W. Grimes 		/*
13591c7c3c6aSMatthew Dillon 		 * synchronous
13601c7c3c6aSMatthew Dillon 		 *
1361c37a77eeSPoul-Henning Kamp 		 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
13621c7c3c6aSMatthew Dillon 		 */
13632c840b1fSAlan Cox 		bp->b_iodone = bdone;
13644b03903aSPoul-Henning Kamp 		swp_pager_strategy(bp);
13651c7c3c6aSMatthew Dillon 
13661c7c3c6aSMatthew Dillon 		/*
13671c7c3c6aSMatthew Dillon 		 * Wait for the sync I/O to complete, then update rtvals.
13681c7c3c6aSMatthew Dillon 		 * We just set the rtvals[] to VM_PAGER_PEND so we can call
13691c7c3c6aSMatthew Dillon 		 * our async completion routine at the end, thus avoiding a
13701c7c3c6aSMatthew Dillon 		 * double-free.
137126f9a767SRodney W. Grimes 		 */
13724dcc5c2dSMatthew Dillon 		s = splbio();
13732c840b1fSAlan Cox 		bwait(bp, PVM, "swwrt");
13741c7c3c6aSMatthew Dillon 		for (j = 0; j < n; ++j)
13751c7c3c6aSMatthew Dillon 			rtvals[i+j] = VM_PAGER_PEND;
13761c7c3c6aSMatthew Dillon 		/*
13771c7c3c6aSMatthew Dillon 		 * Now that we are through with the bp, we can call the
13781c7c3c6aSMatthew Dillon 		 * normal async completion, which frees everything up.
13791c7c3c6aSMatthew Dillon 		 */
13801c7c3c6aSMatthew Dillon 		swp_pager_async_iodone(bp);
138126f9a767SRodney W. Grimes 		splx(s);
13821c7c3c6aSMatthew Dillon 	}
13832e3b314dSAlan Cox 	VM_OBJECT_LOCK(object);
13841c7c3c6aSMatthew Dillon }
13851c7c3c6aSMatthew Dillon 
13861c7c3c6aSMatthew Dillon /*
13871c7c3c6aSMatthew Dillon  *	swp_pager_async_iodone:
13881c7c3c6aSMatthew Dillon  *
13891c7c3c6aSMatthew Dillon  *	Completion routine for asynchronous reads and writes from/to swap.
13901c7c3c6aSMatthew Dillon  *	Also called manually by synchronous code to finish up a bp.
13911c7c3c6aSMatthew Dillon  *
13921c7c3c6aSMatthew Dillon  *	For READ operations, the pages are PG_BUSY'd.  For WRITE operations,
13931c7c3c6aSMatthew Dillon  *	the pages are vm_page_t->busy'd.  For READ operations, we PG_BUSY
13941c7c3c6aSMatthew Dillon  *	unbusy all pages except the 'main' request page.  For WRITE
13951c7c3c6aSMatthew Dillon  *	operations, we vm_page_t->busy'd unbusy all pages ( we can do this
13961c7c3c6aSMatthew Dillon  *	because we marked them all VM_PAGER_PEND on return from putpages ).
13971c7c3c6aSMatthew Dillon  *
13981c7c3c6aSMatthew Dillon  *	This routine may not block.
13994dcc5c2dSMatthew Dillon  *	This routine is called at splbio() or better
14004dcc5c2dSMatthew Dillon  *
14014dcc5c2dSMatthew Dillon  *	We up ourselves to splvm() as required for various vm_page related
14024dcc5c2dSMatthew Dillon  *	calls.
14031c7c3c6aSMatthew Dillon  */
14041c7c3c6aSMatthew Dillon static void
14052f249180SPoul-Henning Kamp swp_pager_async_iodone(struct buf *bp)
14061c7c3c6aSMatthew Dillon {
14071c7c3c6aSMatthew Dillon 	int s;
14081c7c3c6aSMatthew Dillon 	int i;
14091c7c3c6aSMatthew Dillon 	vm_object_t object = NULL;
14101c7c3c6aSMatthew Dillon 
14111c7c3c6aSMatthew Dillon 	bp->b_flags |= B_DONE;
14121c7c3c6aSMatthew Dillon 
14131c7c3c6aSMatthew Dillon 	/*
14141c7c3c6aSMatthew Dillon 	 * report error
14151c7c3c6aSMatthew Dillon 	 */
1416c244d2deSPoul-Henning Kamp 	if (bp->b_ioflags & BIO_ERROR) {
14171c7c3c6aSMatthew Dillon 		printf(
14181c7c3c6aSMatthew Dillon 		    "swap_pager: I/O error - %s failed; blkno %ld,"
14191c7c3c6aSMatthew Dillon 			"size %ld, error %d\n",
142021144e3bSPoul-Henning Kamp 		    ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
14211c7c3c6aSMatthew Dillon 		    (long)bp->b_blkno,
14221c7c3c6aSMatthew Dillon 		    (long)bp->b_bcount,
14231c7c3c6aSMatthew Dillon 		    bp->b_error
14241c7c3c6aSMatthew Dillon 		);
14251c7c3c6aSMatthew Dillon 	}
14261c7c3c6aSMatthew Dillon 
14271c7c3c6aSMatthew Dillon 	/*
14284dcc5c2dSMatthew Dillon 	 * set object, raise to splvm().
14291c7c3c6aSMatthew Dillon 	 */
14304dcc5c2dSMatthew Dillon 	s = splvm();
143126f9a767SRodney W. Grimes 
143226f9a767SRodney W. Grimes 	/*
143326f9a767SRodney W. Grimes 	 * remove the mapping for kernel virtual
143426f9a767SRodney W. Grimes 	 */
14351c7c3c6aSMatthew Dillon 	pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
143626f9a767SRodney W. Grimes 
143733a609ecSAlan Cox 	if (bp->b_npages) {
143833a609ecSAlan Cox 		object = bp->b_pages[0]->object;
143933a609ecSAlan Cox 		VM_OBJECT_LOCK(object);
144033a609ecSAlan Cox 	}
144140eab1e9SAlan Cox 	vm_page_lock_queues();
144226f9a767SRodney W. Grimes 	/*
14431c7c3c6aSMatthew Dillon 	 * cleanup pages.  If an error occurs writing to swap, we are in
14441c7c3c6aSMatthew Dillon 	 * very serious trouble.  If it happens to be a disk error, though,
14451c7c3c6aSMatthew Dillon 	 * we may be able to recover by reassigning the swap later on.  So
14461c7c3c6aSMatthew Dillon 	 * in this case we remove the m->swapblk assignment for the page
14471c7c3c6aSMatthew Dillon 	 * but do not free it in the rlist.  The errornous block(s) are thus
14481c7c3c6aSMatthew Dillon 	 * never reallocated as swap.  Redirty the page and continue.
144926f9a767SRodney W. Grimes 	 */
14501c7c3c6aSMatthew Dillon 	for (i = 0; i < bp->b_npages; ++i) {
14511c7c3c6aSMatthew Dillon 		vm_page_t m = bp->b_pages[i];
1452e47ed70bSJohn Dyson 
14531c7c3c6aSMatthew Dillon 		vm_page_flag_clear(m, PG_SWAPINPROG);
1454e47ed70bSJohn Dyson 
1455c244d2deSPoul-Henning Kamp 		if (bp->b_ioflags & BIO_ERROR) {
1456ffc82b0aSJohn Dyson 			/*
14571c7c3c6aSMatthew Dillon 			 * If an error occurs I'd love to throw the swapblk
14581c7c3c6aSMatthew Dillon 			 * away without freeing it back to swapspace, so it
14591c7c3c6aSMatthew Dillon 			 * can never be used again.  But I can't from an
14601c7c3c6aSMatthew Dillon 			 * interrupt.
1461ffc82b0aSJohn Dyson 			 */
146221144e3bSPoul-Henning Kamp 			if (bp->b_iocmd == BIO_READ) {
14631c7c3c6aSMatthew Dillon 				/*
14641c7c3c6aSMatthew Dillon 				 * When reading, reqpage needs to stay
14651c7c3c6aSMatthew Dillon 				 * locked for the parent, but all other
14661c7c3c6aSMatthew Dillon 				 * pages can be freed.  We still want to
14671c7c3c6aSMatthew Dillon 				 * wakeup the parent waiting on the page,
14681c7c3c6aSMatthew Dillon 				 * though.  ( also: pg_reqpage can be -1 and
14691c7c3c6aSMatthew Dillon 				 * not match anything ).
14701c7c3c6aSMatthew Dillon 				 *
14711c7c3c6aSMatthew Dillon 				 * We have to wake specifically requested pages
14721c7c3c6aSMatthew Dillon 				 * up too because we cleared PG_SWAPINPROG and
14731c7c3c6aSMatthew Dillon 				 * someone may be waiting for that.
14741c7c3c6aSMatthew Dillon 				 *
14751c7c3c6aSMatthew Dillon 				 * NOTE: for reads, m->dirty will probably
1476956f3135SPhilippe Charnier 				 * be overridden by the original caller of
14771c7c3c6aSMatthew Dillon 				 * getpages so don't play cute tricks here.
14781c7c3c6aSMatthew Dillon 				 *
1479279d7226SMatthew Dillon 				 * XXX IT IS NOT LEGAL TO FREE THE PAGE HERE
1480279d7226SMatthew Dillon 				 * AS THIS MESSES WITH object->memq, and it is
1481279d7226SMatthew Dillon 				 * not legal to mess with object->memq from an
1482279d7226SMatthew Dillon 				 * interrupt.
14831c7c3c6aSMatthew Dillon 				 */
14841c7c3c6aSMatthew Dillon 				m->valid = 0;
14851c7c3c6aSMatthew Dillon 				if (i != bp->b_pager.pg_reqpage)
14861c7c3c6aSMatthew Dillon 					vm_page_free(m);
14871c7c3c6aSMatthew Dillon 				else
14881c7c3c6aSMatthew Dillon 					vm_page_flash(m);
14891c7c3c6aSMatthew Dillon 				/*
14901c7c3c6aSMatthew Dillon 				 * If i == bp->b_pager.pg_reqpage, do not wake
14911c7c3c6aSMatthew Dillon 				 * the page up.  The caller needs to.
14921c7c3c6aSMatthew Dillon 				 */
14931c7c3c6aSMatthew Dillon 			} else {
14941c7c3c6aSMatthew Dillon 				/*
14951c7c3c6aSMatthew Dillon 				 * If a write error occurs, reactivate page
14961c7c3c6aSMatthew Dillon 				 * so it doesn't clog the inactive list,
14971c7c3c6aSMatthew Dillon 				 * then finish the I/O.
14981c7c3c6aSMatthew Dillon 				 */
14997dbf82dcSMatthew Dillon 				vm_page_dirty(m);
15001c7c3c6aSMatthew Dillon 				vm_page_activate(m);
15011c7c3c6aSMatthew Dillon 				vm_page_io_finish(m);
15021c7c3c6aSMatthew Dillon 			}
150321144e3bSPoul-Henning Kamp 		} else if (bp->b_iocmd == BIO_READ) {
15041c7c3c6aSMatthew Dillon 			/*
15051c7c3c6aSMatthew Dillon 			 * For read success, clear dirty bits.  Nobody should
15061c7c3c6aSMatthew Dillon 			 * have this page mapped but don't take any chances,
15071c7c3c6aSMatthew Dillon 			 * make sure the pmap modify bits are also cleared.
15081c7c3c6aSMatthew Dillon 			 *
15091c7c3c6aSMatthew Dillon 			 * NOTE: for reads, m->dirty will probably be
1510956f3135SPhilippe Charnier 			 * overridden by the original caller of getpages so
15111c7c3c6aSMatthew Dillon 			 * we cannot set them in order to free the underlying
15121c7c3c6aSMatthew Dillon 			 * swap in a low-swap situation.  I don't think we'd
15131c7c3c6aSMatthew Dillon 			 * want to do that anyway, but it was an optimization
15141c7c3c6aSMatthew Dillon 			 * that existed in the old swapper for a time before
15151c7c3c6aSMatthew Dillon 			 * it got ripped out due to precisely this problem.
15161c7c3c6aSMatthew Dillon 			 *
15171c7c3c6aSMatthew Dillon 			 * If not the requested page then deactivate it.
15181c7c3c6aSMatthew Dillon 			 *
15191c7c3c6aSMatthew Dillon 			 * Note that the requested page, reqpage, is left
15201c7c3c6aSMatthew Dillon 			 * busied, but we still have to wake it up.  The
15211c7c3c6aSMatthew Dillon 			 * other pages are released (unbusied) by
15221c7c3c6aSMatthew Dillon 			 * vm_page_wakeup().  We do not set reqpage's
15231c7c3c6aSMatthew Dillon 			 * valid bits here, it is up to the caller.
15241c7c3c6aSMatthew Dillon 			 */
15250385347cSPeter Wemm 			pmap_clear_modify(m);
15261c7c3c6aSMatthew Dillon 			m->valid = VM_PAGE_BITS_ALL;
15272c28a105SAlan Cox 			vm_page_undirty(m);
15281c7c3c6aSMatthew Dillon 
15291c7c3c6aSMatthew Dillon 			/*
15301c7c3c6aSMatthew Dillon 			 * We have to wake specifically requested pages
15311c7c3c6aSMatthew Dillon 			 * up too because we cleared PG_SWAPINPROG and
15321c7c3c6aSMatthew Dillon 			 * could be waiting for it in getpages.  However,
15331c7c3c6aSMatthew Dillon 			 * be sure to not unbusy getpages specifically
15341c7c3c6aSMatthew Dillon 			 * requested page - getpages expects it to be
15351c7c3c6aSMatthew Dillon 			 * left busy.
15361c7c3c6aSMatthew Dillon 			 */
15371c7c3c6aSMatthew Dillon 			if (i != bp->b_pager.pg_reqpage) {
15381c7c3c6aSMatthew Dillon 				vm_page_deactivate(m);
15391c7c3c6aSMatthew Dillon 				vm_page_wakeup(m);
15401c7c3c6aSMatthew Dillon 			} else {
15411c7c3c6aSMatthew Dillon 				vm_page_flash(m);
15421c7c3c6aSMatthew Dillon 			}
15431c7c3c6aSMatthew Dillon 		} else {
15441c7c3c6aSMatthew Dillon 			/*
15451c7c3c6aSMatthew Dillon 			 * For write success, clear the modify and dirty
15461c7c3c6aSMatthew Dillon 			 * status, then finish the I/O ( which decrements the
15471c7c3c6aSMatthew Dillon 			 * busy count and possibly wakes waiter's up ).
15481c7c3c6aSMatthew Dillon 			 */
15490385347cSPeter Wemm 			pmap_clear_modify(m);
1550c52e7044SAlan Cox 			vm_page_undirty(m);
15511c7c3c6aSMatthew Dillon 			vm_page_io_finish(m);
15522c840b1fSAlan Cox 			if (vm_page_count_severe())
15532c840b1fSAlan Cox 				vm_page_try_to_cache(m);
1554ffc82b0aSJohn Dyson 		}
1555df8bae1dSRodney W. Grimes 	}
155640eab1e9SAlan Cox 	vm_page_unlock_queues();
155726f9a767SRodney W. Grimes 
15581c7c3c6aSMatthew Dillon 	/*
15591c7c3c6aSMatthew Dillon 	 * adjust pip.  NOTE: the original parent may still have its own
15601c7c3c6aSMatthew Dillon 	 * pip refs on the object.
15611c7c3c6aSMatthew Dillon 	 */
15620d420ad3SAlan Cox 	if (object != NULL) {
15631c7c3c6aSMatthew Dillon 		vm_object_pip_wakeupn(object, bp->b_npages);
15640d420ad3SAlan Cox 		VM_OBJECT_UNLOCK(object);
15650d420ad3SAlan Cox 	}
156626f9a767SRodney W. Grimes 
15671c7c3c6aSMatthew Dillon 	/*
15681c7c3c6aSMatthew Dillon 	 * release the physical I/O buffer
15691c7c3c6aSMatthew Dillon 	 */
1570327f4e83SMatthew Dillon 	relpbuf(
1571327f4e83SMatthew Dillon 	    bp,
157221144e3bSPoul-Henning Kamp 	    ((bp->b_iocmd == BIO_READ) ? &nsw_rcount :
1573327f4e83SMatthew Dillon 		((bp->b_flags & B_ASYNC) ?
1574327f4e83SMatthew Dillon 		    &nsw_wcount_async :
1575327f4e83SMatthew Dillon 		    &nsw_wcount_sync
1576327f4e83SMatthew Dillon 		)
1577327f4e83SMatthew Dillon 	    )
1578327f4e83SMatthew Dillon 	);
157926f9a767SRodney W. Grimes 	splx(s);
158026f9a767SRodney W. Grimes }
15811c7c3c6aSMatthew Dillon 
158292da00bbSMatthew Dillon /*
158392da00bbSMatthew Dillon  *	swap_pager_isswapped:
158492da00bbSMatthew Dillon  *
158592da00bbSMatthew Dillon  *	Return 1 if at least one page in the given object is paged
158692da00bbSMatthew Dillon  *	out to the given swap device.
158792da00bbSMatthew Dillon  *
158892da00bbSMatthew Dillon  *	This routine may not block.
158992da00bbSMatthew Dillon  */
15908f60c087SPoul-Henning Kamp int
15918f60c087SPoul-Henning Kamp swap_pager_isswapped(vm_object_t object, struct swdevt *sp)
15928f60c087SPoul-Henning Kamp {
159392da00bbSMatthew Dillon 	daddr_t index = 0;
159492da00bbSMatthew Dillon 	int bcount;
159592da00bbSMatthew Dillon 	int i;
159692da00bbSMatthew Dillon 
159717cd3642SAlan Cox 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
159892da00bbSMatthew Dillon 	for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) {
159992da00bbSMatthew Dillon 		struct swblock *swap;
160092da00bbSMatthew Dillon 
16017827d9b0SAlan Cox 		mtx_lock(&swhash_mtx);
160292da00bbSMatthew Dillon 		if ((swap = *swp_pager_hash(object, index)) != NULL) {
160392da00bbSMatthew Dillon 			for (i = 0; i < SWAP_META_PAGES; ++i) {
160492da00bbSMatthew Dillon 				daddr_t v = swap->swb_pages[i];
16058f60c087SPoul-Henning Kamp 				if (v == SWAPBLK_NONE)
16068f60c087SPoul-Henning Kamp 					continue;
16077827d9b0SAlan Cox 				if (swp_pager_find_dev(v) == sp) {
16087827d9b0SAlan Cox 					mtx_unlock(&swhash_mtx);
160992da00bbSMatthew Dillon 					return 1;
161092da00bbSMatthew Dillon 				}
161192da00bbSMatthew Dillon 			}
16127827d9b0SAlan Cox 		}
16137827d9b0SAlan Cox 		mtx_unlock(&swhash_mtx);
161492da00bbSMatthew Dillon 		index += SWAP_META_PAGES;
161592da00bbSMatthew Dillon 		if (index > 0x20000000)
161692da00bbSMatthew Dillon 			panic("swap_pager_isswapped: failed to locate all swap meta blocks");
161792da00bbSMatthew Dillon 	}
161892da00bbSMatthew Dillon 	return 0;
161992da00bbSMatthew Dillon }
162092da00bbSMatthew Dillon 
162192da00bbSMatthew Dillon /*
162292da00bbSMatthew Dillon  * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in
162392da00bbSMatthew Dillon  *
162492da00bbSMatthew Dillon  *	This routine dissociates the page at the given index within a
162592da00bbSMatthew Dillon  *	swap block from its backing store, paging it in if necessary.
162692da00bbSMatthew Dillon  *	If the page is paged in, it is placed in the inactive queue,
162792da00bbSMatthew Dillon  *	since it had its backing store ripped out from under it.
162892da00bbSMatthew Dillon  *	We also attempt to swap in all other pages in the swap block,
162992da00bbSMatthew Dillon  *	we only guarantee that the one at the specified index is
163092da00bbSMatthew Dillon  *	paged in.
163192da00bbSMatthew Dillon  *
163292da00bbSMatthew Dillon  *	XXX - The code to page the whole block in doesn't work, so we
163392da00bbSMatthew Dillon  *	      revert to the one-by-one behavior for now.  Sigh.
163492da00bbSMatthew Dillon  */
163592da00bbSMatthew Dillon static __inline void
163692da00bbSMatthew Dillon swp_pager_force_pagein(struct swblock *swap, int idx)
163792da00bbSMatthew Dillon {
163892da00bbSMatthew Dillon 	vm_object_t object;
163992da00bbSMatthew Dillon 	vm_page_t m;
164092da00bbSMatthew Dillon 	vm_pindex_t pindex;
164192da00bbSMatthew Dillon 
164292da00bbSMatthew Dillon 	object = swap->swb_object;
164392da00bbSMatthew Dillon 	pindex = swap->swb_index;
1644d536c58fSAlan Cox 	mtx_unlock(&swhash_mtx);
164592da00bbSMatthew Dillon 
1646d22bc710SAlan Cox 	VM_OBJECT_LOCK(object);
164792da00bbSMatthew Dillon 	vm_object_pip_add(object, 1);
164892da00bbSMatthew Dillon 	m = vm_page_grab(object, pindex + idx, VM_ALLOC_NORMAL|VM_ALLOC_RETRY);
164992da00bbSMatthew Dillon 	if (m->valid == VM_PAGE_BITS_ALL) {
165092da00bbSMatthew Dillon 		vm_object_pip_subtract(object, 1);
165192da00bbSMatthew Dillon 		vm_page_lock_queues();
165292da00bbSMatthew Dillon 		vm_page_activate(m);
165392da00bbSMatthew Dillon 		vm_page_dirty(m);
165492da00bbSMatthew Dillon 		vm_page_wakeup(m);
165592da00bbSMatthew Dillon 		vm_page_unlock_queues();
165692da00bbSMatthew Dillon 		vm_pager_page_unswapped(m);
16572e3b314dSAlan Cox 		VM_OBJECT_UNLOCK(object);
165892da00bbSMatthew Dillon 		return;
165992da00bbSMatthew Dillon 	}
166092da00bbSMatthew Dillon 
166192da00bbSMatthew Dillon 	if (swap_pager_getpages(object, &m, 1, 0) !=
166292da00bbSMatthew Dillon 	    VM_PAGER_OK)
166392da00bbSMatthew Dillon 		panic("swap_pager_force_pagein: read from swap failed");/*XXX*/
166492da00bbSMatthew Dillon 	vm_object_pip_subtract(object, 1);
166592da00bbSMatthew Dillon 	vm_page_lock_queues();
166692da00bbSMatthew Dillon 	vm_page_dirty(m);
166792da00bbSMatthew Dillon 	vm_page_dontneed(m);
166892da00bbSMatthew Dillon 	vm_page_wakeup(m);
166992da00bbSMatthew Dillon 	vm_page_unlock_queues();
167092da00bbSMatthew Dillon 	vm_pager_page_unswapped(m);
16712e3b314dSAlan Cox 	VM_OBJECT_UNLOCK(object);
167292da00bbSMatthew Dillon }
167392da00bbSMatthew Dillon 
167492da00bbSMatthew Dillon 
167592da00bbSMatthew Dillon /*
167692da00bbSMatthew Dillon  *	swap_pager_swapoff:
167792da00bbSMatthew Dillon  *
167892da00bbSMatthew Dillon  *	Page in all of the pages that have been paged out to the
167992da00bbSMatthew Dillon  *	given device.  The corresponding blocks in the bitmap must be
168092da00bbSMatthew Dillon  *	marked as allocated and the device must be flagged SW_CLOSING.
168192da00bbSMatthew Dillon  *	There may be no processes swapped out to the device.
168292da00bbSMatthew Dillon  *
168392da00bbSMatthew Dillon  *	The sw_used parameter points to the field in the swdev structure
168492da00bbSMatthew Dillon  *	that contains a count of the number of blocks still allocated
168592da00bbSMatthew Dillon  *	on the device.  If we encounter objects with a nonzero pip count
168692da00bbSMatthew Dillon  *	in our scan, we use this number to determine if we're really done.
168792da00bbSMatthew Dillon  *
168892da00bbSMatthew Dillon  *	This routine may block.
168992da00bbSMatthew Dillon  */
1690e9c0cc15SPoul-Henning Kamp static void
16918f60c087SPoul-Henning Kamp swap_pager_swapoff(struct swdevt *sp, int *sw_used)
169292da00bbSMatthew Dillon {
169392da00bbSMatthew Dillon 	struct swblock **pswap;
169492da00bbSMatthew Dillon 	struct swblock *swap;
169592da00bbSMatthew Dillon 	vm_object_t waitobj;
169692da00bbSMatthew Dillon 	daddr_t v;
169792da00bbSMatthew Dillon 	int i, j;
169892da00bbSMatthew Dillon 
169992da00bbSMatthew Dillon 	GIANT_REQUIRED;
170092da00bbSMatthew Dillon 
170192da00bbSMatthew Dillon full_rescan:
170292da00bbSMatthew Dillon 	waitobj = NULL;
170392da00bbSMatthew Dillon 	for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */
170492da00bbSMatthew Dillon restart:
170592da00bbSMatthew Dillon 		pswap = &swhash[i];
1706d536c58fSAlan Cox 		mtx_lock(&swhash_mtx);
170792da00bbSMatthew Dillon 		while ((swap = *pswap) != NULL) {
170892da00bbSMatthew Dillon                         for (j = 0; j < SWAP_META_PAGES; ++j) {
170992da00bbSMatthew Dillon                                 v = swap->swb_pages[j];
171092da00bbSMatthew Dillon                                 if (v != SWAPBLK_NONE &&
17114b03903aSPoul-Henning Kamp 				    swp_pager_find_dev(v) == sp)
171292da00bbSMatthew Dillon                                         break;
171392da00bbSMatthew Dillon                         }
171492da00bbSMatthew Dillon 			if (j < SWAP_META_PAGES) {
171592da00bbSMatthew Dillon 				swp_pager_force_pagein(swap, j);
171692da00bbSMatthew Dillon 				goto restart;
171792da00bbSMatthew Dillon 			} else if (swap->swb_object->paging_in_progress) {
171892da00bbSMatthew Dillon 				if (!waitobj)
171992da00bbSMatthew Dillon 					waitobj = swap->swb_object;
172092da00bbSMatthew Dillon 			}
172192da00bbSMatthew Dillon 			pswap = &swap->swb_hnext;
172292da00bbSMatthew Dillon 		}
1723d536c58fSAlan Cox 		mtx_unlock(&swhash_mtx);
172492da00bbSMatthew Dillon 	}
172592da00bbSMatthew Dillon 	if (waitobj && *sw_used) {
172692da00bbSMatthew Dillon 	    /*
172792da00bbSMatthew Dillon 	     * We wait on an arbitrary object to clock our rescans
172892da00bbSMatthew Dillon 	     * to the rate of paging completion.
172992da00bbSMatthew Dillon 	     */
17301ca58953SAlan Cox 	    VM_OBJECT_LOCK(waitobj);
173192da00bbSMatthew Dillon 	    vm_object_pip_wait(waitobj, "swpoff");
17321ca58953SAlan Cox 	    VM_OBJECT_UNLOCK(waitobj);
173392da00bbSMatthew Dillon 	    goto full_rescan;
173492da00bbSMatthew Dillon 	}
173592da00bbSMatthew Dillon 	if (*sw_used)
173692da00bbSMatthew Dillon 	    panic("swapoff: failed to locate %d swap blocks", *sw_used);
173792da00bbSMatthew Dillon }
173892da00bbSMatthew Dillon 
17391c7c3c6aSMatthew Dillon /************************************************************************
17401c7c3c6aSMatthew Dillon  *				SWAP META DATA 				*
17411c7c3c6aSMatthew Dillon  ************************************************************************
17421c7c3c6aSMatthew Dillon  *
17431c7c3c6aSMatthew Dillon  *	These routines manipulate the swap metadata stored in the
17444dcc5c2dSMatthew Dillon  *	OBJT_SWAP object.  All swp_*() routines must be called at
17454dcc5c2dSMatthew Dillon  *	splvm() because swap can be freed up by the low level vm_page
17464dcc5c2dSMatthew Dillon  *	code which might be called from interrupts beyond what splbio() covers.
17471c7c3c6aSMatthew Dillon  *
17484dcc5c2dSMatthew Dillon  *	Swap metadata is implemented with a global hash and not directly
17494dcc5c2dSMatthew Dillon  *	linked into the object.  Instead the object simply contains
17504dcc5c2dSMatthew Dillon  *	appropriate tracking counters.
17511c7c3c6aSMatthew Dillon  */
17521c7c3c6aSMatthew Dillon 
17531c7c3c6aSMatthew Dillon /*
17541c7c3c6aSMatthew Dillon  * SWP_PAGER_META_BUILD() -	add swap block to swap meta data for object
17551c7c3c6aSMatthew Dillon  *
17561c7c3c6aSMatthew Dillon  *	We first convert the object to a swap object if it is a default
17571c7c3c6aSMatthew Dillon  *	object.
17581c7c3c6aSMatthew Dillon  *
17591c7c3c6aSMatthew Dillon  *	The specified swapblk is added to the object's swap metadata.  If
17601c7c3c6aSMatthew Dillon  *	the swapblk is not valid, it is freed instead.  Any previously
17611c7c3c6aSMatthew Dillon  *	assigned swapblk is freed.
17624dcc5c2dSMatthew Dillon  *
17634dcc5c2dSMatthew Dillon  *	This routine must be called at splvm(), except when used to convert
17644dcc5c2dSMatthew Dillon  *	an OBJT_DEFAULT object into an OBJT_SWAP object.
17651c7c3c6aSMatthew Dillon  */
17661c7c3c6aSMatthew Dillon static void
17672f249180SPoul-Henning Kamp swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
17682f249180SPoul-Henning Kamp {
17691c7c3c6aSMatthew Dillon 	struct swblock *swap;
17701c7c3c6aSMatthew Dillon 	struct swblock **pswap;
177123f09d50SIan Dowse 	int idx;
17721c7c3c6aSMatthew Dillon 
1773ee3dc7d7SAlan Cox 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
17741c7c3c6aSMatthew Dillon 	/*
17751c7c3c6aSMatthew Dillon 	 * Convert default object to swap object if necessary
17761c7c3c6aSMatthew Dillon 	 */
17771c7c3c6aSMatthew Dillon 	if (object->type != OBJT_SWAP) {
17781c7c3c6aSMatthew Dillon 		object->type = OBJT_SWAP;
17791c7c3c6aSMatthew Dillon 		object->un_pager.swp.swp_bcount = 0;
17801c7c3c6aSMatthew Dillon 
17811c7c3c6aSMatthew Dillon 		if (object->handle != NULL) {
1782bd228075SAlan Cox 			mtx_lock(&sw_alloc_mtx);
17831c7c3c6aSMatthew Dillon 			TAILQ_INSERT_TAIL(
17841c7c3c6aSMatthew Dillon 			    NOBJLIST(object->handle),
17851c7c3c6aSMatthew Dillon 			    object,
17861c7c3c6aSMatthew Dillon 			    pager_object_list
17871c7c3c6aSMatthew Dillon 			);
1788a9fa2c05SAlfred Perlstein 			mtx_unlock(&sw_alloc_mtx);
17891c7c3c6aSMatthew Dillon 		}
1790bd228075SAlan Cox 	}
17911c7c3c6aSMatthew Dillon 
17921c7c3c6aSMatthew Dillon 	/*
17931c7c3c6aSMatthew Dillon 	 * Locate hash entry.  If not found create, but if we aren't adding
17944dcc5c2dSMatthew Dillon 	 * anything just return.  If we run out of space in the map we wait
17954dcc5c2dSMatthew Dillon 	 * and, since the hash table may have changed, retry.
17961c7c3c6aSMatthew Dillon 	 */
17974dcc5c2dSMatthew Dillon retry:
17987827d9b0SAlan Cox 	mtx_lock(&swhash_mtx);
179923f09d50SIan Dowse 	pswap = swp_pager_hash(object, pindex);
18001c7c3c6aSMatthew Dillon 
18011c7c3c6aSMatthew Dillon 	if ((swap = *pswap) == NULL) {
18021c7c3c6aSMatthew Dillon 		int i;
18031c7c3c6aSMatthew Dillon 
18041c7c3c6aSMatthew Dillon 		if (swapblk == SWAPBLK_NONE)
18057827d9b0SAlan Cox 			goto done;
18061c7c3c6aSMatthew Dillon 
1807670d17b5SJeff Roberson 		swap = *pswap = uma_zalloc(swap_zone, M_NOWAIT);
18084dcc5c2dSMatthew Dillon 		if (swap == NULL) {
18097827d9b0SAlan Cox 			mtx_unlock(&swhash_mtx);
1810ee3dc7d7SAlan Cox 			VM_OBJECT_UNLOCK(object);
18114dcc5c2dSMatthew Dillon 			VM_WAIT;
1812ee3dc7d7SAlan Cox 			VM_OBJECT_LOCK(object);
18134dcc5c2dSMatthew Dillon 			goto retry;
18144dcc5c2dSMatthew Dillon 		}
1815670d17b5SJeff Roberson 
18161c7c3c6aSMatthew Dillon 		swap->swb_hnext = NULL;
18171c7c3c6aSMatthew Dillon 		swap->swb_object = object;
181823f09d50SIan Dowse 		swap->swb_index = pindex & ~(vm_pindex_t)SWAP_META_MASK;
18191c7c3c6aSMatthew Dillon 		swap->swb_count = 0;
18201c7c3c6aSMatthew Dillon 
18211c7c3c6aSMatthew Dillon 		++object->un_pager.swp.swp_bcount;
18221c7c3c6aSMatthew Dillon 
18231c7c3c6aSMatthew Dillon 		for (i = 0; i < SWAP_META_PAGES; ++i)
18241c7c3c6aSMatthew Dillon 			swap->swb_pages[i] = SWAPBLK_NONE;
18251c7c3c6aSMatthew Dillon 	}
18261c7c3c6aSMatthew Dillon 
18271c7c3c6aSMatthew Dillon 	/*
18281c7c3c6aSMatthew Dillon 	 * Delete prior contents of metadata
18291c7c3c6aSMatthew Dillon 	 */
183023f09d50SIan Dowse 	idx = pindex & SWAP_META_MASK;
18311c7c3c6aSMatthew Dillon 
183223f09d50SIan Dowse 	if (swap->swb_pages[idx] != SWAPBLK_NONE) {
183323f09d50SIan Dowse 		swp_pager_freeswapspace(swap->swb_pages[idx], 1);
18341c7c3c6aSMatthew Dillon 		--swap->swb_count;
18351c7c3c6aSMatthew Dillon 	}
18361c7c3c6aSMatthew Dillon 
18371c7c3c6aSMatthew Dillon 	/*
18381c7c3c6aSMatthew Dillon 	 * Enter block into metadata
18391c7c3c6aSMatthew Dillon 	 */
184023f09d50SIan Dowse 	swap->swb_pages[idx] = swapblk;
18414dcc5c2dSMatthew Dillon 	if (swapblk != SWAPBLK_NONE)
18421c7c3c6aSMatthew Dillon 		++swap->swb_count;
18437827d9b0SAlan Cox done:
18447827d9b0SAlan Cox 	mtx_unlock(&swhash_mtx);
18451c7c3c6aSMatthew Dillon }
18461c7c3c6aSMatthew Dillon 
18471c7c3c6aSMatthew Dillon /*
18481c7c3c6aSMatthew Dillon  * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
18491c7c3c6aSMatthew Dillon  *
18501c7c3c6aSMatthew Dillon  *	The requested range of blocks is freed, with any associated swap
18511c7c3c6aSMatthew Dillon  *	returned to the swap bitmap.
18521c7c3c6aSMatthew Dillon  *
18531c7c3c6aSMatthew Dillon  *	This routine will free swap metadata structures as they are cleaned
18541c7c3c6aSMatthew Dillon  *	out.  This routine does *NOT* operate on swap metadata associated
18551c7c3c6aSMatthew Dillon  *	with resident pages.
18561c7c3c6aSMatthew Dillon  *
18571c7c3c6aSMatthew Dillon  *	This routine must be called at splvm()
18581c7c3c6aSMatthew Dillon  */
18591c7c3c6aSMatthew Dillon static void
18604dcc5c2dSMatthew Dillon swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count)
18611c7c3c6aSMatthew Dillon {
18622928cef7SAlan Cox 
1863ee3dc7d7SAlan Cox 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
18641c7c3c6aSMatthew Dillon 	if (object->type != OBJT_SWAP)
18651c7c3c6aSMatthew Dillon 		return;
18661c7c3c6aSMatthew Dillon 
18671c7c3c6aSMatthew Dillon 	while (count > 0) {
18681c7c3c6aSMatthew Dillon 		struct swblock **pswap;
18691c7c3c6aSMatthew Dillon 		struct swblock *swap;
18701c7c3c6aSMatthew Dillon 
18717827d9b0SAlan Cox 		mtx_lock(&swhash_mtx);
18721c7c3c6aSMatthew Dillon 		pswap = swp_pager_hash(object, index);
18731c7c3c6aSMatthew Dillon 
18741c7c3c6aSMatthew Dillon 		if ((swap = *pswap) != NULL) {
18751c7c3c6aSMatthew Dillon 			daddr_t v = swap->swb_pages[index & SWAP_META_MASK];
18761c7c3c6aSMatthew Dillon 
18771c7c3c6aSMatthew Dillon 			if (v != SWAPBLK_NONE) {
18781c7c3c6aSMatthew Dillon 				swp_pager_freeswapspace(v, 1);
18791c7c3c6aSMatthew Dillon 				swap->swb_pages[index & SWAP_META_MASK] =
18801c7c3c6aSMatthew Dillon 					SWAPBLK_NONE;
18811c7c3c6aSMatthew Dillon 				if (--swap->swb_count == 0) {
18821c7c3c6aSMatthew Dillon 					*pswap = swap->swb_hnext;
1883670d17b5SJeff Roberson 					uma_zfree(swap_zone, swap);
18841c7c3c6aSMatthew Dillon 					--object->un_pager.swp.swp_bcount;
18851c7c3c6aSMatthew Dillon 				}
18861c7c3c6aSMatthew Dillon 			}
18871c7c3c6aSMatthew Dillon 			--count;
18881c7c3c6aSMatthew Dillon 			++index;
18891c7c3c6aSMatthew Dillon 		} else {
18904dcc5c2dSMatthew Dillon 			int n = SWAP_META_PAGES - (index & SWAP_META_MASK);
18911c7c3c6aSMatthew Dillon 			count -= n;
18921c7c3c6aSMatthew Dillon 			index += n;
18931c7c3c6aSMatthew Dillon 		}
18947827d9b0SAlan Cox 		mtx_unlock(&swhash_mtx);
18951c7c3c6aSMatthew Dillon 	}
18961c7c3c6aSMatthew Dillon }
18971c7c3c6aSMatthew Dillon 
18981c7c3c6aSMatthew Dillon /*
18991c7c3c6aSMatthew Dillon  * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
19001c7c3c6aSMatthew Dillon  *
19011c7c3c6aSMatthew Dillon  *	This routine locates and destroys all swap metadata associated with
19021c7c3c6aSMatthew Dillon  *	an object.
19034dcc5c2dSMatthew Dillon  *
19044dcc5c2dSMatthew Dillon  *	This routine must be called at splvm()
19051c7c3c6aSMatthew Dillon  */
19061c7c3c6aSMatthew Dillon static void
19071c7c3c6aSMatthew Dillon swp_pager_meta_free_all(vm_object_t object)
19081c7c3c6aSMatthew Dillon {
19091c7c3c6aSMatthew Dillon 	daddr_t index = 0;
19101c7c3c6aSMatthew Dillon 
1911ee3dc7d7SAlan Cox 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
19121c7c3c6aSMatthew Dillon 	if (object->type != OBJT_SWAP)
19131c7c3c6aSMatthew Dillon 		return;
19141c7c3c6aSMatthew Dillon 
19151c7c3c6aSMatthew Dillon 	while (object->un_pager.swp.swp_bcount) {
19161c7c3c6aSMatthew Dillon 		struct swblock **pswap;
19171c7c3c6aSMatthew Dillon 		struct swblock *swap;
19181c7c3c6aSMatthew Dillon 
19197827d9b0SAlan Cox 		mtx_lock(&swhash_mtx);
19201c7c3c6aSMatthew Dillon 		pswap = swp_pager_hash(object, index);
19211c7c3c6aSMatthew Dillon 		if ((swap = *pswap) != NULL) {
19221c7c3c6aSMatthew Dillon 			int i;
19231c7c3c6aSMatthew Dillon 
19241c7c3c6aSMatthew Dillon 			for (i = 0; i < SWAP_META_PAGES; ++i) {
19251c7c3c6aSMatthew Dillon 				daddr_t v = swap->swb_pages[i];
19261c7c3c6aSMatthew Dillon 				if (v != SWAPBLK_NONE) {
19271c7c3c6aSMatthew Dillon 					--swap->swb_count;
19284dcc5c2dSMatthew Dillon 					swp_pager_freeswapspace(v, 1);
19291c7c3c6aSMatthew Dillon 				}
19301c7c3c6aSMatthew Dillon 			}
19311c7c3c6aSMatthew Dillon 			if (swap->swb_count != 0)
19321c7c3c6aSMatthew Dillon 				panic("swap_pager_meta_free_all: swb_count != 0");
19331c7c3c6aSMatthew Dillon 			*pswap = swap->swb_hnext;
1934670d17b5SJeff Roberson 			uma_zfree(swap_zone, swap);
19351c7c3c6aSMatthew Dillon 			--object->un_pager.swp.swp_bcount;
19361c7c3c6aSMatthew Dillon 		}
19377827d9b0SAlan Cox 		mtx_unlock(&swhash_mtx);
19381c7c3c6aSMatthew Dillon 		index += SWAP_META_PAGES;
19391c7c3c6aSMatthew Dillon 		if (index > 0x20000000)
19401c7c3c6aSMatthew Dillon 			panic("swp_pager_meta_free_all: failed to locate all swap meta blocks");
19411c7c3c6aSMatthew Dillon 	}
19421c7c3c6aSMatthew Dillon }
19431c7c3c6aSMatthew Dillon 
19441c7c3c6aSMatthew Dillon /*
19451c7c3c6aSMatthew Dillon  * SWP_PAGER_METACTL() -  misc control of swap and vm_page_t meta data.
19461c7c3c6aSMatthew Dillon  *
19471c7c3c6aSMatthew Dillon  *	This routine is capable of looking up, popping, or freeing
19481c7c3c6aSMatthew Dillon  *	swapblk assignments in the swap meta data or in the vm_page_t.
19491c7c3c6aSMatthew Dillon  *	The routine typically returns the swapblk being looked-up, or popped,
19501c7c3c6aSMatthew Dillon  *	or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block
19511c7c3c6aSMatthew Dillon  *	was invalid.  This routine will automatically free any invalid
19521c7c3c6aSMatthew Dillon  *	meta-data swapblks.
19531c7c3c6aSMatthew Dillon  *
19541c7c3c6aSMatthew Dillon  *	It is not possible to store invalid swapblks in the swap meta data
19551c7c3c6aSMatthew Dillon  *	(other then a literal 'SWAPBLK_NONE'), so we don't bother checking.
19561c7c3c6aSMatthew Dillon  *
19571c7c3c6aSMatthew Dillon  *	When acting on a busy resident page and paging is in progress, we
19581c7c3c6aSMatthew Dillon  *	have to wait until paging is complete but otherwise can act on the
19591c7c3c6aSMatthew Dillon  *	busy page.
19601c7c3c6aSMatthew Dillon  *
19614dcc5c2dSMatthew Dillon  *	This routine must be called at splvm().
19621c7c3c6aSMatthew Dillon  *
19634dcc5c2dSMatthew Dillon  *	SWM_FREE	remove and free swap block from metadata
19641c7c3c6aSMatthew Dillon  *	SWM_POP		remove from meta data but do not free.. pop it out
19651c7c3c6aSMatthew Dillon  */
19661c7c3c6aSMatthew Dillon static daddr_t
19672f249180SPoul-Henning Kamp swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
19682f249180SPoul-Henning Kamp {
19694dcc5c2dSMatthew Dillon 	struct swblock **pswap;
19704dcc5c2dSMatthew Dillon 	struct swblock *swap;
19714dcc5c2dSMatthew Dillon 	daddr_t r1;
197223f09d50SIan Dowse 	int idx;
19734dcc5c2dSMatthew Dillon 
1974c7c8dd7eSAlan Cox 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
19751c7c3c6aSMatthew Dillon 	/*
19761c7c3c6aSMatthew Dillon 	 * The meta data only exists of the object is OBJT_SWAP
19771c7c3c6aSMatthew Dillon 	 * and even then might not be allocated yet.
19781c7c3c6aSMatthew Dillon 	 */
19794dcc5c2dSMatthew Dillon 	if (object->type != OBJT_SWAP)
19801c7c3c6aSMatthew Dillon 		return (SWAPBLK_NONE);
19811c7c3c6aSMatthew Dillon 
19824dcc5c2dSMatthew Dillon 	r1 = SWAPBLK_NONE;
19837827d9b0SAlan Cox 	mtx_lock(&swhash_mtx);
198423f09d50SIan Dowse 	pswap = swp_pager_hash(object, pindex);
19851c7c3c6aSMatthew Dillon 
19861c7c3c6aSMatthew Dillon 	if ((swap = *pswap) != NULL) {
198723f09d50SIan Dowse 		idx = pindex & SWAP_META_MASK;
198823f09d50SIan Dowse 		r1 = swap->swb_pages[idx];
19891c7c3c6aSMatthew Dillon 
19901c7c3c6aSMatthew Dillon 		if (r1 != SWAPBLK_NONE) {
19911c7c3c6aSMatthew Dillon 			if (flags & SWM_FREE) {
19924dcc5c2dSMatthew Dillon 				swp_pager_freeswapspace(r1, 1);
19931c7c3c6aSMatthew Dillon 				r1 = SWAPBLK_NONE;
19941c7c3c6aSMatthew Dillon 			}
19951c7c3c6aSMatthew Dillon 			if (flags & (SWM_FREE|SWM_POP)) {
199623f09d50SIan Dowse 				swap->swb_pages[idx] = SWAPBLK_NONE;
19971c7c3c6aSMatthew Dillon 				if (--swap->swb_count == 0) {
19981c7c3c6aSMatthew Dillon 					*pswap = swap->swb_hnext;
1999670d17b5SJeff Roberson 					uma_zfree(swap_zone, swap);
20001c7c3c6aSMatthew Dillon 					--object->un_pager.swp.swp_bcount;
20011c7c3c6aSMatthew Dillon 				}
20021c7c3c6aSMatthew Dillon 			}
20031c7c3c6aSMatthew Dillon 		}
20041c7c3c6aSMatthew Dillon 	}
20057827d9b0SAlan Cox 	mtx_unlock(&swhash_mtx);
20061c7c3c6aSMatthew Dillon 	return (r1);
20071c7c3c6aSMatthew Dillon }
20081c7c3c6aSMatthew Dillon 
2009e9c0cc15SPoul-Henning Kamp /*
2010e9c0cc15SPoul-Henning Kamp  * System call swapon(name) enables swapping on device name,
2011e9c0cc15SPoul-Henning Kamp  * which must be in the swdevsw.  Return EBUSY
2012e9c0cc15SPoul-Henning Kamp  * if already swapping on this device.
2013e9c0cc15SPoul-Henning Kamp  */
2014e9c0cc15SPoul-Henning Kamp #ifndef _SYS_SYSPROTO_H_
2015e9c0cc15SPoul-Henning Kamp struct swapon_args {
2016e9c0cc15SPoul-Henning Kamp 	char *name;
2017e9c0cc15SPoul-Henning Kamp };
2018e9c0cc15SPoul-Henning Kamp #endif
2019e9c0cc15SPoul-Henning Kamp 
2020e9c0cc15SPoul-Henning Kamp /*
2021e9c0cc15SPoul-Henning Kamp  * MPSAFE
2022e9c0cc15SPoul-Henning Kamp  */
2023e9c0cc15SPoul-Henning Kamp /* ARGSUSED */
2024e9c0cc15SPoul-Henning Kamp int
20252f249180SPoul-Henning Kamp swapon(struct thread *td, struct swapon_args *uap)
2026e9c0cc15SPoul-Henning Kamp {
2027e9c0cc15SPoul-Henning Kamp 	struct vattr attr;
2028e9c0cc15SPoul-Henning Kamp 	struct vnode *vp;
2029e9c0cc15SPoul-Henning Kamp 	struct nameidata nd;
2030e9c0cc15SPoul-Henning Kamp 	int error;
2031e9c0cc15SPoul-Henning Kamp 
2032e9c0cc15SPoul-Henning Kamp 	mtx_lock(&Giant);
2033e9c0cc15SPoul-Henning Kamp 	error = suser(td);
2034e9c0cc15SPoul-Henning Kamp 	if (error)
2035e9c0cc15SPoul-Henning Kamp 		goto done2;
2036e9c0cc15SPoul-Henning Kamp 
2037e9c0cc15SPoul-Henning Kamp 	while (swdev_syscall_active)
2038e9c0cc15SPoul-Henning Kamp 	    tsleep(&swdev_syscall_active, PUSER - 1, "swpon", 0);
2039e9c0cc15SPoul-Henning Kamp 	swdev_syscall_active = 1;
2040e9c0cc15SPoul-Henning Kamp 
2041e9c0cc15SPoul-Henning Kamp 	/*
2042e9c0cc15SPoul-Henning Kamp 	 * Swap metadata may not fit in the KVM if we have physical
2043e9c0cc15SPoul-Henning Kamp 	 * memory of >1GB.
2044e9c0cc15SPoul-Henning Kamp 	 */
2045e9c0cc15SPoul-Henning Kamp 	if (swap_zone == NULL) {
2046e9c0cc15SPoul-Henning Kamp 		error = ENOMEM;
2047e9c0cc15SPoul-Henning Kamp 		goto done;
2048e9c0cc15SPoul-Henning Kamp 	}
2049e9c0cc15SPoul-Henning Kamp 
2050e9c0cc15SPoul-Henning Kamp 	NDINIT(&nd, LOOKUP, FOLLOW, UIO_USERSPACE, uap->name, td);
2051e9c0cc15SPoul-Henning Kamp 	error = namei(&nd);
2052e9c0cc15SPoul-Henning Kamp 	if (error)
2053e9c0cc15SPoul-Henning Kamp 		goto done;
2054e9c0cc15SPoul-Henning Kamp 
2055e9c0cc15SPoul-Henning Kamp 	NDFREE(&nd, NDF_ONLY_PNBUF);
2056e9c0cc15SPoul-Henning Kamp 	vp = nd.ni_vp;
2057e9c0cc15SPoul-Henning Kamp 
205820da9c2eSPoul-Henning Kamp 	if (vn_isdisk(vp, &error)) {
2059dee34ca4SPoul-Henning Kamp 		error = swapongeom(td, vp);
206020da9c2eSPoul-Henning Kamp 	} else if (vp->v_type == VREG &&
2061e9c0cc15SPoul-Henning Kamp 	    (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
2062e9c0cc15SPoul-Henning Kamp 	    (error = VOP_GETATTR(vp, &attr, td->td_ucred, td)) == 0) {
2063e9c0cc15SPoul-Henning Kamp 		/*
2064e9c0cc15SPoul-Henning Kamp 		 * Allow direct swapping to NFS regular files in the same
2065e9c0cc15SPoul-Henning Kamp 		 * way that nfs_mountroot() sets up diskless swapping.
2066e9c0cc15SPoul-Henning Kamp 		 */
206759efee01SPoul-Henning Kamp 		error = swaponvp(td, vp, attr.va_size / DEV_BSIZE);
2068e9c0cc15SPoul-Henning Kamp 	}
2069e9c0cc15SPoul-Henning Kamp 
2070e9c0cc15SPoul-Henning Kamp 	if (error)
2071e9c0cc15SPoul-Henning Kamp 		vrele(vp);
2072e9c0cc15SPoul-Henning Kamp done:
2073e9c0cc15SPoul-Henning Kamp 	swdev_syscall_active = 0;
2074e9c0cc15SPoul-Henning Kamp 	wakeup_one(&swdev_syscall_active);
2075e9c0cc15SPoul-Henning Kamp done2:
2076e9c0cc15SPoul-Henning Kamp 	mtx_unlock(&Giant);
2077e9c0cc15SPoul-Henning Kamp 	return (error);
2078e9c0cc15SPoul-Henning Kamp }
2079e9c0cc15SPoul-Henning Kamp 
208059efee01SPoul-Henning Kamp static void
2081f3732fd1SPoul-Henning Kamp swaponsomething(struct vnode *vp, void *id, u_long nblks, sw_strategy_t *strategy, sw_close_t *close, dev_t dev)
2082e9c0cc15SPoul-Henning Kamp {
20832d9974c1SAlan Cox 	struct swdevt *sp, *tsp;
2084e9c0cc15SPoul-Henning Kamp 	swblk_t dvbase;
20858f60c087SPoul-Henning Kamp 	u_long mblocks;
2086e9c0cc15SPoul-Henning Kamp 
2087e9c0cc15SPoul-Henning Kamp 	/*
2088e9c0cc15SPoul-Henning Kamp 	 * If we go beyond this, we get overflows in the radix
2089e9c0cc15SPoul-Henning Kamp 	 * tree bitmap code.
2090e9c0cc15SPoul-Henning Kamp 	 */
20918f60c087SPoul-Henning Kamp 	mblocks = 0x40000000 / BLIST_META_RADIX;
2092d3dd89abSPoul-Henning Kamp 	if (nblks > mblocks) {
209385fdafb9SPoul-Henning Kamp 		printf("WARNING: reducing size to maximum of %lu blocks per swap unit\n",
2094d3dd89abSPoul-Henning Kamp 			mblocks);
2095d3dd89abSPoul-Henning Kamp 		nblks = mblocks;
2096e9c0cc15SPoul-Henning Kamp 	}
2097e9c0cc15SPoul-Henning Kamp 	/*
2098e9c0cc15SPoul-Henning Kamp 	 * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks.
2099e9c0cc15SPoul-Henning Kamp 	 * First chop nblks off to page-align it, then convert.
2100e9c0cc15SPoul-Henning Kamp 	 *
2101e9c0cc15SPoul-Henning Kamp 	 * sw->sw_nblks is in page-sized chunks now too.
2102e9c0cc15SPoul-Henning Kamp 	 */
2103e9c0cc15SPoul-Henning Kamp 	nblks &= ~(ctodb(1) - 1);
2104e9c0cc15SPoul-Henning Kamp 	nblks = dbtoc(nblks);
2105e9c0cc15SPoul-Henning Kamp 
21068f60c087SPoul-Henning Kamp 	sp = malloc(sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO);
2107dee34ca4SPoul-Henning Kamp 	sp->sw_vp = vp;
2108dee34ca4SPoul-Henning Kamp 	sp->sw_id = id;
2109f3732fd1SPoul-Henning Kamp 	sp->sw_dev = dev;
21108d677ef9SPoul-Henning Kamp 	sp->sw_flags = 0;
2111e9c0cc15SPoul-Henning Kamp 	sp->sw_nblks = nblks;
2112e9c0cc15SPoul-Henning Kamp 	sp->sw_used = 0;
211359efee01SPoul-Henning Kamp 	sp->sw_strategy = strategy;
2114dee34ca4SPoul-Henning Kamp 	sp->sw_close = close;
2115e9c0cc15SPoul-Henning Kamp 
21168f60c087SPoul-Henning Kamp 	sp->sw_blist = blist_create(nblks);
2117e9c0cc15SPoul-Henning Kamp 	/*
2118ef3c5abdSPoul-Henning Kamp 	 * Do not free the first two block in order to avoid overwriting
21198f60c087SPoul-Henning Kamp 	 * any bsd label at the front of the partition
2120e9c0cc15SPoul-Henning Kamp 	 */
2121ef3c5abdSPoul-Henning Kamp 	blist_free(sp->sw_blist, 2, nblks - 2);
2122e9c0cc15SPoul-Henning Kamp 
21232d9974c1SAlan Cox 	dvbase = 0;
212420da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
21252d9974c1SAlan Cox 	TAILQ_FOREACH(tsp, &swtailq, sw_list) {
21262d9974c1SAlan Cox 		if (tsp->sw_end >= dvbase) {
21272d9974c1SAlan Cox 			/*
21282d9974c1SAlan Cox 			 * We put one uncovered page between the devices
21292d9974c1SAlan Cox 			 * in order to definitively prevent any cross-device
21302d9974c1SAlan Cox 			 * I/O requests
21312d9974c1SAlan Cox 			 */
21322d9974c1SAlan Cox 			dvbase = tsp->sw_end + 1;
21332d9974c1SAlan Cox 		}
21342d9974c1SAlan Cox 	}
21352d9974c1SAlan Cox 	sp->sw_first = dvbase;
21362d9974c1SAlan Cox 	sp->sw_end = dvbase + nblks;
21378f60c087SPoul-Henning Kamp 	TAILQ_INSERT_TAIL(&swtailq, sp, sw_list);
21388f60c087SPoul-Henning Kamp 	nswapdev++;
21398f60c087SPoul-Henning Kamp 	swap_pager_avail += nblks;
2140d05bc129SAlan Cox 	swp_sizecheck();
2141d05bc129SAlan Cox 	mtx_unlock(&sw_dev_mtx);
214259efee01SPoul-Henning Kamp }
2143e9c0cc15SPoul-Henning Kamp 
2144e9c0cc15SPoul-Henning Kamp /*
2145e9c0cc15SPoul-Henning Kamp  * SYSCALL: swapoff(devname)
2146e9c0cc15SPoul-Henning Kamp  *
2147e9c0cc15SPoul-Henning Kamp  * Disable swapping on the given device.
2148dee34ca4SPoul-Henning Kamp  *
2149dee34ca4SPoul-Henning Kamp  * XXX: Badly designed system call: it should use a device index
2150dee34ca4SPoul-Henning Kamp  * rather than filename as specification.  We keep sw_vp around
2151dee34ca4SPoul-Henning Kamp  * only to make this work.
2152e9c0cc15SPoul-Henning Kamp  */
2153e9c0cc15SPoul-Henning Kamp #ifndef _SYS_SYSPROTO_H_
2154e9c0cc15SPoul-Henning Kamp struct swapoff_args {
2155e9c0cc15SPoul-Henning Kamp 	char *name;
2156e9c0cc15SPoul-Henning Kamp };
2157e9c0cc15SPoul-Henning Kamp #endif
2158e9c0cc15SPoul-Henning Kamp 
2159e9c0cc15SPoul-Henning Kamp /*
2160e9c0cc15SPoul-Henning Kamp  * MPSAFE
2161e9c0cc15SPoul-Henning Kamp  */
2162e9c0cc15SPoul-Henning Kamp /* ARGSUSED */
2163e9c0cc15SPoul-Henning Kamp int
21642f249180SPoul-Henning Kamp swapoff(struct thread *td, struct swapoff_args *uap)
2165e9c0cc15SPoul-Henning Kamp {
2166e9c0cc15SPoul-Henning Kamp 	struct vnode *vp;
2167e9c0cc15SPoul-Henning Kamp 	struct nameidata nd;
2168e9c0cc15SPoul-Henning Kamp 	struct swdevt *sp;
21698f60c087SPoul-Henning Kamp 	u_long nblks, dvbase;
21708f60c087SPoul-Henning Kamp 	int error;
2171e9c0cc15SPoul-Henning Kamp 
2172e9c0cc15SPoul-Henning Kamp 	mtx_lock(&Giant);
2173e9c0cc15SPoul-Henning Kamp 
2174e9c0cc15SPoul-Henning Kamp 	error = suser(td);
2175e9c0cc15SPoul-Henning Kamp 	if (error)
2176e9c0cc15SPoul-Henning Kamp 		goto done2;
2177e9c0cc15SPoul-Henning Kamp 
2178e9c0cc15SPoul-Henning Kamp 	while (swdev_syscall_active)
2179e9c0cc15SPoul-Henning Kamp 	    tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
2180e9c0cc15SPoul-Henning Kamp 	swdev_syscall_active = 1;
2181e9c0cc15SPoul-Henning Kamp 
2182e9c0cc15SPoul-Henning Kamp 	NDINIT(&nd, LOOKUP, FOLLOW, UIO_USERSPACE, uap->name, td);
2183e9c0cc15SPoul-Henning Kamp 	error = namei(&nd);
2184e9c0cc15SPoul-Henning Kamp 	if (error)
2185e9c0cc15SPoul-Henning Kamp 		goto done;
2186e9c0cc15SPoul-Henning Kamp 	NDFREE(&nd, NDF_ONLY_PNBUF);
2187e9c0cc15SPoul-Henning Kamp 	vp = nd.ni_vp;
2188e9c0cc15SPoul-Henning Kamp 
218920da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
21908f60c087SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2191dee34ca4SPoul-Henning Kamp 		if (sp->sw_vp == vp)
2192e9c0cc15SPoul-Henning Kamp 			goto found;
2193e9c0cc15SPoul-Henning Kamp 	}
219420da9c2eSPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
2195e9c0cc15SPoul-Henning Kamp 	error = EINVAL;
2196e9c0cc15SPoul-Henning Kamp 	goto done;
2197e9c0cc15SPoul-Henning Kamp found:
219820da9c2eSPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
2199e9c0cc15SPoul-Henning Kamp #ifdef MAC
2200e9c0cc15SPoul-Henning Kamp 	(void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
2201e9c0cc15SPoul-Henning Kamp 	error = mac_check_system_swapoff(td->td_ucred, vp);
2202e9c0cc15SPoul-Henning Kamp 	(void) VOP_UNLOCK(vp, 0, td);
2203e9c0cc15SPoul-Henning Kamp 	if (error != 0)
2204e9c0cc15SPoul-Henning Kamp 		goto done;
2205e9c0cc15SPoul-Henning Kamp #endif
2206e9c0cc15SPoul-Henning Kamp 
2207e9c0cc15SPoul-Henning Kamp 	nblks = sp->sw_nblks;
2208e9c0cc15SPoul-Henning Kamp 
2209e9c0cc15SPoul-Henning Kamp 	/*
2210e9c0cc15SPoul-Henning Kamp 	 * We can turn off this swap device safely only if the
2211e9c0cc15SPoul-Henning Kamp 	 * available virtual memory in the system will fit the amount
2212e9c0cc15SPoul-Henning Kamp 	 * of data we will have to page back in, plus an epsilon so
2213e9c0cc15SPoul-Henning Kamp 	 * the system doesn't become critically low on swap space.
2214e9c0cc15SPoul-Henning Kamp 	 */
22158f60c087SPoul-Henning Kamp 	if (cnt.v_free_count + cnt.v_cache_count + swap_pager_avail <
2216e9c0cc15SPoul-Henning Kamp 	    nblks + nswap_lowat) {
2217e9c0cc15SPoul-Henning Kamp 		error = ENOMEM;
2218e9c0cc15SPoul-Henning Kamp 		goto done;
2219e9c0cc15SPoul-Henning Kamp 	}
2220e9c0cc15SPoul-Henning Kamp 
2221e9c0cc15SPoul-Henning Kamp 	/*
2222e9c0cc15SPoul-Henning Kamp 	 * Prevent further allocations on this device.
2223e9c0cc15SPoul-Henning Kamp 	 */
22242928cef7SAlan Cox 	mtx_lock(&sw_dev_mtx);
2225e9c0cc15SPoul-Henning Kamp 	sp->sw_flags |= SW_CLOSING;
22268f60c087SPoul-Henning Kamp 	for (dvbase = 0; dvbase < sp->sw_end; dvbase += dmmax) {
22278f60c087SPoul-Henning Kamp 		swap_pager_avail -= blist_fill(sp->sw_blist,
22288f60c087SPoul-Henning Kamp 		     dvbase, dmmax);
2229e9c0cc15SPoul-Henning Kamp 	}
22302928cef7SAlan Cox 	mtx_unlock(&sw_dev_mtx);
2231e9c0cc15SPoul-Henning Kamp 
2232e9c0cc15SPoul-Henning Kamp 	/*
2233e9c0cc15SPoul-Henning Kamp 	 * Page in the contents of the device and close it.
2234e9c0cc15SPoul-Henning Kamp 	 */
2235e9c0cc15SPoul-Henning Kamp #ifndef NO_SWAPPING
22368f60c087SPoul-Henning Kamp        	vm_proc_swapin_all(sp);
2237e9c0cc15SPoul-Henning Kamp #endif /* !NO_SWAPPING */
22388f60c087SPoul-Henning Kamp 	swap_pager_swapoff(sp, &sp->sw_used);
2239e9c0cc15SPoul-Henning Kamp 
2240dee34ca4SPoul-Henning Kamp 	sp->sw_close(td, sp);
224159efee01SPoul-Henning Kamp 	sp->sw_id = NULL;
224220da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
22438f60c087SPoul-Henning Kamp 	TAILQ_REMOVE(&swtailq, sp, sw_list);
22440676a140SAlan Cox 	nswapdev--;
22457dea2c2eSAlan Cox 	if (nswapdev == 0) {
22467dea2c2eSAlan Cox 		swap_pager_full = 2;
22477dea2c2eSAlan Cox 		swap_pager_almost_full = 1;
22487dea2c2eSAlan Cox 	}
22498f60c087SPoul-Henning Kamp 	if (swdevhd == sp)
22508f60c087SPoul-Henning Kamp 		swdevhd = NULL;
2251d05bc129SAlan Cox 	mtx_unlock(&sw_dev_mtx);
22528f60c087SPoul-Henning Kamp 	blist_destroy(sp->sw_blist);
22538f60c087SPoul-Henning Kamp 	free(sp, M_VMPGDATA);
2254e9c0cc15SPoul-Henning Kamp 
2255e9c0cc15SPoul-Henning Kamp done:
2256e9c0cc15SPoul-Henning Kamp 	swdev_syscall_active = 0;
2257e9c0cc15SPoul-Henning Kamp 	wakeup_one(&swdev_syscall_active);
2258e9c0cc15SPoul-Henning Kamp done2:
2259e9c0cc15SPoul-Henning Kamp 	mtx_unlock(&Giant);
2260e9c0cc15SPoul-Henning Kamp 	return (error);
2261e9c0cc15SPoul-Henning Kamp }
2262e9c0cc15SPoul-Henning Kamp 
2263567104a1SPoul-Henning Kamp void
2264567104a1SPoul-Henning Kamp swap_pager_status(int *total, int *used)
2265567104a1SPoul-Henning Kamp {
2266567104a1SPoul-Henning Kamp 	struct swdevt *sp;
2267567104a1SPoul-Henning Kamp 
2268567104a1SPoul-Henning Kamp 	*total = 0;
2269567104a1SPoul-Henning Kamp 	*used = 0;
227020da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
22718f60c087SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2272567104a1SPoul-Henning Kamp 		*total += sp->sw_nblks;
2273567104a1SPoul-Henning Kamp 		*used += sp->sw_used;
2274567104a1SPoul-Henning Kamp 	}
227520da9c2eSPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
2276567104a1SPoul-Henning Kamp }
2277567104a1SPoul-Henning Kamp 
2278e9c0cc15SPoul-Henning Kamp static int
2279e9c0cc15SPoul-Henning Kamp sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS)
2280e9c0cc15SPoul-Henning Kamp {
2281e9c0cc15SPoul-Henning Kamp 	int	*name = (int *)arg1;
22828f60c087SPoul-Henning Kamp 	int	error, n;
2283e9c0cc15SPoul-Henning Kamp 	struct xswdev xs;
2284e9c0cc15SPoul-Henning Kamp 	struct swdevt *sp;
2285e9c0cc15SPoul-Henning Kamp 
2286e9c0cc15SPoul-Henning Kamp 	if (arg2 != 1) /* name length */
2287e9c0cc15SPoul-Henning Kamp 		return (EINVAL);
2288e9c0cc15SPoul-Henning Kamp 
22898f60c087SPoul-Henning Kamp 	n = 0;
229020da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
22918f60c087SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2292e9c0cc15SPoul-Henning Kamp 		if (n == *name) {
229320da9c2eSPoul-Henning Kamp 			mtx_unlock(&sw_dev_mtx);
2294e9c0cc15SPoul-Henning Kamp 			xs.xsw_version = XSWDEV_VERSION;
2295f3732fd1SPoul-Henning Kamp 			xs.xsw_dev = sp->sw_dev;
2296e9c0cc15SPoul-Henning Kamp 			xs.xsw_flags = sp->sw_flags;
2297e9c0cc15SPoul-Henning Kamp 			xs.xsw_nblks = sp->sw_nblks;
2298e9c0cc15SPoul-Henning Kamp 			xs.xsw_used = sp->sw_used;
2299e9c0cc15SPoul-Henning Kamp 
2300e9c0cc15SPoul-Henning Kamp 			error = SYSCTL_OUT(req, &xs, sizeof(xs));
2301e9c0cc15SPoul-Henning Kamp 			return (error);
2302e9c0cc15SPoul-Henning Kamp 		}
2303e9c0cc15SPoul-Henning Kamp 		n++;
2304e9c0cc15SPoul-Henning Kamp 	}
230520da9c2eSPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
2306e9c0cc15SPoul-Henning Kamp 	return (ENOENT);
2307e9c0cc15SPoul-Henning Kamp }
2308e9c0cc15SPoul-Henning Kamp 
23098f60c087SPoul-Henning Kamp SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, &nswapdev, 0,
2310e9c0cc15SPoul-Henning Kamp     "Number of swap devices");
2311e9c0cc15SPoul-Henning Kamp SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD, sysctl_vm_swap_info,
2312e9c0cc15SPoul-Henning Kamp     "Swap statistics by device");
2313ec38b344SPoul-Henning Kamp 
2314ec38b344SPoul-Henning Kamp /*
2315ec38b344SPoul-Henning Kamp  * vmspace_swap_count() - count the approximate swap useage in pages for a
2316ec38b344SPoul-Henning Kamp  *			  vmspace.
2317ec38b344SPoul-Henning Kamp  *
2318ec38b344SPoul-Henning Kamp  *	The map must be locked.
2319ec38b344SPoul-Henning Kamp  *
2320ec38b344SPoul-Henning Kamp  *	Swap useage is determined by taking the proportional swap used by
2321ec38b344SPoul-Henning Kamp  *	VM objects backing the VM map.  To make up for fractional losses,
2322ec38b344SPoul-Henning Kamp  *	if the VM object has any swap use at all the associated map entries
2323ec38b344SPoul-Henning Kamp  *	count for at least 1 swap page.
2324ec38b344SPoul-Henning Kamp  */
2325ec38b344SPoul-Henning Kamp int
2326ec38b344SPoul-Henning Kamp vmspace_swap_count(struct vmspace *vmspace)
2327ec38b344SPoul-Henning Kamp {
2328ec38b344SPoul-Henning Kamp 	vm_map_t map = &vmspace->vm_map;
2329ec38b344SPoul-Henning Kamp 	vm_map_entry_t cur;
2330ec38b344SPoul-Henning Kamp 	int count = 0;
2331ec38b344SPoul-Henning Kamp 
2332ec38b344SPoul-Henning Kamp 	for (cur = map->header.next; cur != &map->header; cur = cur->next) {
2333ec38b344SPoul-Henning Kamp 		vm_object_t object;
2334ec38b344SPoul-Henning Kamp 
2335ec38b344SPoul-Henning Kamp 		if ((cur->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
2336ec38b344SPoul-Henning Kamp 		    (object = cur->object.vm_object) != NULL) {
2337ec38b344SPoul-Henning Kamp 			VM_OBJECT_LOCK(object);
2338ec38b344SPoul-Henning Kamp 			if (object->type == OBJT_SWAP &&
2339ec38b344SPoul-Henning Kamp 			    object->un_pager.swp.swp_bcount != 0) {
2340ec38b344SPoul-Henning Kamp 				int n = (cur->end - cur->start) / PAGE_SIZE;
2341ec38b344SPoul-Henning Kamp 
2342ec38b344SPoul-Henning Kamp 				count += object->un_pager.swp.swp_bcount *
2343ec38b344SPoul-Henning Kamp 				    SWAP_META_PAGES * n / object->size + 1;
2344ec38b344SPoul-Henning Kamp 			}
2345ec38b344SPoul-Henning Kamp 			VM_OBJECT_UNLOCK(object);
2346ec38b344SPoul-Henning Kamp 		}
2347ec38b344SPoul-Henning Kamp 	}
2348ec38b344SPoul-Henning Kamp 	return (count);
2349ec38b344SPoul-Henning Kamp }
2350dee34ca4SPoul-Henning Kamp 
2351dee34ca4SPoul-Henning Kamp /*
2352dee34ca4SPoul-Henning Kamp  * GEOM backend
2353dee34ca4SPoul-Henning Kamp  *
2354dee34ca4SPoul-Henning Kamp  * Swapping onto disk devices.
2355dee34ca4SPoul-Henning Kamp  *
2356dee34ca4SPoul-Henning Kamp  */
2357dee34ca4SPoul-Henning Kamp 
23585721c9c7SPoul-Henning Kamp static g_orphan_t swapgeom_orphan;
23595721c9c7SPoul-Henning Kamp 
2360dee34ca4SPoul-Henning Kamp static struct g_class g_swap_class = {
2361dee34ca4SPoul-Henning Kamp 	.name = "SWAP",
23625721c9c7SPoul-Henning Kamp 	.version = G_VERSION,
23635721c9c7SPoul-Henning Kamp 	.orphan = swapgeom_orphan,
2364dee34ca4SPoul-Henning Kamp };
2365dee34ca4SPoul-Henning Kamp 
2366dee34ca4SPoul-Henning Kamp DECLARE_GEOM_CLASS(g_swap_class, g_class);
2367dee34ca4SPoul-Henning Kamp 
2368dee34ca4SPoul-Henning Kamp 
2369dee34ca4SPoul-Henning Kamp static void
2370dee34ca4SPoul-Henning Kamp swapgeom_done(struct bio *bp2)
2371dee34ca4SPoul-Henning Kamp {
2372dee34ca4SPoul-Henning Kamp 	struct buf *bp;
2373dee34ca4SPoul-Henning Kamp 
2374dee34ca4SPoul-Henning Kamp 	bp = bp2->bio_caller2;
2375dee34ca4SPoul-Henning Kamp 	if (bp2->bio_error)
2376dee34ca4SPoul-Henning Kamp 		bp->b_ioflags |= BIO_ERROR;
2377dee34ca4SPoul-Henning Kamp 	bufdone(bp);
2378dee34ca4SPoul-Henning Kamp 	g_destroy_bio(bp2);
2379dee34ca4SPoul-Henning Kamp }
2380dee34ca4SPoul-Henning Kamp 
2381dee34ca4SPoul-Henning Kamp static void
2382dee34ca4SPoul-Henning Kamp swapgeom_strategy(struct buf *bp, struct swdevt *sp)
2383dee34ca4SPoul-Henning Kamp {
2384dee34ca4SPoul-Henning Kamp 	struct bio *bio;
2385dee34ca4SPoul-Henning Kamp 	struct g_consumer *cp;
2386dee34ca4SPoul-Henning Kamp 
2387dee34ca4SPoul-Henning Kamp 	cp = sp->sw_id;
2388dee34ca4SPoul-Henning Kamp 	if (cp == NULL) {
2389dee34ca4SPoul-Henning Kamp 		bp->b_error = ENXIO;
2390dee34ca4SPoul-Henning Kamp 		bp->b_ioflags |= BIO_ERROR;
2391dee34ca4SPoul-Henning Kamp 		bufdone(bp);
2392dee34ca4SPoul-Henning Kamp 		return;
2393dee34ca4SPoul-Henning Kamp 	}
2394dee34ca4SPoul-Henning Kamp 	bio = g_clone_bio(&bp->b_io);
23953e5b6861SPoul-Henning Kamp 	if (bio == NULL) {
23963e5b6861SPoul-Henning Kamp 		/*
23973e5b6861SPoul-Henning Kamp 		 * XXX: This is better than panicing, but not much better.
23983e5b6861SPoul-Henning Kamp 		 * XXX: Somehow this should be retried.  A more generic
23993e5b6861SPoul-Henning Kamp 		 * XXX: implementation of ENOMEM in geom may be able to cope.
24003e5b6861SPoul-Henning Kamp 		 */
24013e5b6861SPoul-Henning Kamp 		bp->b_error = ENOMEM;
24023e5b6861SPoul-Henning Kamp 		bp->b_ioflags |= BIO_ERROR;
24033e5b6861SPoul-Henning Kamp 		bufdone(bp);
24043e5b6861SPoul-Henning Kamp 		return;
24053e5b6861SPoul-Henning Kamp 	}
2406dee34ca4SPoul-Henning Kamp 	bio->bio_caller2 = bp;
2407dee34ca4SPoul-Henning Kamp 	bio->bio_offset = (bp->b_blkno - sp->sw_first) * PAGE_SIZE;
2408dee34ca4SPoul-Henning Kamp 	bio->bio_length = bp->b_bcount;
2409dee34ca4SPoul-Henning Kamp 	bio->bio_done = swapgeom_done;
2410dee34ca4SPoul-Henning Kamp 	g_io_request(bio, cp);
2411dee34ca4SPoul-Henning Kamp 	return;
2412dee34ca4SPoul-Henning Kamp }
2413dee34ca4SPoul-Henning Kamp 
2414dee34ca4SPoul-Henning Kamp static void
2415dee34ca4SPoul-Henning Kamp swapgeom_orphan(struct g_consumer *cp)
2416dee34ca4SPoul-Henning Kamp {
2417dee34ca4SPoul-Henning Kamp 	struct swdevt *sp;
2418dee34ca4SPoul-Henning Kamp 
2419dee34ca4SPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
2420dee34ca4SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list)
2421dee34ca4SPoul-Henning Kamp 		if (sp->sw_id == cp)
2422dee34ca4SPoul-Henning Kamp 			sp->sw_id = NULL;
2423dee34ca4SPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
2424dee34ca4SPoul-Henning Kamp }
2425dee34ca4SPoul-Henning Kamp 
2426dee34ca4SPoul-Henning Kamp static void
2427dee34ca4SPoul-Henning Kamp swapgeom_close_ev(void *arg, int flags)
2428dee34ca4SPoul-Henning Kamp {
2429dee34ca4SPoul-Henning Kamp 	struct g_consumer *cp;
2430dee34ca4SPoul-Henning Kamp 
2431dee34ca4SPoul-Henning Kamp 	cp = arg;
2432d2bae332SPoul-Henning Kamp 	g_access(cp, -1, -1, 0);
2433dee34ca4SPoul-Henning Kamp 	g_detach(cp);
2434dee34ca4SPoul-Henning Kamp 	g_destroy_consumer(cp);
2435dee34ca4SPoul-Henning Kamp }
2436dee34ca4SPoul-Henning Kamp 
2437dee34ca4SPoul-Henning Kamp static void
2438dee34ca4SPoul-Henning Kamp swapgeom_close(struct thread *td, struct swdevt *sw)
2439dee34ca4SPoul-Henning Kamp {
2440dee34ca4SPoul-Henning Kamp 
2441dee34ca4SPoul-Henning Kamp 	/* XXX: direct call when Giant untangled */
2442dee34ca4SPoul-Henning Kamp 	g_waitfor_event(swapgeom_close_ev, sw->sw_id, M_WAITOK, NULL);
2443dee34ca4SPoul-Henning Kamp }
2444dee34ca4SPoul-Henning Kamp 
2445dee34ca4SPoul-Henning Kamp 
2446dee34ca4SPoul-Henning Kamp struct swh0h0 {
244789c9c53dSPoul-Henning Kamp 	struct cdev *dev;
2448dee34ca4SPoul-Henning Kamp 	struct vnode *vp;
2449dee34ca4SPoul-Henning Kamp 	int	error;
2450dee34ca4SPoul-Henning Kamp };
2451dee34ca4SPoul-Henning Kamp 
2452dee34ca4SPoul-Henning Kamp static void
2453dee34ca4SPoul-Henning Kamp swapongeom_ev(void *arg, int flags)
2454dee34ca4SPoul-Henning Kamp {
2455dee34ca4SPoul-Henning Kamp 	struct swh0h0 *swh;
2456dee34ca4SPoul-Henning Kamp 	struct g_provider *pp;
2457dee34ca4SPoul-Henning Kamp 	struct g_consumer *cp;
2458dee34ca4SPoul-Henning Kamp 	static struct g_geom *gp;
2459dee34ca4SPoul-Henning Kamp 	struct swdevt *sp;
2460dee34ca4SPoul-Henning Kamp 	u_long nblks;
2461dee34ca4SPoul-Henning Kamp 	int error;
2462dee34ca4SPoul-Henning Kamp 
2463dee34ca4SPoul-Henning Kamp 	swh = arg;
2464dee34ca4SPoul-Henning Kamp 	swh->error = 0;
2465dee34ca4SPoul-Henning Kamp 	pp = g_dev_getprovider(swh->dev);
2466dee34ca4SPoul-Henning Kamp 	if (pp == NULL) {
2467dee34ca4SPoul-Henning Kamp 		swh->error = ENODEV;
2468dee34ca4SPoul-Henning Kamp 		return;
2469dee34ca4SPoul-Henning Kamp 	}
2470dee34ca4SPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
2471dee34ca4SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2472dee34ca4SPoul-Henning Kamp 		cp = sp->sw_id;
2473dee34ca4SPoul-Henning Kamp 		if (cp != NULL && cp->provider == pp) {
2474dee34ca4SPoul-Henning Kamp 			mtx_unlock(&sw_dev_mtx);
2475dee34ca4SPoul-Henning Kamp 			swh->error = EBUSY;
2476dee34ca4SPoul-Henning Kamp 			return;
2477dee34ca4SPoul-Henning Kamp 		}
2478dee34ca4SPoul-Henning Kamp 	}
2479dee34ca4SPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
24805721c9c7SPoul-Henning Kamp 	if (gp == NULL)
2481dee34ca4SPoul-Henning Kamp 		gp = g_new_geomf(&g_swap_class, "swap", NULL);
2482dee34ca4SPoul-Henning Kamp 	cp = g_new_consumer(gp);
2483dee34ca4SPoul-Henning Kamp 	g_attach(cp, pp);
2484afeb65e6SPoul-Henning Kamp 	/*
2485afeb65e6SPoul-Henning Kamp 	 * XXX: Everytime you think you can improve the margin for
2486afeb65e6SPoul-Henning Kamp 	 * footshooting, somebody depends on the ability to do so:
2487afeb65e6SPoul-Henning Kamp 	 * savecore(8) wants to write to our swapdev so we cannot
2488afeb65e6SPoul-Henning Kamp 	 * set an exclusive count :-(
2489afeb65e6SPoul-Henning Kamp 	 */
2490d2bae332SPoul-Henning Kamp 	error = g_access(cp, 1, 1, 0);
2491dee34ca4SPoul-Henning Kamp 	if (error) {
2492dee34ca4SPoul-Henning Kamp 		g_detach(cp);
2493dee34ca4SPoul-Henning Kamp 		g_destroy_consumer(cp);
2494dee34ca4SPoul-Henning Kamp 		swh->error = error;
2495dee34ca4SPoul-Henning Kamp 		return;
2496dee34ca4SPoul-Henning Kamp 	}
2497dee34ca4SPoul-Henning Kamp 	nblks = pp->mediasize / DEV_BSIZE;
2498dee34ca4SPoul-Henning Kamp 	swaponsomething(swh->vp, cp, nblks, swapgeom_strategy,
2499dee34ca4SPoul-Henning Kamp 	    swapgeom_close, dev2udev(swh->dev));
2500dee34ca4SPoul-Henning Kamp 	swh->error = 0;
2501dee34ca4SPoul-Henning Kamp 	return;
2502dee34ca4SPoul-Henning Kamp }
2503dee34ca4SPoul-Henning Kamp 
2504dee34ca4SPoul-Henning Kamp static int
2505dee34ca4SPoul-Henning Kamp swapongeom(struct thread *td, struct vnode *vp)
2506dee34ca4SPoul-Henning Kamp {
2507dee34ca4SPoul-Henning Kamp 	int error;
2508dee34ca4SPoul-Henning Kamp 	struct swh0h0 swh;
2509dee34ca4SPoul-Henning Kamp 
2510dee34ca4SPoul-Henning Kamp 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
2511dee34ca4SPoul-Henning Kamp 
2512dee34ca4SPoul-Henning Kamp 	swh.dev = vp->v_rdev;
2513dee34ca4SPoul-Henning Kamp 	swh.vp = vp;
2514dee34ca4SPoul-Henning Kamp 	swh.error = 0;
2515dee34ca4SPoul-Henning Kamp 	/* XXX: direct call when Giant untangled */
2516dee34ca4SPoul-Henning Kamp 	error = g_waitfor_event(swapongeom_ev, &swh, M_WAITOK, NULL);
2517dee34ca4SPoul-Henning Kamp 	if (!error)
2518dee34ca4SPoul-Henning Kamp 		error = swh.error;
2519dee34ca4SPoul-Henning Kamp 	VOP_UNLOCK(vp, 0, td);
2520dee34ca4SPoul-Henning Kamp 	return (error);
2521dee34ca4SPoul-Henning Kamp }
2522dee34ca4SPoul-Henning Kamp 
2523dee34ca4SPoul-Henning Kamp /*
2524dee34ca4SPoul-Henning Kamp  * VNODE backend
2525dee34ca4SPoul-Henning Kamp  *
2526dee34ca4SPoul-Henning Kamp  * This is used mainly for network filesystem (read: probably only tested
2527dee34ca4SPoul-Henning Kamp  * with NFS) swapfiles.
2528dee34ca4SPoul-Henning Kamp  *
2529dee34ca4SPoul-Henning Kamp  */
2530dee34ca4SPoul-Henning Kamp 
2531dee34ca4SPoul-Henning Kamp static void
2532dee34ca4SPoul-Henning Kamp swapdev_strategy(struct buf *bp, struct swdevt *sp)
2533dee34ca4SPoul-Henning Kamp {
2534dee34ca4SPoul-Henning Kamp 	int s;
2535dee34ca4SPoul-Henning Kamp 	struct vnode *vp, *vp2;
2536dee34ca4SPoul-Henning Kamp 
2537f3732fd1SPoul-Henning Kamp 	bp->b_dev = NULL;
2538dee34ca4SPoul-Henning Kamp 	bp->b_blkno = ctodb(bp->b_blkno - sp->sw_first);
2539dee34ca4SPoul-Henning Kamp 
2540dee34ca4SPoul-Henning Kamp 	vp2 = sp->sw_id;
2541dee34ca4SPoul-Henning Kamp 	vhold(vp2);
2542dee34ca4SPoul-Henning Kamp 	s = splvm();
2543dee34ca4SPoul-Henning Kamp 	if (bp->b_iocmd == BIO_WRITE) {
2544dee34ca4SPoul-Henning Kamp 		vp = bp->b_vp;
2545dee34ca4SPoul-Henning Kamp 		if (vp) {
2546dee34ca4SPoul-Henning Kamp 			VI_LOCK(vp);
2547dee34ca4SPoul-Henning Kamp 			vp->v_numoutput--;
2548dee34ca4SPoul-Henning Kamp 			if ((vp->v_iflag & VI_BWAIT) && vp->v_numoutput <= 0) {
2549dee34ca4SPoul-Henning Kamp 				vp->v_iflag &= ~VI_BWAIT;
2550dee34ca4SPoul-Henning Kamp 				wakeup(&vp->v_numoutput);
2551dee34ca4SPoul-Henning Kamp 			}
2552dee34ca4SPoul-Henning Kamp 			VI_UNLOCK(vp);
2553dee34ca4SPoul-Henning Kamp 		}
2554dee34ca4SPoul-Henning Kamp 		VI_LOCK(vp2);
2555dee34ca4SPoul-Henning Kamp 		vp2->v_numoutput++;
2556dee34ca4SPoul-Henning Kamp 		VI_UNLOCK(vp2);
2557dee34ca4SPoul-Henning Kamp 	}
2558dee34ca4SPoul-Henning Kamp 	bp->b_vp = vp2;
2559dee34ca4SPoul-Henning Kamp 	splx(s);
25602c18019fSPoul-Henning Kamp 	bp->b_iooffset = dbtob(bp->b_blkno);
2561dee34ca4SPoul-Henning Kamp 	VOP_STRATEGY(vp2, bp);
2562dee34ca4SPoul-Henning Kamp 	return;
2563dee34ca4SPoul-Henning Kamp }
2564dee34ca4SPoul-Henning Kamp 
2565dee34ca4SPoul-Henning Kamp static void
2566dee34ca4SPoul-Henning Kamp swapdev_close(struct thread *td, struct swdevt *sp)
2567dee34ca4SPoul-Henning Kamp {
2568dee34ca4SPoul-Henning Kamp 
2569dee34ca4SPoul-Henning Kamp 	VOP_CLOSE(sp->sw_vp, FREAD | FWRITE, td->td_ucred, td);
2570dee34ca4SPoul-Henning Kamp 	vrele(sp->sw_vp);
2571dee34ca4SPoul-Henning Kamp }
2572dee34ca4SPoul-Henning Kamp 
2573dee34ca4SPoul-Henning Kamp 
2574dee34ca4SPoul-Henning Kamp static int
2575dee34ca4SPoul-Henning Kamp swaponvp(struct thread *td, struct vnode *vp, u_long nblks)
2576dee34ca4SPoul-Henning Kamp {
2577dee34ca4SPoul-Henning Kamp 	struct swdevt *sp;
2578dee34ca4SPoul-Henning Kamp 	int error;
2579dee34ca4SPoul-Henning Kamp 
2580dee34ca4SPoul-Henning Kamp 	if (nblks == 0)
2581dee34ca4SPoul-Henning Kamp 		return (ENXIO);
2582dee34ca4SPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
2583dee34ca4SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2584dee34ca4SPoul-Henning Kamp 		if (sp->sw_id == vp) {
2585dee34ca4SPoul-Henning Kamp 			mtx_unlock(&sw_dev_mtx);
2586dee34ca4SPoul-Henning Kamp 			return (EBUSY);
2587dee34ca4SPoul-Henning Kamp 		}
2588dee34ca4SPoul-Henning Kamp 	}
2589dee34ca4SPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
2590dee34ca4SPoul-Henning Kamp 
2591dee34ca4SPoul-Henning Kamp 	(void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
2592dee34ca4SPoul-Henning Kamp #ifdef MAC
2593dee34ca4SPoul-Henning Kamp 	error = mac_check_system_swapon(td->td_ucred, vp);
2594dee34ca4SPoul-Henning Kamp 	if (error == 0)
2595dee34ca4SPoul-Henning Kamp #endif
2596dee34ca4SPoul-Henning Kamp 		error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, -1);
2597dee34ca4SPoul-Henning Kamp 	(void) VOP_UNLOCK(vp, 0, td);
2598dee34ca4SPoul-Henning Kamp 	if (error)
2599dee34ca4SPoul-Henning Kamp 		return (error);
2600dee34ca4SPoul-Henning Kamp 
2601dee34ca4SPoul-Henning Kamp 	swaponsomething(vp, vp, nblks, swapdev_strategy, swapdev_close,
2602f3732fd1SPoul-Henning Kamp 	    NODEV);
2603dee34ca4SPoul-Henning Kamp 	return (0);
2604dee34ca4SPoul-Henning Kamp }
2605