xref: /freebsd/sys/vm/swap_pager.c (revision 4b5c9cf62f33d8359a272dda846930aaa7c635f8)
160727d8bSWarner Losh /*-
21c7c3c6aSMatthew Dillon  * Copyright (c) 1998 Matthew Dillon,
326f9a767SRodney W. Grimes  * Copyright (c) 1994 John S. Dyson
4df8bae1dSRodney W. Grimes  * Copyright (c) 1990 University of Utah.
5e9c0cc15SPoul-Henning Kamp  * Copyright (c) 1982, 1986, 1989, 1993
6df8bae1dSRodney W. Grimes  *	The Regents of the University of California.  All rights reserved.
7df8bae1dSRodney W. Grimes  *
8df8bae1dSRodney W. Grimes  * This code is derived from software contributed to Berkeley by
9df8bae1dSRodney W. Grimes  * the Systems Programming Group of the University of Utah Computer
10df8bae1dSRodney W. Grimes  * Science Department.
11df8bae1dSRodney W. Grimes  *
12df8bae1dSRodney W. Grimes  * Redistribution and use in source and binary forms, with or without
13df8bae1dSRodney W. Grimes  * modification, are permitted provided that the following conditions
14df8bae1dSRodney W. Grimes  * are met:
15df8bae1dSRodney W. Grimes  * 1. Redistributions of source code must retain the above copyright
16df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer.
17df8bae1dSRodney W. Grimes  * 2. Redistributions in binary form must reproduce the above copyright
18df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer in the
19df8bae1dSRodney W. Grimes  *    documentation and/or other materials provided with the distribution.
20df8bae1dSRodney W. Grimes  * 3. All advertising materials mentioning features or use of this software
215929bcfaSPhilippe Charnier  *    must display the following acknowledgement:
22df8bae1dSRodney W. Grimes  *	This product includes software developed by the University of
23df8bae1dSRodney W. Grimes  *	California, Berkeley and its contributors.
24df8bae1dSRodney W. Grimes  * 4. Neither the name of the University nor the names of its contributors
25df8bae1dSRodney W. Grimes  *    may be used to endorse or promote products derived from this software
26df8bae1dSRodney W. Grimes  *    without specific prior written permission.
27df8bae1dSRodney W. Grimes  *
28df8bae1dSRodney W. Grimes  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
29df8bae1dSRodney W. Grimes  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30df8bae1dSRodney W. Grimes  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31df8bae1dSRodney W. Grimes  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
32df8bae1dSRodney W. Grimes  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33df8bae1dSRodney W. Grimes  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34df8bae1dSRodney W. Grimes  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35df8bae1dSRodney W. Grimes  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36df8bae1dSRodney W. Grimes  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37df8bae1dSRodney W. Grimes  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38df8bae1dSRodney W. Grimes  * SUCH DAMAGE.
39df8bae1dSRodney W. Grimes  *
401c7c3c6aSMatthew Dillon  *				New Swap System
411c7c3c6aSMatthew Dillon  *				Matthew Dillon
421c7c3c6aSMatthew Dillon  *
431c7c3c6aSMatthew Dillon  * Radix Bitmap 'blists'.
441c7c3c6aSMatthew Dillon  *
451c7c3c6aSMatthew Dillon  *	- The new swapper uses the new radix bitmap code.  This should scale
461c7c3c6aSMatthew Dillon  *	  to arbitrarily small or arbitrarily large swap spaces and an almost
471c7c3c6aSMatthew Dillon  *	  arbitrary degree of fragmentation.
481c7c3c6aSMatthew Dillon  *
491c7c3c6aSMatthew Dillon  * Features:
501c7c3c6aSMatthew Dillon  *
511c7c3c6aSMatthew Dillon  *	- on the fly reallocation of swap during putpages.  The new system
521c7c3c6aSMatthew Dillon  *	  does not try to keep previously allocated swap blocks for dirty
531c7c3c6aSMatthew Dillon  *	  pages.
541c7c3c6aSMatthew Dillon  *
551c7c3c6aSMatthew Dillon  *	- on the fly deallocation of swap
561c7c3c6aSMatthew Dillon  *
571c7c3c6aSMatthew Dillon  *	- No more garbage collection required.  Unnecessarily allocated swap
581c7c3c6aSMatthew Dillon  *	  blocks only exist for dirty vm_page_t's now and these are already
591c7c3c6aSMatthew Dillon  *	  cycled (in a high-load system) by the pager.  We also do on-the-fly
601c7c3c6aSMatthew Dillon  *	  removal of invalidated swap blocks when a page is destroyed
611c7c3c6aSMatthew Dillon  *	  or renamed.
621c7c3c6aSMatthew Dillon  *
63df8bae1dSRodney W. Grimes  * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
64df8bae1dSRodney W. Grimes  *
65df8bae1dSRodney W. Grimes  *	@(#)swap_pager.c	8.9 (Berkeley) 3/21/94
66e9c0cc15SPoul-Henning Kamp  *	@(#)vm_swap.c	8.5 (Berkeley) 2/17/94
67df8bae1dSRodney W. Grimes  */
68df8bae1dSRodney W. Grimes 
69874651b1SDavid E. O'Brien #include <sys/cdefs.h>
70874651b1SDavid E. O'Brien __FBSDID("$FreeBSD$");
71874651b1SDavid E. O'Brien 
72e9c0cc15SPoul-Henning Kamp #include "opt_swap.h"
73e9c0cc15SPoul-Henning Kamp #include "opt_vm.h"
74e9c0cc15SPoul-Henning Kamp 
75df8bae1dSRodney W. Grimes #include <sys/param.h>
76df8bae1dSRodney W. Grimes #include <sys/systm.h>
77af647ddeSBruce Evans #include <sys/conf.h>
7864abb5a5SDavid Greenman #include <sys/kernel.h>
79acd3428bSRobert Watson #include <sys/priv.h>
80df8bae1dSRodney W. Grimes #include <sys/proc.h>
819626b608SPoul-Henning Kamp #include <sys/bio.h>
82df8bae1dSRodney W. Grimes #include <sys/buf.h>
83e9c0cc15SPoul-Henning Kamp #include <sys/disk.h>
84e9c0cc15SPoul-Henning Kamp #include <sys/fcntl.h>
85e9c0cc15SPoul-Henning Kamp #include <sys/mount.h>
86e9c0cc15SPoul-Henning Kamp #include <sys/namei.h>
87df8bae1dSRodney W. Grimes #include <sys/vnode.h>
88df8bae1dSRodney W. Grimes #include <sys/malloc.h>
891ba5ad42SEdward Tomasz Napierala #include <sys/racct.h>
903364c323SKonstantin Belousov #include <sys/resource.h>
913364c323SKonstantin Belousov #include <sys/resourcevar.h>
9289f6b863SAttilio Rao #include <sys/rwlock.h>
93327f4e83SMatthew Dillon #include <sys/sysctl.h>
94e9c0cc15SPoul-Henning Kamp #include <sys/sysproto.h>
951c7c3c6aSMatthew Dillon #include <sys/blist.h>
961c7c3c6aSMatthew Dillon #include <sys/lock.h>
970cddd8f0SMatthew Dillon #include <sys/sx.h>
98936524aaSMatthew Dillon #include <sys/vmmeter.h>
99df8bae1dSRodney W. Grimes 
100aed55708SRobert Watson #include <security/mac/mac_framework.h>
101aed55708SRobert Watson 
102df8bae1dSRodney W. Grimes #include <vm/vm.h>
10321cd6e62SSeigo Tanimura #include <vm/pmap.h>
10421cd6e62SSeigo Tanimura #include <vm/vm_map.h>
10521cd6e62SSeigo Tanimura #include <vm/vm_kern.h>
106efeaf95aSDavid Greenman #include <vm/vm_object.h>
107df8bae1dSRodney W. Grimes #include <vm/vm_page.h>
108efeaf95aSDavid Greenman #include <vm/vm_pager.h>
109df8bae1dSRodney W. Grimes #include <vm/vm_pageout.h>
110e9c0cc15SPoul-Henning Kamp #include <vm/vm_param.h>
111df8bae1dSRodney W. Grimes #include <vm/swap_pager.h>
112efeaf95aSDavid Greenman #include <vm/vm_extern.h>
113670d17b5SJeff Roberson #include <vm/uma.h>
114df8bae1dSRodney W. Grimes 
115dee34ca4SPoul-Henning Kamp #include <geom/geom.h>
116dee34ca4SPoul-Henning Kamp 
117ec38b344SPoul-Henning Kamp /*
11815523cf7SKonstantin Belousov  * SWB_NPAGES must be a power of 2.  It may be set to 1, 2, 4, 8, 16
11915523cf7SKonstantin Belousov  * or 32 pages per allocation.
12015523cf7SKonstantin Belousov  * The 32-page limit is due to the radix code (kern/subr_blist.c).
121ec38b344SPoul-Henning Kamp  */
122ec38b344SPoul-Henning Kamp #ifndef MAX_PAGEOUT_CLUSTER
123ec38b344SPoul-Henning Kamp #define MAX_PAGEOUT_CLUSTER 16
124ec38b344SPoul-Henning Kamp #endif
125ec38b344SPoul-Henning Kamp 
126ec38b344SPoul-Henning Kamp #if !defined(SWB_NPAGES)
127ec38b344SPoul-Henning Kamp #define SWB_NPAGES	MAX_PAGEOUT_CLUSTER
128ec38b344SPoul-Henning Kamp #endif
129ec38b344SPoul-Henning Kamp 
130ec38b344SPoul-Henning Kamp /*
13115523cf7SKonstantin Belousov  * The swblock structure maps an object and a small, fixed-size range
13215523cf7SKonstantin Belousov  * of page indices to disk addresses within a swap area.
13315523cf7SKonstantin Belousov  * The collection of these mappings is implemented as a hash table.
13415523cf7SKonstantin Belousov  * Unused disk addresses within a swap area are allocated and managed
13515523cf7SKonstantin Belousov  * using a blist.
136ec38b344SPoul-Henning Kamp  */
137ec38b344SPoul-Henning Kamp #define SWCORRECT(n) (sizeof(void *) * (n) / sizeof(daddr_t))
138ec38b344SPoul-Henning Kamp #define SWAP_META_PAGES		(SWB_NPAGES * 2)
139ec38b344SPoul-Henning Kamp #define SWAP_META_MASK		(SWAP_META_PAGES - 1)
140ec38b344SPoul-Henning Kamp 
141e9c0cc15SPoul-Henning Kamp struct swblock {
142e9c0cc15SPoul-Henning Kamp 	struct swblock	*swb_hnext;
143e9c0cc15SPoul-Henning Kamp 	vm_object_t	swb_object;
144e9c0cc15SPoul-Henning Kamp 	vm_pindex_t	swb_index;
145e9c0cc15SPoul-Henning Kamp 	int		swb_count;
146e9c0cc15SPoul-Henning Kamp 	daddr_t		swb_pages[SWAP_META_PAGES];
147e9c0cc15SPoul-Henning Kamp };
148e9c0cc15SPoul-Henning Kamp 
1492c4992dbSAlan Cox static MALLOC_DEFINE(M_VMPGDATA, "vm_pgdata", "swap pager private data");
15020da9c2eSPoul-Henning Kamp static struct mtx sw_dev_mtx;
1518f60c087SPoul-Henning Kamp static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq);
1528f60c087SPoul-Henning Kamp static struct swdevt *swdevhd;	/* Allocate from here next */
1538f60c087SPoul-Henning Kamp static int nswapdev;		/* Number of swap devices */
1548f60c087SPoul-Henning Kamp int swap_pager_avail;
155e9c0cc15SPoul-Henning Kamp static int swdev_syscall_active = 0; /* serialize swap(on|off) */
156e9c0cc15SPoul-Henning Kamp 
1573364c323SKonstantin Belousov static vm_ooffset_t swap_total;
1588a9c731fSIvan Voras SYSCTL_QUAD(_vm, OID_AUTO, swap_total, CTLFLAG_RD, &swap_total, 0,
1598a9c731fSIvan Voras     "Total amount of available swap storage.");
1603364c323SKonstantin Belousov static vm_ooffset_t swap_reserved;
1618a9c731fSIvan Voras SYSCTL_QUAD(_vm, OID_AUTO, swap_reserved, CTLFLAG_RD, &swap_reserved, 0,
1628a9c731fSIvan Voras     "Amount of swap storage needed to back all allocated anonymous memory.");
1633364c323SKonstantin Belousov static int overcommit = 0;
1648a9c731fSIvan Voras SYSCTL_INT(_vm, OID_AUTO, overcommit, CTLFLAG_RW, &overcommit, 0,
1658a9c731fSIvan Voras     "Configure virtual memory overcommit behavior. See tuning(7) "
1668a9c731fSIvan Voras     "for details.");
16761203277SDag-Erling Smørgrav static unsigned long swzone;
16861203277SDag-Erling Smørgrav SYSCTL_ULONG(_vm, OID_AUTO, swzone, CTLFLAG_RD, &swzone, 0,
16961203277SDag-Erling Smørgrav     "Actual size of swap metadata zone");
17061203277SDag-Erling Smørgrav static unsigned long swap_maxpages;
17161203277SDag-Erling Smørgrav SYSCTL_ULONG(_vm, OID_AUTO, swap_maxpages, CTLFLAG_RD, &swap_maxpages, 0,
17261203277SDag-Erling Smørgrav     "Maximum amount of swap supported");
1733364c323SKonstantin Belousov 
1743364c323SKonstantin Belousov /* bits from overcommit */
1753364c323SKonstantin Belousov #define	SWAP_RESERVE_FORCE_ON		(1 << 0)
1763364c323SKonstantin Belousov #define	SWAP_RESERVE_RLIMIT_ON		(1 << 1)
1773364c323SKonstantin Belousov #define	SWAP_RESERVE_ALLOW_NONWIRED	(1 << 2)
1783364c323SKonstantin Belousov 
1793364c323SKonstantin Belousov int
1803364c323SKonstantin Belousov swap_reserve(vm_ooffset_t incr)
1813364c323SKonstantin Belousov {
1823364c323SKonstantin Belousov 
183ef694c1aSEdward Tomasz Napierala 	return (swap_reserve_by_cred(incr, curthread->td_ucred));
1843364c323SKonstantin Belousov }
1853364c323SKonstantin Belousov 
1863364c323SKonstantin Belousov int
187ef694c1aSEdward Tomasz Napierala swap_reserve_by_cred(vm_ooffset_t incr, struct ucred *cred)
1883364c323SKonstantin Belousov {
1895c0e1c11SKonstantin Belousov 	vm_ooffset_t r, s;
1903364c323SKonstantin Belousov 	int res, error;
1913364c323SKonstantin Belousov 	static int curfail;
1923364c323SKonstantin Belousov 	static struct timeval lastfail;
193ef694c1aSEdward Tomasz Napierala 	struct uidinfo *uip;
194ef694c1aSEdward Tomasz Napierala 
195ef694c1aSEdward Tomasz Napierala 	uip = cred->cr_ruidinfo;
1963364c323SKonstantin Belousov 
1973364c323SKonstantin Belousov 	if (incr & PAGE_MASK)
1983364c323SKonstantin Belousov 		panic("swap_reserve: & PAGE_MASK");
1993364c323SKonstantin Belousov 
200afcc55f3SEdward Tomasz Napierala #ifdef RACCT
201*4b5c9cf6SEdward Tomasz Napierala 	if (racct_enable) {
2021ba5ad42SEdward Tomasz Napierala 		PROC_LOCK(curproc);
2031ba5ad42SEdward Tomasz Napierala 		error = racct_add(curproc, RACCT_SWAP, incr);
2041ba5ad42SEdward Tomasz Napierala 		PROC_UNLOCK(curproc);
2051ba5ad42SEdward Tomasz Napierala 		if (error != 0)
2061ba5ad42SEdward Tomasz Napierala 			return (0);
207*4b5c9cf6SEdward Tomasz Napierala 	}
208afcc55f3SEdward Tomasz Napierala #endif
2091ba5ad42SEdward Tomasz Napierala 
2103364c323SKonstantin Belousov 	res = 0;
2113364c323SKonstantin Belousov 	mtx_lock(&sw_dev_mtx);
2123364c323SKonstantin Belousov 	r = swap_reserved + incr;
2133364c323SKonstantin Belousov 	if (overcommit & SWAP_RESERVE_ALLOW_NONWIRED) {
21444f1c916SBryan Drewery 		s = vm_cnt.v_page_count - vm_cnt.v_free_reserved - vm_cnt.v_wire_count;
2153364c323SKonstantin Belousov 		s *= PAGE_SIZE;
2163364c323SKonstantin Belousov 	} else
2173364c323SKonstantin Belousov 		s = 0;
2183364c323SKonstantin Belousov 	s += swap_total;
2193364c323SKonstantin Belousov 	if ((overcommit & SWAP_RESERVE_FORCE_ON) == 0 || r <= s ||
2203364c323SKonstantin Belousov 	    (error = priv_check(curthread, PRIV_VM_SWAP_NOQUOTA)) == 0) {
2213364c323SKonstantin Belousov 		res = 1;
2223364c323SKonstantin Belousov 		swap_reserved = r;
2233364c323SKonstantin Belousov 	}
2243364c323SKonstantin Belousov 	mtx_unlock(&sw_dev_mtx);
2253364c323SKonstantin Belousov 
2263364c323SKonstantin Belousov 	if (res) {
2273364c323SKonstantin Belousov 		PROC_LOCK(curproc);
2283364c323SKonstantin Belousov 		UIDINFO_VMSIZE_LOCK(uip);
2295c0e1c11SKonstantin Belousov 		if ((overcommit & SWAP_RESERVE_RLIMIT_ON) != 0 &&
2305c0e1c11SKonstantin Belousov 		    uip->ui_vmsize + incr > lim_cur(curproc, RLIMIT_SWAP) &&
2315c0e1c11SKonstantin Belousov 		    priv_check(curthread, PRIV_VM_SWAP_NORLIMIT))
2323364c323SKonstantin Belousov 			res = 0;
2333364c323SKonstantin Belousov 		else
2343364c323SKonstantin Belousov 			uip->ui_vmsize += incr;
2353364c323SKonstantin Belousov 		UIDINFO_VMSIZE_UNLOCK(uip);
2363364c323SKonstantin Belousov 		PROC_UNLOCK(curproc);
2373364c323SKonstantin Belousov 		if (!res) {
2383364c323SKonstantin Belousov 			mtx_lock(&sw_dev_mtx);
2393364c323SKonstantin Belousov 			swap_reserved -= incr;
2403364c323SKonstantin Belousov 			mtx_unlock(&sw_dev_mtx);
2413364c323SKonstantin Belousov 		}
2423364c323SKonstantin Belousov 	}
2433364c323SKonstantin Belousov 	if (!res && ppsratecheck(&lastfail, &curfail, 1)) {
2443364c323SKonstantin Belousov 		printf("uid %d, pid %d: swap reservation for %jd bytes failed\n",
245134465d7SKonstantin Belousov 		    uip->ui_uid, curproc->p_pid, incr);
2463364c323SKonstantin Belousov 	}
2473364c323SKonstantin Belousov 
248afcc55f3SEdward Tomasz Napierala #ifdef RACCT
2491ba5ad42SEdward Tomasz Napierala 	if (!res) {
2501ba5ad42SEdward Tomasz Napierala 		PROC_LOCK(curproc);
2511ba5ad42SEdward Tomasz Napierala 		racct_sub(curproc, RACCT_SWAP, incr);
2521ba5ad42SEdward Tomasz Napierala 		PROC_UNLOCK(curproc);
2531ba5ad42SEdward Tomasz Napierala 	}
254afcc55f3SEdward Tomasz Napierala #endif
2551ba5ad42SEdward Tomasz Napierala 
2563364c323SKonstantin Belousov 	return (res);
2573364c323SKonstantin Belousov }
2583364c323SKonstantin Belousov 
2593364c323SKonstantin Belousov void
2603364c323SKonstantin Belousov swap_reserve_force(vm_ooffset_t incr)
2613364c323SKonstantin Belousov {
2623364c323SKonstantin Belousov 	struct uidinfo *uip;
2633364c323SKonstantin Belousov 
2643364c323SKonstantin Belousov 	mtx_lock(&sw_dev_mtx);
2653364c323SKonstantin Belousov 	swap_reserved += incr;
2663364c323SKonstantin Belousov 	mtx_unlock(&sw_dev_mtx);
2673364c323SKonstantin Belousov 
268afcc55f3SEdward Tomasz Napierala #ifdef RACCT
2691ba5ad42SEdward Tomasz Napierala 	PROC_LOCK(curproc);
2701ba5ad42SEdward Tomasz Napierala 	racct_add_force(curproc, RACCT_SWAP, incr);
2711ba5ad42SEdward Tomasz Napierala 	PROC_UNLOCK(curproc);
272afcc55f3SEdward Tomasz Napierala #endif
2731ba5ad42SEdward Tomasz Napierala 
2743364c323SKonstantin Belousov 	uip = curthread->td_ucred->cr_ruidinfo;
2753364c323SKonstantin Belousov 	PROC_LOCK(curproc);
2763364c323SKonstantin Belousov 	UIDINFO_VMSIZE_LOCK(uip);
2773364c323SKonstantin Belousov 	uip->ui_vmsize += incr;
2783364c323SKonstantin Belousov 	UIDINFO_VMSIZE_UNLOCK(uip);
2793364c323SKonstantin Belousov 	PROC_UNLOCK(curproc);
2803364c323SKonstantin Belousov }
2813364c323SKonstantin Belousov 
2823364c323SKonstantin Belousov void
2833364c323SKonstantin Belousov swap_release(vm_ooffset_t decr)
2843364c323SKonstantin Belousov {
285ef694c1aSEdward Tomasz Napierala 	struct ucred *cred;
2863364c323SKonstantin Belousov 
2873364c323SKonstantin Belousov 	PROC_LOCK(curproc);
288ef694c1aSEdward Tomasz Napierala 	cred = curthread->td_ucred;
289ef694c1aSEdward Tomasz Napierala 	swap_release_by_cred(decr, cred);
2903364c323SKonstantin Belousov 	PROC_UNLOCK(curproc);
2913364c323SKonstantin Belousov }
2923364c323SKonstantin Belousov 
2933364c323SKonstantin Belousov void
294ef694c1aSEdward Tomasz Napierala swap_release_by_cred(vm_ooffset_t decr, struct ucred *cred)
2953364c323SKonstantin Belousov {
296ef694c1aSEdward Tomasz Napierala  	struct uidinfo *uip;
297ef694c1aSEdward Tomasz Napierala 
298ef694c1aSEdward Tomasz Napierala 	uip = cred->cr_ruidinfo;
2993364c323SKonstantin Belousov 
3003364c323SKonstantin Belousov 	if (decr & PAGE_MASK)
3013364c323SKonstantin Belousov 		panic("swap_release: & PAGE_MASK");
3023364c323SKonstantin Belousov 
3033364c323SKonstantin Belousov 	mtx_lock(&sw_dev_mtx);
3043364c323SKonstantin Belousov 	if (swap_reserved < decr)
3053364c323SKonstantin Belousov 		panic("swap_reserved < decr");
3063364c323SKonstantin Belousov 	swap_reserved -= decr;
3073364c323SKonstantin Belousov 	mtx_unlock(&sw_dev_mtx);
3083364c323SKonstantin Belousov 
3093364c323SKonstantin Belousov 	UIDINFO_VMSIZE_LOCK(uip);
3103364c323SKonstantin Belousov 	if (uip->ui_vmsize < decr)
3113364c323SKonstantin Belousov 		printf("negative vmsize for uid = %d\n", uip->ui_uid);
3123364c323SKonstantin Belousov 	uip->ui_vmsize -= decr;
3133364c323SKonstantin Belousov 	UIDINFO_VMSIZE_UNLOCK(uip);
3141ba5ad42SEdward Tomasz Napierala 
3151ba5ad42SEdward Tomasz Napierala 	racct_sub_cred(cred, RACCT_SWAP, decr);
3163364c323SKonstantin Belousov }
3173364c323SKonstantin Belousov 
3184b03903aSPoul-Henning Kamp static void swapdev_strategy(struct buf *, struct swdevt *sw);
319e9c0cc15SPoul-Henning Kamp 
3201c7c3c6aSMatthew Dillon #define SWM_FREE	0x02	/* free, period			*/
3211c7c3c6aSMatthew Dillon #define SWM_POP		0x04	/* pop out			*/
32226f9a767SRodney W. Grimes 
3237dea2c2eSAlan Cox int swap_pager_full = 2;	/* swap space exhaustion (task killing) */
3247dea2c2eSAlan Cox static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/
3251c7c3c6aSMatthew Dillon static int nsw_rcount;		/* free read buffers			*/
326327f4e83SMatthew Dillon static int nsw_wcount_sync;	/* limit write buffers / synchronous	*/
327327f4e83SMatthew Dillon static int nsw_wcount_async;	/* limit write buffers / asynchronous	*/
328327f4e83SMatthew Dillon static int nsw_wcount_async_max;/* assigned maximum			*/
329327f4e83SMatthew Dillon static int nsw_cluster_max;	/* maximum VOP I/O allowed		*/
3301c7c3c6aSMatthew Dillon 
3311c7c3c6aSMatthew Dillon static struct swblock **swhash;
3321c7c3c6aSMatthew Dillon static int swhash_mask;
3337827d9b0SAlan Cox static struct mtx swhash_mtx;
3347827d9b0SAlan Cox 
335327f4e83SMatthew Dillon static int swap_async_max = 4;	/* maximum in-progress async I/O's	*/
3360cddd8f0SMatthew Dillon static struct sx sw_alloc_sx;
337327f4e83SMatthew Dillon 
33824e7ab7cSPoul-Henning Kamp 
339327f4e83SMatthew Dillon SYSCTL_INT(_vm, OID_AUTO, swap_async_max,
340327f4e83SMatthew Dillon 	CTLFLAG_RW, &swap_async_max, 0, "Maximum running async swap ops");
3411c7c3c6aSMatthew Dillon 
3421c7c3c6aSMatthew Dillon /*
3431c7c3c6aSMatthew Dillon  * "named" and "unnamed" anon region objects.  Try to reduce the overhead
3441c7c3c6aSMatthew Dillon  * of searching a named list by hashing it just a little.
3451c7c3c6aSMatthew Dillon  */
3461c7c3c6aSMatthew Dillon 
3471c7c3c6aSMatthew Dillon #define NOBJLISTS		8
3481c7c3c6aSMatthew Dillon 
3491c7c3c6aSMatthew Dillon #define NOBJLIST(handle)	\
350af647ddeSBruce Evans 	(&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)])
3511c7c3c6aSMatthew Dillon 
352a9fa2c05SAlfred Perlstein static struct mtx sw_alloc_mtx;	/* protect list manipulation */
3531c7c3c6aSMatthew Dillon static struct pagerlst	swap_pager_object_list[NOBJLISTS];
354e9c0cc15SPoul-Henning Kamp static uma_zone_t	swap_zone;
3551c7c3c6aSMatthew Dillon 
3561c7c3c6aSMatthew Dillon /*
3571c7c3c6aSMatthew Dillon  * pagerops for OBJT_SWAP - "swap pager".  Some ops are also global procedure
3581c7c3c6aSMatthew Dillon  * calls hooked from other parts of the VM system and do not appear here.
3591c7c3c6aSMatthew Dillon  * (see vm/swap_pager.h).
3601c7c3c6aSMatthew Dillon  */
361ff98689dSBruce Evans static vm_object_t
36211caded3SAlfred Perlstein 		swap_pager_alloc(void *handle, vm_ooffset_t size,
3633364c323SKonstantin Belousov 		    vm_prot_t prot, vm_ooffset_t offset, struct ucred *);
36411caded3SAlfred Perlstein static void	swap_pager_dealloc(vm_object_t object);
36511caded3SAlfred Perlstein static int	swap_pager_getpages(vm_object_t, vm_page_t *, int, int);
36690effb23SGleb Smirnoff static int	swap_pager_getpages_async(vm_object_t, vm_page_t *, int, int,
36790effb23SGleb Smirnoff     pgo_getpages_iodone_t, void *);
368751221fdSPoul-Henning Kamp static void	swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *);
3695ea4972cSAlan Cox static boolean_t
3705ea4972cSAlan Cox 		swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
37111caded3SAlfred Perlstein static void	swap_pager_init(void);
37211caded3SAlfred Perlstein static void	swap_pager_unswapped(vm_page_t);
373b3fed13eSDavid Schultz static void	swap_pager_swapoff(struct swdevt *sp);
374f708ef1bSPoul-Henning Kamp 
375df8bae1dSRodney W. Grimes struct pagerops swappagerops = {
376e04e4bacSPoul-Henning Kamp 	.pgo_init =	swap_pager_init,	/* early system initialization of pager	*/
377e04e4bacSPoul-Henning Kamp 	.pgo_alloc =	swap_pager_alloc,	/* allocate an OBJT_SWAP object		*/
378e04e4bacSPoul-Henning Kamp 	.pgo_dealloc =	swap_pager_dealloc,	/* deallocate an OBJT_SWAP object	*/
379e04e4bacSPoul-Henning Kamp 	.pgo_getpages =	swap_pager_getpages,	/* pagein				*/
38090effb23SGleb Smirnoff 	.pgo_getpages_async = swap_pager_getpages_async, /* pagein (async)		*/
381e04e4bacSPoul-Henning Kamp 	.pgo_putpages =	swap_pager_putpages,	/* pageout				*/
382e04e4bacSPoul-Henning Kamp 	.pgo_haspage =	swap_pager_haspage,	/* get backing store status for page	*/
383e04e4bacSPoul-Henning Kamp 	.pgo_pageunswapped = swap_pager_unswapped,	/* remove swap related to page		*/
384df8bae1dSRodney W. Grimes };
385df8bae1dSRodney W. Grimes 
3861c7c3c6aSMatthew Dillon /*
3871c7c3c6aSMatthew Dillon  * dmmax is in page-sized chunks with the new swap system.  It was
38864bcb9c8SMatthew Dillon  * dev-bsized chunks in the old.  dmmax is always a power of 2.
3891c7c3c6aSMatthew Dillon  *
3901c7c3c6aSMatthew Dillon  * swap_*() routines are externally accessible.  swp_*() routines are
3911c7c3c6aSMatthew Dillon  * internal.
3921c7c3c6aSMatthew Dillon  */
393751221fdSPoul-Henning Kamp static int dmmax;
394e9c0cc15SPoul-Henning Kamp static int nswap_lowat = 128;	/* in pages, swap_pager_almost_full warn */
395e9c0cc15SPoul-Henning Kamp static int nswap_hiwat = 512;	/* in pages, swap_pager_almost_full warn */
39626f9a767SRodney W. Grimes 
397cee313c4SRobert Watson SYSCTL_INT(_vm, OID_AUTO, dmmax,
398cee313c4SRobert Watson 	CTLFLAG_RD, &dmmax, 0, "Maximum size of a swap block");
399cee313c4SRobert Watson 
400a5edd34aSPoul-Henning Kamp static void	swp_sizecheck(void);
40111caded3SAlfred Perlstein static void	swp_pager_async_iodone(struct buf *bp);
402dee34ca4SPoul-Henning Kamp static int	swapongeom(struct thread *, struct vnode *);
40359efee01SPoul-Henning Kamp static int	swaponvp(struct thread *, struct vnode *, u_long);
40435918c55SChristian S.J. Peron static int	swapoff_one(struct swdevt *sp, struct ucred *cred);
40524a1cce3SDavid Greenman 
4061c7c3c6aSMatthew Dillon /*
4071c7c3c6aSMatthew Dillon  * Swap bitmap functions
4081c7c3c6aSMatthew Dillon  */
409a5edd34aSPoul-Henning Kamp static void	swp_pager_freeswapspace(daddr_t blk, int npages);
410a5edd34aSPoul-Henning Kamp static daddr_t	swp_pager_getswapspace(int npages);
4111c7c3c6aSMatthew Dillon 
4121c7c3c6aSMatthew Dillon /*
4131c7c3c6aSMatthew Dillon  * Metadata functions
4141c7c3c6aSMatthew Dillon  */
415a5edd34aSPoul-Henning Kamp static struct swblock **swp_pager_hash(vm_object_t object, vm_pindex_t index);
41611caded3SAlfred Perlstein static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
41711caded3SAlfred Perlstein static void swp_pager_meta_free(vm_object_t, vm_pindex_t, daddr_t);
41811caded3SAlfred Perlstein static void swp_pager_meta_free_all(vm_object_t);
41911caded3SAlfred Perlstein static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);
4201c7c3c6aSMatthew Dillon 
421e86a87e9SKonstantin Belousov static void
422e86a87e9SKonstantin Belousov swp_pager_free_nrpage(vm_page_t m)
423e86a87e9SKonstantin Belousov {
424e86a87e9SKonstantin Belousov 
4253c4a2440SAlan Cox 	vm_page_lock(m);
426e86a87e9SKonstantin Belousov 	if (m->wire_count == 0)
427e86a87e9SKonstantin Belousov 		vm_page_free(m);
4283c4a2440SAlan Cox 	vm_page_unlock(m);
429e86a87e9SKonstantin Belousov }
430e86a87e9SKonstantin Belousov 
4311c7c3c6aSMatthew Dillon /*
4321c7c3c6aSMatthew Dillon  * SWP_SIZECHECK() -	update swap_pager_full indication
4331c7c3c6aSMatthew Dillon  *
43420d3034fSMatthew Dillon  *	update the swap_pager_almost_full indication and warn when we are
43520d3034fSMatthew Dillon  *	about to run out of swap space, using lowat/hiwat hysteresis.
43620d3034fSMatthew Dillon  *
43720d3034fSMatthew Dillon  *	Clear swap_pager_full ( task killing ) indication when lowat is met.
4381c7c3c6aSMatthew Dillon  *
4391c7c3c6aSMatthew Dillon  *	No restrictions on call
4401c7c3c6aSMatthew Dillon  *	This routine may not block.
4411c7c3c6aSMatthew Dillon  */
442a5edd34aSPoul-Henning Kamp static void
4432f249180SPoul-Henning Kamp swp_sizecheck(void)
4440d94caffSDavid Greenman {
44523955314SAlfred Perlstein 
4468f60c087SPoul-Henning Kamp 	if (swap_pager_avail < nswap_lowat) {
44720d3034fSMatthew Dillon 		if (swap_pager_almost_full == 0) {
4481af87c92SDavid Greenman 			printf("swap_pager: out of swap space\n");
44920d3034fSMatthew Dillon 			swap_pager_almost_full = 1;
4502b0d37a4SMatthew Dillon 		}
45120d3034fSMatthew Dillon 	} else {
45226f9a767SRodney W. Grimes 		swap_pager_full = 0;
4538f60c087SPoul-Henning Kamp 		if (swap_pager_avail > nswap_hiwat)
45420d3034fSMatthew Dillon 			swap_pager_almost_full = 0;
45526f9a767SRodney W. Grimes 	}
4561c7c3c6aSMatthew Dillon }
4571c7c3c6aSMatthew Dillon 
4581c7c3c6aSMatthew Dillon /*
459da5fd145SPeter Wemm  * SWP_PAGER_HASH() -	hash swap meta data
460da5fd145SPeter Wemm  *
461a5edd34aSPoul-Henning Kamp  *	This is an helper function which hashes the swapblk given
462da5fd145SPeter Wemm  *	the object and page index.  It returns a pointer to a pointer
463da5fd145SPeter Wemm  *	to the object, or a pointer to a NULL pointer if it could not
464da5fd145SPeter Wemm  *	find a swapblk.
465da5fd145SPeter Wemm  */
466a5edd34aSPoul-Henning Kamp static struct swblock **
467da5fd145SPeter Wemm swp_pager_hash(vm_object_t object, vm_pindex_t index)
468da5fd145SPeter Wemm {
469da5fd145SPeter Wemm 	struct swblock **pswap;
470da5fd145SPeter Wemm 	struct swblock *swap;
471da5fd145SPeter Wemm 
472da5fd145SPeter Wemm 	index &= ~(vm_pindex_t)SWAP_META_MASK;
473da5fd145SPeter Wemm 	pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask];
474da5fd145SPeter Wemm 	while ((swap = *pswap) != NULL) {
475da5fd145SPeter Wemm 		if (swap->swb_object == object &&
476da5fd145SPeter Wemm 		    swap->swb_index == index
477da5fd145SPeter Wemm 		) {
478da5fd145SPeter Wemm 			break;
479da5fd145SPeter Wemm 		}
480da5fd145SPeter Wemm 		pswap = &swap->swb_hnext;
481da5fd145SPeter Wemm 	}
482da5fd145SPeter Wemm 	return (pswap);
483da5fd145SPeter Wemm }
484da5fd145SPeter Wemm 
485da5fd145SPeter Wemm /*
4861c7c3c6aSMatthew Dillon  * SWAP_PAGER_INIT() -	initialize the swap pager!
4871c7c3c6aSMatthew Dillon  *
4881c7c3c6aSMatthew Dillon  *	Expected to be started from system init.  NOTE:  This code is run
4891c7c3c6aSMatthew Dillon  *	before much else so be careful what you depend on.  Most of the VM
4901c7c3c6aSMatthew Dillon  *	system has yet to be initialized at this point.
4911c7c3c6aSMatthew Dillon  */
492f5a12711SPoul-Henning Kamp static void
4932f249180SPoul-Henning Kamp swap_pager_init(void)
494df8bae1dSRodney W. Grimes {
4951c7c3c6aSMatthew Dillon 	/*
4961c7c3c6aSMatthew Dillon 	 * Initialize object lists
4971c7c3c6aSMatthew Dillon 	 */
4981c7c3c6aSMatthew Dillon 	int i;
4991c7c3c6aSMatthew Dillon 
5001c7c3c6aSMatthew Dillon 	for (i = 0; i < NOBJLISTS; ++i)
5011c7c3c6aSMatthew Dillon 		TAILQ_INIT(&swap_pager_object_list[i]);
5026008862bSJohn Baldwin 	mtx_init(&sw_alloc_mtx, "swap_pager list", NULL, MTX_DEF);
50320da9c2eSPoul-Henning Kamp 	mtx_init(&sw_dev_mtx, "swapdev", NULL, MTX_DEF);
504df8bae1dSRodney W. Grimes 
505df8bae1dSRodney W. Grimes 	/*
5061c7c3c6aSMatthew Dillon 	 * Device Stripe, in PAGE_SIZE'd blocks
507df8bae1dSRodney W. Grimes 	 */
5081c7c3c6aSMatthew Dillon 	dmmax = SWB_NPAGES * 2;
5091c7c3c6aSMatthew Dillon }
51026f9a767SRodney W. Grimes 
511df8bae1dSRodney W. Grimes /*
5121c7c3c6aSMatthew Dillon  * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process
5131c7c3c6aSMatthew Dillon  *
5141c7c3c6aSMatthew Dillon  *	Expected to be started from pageout process once, prior to entering
5151c7c3c6aSMatthew Dillon  *	its main loop.
516df8bae1dSRodney W. Grimes  */
51724a1cce3SDavid Greenman void
5182f249180SPoul-Henning Kamp swap_pager_swap_init(void)
519df8bae1dSRodney W. Grimes {
52061203277SDag-Erling Smørgrav 	unsigned long n, n2;
5210d94caffSDavid Greenman 
52226f9a767SRodney W. Grimes 	/*
5231c7c3c6aSMatthew Dillon 	 * Number of in-transit swap bp operations.  Don't
5241c7c3c6aSMatthew Dillon 	 * exhaust the pbufs completely.  Make sure we
5251c7c3c6aSMatthew Dillon 	 * initialize workable values (0 will work for hysteresis
5261c7c3c6aSMatthew Dillon 	 * but it isn't very efficient).
5271c7c3c6aSMatthew Dillon 	 *
528327f4e83SMatthew Dillon 	 * The nsw_cluster_max is constrained by the bp->b_pages[]
5291c7c3c6aSMatthew Dillon 	 * array (MAXPHYS/PAGE_SIZE) and our locally defined
5301c7c3c6aSMatthew Dillon 	 * MAX_PAGEOUT_CLUSTER.   Also be aware that swap ops are
5311c7c3c6aSMatthew Dillon 	 * constrained by the swap device interleave stripe size.
532327f4e83SMatthew Dillon 	 *
533327f4e83SMatthew Dillon 	 * Currently we hardwire nsw_wcount_async to 4.  This limit is
534327f4e83SMatthew Dillon 	 * designed to prevent other I/O from having high latencies due to
535327f4e83SMatthew Dillon 	 * our pageout I/O.  The value 4 works well for one or two active swap
536327f4e83SMatthew Dillon 	 * devices but is probably a little low if you have more.  Even so,
537327f4e83SMatthew Dillon 	 * a higher value would probably generate only a limited improvement
538327f4e83SMatthew Dillon 	 * with three or four active swap devices since the system does not
539327f4e83SMatthew Dillon 	 * typically have to pageout at extreme bandwidths.   We will want
540327f4e83SMatthew Dillon 	 * at least 2 per swap devices, and 4 is a pretty good value if you
541327f4e83SMatthew Dillon 	 * have one NFS swap device due to the command/ack latency over NFS.
542327f4e83SMatthew Dillon 	 * So it all works out pretty well.
54326f9a767SRodney W. Grimes 	 */
544ad3cce20SMatthew Dillon 	nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER);
545327f4e83SMatthew Dillon 
5466d541bf1SJohn Baldwin 	mtx_lock(&pbuf_mtx);
5471c7c3c6aSMatthew Dillon 	nsw_rcount = (nswbuf + 1) / 2;
548327f4e83SMatthew Dillon 	nsw_wcount_sync = (nswbuf + 3) / 4;
549327f4e83SMatthew Dillon 	nsw_wcount_async = 4;
550327f4e83SMatthew Dillon 	nsw_wcount_async_max = nsw_wcount_async;
5516d541bf1SJohn Baldwin 	mtx_unlock(&pbuf_mtx);
55224a1cce3SDavid Greenman 
5531c7c3c6aSMatthew Dillon 	/*
5541c7c3c6aSMatthew Dillon 	 * Initialize our zone.  Right now I'm just guessing on the number
5551c7c3c6aSMatthew Dillon 	 * we need based on the number of pages in the system.  Each swblock
55661203277SDag-Erling Smørgrav 	 * can hold 32 pages, so this is probably overkill.  This reservation
557ec61f55dSMatthew Dillon 	 * is typically limited to around 32MB by default.
5581c7c3c6aSMatthew Dillon 	 */
55944f1c916SBryan Drewery 	n = vm_cnt.v_page_count / 2;
5602f9e4e80SMatthew Dillon 	if (maxswzone && n > maxswzone / sizeof(struct swblock))
5612f9e4e80SMatthew Dillon 		n = maxswzone / sizeof(struct swblock);
56222a5e6b9SDag-Erling Smørgrav 	n2 = n;
563670d17b5SJeff Roberson 	swap_zone = uma_zcreate("SWAPMETA", sizeof(struct swblock), NULL, NULL,
5647827d9b0SAlan Cox 	    NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM);
565010b1ca1SDavid Schultz 	if (swap_zone == NULL)
566010b1ca1SDavid Schultz 		panic("failed to create swap_zone.");
5678355f576SJeff Roberson 	do {
568a4915c21SAttilio Rao 		if (uma_zone_reserve_kva(swap_zone, n))
56961ce6eeeSAlfred Perlstein 			break;
57061ce6eeeSAlfred Perlstein 		/*
57161ce6eeeSAlfred Perlstein 		 * if the allocation failed, try a zone two thirds the
57261ce6eeeSAlfred Perlstein 		 * size of the previous attempt.
57361ce6eeeSAlfred Perlstein 		 */
57461ce6eeeSAlfred Perlstein 		n -= ((n + 2) / 3);
57561ce6eeeSAlfred Perlstein 	} while (n > 0);
57621cd6e62SSeigo Tanimura 	if (n2 != n)
57761203277SDag-Erling Smørgrav 		printf("Swap zone entries reduced from %lu to %lu.\n", n2, n);
57861203277SDag-Erling Smørgrav 	swap_maxpages = n * SWAP_META_PAGES;
57961203277SDag-Erling Smørgrav 	swzone = n * sizeof(struct swblock);
58022a5e6b9SDag-Erling Smørgrav 	n2 = n;
58124a1cce3SDavid Greenman 
5821c7c3c6aSMatthew Dillon 	/*
5831c7c3c6aSMatthew Dillon 	 * Initialize our meta-data hash table.  The swapper does not need to
5841c7c3c6aSMatthew Dillon 	 * be quite as efficient as the VM system, so we do not use an
5851c7c3c6aSMatthew Dillon 	 * oversized hash table.
5861c7c3c6aSMatthew Dillon 	 *
5871c7c3c6aSMatthew Dillon 	 * 	n: 		size of hash table, must be power of 2
5881c7c3c6aSMatthew Dillon 	 *	swhash_mask:	hash table index mask
5891c7c3c6aSMatthew Dillon 	 */
59061ce6eeeSAlfred Perlstein 	for (n = 1; n < n2 / 8; n *= 2)
5911c7c3c6aSMatthew Dillon 		;
592a163d034SWarner Losh 	swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO);
5931c7c3c6aSMatthew Dillon 	swhash_mask = n - 1;
5947827d9b0SAlan Cox 	mtx_init(&swhash_mtx, "swap_pager swhash", NULL, MTX_DEF);
59524a1cce3SDavid Greenman }
59624a1cce3SDavid Greenman 
59724a1cce3SDavid Greenman /*
5981c7c3c6aSMatthew Dillon  * SWAP_PAGER_ALLOC() -	allocate a new OBJT_SWAP VM object and instantiate
5991c7c3c6aSMatthew Dillon  *			its metadata structures.
6001c7c3c6aSMatthew Dillon  *
6011c7c3c6aSMatthew Dillon  *	This routine is called from the mmap and fork code to create a new
6021c7c3c6aSMatthew Dillon  *	OBJT_SWAP object.  We do this by creating an OBJT_DEFAULT object
6031c7c3c6aSMatthew Dillon  *	and then converting it with swp_pager_meta_build().
6041c7c3c6aSMatthew Dillon  *
6051c7c3c6aSMatthew Dillon  *	This routine may block in vm_object_allocate() and create a named
606cec9f109SDavid E. O'Brien  *	object lookup race, so we must interlock.
60724c46d03SAlan Cox  *
60824c46d03SAlan Cox  * MPSAFE
60924a1cce3SDavid Greenman  */
610f5a12711SPoul-Henning Kamp static vm_object_t
6116cde7a16SDavid Greenman swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
6123364c323SKonstantin Belousov     vm_ooffset_t offset, struct ucred *cred)
61324a1cce3SDavid Greenman {
61424a1cce3SDavid Greenman 	vm_object_t object;
6152f7af3dbSAlan Cox 	vm_pindex_t pindex;
6162f7af3dbSAlan Cox 
6172f7af3dbSAlan Cox 	pindex = OFF_TO_IDX(offset + PAGE_MASK + size);
61824a1cce3SDavid Greenman 	if (handle) {
619e793b779SAlan Cox 		mtx_lock(&Giant);
6201c7c3c6aSMatthew Dillon 		/*
6211c7c3c6aSMatthew Dillon 		 * Reference existing named region or allocate new one.  There
6221c7c3c6aSMatthew Dillon 		 * should not be a race here against swp_pager_meta_build()
6231c7c3c6aSMatthew Dillon 		 * as called from vm_page_remove() in regards to the lookup
6241c7c3c6aSMatthew Dillon 		 * of the handle.
6251c7c3c6aSMatthew Dillon 		 */
6260cddd8f0SMatthew Dillon 		sx_xlock(&sw_alloc_sx);
6271c7c3c6aSMatthew Dillon 		object = vm_pager_object_lookup(NOBJLIST(handle), handle);
628b5e8f167SAlan Cox 		if (object == NULL) {
6293364c323SKonstantin Belousov 			if (cred != NULL) {
630ef694c1aSEdward Tomasz Napierala 				if (!swap_reserve_by_cred(size, cred)) {
6313364c323SKonstantin Belousov 					sx_xunlock(&sw_alloc_sx);
6323364c323SKonstantin Belousov 					mtx_unlock(&Giant);
6333364c323SKonstantin Belousov 					return (NULL);
6343364c323SKonstantin Belousov 				}
635ef694c1aSEdward Tomasz Napierala 				crhold(cred);
6363364c323SKonstantin Belousov 			}
6372f7af3dbSAlan Cox 			object = vm_object_allocate(OBJT_DEFAULT, pindex);
63889f6b863SAttilio Rao 			VM_OBJECT_WLOCK(object);
6393364c323SKonstantin Belousov 			object->handle = handle;
6403364c323SKonstantin Belousov 			if (cred != NULL) {
641ef694c1aSEdward Tomasz Napierala 				object->cred = cred;
6423364c323SKonstantin Belousov 				object->charge = size;
6433364c323SKonstantin Belousov 			}
6444dcc5c2dSMatthew Dillon 			swp_pager_meta_build(object, 0, SWAPBLK_NONE);
64589f6b863SAttilio Rao 			VM_OBJECT_WUNLOCK(object);
64624a1cce3SDavid Greenman 		}
6470cddd8f0SMatthew Dillon 		sx_xunlock(&sw_alloc_sx);
648e793b779SAlan Cox 		mtx_unlock(&Giant);
64924a1cce3SDavid Greenman 	} else {
6503364c323SKonstantin Belousov 		if (cred != NULL) {
651ef694c1aSEdward Tomasz Napierala 			if (!swap_reserve_by_cred(size, cred))
6523364c323SKonstantin Belousov 				return (NULL);
653ef694c1aSEdward Tomasz Napierala 			crhold(cred);
6543364c323SKonstantin Belousov 		}
6552f7af3dbSAlan Cox 		object = vm_object_allocate(OBJT_DEFAULT, pindex);
65689f6b863SAttilio Rao 		VM_OBJECT_WLOCK(object);
6573364c323SKonstantin Belousov 		if (cred != NULL) {
658ef694c1aSEdward Tomasz Napierala 			object->cred = cred;
6593364c323SKonstantin Belousov 			object->charge = size;
6603364c323SKonstantin Belousov 		}
6614dcc5c2dSMatthew Dillon 		swp_pager_meta_build(object, 0, SWAPBLK_NONE);
66289f6b863SAttilio Rao 		VM_OBJECT_WUNLOCK(object);
66324a1cce3SDavid Greenman 	}
66424a1cce3SDavid Greenman 	return (object);
665df8bae1dSRodney W. Grimes }
666df8bae1dSRodney W. Grimes 
66726f9a767SRodney W. Grimes /*
6681c7c3c6aSMatthew Dillon  * SWAP_PAGER_DEALLOC() -	remove swap metadata from object
6691c7c3c6aSMatthew Dillon  *
6701c7c3c6aSMatthew Dillon  *	The swap backing for the object is destroyed.  The code is
6711c7c3c6aSMatthew Dillon  *	designed such that we can reinstantiate it later, but this
6721c7c3c6aSMatthew Dillon  *	routine is typically called only when the entire object is
6731c7c3c6aSMatthew Dillon  *	about to be destroyed.
6741c7c3c6aSMatthew Dillon  *
67515523cf7SKonstantin Belousov  *	The object must be locked.
67626f9a767SRodney W. Grimes  */
677df8bae1dSRodney W. Grimes static void
6782f249180SPoul-Henning Kamp swap_pager_dealloc(vm_object_t object)
67926f9a767SRodney W. Grimes {
6804dcc5c2dSMatthew Dillon 
68126f9a767SRodney W. Grimes 	/*
6821c7c3c6aSMatthew Dillon 	 * Remove from list right away so lookups will fail if we block for
6831c7c3c6aSMatthew Dillon 	 * pageout completion.
68426f9a767SRodney W. Grimes 	 */
685bd228075SAlan Cox 	if (object->handle != NULL) {
686a9fa2c05SAlfred Perlstein 		mtx_lock(&sw_alloc_mtx);
6871c7c3c6aSMatthew Dillon 		TAILQ_REMOVE(NOBJLIST(object->handle), object, pager_object_list);
688a9fa2c05SAlfred Perlstein 		mtx_unlock(&sw_alloc_mtx);
689bd228075SAlan Cox 	}
6901c7c3c6aSMatthew Dillon 
69189f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
6921c7c3c6aSMatthew Dillon 	vm_object_pip_wait(object, "swpdea");
6931c7c3c6aSMatthew Dillon 
6941c7c3c6aSMatthew Dillon 	/*
6951c7c3c6aSMatthew Dillon 	 * Free all remaining metadata.  We only bother to free it from
6961c7c3c6aSMatthew Dillon 	 * the swap meta data.  We do not attempt to free swapblk's still
6971c7c3c6aSMatthew Dillon 	 * associated with vm_page_t's for this object.  We do not care
6981c7c3c6aSMatthew Dillon 	 * if paging is still in progress on some objects.
6991c7c3c6aSMatthew Dillon 	 */
7001c7c3c6aSMatthew Dillon 	swp_pager_meta_free_all(object);
7011c7c3c6aSMatthew Dillon }
7021c7c3c6aSMatthew Dillon 
7031c7c3c6aSMatthew Dillon /************************************************************************
7041c7c3c6aSMatthew Dillon  *			SWAP PAGER BITMAP ROUTINES			*
7051c7c3c6aSMatthew Dillon  ************************************************************************/
7061c7c3c6aSMatthew Dillon 
7071c7c3c6aSMatthew Dillon /*
7081c7c3c6aSMatthew Dillon  * SWP_PAGER_GETSWAPSPACE() -	allocate raw swap space
7091c7c3c6aSMatthew Dillon  *
7101c7c3c6aSMatthew Dillon  *	Allocate swap for the requested number of pages.  The starting
7111c7c3c6aSMatthew Dillon  *	swap block number (a page index) is returned or SWAPBLK_NONE
7121c7c3c6aSMatthew Dillon  *	if the allocation failed.
7131c7c3c6aSMatthew Dillon  *
7141c7c3c6aSMatthew Dillon  *	Also has the side effect of advising that somebody made a mistake
7151c7c3c6aSMatthew Dillon  *	when they configured swap and didn't configure enough.
7161c7c3c6aSMatthew Dillon  *
71715523cf7SKonstantin Belousov  *	This routine may not sleep.
7188f60c087SPoul-Henning Kamp  *
7198f60c087SPoul-Henning Kamp  *	We allocate in round-robin fashion from the configured devices.
7201c7c3c6aSMatthew Dillon  */
721a5edd34aSPoul-Henning Kamp static daddr_t
7222f249180SPoul-Henning Kamp swp_pager_getswapspace(int npages)
7231c7c3c6aSMatthew Dillon {
7241c7c3c6aSMatthew Dillon 	daddr_t blk;
7258f60c087SPoul-Henning Kamp 	struct swdevt *sp;
7268f60c087SPoul-Henning Kamp 	int i;
7271c7c3c6aSMatthew Dillon 
7288f60c087SPoul-Henning Kamp 	blk = SWAPBLK_NONE;
72920da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
7308f60c087SPoul-Henning Kamp 	sp = swdevhd;
7318f60c087SPoul-Henning Kamp 	for (i = 0; i < nswapdev; i++) {
7328f60c087SPoul-Henning Kamp 		if (sp == NULL)
7338f60c087SPoul-Henning Kamp 			sp = TAILQ_FIRST(&swtailq);
7348f60c087SPoul-Henning Kamp 		if (!(sp->sw_flags & SW_CLOSING)) {
7358f60c087SPoul-Henning Kamp 			blk = blist_alloc(sp->sw_blist, npages);
7368f60c087SPoul-Henning Kamp 			if (blk != SWAPBLK_NONE) {
7378f60c087SPoul-Henning Kamp 				blk += sp->sw_first;
7388f60c087SPoul-Henning Kamp 				sp->sw_used += npages;
739d05bc129SAlan Cox 				swap_pager_avail -= npages;
7408f60c087SPoul-Henning Kamp 				swp_sizecheck();
7418f60c087SPoul-Henning Kamp 				swdevhd = TAILQ_NEXT(sp, sw_list);
742d05bc129SAlan Cox 				goto done;
7438f60c087SPoul-Henning Kamp 			}
7448f60c087SPoul-Henning Kamp 		}
7458f60c087SPoul-Henning Kamp 		sp = TAILQ_NEXT(sp, sw_list);
7468f60c087SPoul-Henning Kamp 	}
7472b0d37a4SMatthew Dillon 	if (swap_pager_full != 2) {
74820da9c2eSPoul-Henning Kamp 		printf("swap_pager_getswapspace(%d): failed\n", npages);
7492b0d37a4SMatthew Dillon 		swap_pager_full = 2;
75020d3034fSMatthew Dillon 		swap_pager_almost_full = 1;
7512b0d37a4SMatthew Dillon 	}
7528f60c087SPoul-Henning Kamp 	swdevhd = NULL;
753d05bc129SAlan Cox done:
754d05bc129SAlan Cox 	mtx_unlock(&sw_dev_mtx);
7551c7c3c6aSMatthew Dillon 	return (blk);
75626f9a767SRodney W. Grimes }
75726f9a767SRodney W. Grimes 
758b3fed13eSDavid Schultz static int
759b3fed13eSDavid Schultz swp_pager_isondev(daddr_t blk, struct swdevt *sp)
7608f60c087SPoul-Henning Kamp {
7618f60c087SPoul-Henning Kamp 
762b3fed13eSDavid Schultz 	return (blk >= sp->sw_first && blk < sp->sw_end);
7638f60c087SPoul-Henning Kamp }
7648f60c087SPoul-Henning Kamp 
7654b03903aSPoul-Henning Kamp static void
7664b03903aSPoul-Henning Kamp swp_pager_strategy(struct buf *bp)
7674b03903aSPoul-Henning Kamp {
7684b03903aSPoul-Henning Kamp 	struct swdevt *sp;
7694b03903aSPoul-Henning Kamp 
77020da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
7714b03903aSPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
7724b03903aSPoul-Henning Kamp 		if (bp->b_blkno >= sp->sw_first && bp->b_blkno < sp->sw_end) {
77320da9c2eSPoul-Henning Kamp 			mtx_unlock(&sw_dev_mtx);
7742cc718a1SKonstantin Belousov 			if ((sp->sw_flags & SW_UNMAPPED) != 0 &&
7752cc718a1SKonstantin Belousov 			    unmapped_buf_allowed) {
7762cc718a1SKonstantin Belousov 				bp->b_kvaalloc = bp->b_data;
7772cc718a1SKonstantin Belousov 				bp->b_data = unmapped_buf;
7782cc718a1SKonstantin Belousov 				bp->b_kvabase = unmapped_buf;
7792cc718a1SKonstantin Belousov 				bp->b_offset = 0;
7802cc718a1SKonstantin Belousov 				bp->b_flags |= B_UNMAPPED;
7812cc718a1SKonstantin Belousov 			} else {
7822cc718a1SKonstantin Belousov 				pmap_qenter((vm_offset_t)bp->b_data,
7832cc718a1SKonstantin Belousov 				    &bp->b_pages[0], bp->b_bcount / PAGE_SIZE);
7842cc718a1SKonstantin Belousov 			}
7854b03903aSPoul-Henning Kamp 			sp->sw_strategy(bp, sp);
7864b03903aSPoul-Henning Kamp 			return;
7874b03903aSPoul-Henning Kamp 		}
7884b03903aSPoul-Henning Kamp 	}
78920da9c2eSPoul-Henning Kamp 	panic("Swapdev not found");
7904b03903aSPoul-Henning Kamp }
7914b03903aSPoul-Henning Kamp 
7928f60c087SPoul-Henning Kamp 
79326f9a767SRodney W. Grimes /*
7941c7c3c6aSMatthew Dillon  * SWP_PAGER_FREESWAPSPACE() -	free raw swap space
7951c7c3c6aSMatthew Dillon  *
7961c7c3c6aSMatthew Dillon  *	This routine returns the specified swap blocks back to the bitmap.
7971c7c3c6aSMatthew Dillon  *
79815523cf7SKonstantin Belousov  *	This routine may not sleep.
79926f9a767SRodney W. Grimes  */
800a5edd34aSPoul-Henning Kamp static void
8018f60c087SPoul-Henning Kamp swp_pager_freeswapspace(daddr_t blk, int npages)
8020d94caffSDavid Greenman {
8038f60c087SPoul-Henning Kamp 	struct swdevt *sp;
80492da00bbSMatthew Dillon 
8057645e885SAlan Cox 	mtx_lock(&sw_dev_mtx);
8067645e885SAlan Cox 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
8077645e885SAlan Cox 		if (blk >= sp->sw_first && blk < sp->sw_end) {
80892da00bbSMatthew Dillon 			sp->sw_used -= npages;
80992da00bbSMatthew Dillon 			/*
8107645e885SAlan Cox 			 * If we are attempting to stop swapping on
8117645e885SAlan Cox 			 * this device, we don't want to mark any
8127645e885SAlan Cox 			 * blocks free lest they be reused.
81392da00bbSMatthew Dillon 			 */
8147645e885SAlan Cox 			if ((sp->sw_flags & SW_CLOSING) == 0) {
8157645e885SAlan Cox 				blist_free(sp->sw_blist, blk - sp->sw_first,
8167645e885SAlan Cox 				    npages);
8178f60c087SPoul-Henning Kamp 				swap_pager_avail += npages;
8181c7c3c6aSMatthew Dillon 				swp_sizecheck();
81926f9a767SRodney W. Grimes 			}
8207645e885SAlan Cox 			mtx_unlock(&sw_dev_mtx);
8217645e885SAlan Cox 			return;
8227645e885SAlan Cox 		}
8237645e885SAlan Cox 	}
8247645e885SAlan Cox 	panic("Swapdev not found");
8257645e885SAlan Cox }
8261c7c3c6aSMatthew Dillon 
82726f9a767SRodney W. Grimes /*
8281c7c3c6aSMatthew Dillon  * SWAP_PAGER_FREESPACE() -	frees swap blocks associated with a page
8291c7c3c6aSMatthew Dillon  *				range within an object.
8301c7c3c6aSMatthew Dillon  *
8311c7c3c6aSMatthew Dillon  *	This is a globally accessible routine.
8321c7c3c6aSMatthew Dillon  *
8331c7c3c6aSMatthew Dillon  *	This routine removes swapblk assignments from swap metadata.
8341c7c3c6aSMatthew Dillon  *
8351c7c3c6aSMatthew Dillon  *	The external callers of this routine typically have already destroyed
8361c7c3c6aSMatthew Dillon  *	or renamed vm_page_t's associated with this range in the object so
8371c7c3c6aSMatthew Dillon  *	we should be ok.
838c25673ffSAttilio Rao  *
839c25673ffSAttilio Rao  *	The object must be locked.
84026f9a767SRodney W. Grimes  */
84126f9a767SRodney W. Grimes void
8422f249180SPoul-Henning Kamp swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size)
84326f9a767SRodney W. Grimes {
84423955314SAlfred Perlstein 
8451c7c3c6aSMatthew Dillon 	swp_pager_meta_free(object, start, size);
8464dcc5c2dSMatthew Dillon }
8474dcc5c2dSMatthew Dillon 
8484dcc5c2dSMatthew Dillon /*
8494dcc5c2dSMatthew Dillon  * SWAP_PAGER_RESERVE() - reserve swap blocks in object
8504dcc5c2dSMatthew Dillon  *
8514dcc5c2dSMatthew Dillon  *	Assigns swap blocks to the specified range within the object.  The
85256ce850bSKonstantin Belousov  *	swap blocks are not zeroed.  Any previous swap assignment is destroyed.
8534dcc5c2dSMatthew Dillon  *
8544dcc5c2dSMatthew Dillon  *	Returns 0 on success, -1 on failure.
8554dcc5c2dSMatthew Dillon  */
8564dcc5c2dSMatthew Dillon int
8574dcc5c2dSMatthew Dillon swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
8584dcc5c2dSMatthew Dillon {
8594dcc5c2dSMatthew Dillon 	int n = 0;
8604dcc5c2dSMatthew Dillon 	daddr_t blk = SWAPBLK_NONE;
8614dcc5c2dSMatthew Dillon 	vm_pindex_t beg = start;	/* save start index */
8624dcc5c2dSMatthew Dillon 
86389f6b863SAttilio Rao 	VM_OBJECT_WLOCK(object);
8644dcc5c2dSMatthew Dillon 	while (size) {
8654dcc5c2dSMatthew Dillon 		if (n == 0) {
8664dcc5c2dSMatthew Dillon 			n = BLIST_MAX_ALLOC;
8674dcc5c2dSMatthew Dillon 			while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
8684dcc5c2dSMatthew Dillon 				n >>= 1;
8694dcc5c2dSMatthew Dillon 				if (n == 0) {
8704dcc5c2dSMatthew Dillon 					swp_pager_meta_free(object, beg, start - beg);
87189f6b863SAttilio Rao 					VM_OBJECT_WUNLOCK(object);
8724dcc5c2dSMatthew Dillon 					return (-1);
8734dcc5c2dSMatthew Dillon 				}
8744dcc5c2dSMatthew Dillon 			}
8754dcc5c2dSMatthew Dillon 		}
8764dcc5c2dSMatthew Dillon 		swp_pager_meta_build(object, start, blk);
8774dcc5c2dSMatthew Dillon 		--size;
8784dcc5c2dSMatthew Dillon 		++start;
8794dcc5c2dSMatthew Dillon 		++blk;
8804dcc5c2dSMatthew Dillon 		--n;
8814dcc5c2dSMatthew Dillon 	}
8824dcc5c2dSMatthew Dillon 	swp_pager_meta_free(object, start, n);
88389f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(object);
8844dcc5c2dSMatthew Dillon 	return (0);
88526f9a767SRodney W. Grimes }
88626f9a767SRodney W. Grimes 
8870a47b48bSJohn Dyson /*
8881c7c3c6aSMatthew Dillon  * SWAP_PAGER_COPY() -  copy blocks from source pager to destination pager
8891c7c3c6aSMatthew Dillon  *			and destroy the source.
8901c7c3c6aSMatthew Dillon  *
8911c7c3c6aSMatthew Dillon  *	Copy any valid swapblks from the source to the destination.  In
8921c7c3c6aSMatthew Dillon  *	cases where both the source and destination have a valid swapblk,
8931c7c3c6aSMatthew Dillon  *	we keep the destination's.
8941c7c3c6aSMatthew Dillon  *
89515523cf7SKonstantin Belousov  *	This routine is allowed to sleep.  It may sleep allocating metadata
8961c7c3c6aSMatthew Dillon  *	indirectly through swp_pager_meta_build() or if paging is still in
8971c7c3c6aSMatthew Dillon  *	progress on the source.
8981c7c3c6aSMatthew Dillon  *
8991c7c3c6aSMatthew Dillon  *	The source object contains no vm_page_t's (which is just as well)
9001c7c3c6aSMatthew Dillon  *
9011c7c3c6aSMatthew Dillon  *	The source object is of type OBJT_SWAP.
9021c7c3c6aSMatthew Dillon  *
90315523cf7SKonstantin Belousov  *	The source and destination objects must be locked.
90415523cf7SKonstantin Belousov  *	Both object locks may temporarily be released.
90526f9a767SRodney W. Grimes  */
90626f9a767SRodney W. Grimes void
9072f249180SPoul-Henning Kamp swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
9082f249180SPoul-Henning Kamp     vm_pindex_t offset, int destroysource)
90926f9a767SRodney W. Grimes {
910a316d390SJohn Dyson 	vm_pindex_t i;
9114dcc5c2dSMatthew Dillon 
91289f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(srcobject);
91389f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(dstobject);
9140cddd8f0SMatthew Dillon 
91526f9a767SRodney W. Grimes 	/*
9161c7c3c6aSMatthew Dillon 	 * If destroysource is set, we remove the source object from the
9171c7c3c6aSMatthew Dillon 	 * swap_pager internal queue now.
91826f9a767SRodney W. Grimes 	 */
919cbd8ec09SJohn Dyson 	if (destroysource) {
920bd228075SAlan Cox 		if (srcobject->handle != NULL) {
921a9fa2c05SAlfred Perlstein 			mtx_lock(&sw_alloc_mtx);
9221c7c3c6aSMatthew Dillon 			TAILQ_REMOVE(
9231c7c3c6aSMatthew Dillon 			    NOBJLIST(srcobject->handle),
9241c7c3c6aSMatthew Dillon 			    srcobject,
9251c7c3c6aSMatthew Dillon 			    pager_object_list
9261c7c3c6aSMatthew Dillon 			);
927a9fa2c05SAlfred Perlstein 			mtx_unlock(&sw_alloc_mtx);
928cbd8ec09SJohn Dyson 		}
929bd228075SAlan Cox 	}
93026f9a767SRodney W. Grimes 
9311c7c3c6aSMatthew Dillon 	/*
9321c7c3c6aSMatthew Dillon 	 * transfer source to destination.
9331c7c3c6aSMatthew Dillon 	 */
9341c7c3c6aSMatthew Dillon 	for (i = 0; i < dstobject->size; ++i) {
9351c7c3c6aSMatthew Dillon 		daddr_t dstaddr;
9361c7c3c6aSMatthew Dillon 
9371c7c3c6aSMatthew Dillon 		/*
9381c7c3c6aSMatthew Dillon 		 * Locate (without changing) the swapblk on the destination,
9391c7c3c6aSMatthew Dillon 		 * unless it is invalid in which case free it silently, or
9401c7c3c6aSMatthew Dillon 		 * if the destination is a resident page, in which case the
9411c7c3c6aSMatthew Dillon 		 * source is thrown away.
9421c7c3c6aSMatthew Dillon 		 */
9431c7c3c6aSMatthew Dillon 		dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
9441c7c3c6aSMatthew Dillon 
9451c7c3c6aSMatthew Dillon 		if (dstaddr == SWAPBLK_NONE) {
9461c7c3c6aSMatthew Dillon 			/*
9471c7c3c6aSMatthew Dillon 			 * Destination has no swapblk and is not resident,
9481c7c3c6aSMatthew Dillon 			 * copy source.
9491c7c3c6aSMatthew Dillon 			 */
9501c7c3c6aSMatthew Dillon 			daddr_t srcaddr;
9511c7c3c6aSMatthew Dillon 
9521c7c3c6aSMatthew Dillon 			srcaddr = swp_pager_meta_ctl(
9531c7c3c6aSMatthew Dillon 			    srcobject,
9541c7c3c6aSMatthew Dillon 			    i + offset,
9551c7c3c6aSMatthew Dillon 			    SWM_POP
9561c7c3c6aSMatthew Dillon 			);
9571c7c3c6aSMatthew Dillon 
958ee3dc7d7SAlan Cox 			if (srcaddr != SWAPBLK_NONE) {
959c7c8dd7eSAlan Cox 				/*
960c7c8dd7eSAlan Cox 				 * swp_pager_meta_build() can sleep.
961c7c8dd7eSAlan Cox 				 */
962c7c8dd7eSAlan Cox 				vm_object_pip_add(srcobject, 1);
96389f6b863SAttilio Rao 				VM_OBJECT_WUNLOCK(srcobject);
964c7c8dd7eSAlan Cox 				vm_object_pip_add(dstobject, 1);
9654dcc5c2dSMatthew Dillon 				swp_pager_meta_build(dstobject, i, srcaddr);
966c7c8dd7eSAlan Cox 				vm_object_pip_wakeup(dstobject);
96789f6b863SAttilio Rao 				VM_OBJECT_WLOCK(srcobject);
968c7c8dd7eSAlan Cox 				vm_object_pip_wakeup(srcobject);
969ee3dc7d7SAlan Cox 			}
9701c7c3c6aSMatthew Dillon 		} else {
9711c7c3c6aSMatthew Dillon 			/*
9721c7c3c6aSMatthew Dillon 			 * Destination has valid swapblk or it is represented
9731c7c3c6aSMatthew Dillon 			 * by a resident page.  We destroy the sourceblock.
9741c7c3c6aSMatthew Dillon 			 */
9751c7c3c6aSMatthew Dillon 
9761c7c3c6aSMatthew Dillon 			swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE);
9771c7c3c6aSMatthew Dillon 		}
97826f9a767SRodney W. Grimes 	}
97926f9a767SRodney W. Grimes 
98026f9a767SRodney W. Grimes 	/*
9811c7c3c6aSMatthew Dillon 	 * Free left over swap blocks in source.
9821c7c3c6aSMatthew Dillon 	 *
9831c7c3c6aSMatthew Dillon 	 * We have to revert the type to OBJT_DEFAULT so we do not accidently
9841c7c3c6aSMatthew Dillon 	 * double-remove the object from the swap queues.
98526f9a767SRodney W. Grimes 	 */
986c0877f10SJohn Dyson 	if (destroysource) {
9871c7c3c6aSMatthew Dillon 		swp_pager_meta_free_all(srcobject);
9881c7c3c6aSMatthew Dillon 		/*
9891c7c3c6aSMatthew Dillon 		 * Reverting the type is not necessary, the caller is going
9901c7c3c6aSMatthew Dillon 		 * to destroy srcobject directly, but I'm doing it here
991956f3135SPhilippe Charnier 		 * for consistency since we've removed the object from its
9921c7c3c6aSMatthew Dillon 		 * queues.
9931c7c3c6aSMatthew Dillon 		 */
9941c7c3c6aSMatthew Dillon 		srcobject->type = OBJT_DEFAULT;
995c0877f10SJohn Dyson 	}
99626f9a767SRodney W. Grimes }
99726f9a767SRodney W. Grimes 
998df8bae1dSRodney W. Grimes /*
9991c7c3c6aSMatthew Dillon  * SWAP_PAGER_HASPAGE() -	determine if we have good backing store for
10001c7c3c6aSMatthew Dillon  *				the requested page.
10011c7c3c6aSMatthew Dillon  *
10021c7c3c6aSMatthew Dillon  *	We determine whether good backing store exists for the requested
10031c7c3c6aSMatthew Dillon  *	page and return TRUE if it does, FALSE if it doesn't.
10041c7c3c6aSMatthew Dillon  *
10051c7c3c6aSMatthew Dillon  *	If TRUE, we also try to determine how much valid, contiguous backing
10061c7c3c6aSMatthew Dillon  *	store exists before and after the requested page within a reasonable
10071c7c3c6aSMatthew Dillon  *	distance.  We do not try to restrict it to the swap device stripe
10081c7c3c6aSMatthew Dillon  *	(that is handled in getpages/putpages).  It probably isn't worth
10091c7c3c6aSMatthew Dillon  *	doing here.
1010df8bae1dSRodney W. Grimes  */
10115ea4972cSAlan Cox static boolean_t
10122f249180SPoul-Henning Kamp swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after)
101326f9a767SRodney W. Grimes {
10141c7c3c6aSMatthew Dillon 	daddr_t blk0;
101526f9a767SRodney W. Grimes 
1016c25673ffSAttilio Rao 	VM_OBJECT_ASSERT_LOCKED(object);
10171c7c3c6aSMatthew Dillon 	/*
10181c7c3c6aSMatthew Dillon 	 * do we have good backing store at the requested index ?
10191c7c3c6aSMatthew Dillon 	 */
10201c7c3c6aSMatthew Dillon 	blk0 = swp_pager_meta_ctl(object, pindex, 0);
10211c7c3c6aSMatthew Dillon 
10224dcc5c2dSMatthew Dillon 	if (blk0 == SWAPBLK_NONE) {
10231c7c3c6aSMatthew Dillon 		if (before)
102424a1cce3SDavid Greenman 			*before = 0;
10251c7c3c6aSMatthew Dillon 		if (after)
102624a1cce3SDavid Greenman 			*after = 0;
102726f9a767SRodney W. Grimes 		return (FALSE);
102826f9a767SRodney W. Grimes 	}
102926f9a767SRodney W. Grimes 
103026f9a767SRodney W. Grimes 	/*
10311c7c3c6aSMatthew Dillon 	 * find backwards-looking contiguous good backing store
1032e47ed70bSJohn Dyson 	 */
10331c7c3c6aSMatthew Dillon 	if (before != NULL) {
103426f9a767SRodney W. Grimes 		int i;
10350d94caffSDavid Greenman 
10361c7c3c6aSMatthew Dillon 		for (i = 1; i < (SWB_NPAGES/2); ++i) {
10371c7c3c6aSMatthew Dillon 			daddr_t blk;
10381c7c3c6aSMatthew Dillon 
10391c7c3c6aSMatthew Dillon 			if (i > pindex)
10401c7c3c6aSMatthew Dillon 				break;
10411c7c3c6aSMatthew Dillon 			blk = swp_pager_meta_ctl(object, pindex - i, 0);
10421c7c3c6aSMatthew Dillon 			if (blk != blk0 - i)
10431c7c3c6aSMatthew Dillon 				break;
1044ffc82b0aSJohn Dyson 		}
10451c7c3c6aSMatthew Dillon 		*before = (i - 1);
104626f9a767SRodney W. Grimes 	}
104726f9a767SRodney W. Grimes 
104826f9a767SRodney W. Grimes 	/*
10491c7c3c6aSMatthew Dillon 	 * find forward-looking contiguous good backing store
105026f9a767SRodney W. Grimes 	 */
10511c7c3c6aSMatthew Dillon 	if (after != NULL) {
10521c7c3c6aSMatthew Dillon 		int i;
10531c7c3c6aSMatthew Dillon 
10541c7c3c6aSMatthew Dillon 		for (i = 1; i < (SWB_NPAGES/2); ++i) {
10551c7c3c6aSMatthew Dillon 			daddr_t blk;
10561c7c3c6aSMatthew Dillon 
10571c7c3c6aSMatthew Dillon 			blk = swp_pager_meta_ctl(object, pindex + i, 0);
10581c7c3c6aSMatthew Dillon 			if (blk != blk0 + i)
10591c7c3c6aSMatthew Dillon 				break;
106026f9a767SRodney W. Grimes 		}
10611c7c3c6aSMatthew Dillon 		*after = (i - 1);
10621c7c3c6aSMatthew Dillon 	}
10631c7c3c6aSMatthew Dillon 	return (TRUE);
10641c7c3c6aSMatthew Dillon }
10651c7c3c6aSMatthew Dillon 
10661c7c3c6aSMatthew Dillon /*
10671c7c3c6aSMatthew Dillon  * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page
10681c7c3c6aSMatthew Dillon  *
10691c7c3c6aSMatthew Dillon  *	This removes any associated swap backing store, whether valid or
10701c7c3c6aSMatthew Dillon  *	not, from the page.
10711c7c3c6aSMatthew Dillon  *
10721c7c3c6aSMatthew Dillon  *	This routine is typically called when a page is made dirty, at
10731c7c3c6aSMatthew Dillon  *	which point any associated swap can be freed.  MADV_FREE also
10741c7c3c6aSMatthew Dillon  *	calls us in a special-case situation
10751c7c3c6aSMatthew Dillon  *
10761c7c3c6aSMatthew Dillon  *	NOTE!!!  If the page is clean and the swap was valid, the caller
10771c7c3c6aSMatthew Dillon  *	should make the page dirty before calling this routine.  This routine
10781c7c3c6aSMatthew Dillon  *	does NOT change the m->dirty status of the page.  Also: MADV_FREE
10791c7c3c6aSMatthew Dillon  *	depends on it.
10801c7c3c6aSMatthew Dillon  *
108115523cf7SKonstantin Belousov  *	This routine may not sleep.
1082c25673ffSAttilio Rao  *
1083c25673ffSAttilio Rao  *	The object containing the page must be locked.
10841c7c3c6aSMatthew Dillon  */
10851c7c3c6aSMatthew Dillon static void
10862f249180SPoul-Henning Kamp swap_pager_unswapped(vm_page_t m)
10871c7c3c6aSMatthew Dillon {
10882f249180SPoul-Henning Kamp 
10891c7c3c6aSMatthew Dillon 	swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE);
10901c7c3c6aSMatthew Dillon }
10911c7c3c6aSMatthew Dillon 
10921c7c3c6aSMatthew Dillon /*
10931c7c3c6aSMatthew Dillon  * SWAP_PAGER_GETPAGES() - bring pages in from swap
10941c7c3c6aSMatthew Dillon  *
10951c7c3c6aSMatthew Dillon  *	Attempt to retrieve (m, count) pages from backing store, but make
10961c7c3c6aSMatthew Dillon  *	sure we retrieve at least m[reqpage].  We try to load in as large
10971c7c3c6aSMatthew Dillon  *	a chunk surrounding m[reqpage] as is contiguous in swap and which
10981c7c3c6aSMatthew Dillon  *	belongs to the same object.
10991c7c3c6aSMatthew Dillon  *
11001c7c3c6aSMatthew Dillon  *	The code is designed for asynchronous operation and
11011c7c3c6aSMatthew Dillon  *	immediate-notification of 'reqpage' but tends not to be
11021c7c3c6aSMatthew Dillon  *	used that way.  Please do not optimize-out this algorithmic
11031c7c3c6aSMatthew Dillon  *	feature, I intend to improve on it in the future.
11041c7c3c6aSMatthew Dillon  *
11051c7c3c6aSMatthew Dillon  *	The parent has a single vm_object_pip_add() reference prior to
11061c7c3c6aSMatthew Dillon  *	calling us and we should return with the same.
11071c7c3c6aSMatthew Dillon  *
11081c7c3c6aSMatthew Dillon  *	The parent has BUSY'd the pages.  We should return with 'm'
11091c7c3c6aSMatthew Dillon  *	left busy, but the others adjusted.
11101c7c3c6aSMatthew Dillon  */
1111f708ef1bSPoul-Henning Kamp static int
11122f249180SPoul-Henning Kamp swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage)
1113df8bae1dSRodney W. Grimes {
11141c7c3c6aSMatthew Dillon 	struct buf *bp;
11151c7c3c6aSMatthew Dillon 	vm_page_t mreq;
111626f9a767SRodney W. Grimes 	int i;
111726f9a767SRodney W. Grimes 	int j;
11181c7c3c6aSMatthew Dillon 	daddr_t blk;
11190d94caffSDavid Greenman 
11201c7c3c6aSMatthew Dillon 	mreq = m[reqpage];
11211c7c3c6aSMatthew Dillon 
1122e04e4bacSPoul-Henning Kamp 	KASSERT(mreq->object == object,
1123e04e4bacSPoul-Henning Kamp 	    ("swap_pager_getpages: object mismatch %p/%p",
1124e04e4bacSPoul-Henning Kamp 	    object, mreq->object));
1125e04e4bacSPoul-Henning Kamp 
11261c7c3c6aSMatthew Dillon 	/*
11271c7c3c6aSMatthew Dillon 	 * Calculate range to retrieve.  The pages have already been assigned
1128751221fdSPoul-Henning Kamp 	 * their swapblks.  We require a *contiguous* range but we know it to
1129751221fdSPoul-Henning Kamp 	 * not span devices.   If we do not supply it, bad things
11304dcc5c2dSMatthew Dillon 	 * happen.  Note that blk, iblk & jblk can be SWAPBLK_NONE, but the
11314dcc5c2dSMatthew Dillon 	 * loops are set up such that the case(s) are handled implicitly.
11324dcc5c2dSMatthew Dillon 	 *
11330b6ace47SAlan Cox 	 * The swp_*() calls must be made with the object locked.
11341c7c3c6aSMatthew Dillon 	 */
11351c7c3c6aSMatthew Dillon 	blk = swp_pager_meta_ctl(mreq->object, mreq->pindex, 0);
11361c7c3c6aSMatthew Dillon 
11371c7c3c6aSMatthew Dillon 	for (i = reqpage - 1; i >= 0; --i) {
11381c7c3c6aSMatthew Dillon 		daddr_t iblk;
11391c7c3c6aSMatthew Dillon 
11401c7c3c6aSMatthew Dillon 		iblk = swp_pager_meta_ctl(m[i]->object, m[i]->pindex, 0);
11411c7c3c6aSMatthew Dillon 		if (blk != iblk + (reqpage - i))
114226f9a767SRodney W. Grimes 			break;
114326f9a767SRodney W. Grimes 	}
11441c7c3c6aSMatthew Dillon 	++i;
11451c7c3c6aSMatthew Dillon 
11461c7c3c6aSMatthew Dillon 	for (j = reqpage + 1; j < count; ++j) {
11471c7c3c6aSMatthew Dillon 		daddr_t jblk;
11481c7c3c6aSMatthew Dillon 
11491c7c3c6aSMatthew Dillon 		jblk = swp_pager_meta_ctl(m[j]->object, m[j]->pindex, 0);
11501c7c3c6aSMatthew Dillon 		if (blk != jblk - (j - reqpage))
11511c7c3c6aSMatthew Dillon 			break;
11521c7c3c6aSMatthew Dillon 	}
11531c7c3c6aSMatthew Dillon 
11541c7c3c6aSMatthew Dillon 	/*
11551c7c3c6aSMatthew Dillon 	 * free pages outside our collection range.   Note: we never free
11561c7c3c6aSMatthew Dillon 	 * mreq, it must remain busy throughout.
11571c7c3c6aSMatthew Dillon 	 */
1158071a1710SAlan Cox 	if (0 < i || j < count) {
11591c7c3c6aSMatthew Dillon 		int k;
11601c7c3c6aSMatthew Dillon 
11613c4a2440SAlan Cox 		for (k = 0; k < i; ++k)
1162e86a87e9SKonstantin Belousov 			swp_pager_free_nrpage(m[k]);
11633c4a2440SAlan Cox 		for (k = j; k < count; ++k)
11642965a453SKip Macy 			swp_pager_free_nrpage(m[k]);
1165071a1710SAlan Cox 	}
11661c7c3c6aSMatthew Dillon 
11671c7c3c6aSMatthew Dillon 	/*
11684dcc5c2dSMatthew Dillon 	 * Return VM_PAGER_FAIL if we have nothing to do.  Return mreq
11694dcc5c2dSMatthew Dillon 	 * still busy, but the others unbusied.
11701c7c3c6aSMatthew Dillon 	 */
11714dcc5c2dSMatthew Dillon 	if (blk == SWAPBLK_NONE)
117226f9a767SRodney W. Grimes 		return (VM_PAGER_FAIL);
1173df8bae1dSRodney W. Grimes 
117416f62314SDavid Greenman 	/*
11758630c117SAlan Cox 	 * Getpbuf() can sleep.
11768630c117SAlan Cox 	 */
117789f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(object);
11788630c117SAlan Cox 	/*
117916f62314SDavid Greenman 	 * Get a swap buffer header to perform the IO
118016f62314SDavid Greenman 	 */
11811c7c3c6aSMatthew Dillon 	bp = getpbuf(&nsw_rcount);
11825e04322aSPoul-Henning Kamp 	bp->b_flags |= B_PAGING;
118326f9a767SRodney W. Grimes 
118421144e3bSPoul-Henning Kamp 	bp->b_iocmd = BIO_READ;
11851c7c3c6aSMatthew Dillon 	bp->b_iodone = swp_pager_async_iodone;
1186fdcc1cc0SJohn Baldwin 	bp->b_rcred = crhold(thread0.td_ucred);
1187fdcc1cc0SJohn Baldwin 	bp->b_wcred = crhold(thread0.td_ucred);
11881c7c3c6aSMatthew Dillon 	bp->b_blkno = blk - (reqpage - i);
11891c7c3c6aSMatthew Dillon 	bp->b_bcount = PAGE_SIZE * (j - i);
11901c7c3c6aSMatthew Dillon 	bp->b_bufsize = PAGE_SIZE * (j - i);
11911c7c3c6aSMatthew Dillon 	bp->b_pager.pg_reqpage = reqpage - i;
11921c7c3c6aSMatthew Dillon 
119389f6b863SAttilio Rao 	VM_OBJECT_WLOCK(object);
11941c7c3c6aSMatthew Dillon 	{
11951c7c3c6aSMatthew Dillon 		int k;
11961c7c3c6aSMatthew Dillon 
11971c7c3c6aSMatthew Dillon 		for (k = i; k < j; ++k) {
11981c7c3c6aSMatthew Dillon 			bp->b_pages[k - i] = m[k];
11995786be7cSAlan Cox 			m[k]->oflags |= VPO_SWAPINPROG;
12001c7c3c6aSMatthew Dillon 		}
12011c7c3c6aSMatthew Dillon 	}
12021c7c3c6aSMatthew Dillon 	bp->b_npages = j - i;
120326f9a767SRodney W. Grimes 
1204b4b70819SAttilio Rao 	PCPU_INC(cnt.v_swapin);
1205b4b70819SAttilio Rao 	PCPU_ADD(cnt.v_swappgsin, bp->b_npages);
12061c7c3c6aSMatthew Dillon 
1207df8bae1dSRodney W. Grimes 	/*
12081c7c3c6aSMatthew Dillon 	 * We still hold the lock on mreq, and our automatic completion routine
12091c7c3c6aSMatthew Dillon 	 * does not remove it.
1210df8bae1dSRodney W. Grimes 	 */
1211071a1710SAlan Cox 	vm_object_pip_add(object, bp->b_npages);
121289f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(object);
12131c7c3c6aSMatthew Dillon 
12141c7c3c6aSMatthew Dillon 	/*
12151c7c3c6aSMatthew Dillon 	 * perform the I/O.  NOTE!!!  bp cannot be considered valid after
12161c7c3c6aSMatthew Dillon 	 * this point because we automatically release it on completion.
12171c7c3c6aSMatthew Dillon 	 * Instead, we look at the one page we are interested in which we
12181c7c3c6aSMatthew Dillon 	 * still hold a lock on even through the I/O completion.
12191c7c3c6aSMatthew Dillon 	 *
12201c7c3c6aSMatthew Dillon 	 * The other pages in our m[] array are also released on completion,
12211c7c3c6aSMatthew Dillon 	 * so we cannot assume they are valid anymore either.
12221c7c3c6aSMatthew Dillon 	 *
1223c37a77eeSPoul-Henning Kamp 	 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
12241c7c3c6aSMatthew Dillon 	 */
1225b890cb2cSPeter Wemm 	BUF_KERNPROC(bp);
12264b03903aSPoul-Henning Kamp 	swp_pager_strategy(bp);
122726f9a767SRodney W. Grimes 
122826f9a767SRodney W. Grimes 	/*
12295786be7cSAlan Cox 	 * wait for the page we want to complete.  VPO_SWAPINPROG is always
12301c7c3c6aSMatthew Dillon 	 * cleared on completion.  If an I/O error occurs, SWAPBLK_NONE
12311c7c3c6aSMatthew Dillon 	 * is set in the meta-data.
123226f9a767SRodney W. Grimes 	 */
123389f6b863SAttilio Rao 	VM_OBJECT_WLOCK(object);
12345786be7cSAlan Cox 	while ((mreq->oflags & VPO_SWAPINPROG) != 0) {
1235c7aebda8SAttilio Rao 		mreq->oflags |= VPO_SWAPSLEEP;
1236b4b70819SAttilio Rao 		PCPU_INC(cnt.v_intrans);
1237c7aebda8SAttilio Rao 		if (VM_OBJECT_SLEEP(object, &object->paging_in_progress, PSWP,
1238c7aebda8SAttilio Rao 		    "swread", hz * 20)) {
12399bd86a98SBruce M Simpson 			printf(
1240c5690651SPoul-Henning Kamp "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n",
1241c5690651SPoul-Henning Kamp 			    bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount);
12421c7c3c6aSMatthew Dillon 		}
12431b119d9dSDavid Greenman 	}
124426f9a767SRodney W. Grimes 
124526f9a767SRodney W. Grimes 	/*
1246a1287949SEivind Eklund 	 * mreq is left busied after completion, but all the other pages
12471c7c3c6aSMatthew Dillon 	 * are freed.  If we had an unrecoverable read error the page will
12481c7c3c6aSMatthew Dillon 	 * not be valid.
124926f9a767SRodney W. Grimes 	 */
12501c7c3c6aSMatthew Dillon 	if (mreq->valid != VM_PAGE_BITS_ALL) {
12511c7c3c6aSMatthew Dillon 		return (VM_PAGER_ERROR);
125226f9a767SRodney W. Grimes 	} else {
12531c7c3c6aSMatthew Dillon 		return (VM_PAGER_OK);
125426f9a767SRodney W. Grimes 	}
12551c7c3c6aSMatthew Dillon 
12561c7c3c6aSMatthew Dillon 	/*
12571c7c3c6aSMatthew Dillon 	 * A final note: in a low swap situation, we cannot deallocate swap
12581c7c3c6aSMatthew Dillon 	 * and mark a page dirty here because the caller is likely to mark
12591c7c3c6aSMatthew Dillon 	 * the page clean when we return, causing the page to possibly revert
12601c7c3c6aSMatthew Dillon 	 * to all-zero's later.
12611c7c3c6aSMatthew Dillon 	 */
1262df8bae1dSRodney W. Grimes }
1263df8bae1dSRodney W. Grimes 
12641c7c3c6aSMatthew Dillon /*
126590effb23SGleb Smirnoff  * 	swap_pager_getpages_async():
126690effb23SGleb Smirnoff  *
126790effb23SGleb Smirnoff  *	Right now this is emulation of asynchronous operation on top of
126890effb23SGleb Smirnoff  *	swap_pager_getpages().
126990effb23SGleb Smirnoff  */
127090effb23SGleb Smirnoff static int
127190effb23SGleb Smirnoff swap_pager_getpages_async(vm_object_t object, vm_page_t *m, int count,
127290effb23SGleb Smirnoff     int reqpage, pgo_getpages_iodone_t iodone, void *arg)
127390effb23SGleb Smirnoff {
127490effb23SGleb Smirnoff 	int r, error;
127590effb23SGleb Smirnoff 
127690effb23SGleb Smirnoff 	r = swap_pager_getpages(object, m, count, reqpage);
127790effb23SGleb Smirnoff 	VM_OBJECT_WUNLOCK(object);
127890effb23SGleb Smirnoff 	switch (r) {
127990effb23SGleb Smirnoff 	case VM_PAGER_OK:
128090effb23SGleb Smirnoff 		error = 0;
128190effb23SGleb Smirnoff 		break;
128290effb23SGleb Smirnoff 	case VM_PAGER_ERROR:
128390effb23SGleb Smirnoff 		error = EIO;
128490effb23SGleb Smirnoff 		break;
128590effb23SGleb Smirnoff 	case VM_PAGER_FAIL:
128690effb23SGleb Smirnoff 		error = EINVAL;
128790effb23SGleb Smirnoff 		break;
128890effb23SGleb Smirnoff 	default:
1289d9328101SGleb Smirnoff 		panic("unhandled swap_pager_getpages() error %d", r);
129090effb23SGleb Smirnoff 	}
129190effb23SGleb Smirnoff 	(iodone)(arg, m, count, error);
129290effb23SGleb Smirnoff 	VM_OBJECT_WLOCK(object);
129390effb23SGleb Smirnoff 
129490effb23SGleb Smirnoff 	return (r);
129590effb23SGleb Smirnoff }
129690effb23SGleb Smirnoff 
129790effb23SGleb Smirnoff /*
12981c7c3c6aSMatthew Dillon  *	swap_pager_putpages:
12991c7c3c6aSMatthew Dillon  *
13001c7c3c6aSMatthew Dillon  *	Assign swap (if necessary) and initiate I/O on the specified pages.
13011c7c3c6aSMatthew Dillon  *
13021c7c3c6aSMatthew Dillon  *	We support both OBJT_DEFAULT and OBJT_SWAP objects.  DEFAULT objects
13031c7c3c6aSMatthew Dillon  *	are automatically converted to SWAP objects.
13041c7c3c6aSMatthew Dillon  *
1305ea3aecf5SPeter Wemm  *	In a low memory situation we may block in VOP_STRATEGY(), but the new
13061c7c3c6aSMatthew Dillon  *	vm_page reservation system coupled with properly written VFS devices
13071c7c3c6aSMatthew Dillon  *	should ensure that no low-memory deadlock occurs.  This is an area
13081c7c3c6aSMatthew Dillon  *	which needs work.
13091c7c3c6aSMatthew Dillon  *
13101c7c3c6aSMatthew Dillon  *	The parent has N vm_object_pip_add() references prior to
13111c7c3c6aSMatthew Dillon  *	calling us and will remove references for rtvals[] that are
13121c7c3c6aSMatthew Dillon  *	not set to VM_PAGER_PEND.  We need to remove the rest on I/O
13131c7c3c6aSMatthew Dillon  *	completion.
13141c7c3c6aSMatthew Dillon  *
13151c7c3c6aSMatthew Dillon  *	The parent has soft-busy'd the pages it passes us and will unbusy
13161c7c3c6aSMatthew Dillon  *	those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
13171c7c3c6aSMatthew Dillon  *	We need to unbusy the rest on I/O completion.
13181c7c3c6aSMatthew Dillon  */
1319e4542174SMatthew Dillon void
13202f249180SPoul-Henning Kamp swap_pager_putpages(vm_object_t object, vm_page_t *m, int count,
1321e065e87cSKonstantin Belousov     int flags, int *rtvals)
1322df8bae1dSRodney W. Grimes {
1323e065e87cSKonstantin Belousov 	int i, n;
1324e065e87cSKonstantin Belousov 	boolean_t sync;
1325df8bae1dSRodney W. Grimes 
13261c7c3c6aSMatthew Dillon 	if (count && m[0]->object != object) {
13277036145bSMaxim Konovalov 		panic("swap_pager_putpages: object mismatch %p/%p",
13281c7c3c6aSMatthew Dillon 		    object,
13291c7c3c6aSMatthew Dillon 		    m[0]->object
13301c7c3c6aSMatthew Dillon 		);
13311c7c3c6aSMatthew Dillon 	}
1332ee3dc7d7SAlan Cox 
13331c7c3c6aSMatthew Dillon 	/*
13341c7c3c6aSMatthew Dillon 	 * Step 1
13351c7c3c6aSMatthew Dillon 	 *
13361c7c3c6aSMatthew Dillon 	 * Turn object into OBJT_SWAP
13371c7c3c6aSMatthew Dillon 	 * check for bogus sysops
13381c7c3c6aSMatthew Dillon 	 * force sync if not pageout process
13391c7c3c6aSMatthew Dillon 	 */
13404dcc5c2dSMatthew Dillon 	if (object->type != OBJT_SWAP)
13414dcc5c2dSMatthew Dillon 		swp_pager_meta_build(object, 0, SWAPBLK_NONE);
134289f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(object);
1343e47ed70bSJohn Dyson 
1344e065e87cSKonstantin Belousov 	n = 0;
1345e47ed70bSJohn Dyson 	if (curproc != pageproc)
1346e47ed70bSJohn Dyson 		sync = TRUE;
1347e065e87cSKonstantin Belousov 	else
1348e065e87cSKonstantin Belousov 		sync = (flags & VM_PAGER_PUT_SYNC) != 0;
134926f9a767SRodney W. Grimes 
13501c7c3c6aSMatthew Dillon 	/*
13511c7c3c6aSMatthew Dillon 	 * Step 2
13521c7c3c6aSMatthew Dillon 	 *
1353ad3cce20SMatthew Dillon 	 * Update nsw parameters from swap_async_max sysctl values.
1354ad3cce20SMatthew Dillon 	 * Do not let the sysop crash the machine with bogus numbers.
1355327f4e83SMatthew Dillon 	 */
13566d541bf1SJohn Baldwin 	mtx_lock(&pbuf_mtx);
1357327f4e83SMatthew Dillon 	if (swap_async_max != nsw_wcount_async_max) {
1358327f4e83SMatthew Dillon 		int n;
1359327f4e83SMatthew Dillon 
1360327f4e83SMatthew Dillon 		/*
1361327f4e83SMatthew Dillon 		 * limit range
1362327f4e83SMatthew Dillon 		 */
1363327f4e83SMatthew Dillon 		if ((n = swap_async_max) > nswbuf / 2)
1364327f4e83SMatthew Dillon 			n = nswbuf / 2;
1365327f4e83SMatthew Dillon 		if (n < 1)
1366327f4e83SMatthew Dillon 			n = 1;
1367327f4e83SMatthew Dillon 		swap_async_max = n;
1368327f4e83SMatthew Dillon 
1369327f4e83SMatthew Dillon 		/*
1370327f4e83SMatthew Dillon 		 * Adjust difference ( if possible ).  If the current async
1371327f4e83SMatthew Dillon 		 * count is too low, we may not be able to make the adjustment
1372327f4e83SMatthew Dillon 		 * at this time.
1373327f4e83SMatthew Dillon 		 */
1374327f4e83SMatthew Dillon 		n -= nsw_wcount_async_max;
1375327f4e83SMatthew Dillon 		if (nsw_wcount_async + n >= 0) {
1376327f4e83SMatthew Dillon 			nsw_wcount_async += n;
1377327f4e83SMatthew Dillon 			nsw_wcount_async_max += n;
1378327f4e83SMatthew Dillon 			wakeup(&nsw_wcount_async);
1379327f4e83SMatthew Dillon 		}
1380327f4e83SMatthew Dillon 	}
13816d541bf1SJohn Baldwin 	mtx_unlock(&pbuf_mtx);
1382327f4e83SMatthew Dillon 
1383327f4e83SMatthew Dillon 	/*
1384327f4e83SMatthew Dillon 	 * Step 3
1385327f4e83SMatthew Dillon 	 *
13861c7c3c6aSMatthew Dillon 	 * Assign swap blocks and issue I/O.  We reallocate swap on the fly.
13871c7c3c6aSMatthew Dillon 	 * The page is left dirty until the pageout operation completes
13881c7c3c6aSMatthew Dillon 	 * successfully.
13891c7c3c6aSMatthew Dillon 	 */
13901c7c3c6aSMatthew Dillon 	for (i = 0; i < count; i += n) {
13911c7c3c6aSMatthew Dillon 		int j;
13921c7c3c6aSMatthew Dillon 		struct buf *bp;
1393a316d390SJohn Dyson 		daddr_t blk;
139426f9a767SRodney W. Grimes 
1395df8bae1dSRodney W. Grimes 		/*
13961c7c3c6aSMatthew Dillon 		 * Maximum I/O size is limited by a number of factors.
1397df8bae1dSRodney W. Grimes 		 */
13981c7c3c6aSMatthew Dillon 		n = min(BLIST_MAX_ALLOC, count - i);
1399327f4e83SMatthew Dillon 		n = min(n, nsw_cluster_max);
14001c7c3c6aSMatthew Dillon 
140126f9a767SRodney W. Grimes 		/*
14021c7c3c6aSMatthew Dillon 		 * Get biggest block of swap we can.  If we fail, fall
14031c7c3c6aSMatthew Dillon 		 * back and try to allocate a smaller block.  Don't go
14041c7c3c6aSMatthew Dillon 		 * overboard trying to allocate space if it would overly
14051c7c3c6aSMatthew Dillon 		 * fragment swap.
140626f9a767SRodney W. Grimes 		 */
14071c7c3c6aSMatthew Dillon 		while (
14081c7c3c6aSMatthew Dillon 		    (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE &&
14091c7c3c6aSMatthew Dillon 		    n > 4
14101c7c3c6aSMatthew Dillon 		) {
14111c7c3c6aSMatthew Dillon 			n >>= 1;
141226f9a767SRodney W. Grimes 		}
14131c7c3c6aSMatthew Dillon 		if (blk == SWAPBLK_NONE) {
14144dcc5c2dSMatthew Dillon 			for (j = 0; j < n; ++j)
14151c7c3c6aSMatthew Dillon 				rtvals[i+j] = VM_PAGER_FAIL;
14161c7c3c6aSMatthew Dillon 			continue;
141726f9a767SRodney W. Grimes 		}
141826f9a767SRodney W. Grimes 
141926f9a767SRodney W. Grimes 		/*
14201c7c3c6aSMatthew Dillon 		 * All I/O parameters have been satisfied, build the I/O
14211c7c3c6aSMatthew Dillon 		 * request and assign the swap space.
142226f9a767SRodney W. Grimes 		 */
1423327f4e83SMatthew Dillon 		if (sync == TRUE) {
1424327f4e83SMatthew Dillon 			bp = getpbuf(&nsw_wcount_sync);
1425327f4e83SMatthew Dillon 		} else {
1426327f4e83SMatthew Dillon 			bp = getpbuf(&nsw_wcount_async);
142721144e3bSPoul-Henning Kamp 			bp->b_flags = B_ASYNC;
1428327f4e83SMatthew Dillon 		}
14295e04322aSPoul-Henning Kamp 		bp->b_flags |= B_PAGING;
1430912e4ae9SPoul-Henning Kamp 		bp->b_iocmd = BIO_WRITE;
143126f9a767SRodney W. Grimes 
1432fdcc1cc0SJohn Baldwin 		bp->b_rcred = crhold(thread0.td_ucred);
1433fdcc1cc0SJohn Baldwin 		bp->b_wcred = crhold(thread0.td_ucred);
14341c7c3c6aSMatthew Dillon 		bp->b_bcount = PAGE_SIZE * n;
14351c7c3c6aSMatthew Dillon 		bp->b_bufsize = PAGE_SIZE * n;
14361c7c3c6aSMatthew Dillon 		bp->b_blkno = blk;
1437e47ed70bSJohn Dyson 
143889f6b863SAttilio Rao 		VM_OBJECT_WLOCK(object);
14391c7c3c6aSMatthew Dillon 		for (j = 0; j < n; ++j) {
14401c7c3c6aSMatthew Dillon 			vm_page_t mreq = m[i+j];
14411c7c3c6aSMatthew Dillon 
14421c7c3c6aSMatthew Dillon 			swp_pager_meta_build(
14431c7c3c6aSMatthew Dillon 			    mreq->object,
14441c7c3c6aSMatthew Dillon 			    mreq->pindex,
14454dcc5c2dSMatthew Dillon 			    blk + j
14461c7c3c6aSMatthew Dillon 			);
14477dbf82dcSMatthew Dillon 			vm_page_dirty(mreq);
14481c7c3c6aSMatthew Dillon 			rtvals[i+j] = VM_PAGER_OK;
14491c7c3c6aSMatthew Dillon 
14505786be7cSAlan Cox 			mreq->oflags |= VPO_SWAPINPROG;
14511c7c3c6aSMatthew Dillon 			bp->b_pages[j] = mreq;
14521c7c3c6aSMatthew Dillon 		}
145389f6b863SAttilio Rao 		VM_OBJECT_WUNLOCK(object);
14541c7c3c6aSMatthew Dillon 		bp->b_npages = n;
1455a5296b05SJulian Elischer 		/*
1456a5296b05SJulian Elischer 		 * Must set dirty range for NFS to work.
1457a5296b05SJulian Elischer 		 */
1458a5296b05SJulian Elischer 		bp->b_dirtyoff = 0;
1459a5296b05SJulian Elischer 		bp->b_dirtyend = bp->b_bcount;
14601c7c3c6aSMatthew Dillon 
1461b4b70819SAttilio Rao 		PCPU_INC(cnt.v_swapout);
1462b4b70819SAttilio Rao 		PCPU_ADD(cnt.v_swappgsout, bp->b_npages);
146326f9a767SRodney W. Grimes 
146426f9a767SRodney W. Grimes 		/*
14651c7c3c6aSMatthew Dillon 		 * asynchronous
14661c7c3c6aSMatthew Dillon 		 *
1467c37a77eeSPoul-Henning Kamp 		 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
146826f9a767SRodney W. Grimes 		 */
14691c7c3c6aSMatthew Dillon 		if (sync == FALSE) {
14701c7c3c6aSMatthew Dillon 			bp->b_iodone = swp_pager_async_iodone;
147167812eacSKirk McKusick 			BUF_KERNPROC(bp);
14724b03903aSPoul-Henning Kamp 			swp_pager_strategy(bp);
14731c7c3c6aSMatthew Dillon 
14741c7c3c6aSMatthew Dillon 			for (j = 0; j < n; ++j)
14751c7c3c6aSMatthew Dillon 				rtvals[i+j] = VM_PAGER_PEND;
147623955314SAlfred Perlstein 			/* restart outter loop */
14771c7c3c6aSMatthew Dillon 			continue;
147826f9a767SRodney W. Grimes 		}
1479e47ed70bSJohn Dyson 
148026f9a767SRodney W. Grimes 		/*
14811c7c3c6aSMatthew Dillon 		 * synchronous
14821c7c3c6aSMatthew Dillon 		 *
1483c37a77eeSPoul-Henning Kamp 		 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
14841c7c3c6aSMatthew Dillon 		 */
14852c840b1fSAlan Cox 		bp->b_iodone = bdone;
14864b03903aSPoul-Henning Kamp 		swp_pager_strategy(bp);
14871c7c3c6aSMatthew Dillon 
14881c7c3c6aSMatthew Dillon 		/*
14891c7c3c6aSMatthew Dillon 		 * Wait for the sync I/O to complete, then update rtvals.
14901c7c3c6aSMatthew Dillon 		 * We just set the rtvals[] to VM_PAGER_PEND so we can call
14911c7c3c6aSMatthew Dillon 		 * our async completion routine at the end, thus avoiding a
14921c7c3c6aSMatthew Dillon 		 * double-free.
149326f9a767SRodney W. Grimes 		 */
14942c840b1fSAlan Cox 		bwait(bp, PVM, "swwrt");
14951c7c3c6aSMatthew Dillon 		for (j = 0; j < n; ++j)
14961c7c3c6aSMatthew Dillon 			rtvals[i+j] = VM_PAGER_PEND;
14971c7c3c6aSMatthew Dillon 		/*
14981c7c3c6aSMatthew Dillon 		 * Now that we are through with the bp, we can call the
14991c7c3c6aSMatthew Dillon 		 * normal async completion, which frees everything up.
15001c7c3c6aSMatthew Dillon 		 */
15011c7c3c6aSMatthew Dillon 		swp_pager_async_iodone(bp);
15021c7c3c6aSMatthew Dillon 	}
150389f6b863SAttilio Rao 	VM_OBJECT_WLOCK(object);
15041c7c3c6aSMatthew Dillon }
15051c7c3c6aSMatthew Dillon 
15061c7c3c6aSMatthew Dillon /*
15071c7c3c6aSMatthew Dillon  *	swp_pager_async_iodone:
15081c7c3c6aSMatthew Dillon  *
15091c7c3c6aSMatthew Dillon  *	Completion routine for asynchronous reads and writes from/to swap.
15101c7c3c6aSMatthew Dillon  *	Also called manually by synchronous code to finish up a bp.
15111c7c3c6aSMatthew Dillon  *
151215523cf7SKonstantin Belousov  *	This routine may not sleep.
15131c7c3c6aSMatthew Dillon  */
15141c7c3c6aSMatthew Dillon static void
15152f249180SPoul-Henning Kamp swp_pager_async_iodone(struct buf *bp)
15161c7c3c6aSMatthew Dillon {
15171c7c3c6aSMatthew Dillon 	int i;
15181c7c3c6aSMatthew Dillon 	vm_object_t object = NULL;
15191c7c3c6aSMatthew Dillon 
15201c7c3c6aSMatthew Dillon 	/*
15211c7c3c6aSMatthew Dillon 	 * report error
15221c7c3c6aSMatthew Dillon 	 */
1523c244d2deSPoul-Henning Kamp 	if (bp->b_ioflags & BIO_ERROR) {
15241c7c3c6aSMatthew Dillon 		printf(
15251c7c3c6aSMatthew Dillon 		    "swap_pager: I/O error - %s failed; blkno %ld,"
15261c7c3c6aSMatthew Dillon 			"size %ld, error %d\n",
152721144e3bSPoul-Henning Kamp 		    ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
15281c7c3c6aSMatthew Dillon 		    (long)bp->b_blkno,
15291c7c3c6aSMatthew Dillon 		    (long)bp->b_bcount,
15301c7c3c6aSMatthew Dillon 		    bp->b_error
15311c7c3c6aSMatthew Dillon 		);
15321c7c3c6aSMatthew Dillon 	}
15331c7c3c6aSMatthew Dillon 
15341c7c3c6aSMatthew Dillon 	/*
153526f9a767SRodney W. Grimes 	 * remove the mapping for kernel virtual
153626f9a767SRodney W. Grimes 	 */
15372cc718a1SKonstantin Belousov 	if ((bp->b_flags & B_UNMAPPED) != 0) {
15382cc718a1SKonstantin Belousov 		bp->b_data = bp->b_kvaalloc;
15392cc718a1SKonstantin Belousov 		bp->b_kvabase = bp->b_kvaalloc;
15402cc718a1SKonstantin Belousov 		bp->b_flags &= ~B_UNMAPPED;
15412cc718a1SKonstantin Belousov 	} else
15421c7c3c6aSMatthew Dillon 		pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
154326f9a767SRodney W. Grimes 
154433a609ecSAlan Cox 	if (bp->b_npages) {
154533a609ecSAlan Cox 		object = bp->b_pages[0]->object;
154689f6b863SAttilio Rao 		VM_OBJECT_WLOCK(object);
154733a609ecSAlan Cox 	}
15482965a453SKip Macy 
154926f9a767SRodney W. Grimes 	/*
15501c7c3c6aSMatthew Dillon 	 * cleanup pages.  If an error occurs writing to swap, we are in
15511c7c3c6aSMatthew Dillon 	 * very serious trouble.  If it happens to be a disk error, though,
15521c7c3c6aSMatthew Dillon 	 * we may be able to recover by reassigning the swap later on.  So
15531c7c3c6aSMatthew Dillon 	 * in this case we remove the m->swapblk assignment for the page
15541c7c3c6aSMatthew Dillon 	 * but do not free it in the rlist.  The errornous block(s) are thus
15551c7c3c6aSMatthew Dillon 	 * never reallocated as swap.  Redirty the page and continue.
155626f9a767SRodney W. Grimes 	 */
15571c7c3c6aSMatthew Dillon 	for (i = 0; i < bp->b_npages; ++i) {
15581c7c3c6aSMatthew Dillon 		vm_page_t m = bp->b_pages[i];
1559e47ed70bSJohn Dyson 
15605786be7cSAlan Cox 		m->oflags &= ~VPO_SWAPINPROG;
1561c7aebda8SAttilio Rao 		if (m->oflags & VPO_SWAPSLEEP) {
1562c7aebda8SAttilio Rao 			m->oflags &= ~VPO_SWAPSLEEP;
1563c7aebda8SAttilio Rao 			wakeup(&object->paging_in_progress);
1564c7aebda8SAttilio Rao 		}
1565e47ed70bSJohn Dyson 
1566c244d2deSPoul-Henning Kamp 		if (bp->b_ioflags & BIO_ERROR) {
1567ffc82b0aSJohn Dyson 			/*
15681c7c3c6aSMatthew Dillon 			 * If an error occurs I'd love to throw the swapblk
15691c7c3c6aSMatthew Dillon 			 * away without freeing it back to swapspace, so it
15701c7c3c6aSMatthew Dillon 			 * can never be used again.  But I can't from an
15711c7c3c6aSMatthew Dillon 			 * interrupt.
1572ffc82b0aSJohn Dyson 			 */
157321144e3bSPoul-Henning Kamp 			if (bp->b_iocmd == BIO_READ) {
15741c7c3c6aSMatthew Dillon 				/*
15751c7c3c6aSMatthew Dillon 				 * When reading, reqpage needs to stay
15761c7c3c6aSMatthew Dillon 				 * locked for the parent, but all other
15771c7c3c6aSMatthew Dillon 				 * pages can be freed.  We still want to
15781c7c3c6aSMatthew Dillon 				 * wakeup the parent waiting on the page,
15791c7c3c6aSMatthew Dillon 				 * though.  ( also: pg_reqpage can be -1 and
15801c7c3c6aSMatthew Dillon 				 * not match anything ).
15811c7c3c6aSMatthew Dillon 				 *
15821c7c3c6aSMatthew Dillon 				 * We have to wake specifically requested pages
15835786be7cSAlan Cox 				 * up too because we cleared VPO_SWAPINPROG and
15841c7c3c6aSMatthew Dillon 				 * someone may be waiting for that.
15851c7c3c6aSMatthew Dillon 				 *
15861c7c3c6aSMatthew Dillon 				 * NOTE: for reads, m->dirty will probably
1587956f3135SPhilippe Charnier 				 * be overridden by the original caller of
15881c7c3c6aSMatthew Dillon 				 * getpages so don't play cute tricks here.
15891c7c3c6aSMatthew Dillon 				 */
15901c7c3c6aSMatthew Dillon 				m->valid = 0;
15911c7c3c6aSMatthew Dillon 				if (i != bp->b_pager.pg_reqpage)
1592e86a87e9SKonstantin Belousov 					swp_pager_free_nrpage(m);
1593c7aebda8SAttilio Rao 				else {
1594c7aebda8SAttilio Rao 					vm_page_lock(m);
15951c7c3c6aSMatthew Dillon 					vm_page_flash(m);
1596c7aebda8SAttilio Rao 					vm_page_unlock(m);
1597c7aebda8SAttilio Rao 				}
15981c7c3c6aSMatthew Dillon 				/*
15991c7c3c6aSMatthew Dillon 				 * If i == bp->b_pager.pg_reqpage, do not wake
16001c7c3c6aSMatthew Dillon 				 * the page up.  The caller needs to.
16011c7c3c6aSMatthew Dillon 				 */
16021c7c3c6aSMatthew Dillon 			} else {
16031c7c3c6aSMatthew Dillon 				/*
16041c7c3c6aSMatthew Dillon 				 * If a write error occurs, reactivate page
16051c7c3c6aSMatthew Dillon 				 * so it doesn't clog the inactive list,
16061c7c3c6aSMatthew Dillon 				 * then finish the I/O.
16071c7c3c6aSMatthew Dillon 				 */
16087dbf82dcSMatthew Dillon 				vm_page_dirty(m);
16093c4a2440SAlan Cox 				vm_page_lock(m);
16101c7c3c6aSMatthew Dillon 				vm_page_activate(m);
16113c4a2440SAlan Cox 				vm_page_unlock(m);
1612c7aebda8SAttilio Rao 				vm_page_sunbusy(m);
16131c7c3c6aSMatthew Dillon 			}
161421144e3bSPoul-Henning Kamp 		} else if (bp->b_iocmd == BIO_READ) {
16151c7c3c6aSMatthew Dillon 			/*
16161c7c3c6aSMatthew Dillon 			 * NOTE: for reads, m->dirty will probably be
1617956f3135SPhilippe Charnier 			 * overridden by the original caller of getpages so
16181c7c3c6aSMatthew Dillon 			 * we cannot set them in order to free the underlying
16191c7c3c6aSMatthew Dillon 			 * swap in a low-swap situation.  I don't think we'd
16201c7c3c6aSMatthew Dillon 			 * want to do that anyway, but it was an optimization
16211c7c3c6aSMatthew Dillon 			 * that existed in the old swapper for a time before
16221c7c3c6aSMatthew Dillon 			 * it got ripped out due to precisely this problem.
16231c7c3c6aSMatthew Dillon 			 *
16241c7c3c6aSMatthew Dillon 			 * If not the requested page then deactivate it.
16251c7c3c6aSMatthew Dillon 			 *
16261c7c3c6aSMatthew Dillon 			 * Note that the requested page, reqpage, is left
16271c7c3c6aSMatthew Dillon 			 * busied, but we still have to wake it up.  The
16281c7c3c6aSMatthew Dillon 			 * other pages are released (unbusied) by
1629c7aebda8SAttilio Rao 			 * vm_page_xunbusy().
16301c7c3c6aSMatthew Dillon 			 */
1631016a3c93SAlan Cox 			KASSERT(!pmap_page_is_mapped(m),
1632016a3c93SAlan Cox 			    ("swp_pager_async_iodone: page %p is mapped", m));
16331c7c3c6aSMatthew Dillon 			m->valid = VM_PAGE_BITS_ALL;
1634016a3c93SAlan Cox 			KASSERT(m->dirty == 0,
1635016a3c93SAlan Cox 			    ("swp_pager_async_iodone: page %p is dirty", m));
16361c7c3c6aSMatthew Dillon 
16371c7c3c6aSMatthew Dillon 			/*
16381c7c3c6aSMatthew Dillon 			 * We have to wake specifically requested pages
16395786be7cSAlan Cox 			 * up too because we cleared VPO_SWAPINPROG and
16401c7c3c6aSMatthew Dillon 			 * could be waiting for it in getpages.  However,
16411c7c3c6aSMatthew Dillon 			 * be sure to not unbusy getpages specifically
16421c7c3c6aSMatthew Dillon 			 * requested page - getpages expects it to be
16431c7c3c6aSMatthew Dillon 			 * left busy.
16441c7c3c6aSMatthew Dillon 			 */
16451c7c3c6aSMatthew Dillon 			if (i != bp->b_pager.pg_reqpage) {
16463c4a2440SAlan Cox 				vm_page_lock(m);
16471c7c3c6aSMatthew Dillon 				vm_page_deactivate(m);
16483c4a2440SAlan Cox 				vm_page_unlock(m);
1649c7aebda8SAttilio Rao 				vm_page_xunbusy(m);
1650c7aebda8SAttilio Rao 			} else {
1651c7aebda8SAttilio Rao 				vm_page_lock(m);
16521c7c3c6aSMatthew Dillon 				vm_page_flash(m);
1653c7aebda8SAttilio Rao 				vm_page_unlock(m);
1654c7aebda8SAttilio Rao 			}
16551c7c3c6aSMatthew Dillon 		} else {
16561c7c3c6aSMatthew Dillon 			/*
1657016a3c93SAlan Cox 			 * For write success, clear the dirty
16581c7c3c6aSMatthew Dillon 			 * status, then finish the I/O ( which decrements the
16591c7c3c6aSMatthew Dillon 			 * busy count and possibly wakes waiter's up ).
16601c7c3c6aSMatthew Dillon 			 */
16616031c68dSAlan Cox 			KASSERT(!pmap_page_is_write_mapped(m),
1662016a3c93SAlan Cox 			    ("swp_pager_async_iodone: page %p is not write"
1663016a3c93SAlan Cox 			    " protected", m));
1664c52e7044SAlan Cox 			vm_page_undirty(m);
1665c7aebda8SAttilio Rao 			vm_page_sunbusy(m);
16663c4a2440SAlan Cox 			if (vm_page_count_severe()) {
16673c4a2440SAlan Cox 				vm_page_lock(m);
16682c840b1fSAlan Cox 				vm_page_try_to_cache(m);
16692965a453SKip Macy 				vm_page_unlock(m);
16702965a453SKip Macy 			}
16713c4a2440SAlan Cox 		}
16723c4a2440SAlan Cox 	}
167326f9a767SRodney W. Grimes 
16741c7c3c6aSMatthew Dillon 	/*
16751c7c3c6aSMatthew Dillon 	 * adjust pip.  NOTE: the original parent may still have its own
16761c7c3c6aSMatthew Dillon 	 * pip refs on the object.
16771c7c3c6aSMatthew Dillon 	 */
16780d420ad3SAlan Cox 	if (object != NULL) {
16791c7c3c6aSMatthew Dillon 		vm_object_pip_wakeupn(object, bp->b_npages);
168089f6b863SAttilio Rao 		VM_OBJECT_WUNLOCK(object);
16810d420ad3SAlan Cox 	}
168226f9a767SRodney W. Grimes 
16831c7c3c6aSMatthew Dillon 	/*
1684100650deSOlivier Houchard 	 * swapdev_strategy() manually sets b_vp and b_bufobj before calling
1685100650deSOlivier Houchard 	 * bstrategy(). Set them back to NULL now we're done with it, or we'll
1686100650deSOlivier Houchard 	 * trigger a KASSERT in relpbuf().
1687100650deSOlivier Houchard 	 */
1688100650deSOlivier Houchard 	if (bp->b_vp) {
1689100650deSOlivier Houchard 		    bp->b_vp = NULL;
1690100650deSOlivier Houchard 		    bp->b_bufobj = NULL;
1691100650deSOlivier Houchard 	}
1692100650deSOlivier Houchard 	/*
16931c7c3c6aSMatthew Dillon 	 * release the physical I/O buffer
16941c7c3c6aSMatthew Dillon 	 */
1695327f4e83SMatthew Dillon 	relpbuf(
1696327f4e83SMatthew Dillon 	    bp,
169721144e3bSPoul-Henning Kamp 	    ((bp->b_iocmd == BIO_READ) ? &nsw_rcount :
1698327f4e83SMatthew Dillon 		((bp->b_flags & B_ASYNC) ?
1699327f4e83SMatthew Dillon 		    &nsw_wcount_async :
1700327f4e83SMatthew Dillon 		    &nsw_wcount_sync
1701327f4e83SMatthew Dillon 		)
1702327f4e83SMatthew Dillon 	    )
1703327f4e83SMatthew Dillon 	);
170426f9a767SRodney W. Grimes }
17051c7c3c6aSMatthew Dillon 
170692da00bbSMatthew Dillon /*
170792da00bbSMatthew Dillon  *	swap_pager_isswapped:
170892da00bbSMatthew Dillon  *
170992da00bbSMatthew Dillon  *	Return 1 if at least one page in the given object is paged
171092da00bbSMatthew Dillon  *	out to the given swap device.
171192da00bbSMatthew Dillon  *
171215523cf7SKonstantin Belousov  *	This routine may not sleep.
171392da00bbSMatthew Dillon  */
17148f60c087SPoul-Henning Kamp int
17158f60c087SPoul-Henning Kamp swap_pager_isswapped(vm_object_t object, struct swdevt *sp)
17168f60c087SPoul-Henning Kamp {
171792da00bbSMatthew Dillon 	daddr_t index = 0;
171892da00bbSMatthew Dillon 	int bcount;
171992da00bbSMatthew Dillon 	int i;
172092da00bbSMatthew Dillon 
172189f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
1722f6bcadc4SDavid Schultz 	if (object->type != OBJT_SWAP)
1723f6bcadc4SDavid Schultz 		return (0);
1724f6bcadc4SDavid Schultz 
1725b3fed13eSDavid Schultz 	mtx_lock(&swhash_mtx);
172692da00bbSMatthew Dillon 	for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) {
172792da00bbSMatthew Dillon 		struct swblock *swap;
172892da00bbSMatthew Dillon 
172992da00bbSMatthew Dillon 		if ((swap = *swp_pager_hash(object, index)) != NULL) {
173092da00bbSMatthew Dillon 			for (i = 0; i < SWAP_META_PAGES; ++i) {
1731b3fed13eSDavid Schultz 				if (swp_pager_isondev(swap->swb_pages[i], sp)) {
17327827d9b0SAlan Cox 					mtx_unlock(&swhash_mtx);
1733b3fed13eSDavid Schultz 					return (1);
173492da00bbSMatthew Dillon 				}
173592da00bbSMatthew Dillon 			}
17367827d9b0SAlan Cox 		}
173792da00bbSMatthew Dillon 		index += SWAP_META_PAGES;
173892da00bbSMatthew Dillon 	}
1739b3fed13eSDavid Schultz 	mtx_unlock(&swhash_mtx);
1740b3fed13eSDavid Schultz 	return (0);
174192da00bbSMatthew Dillon }
174292da00bbSMatthew Dillon 
174392da00bbSMatthew Dillon /*
174492da00bbSMatthew Dillon  * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in
174592da00bbSMatthew Dillon  *
174692da00bbSMatthew Dillon  *	This routine dissociates the page at the given index within a
174792da00bbSMatthew Dillon  *	swap block from its backing store, paging it in if necessary.
174892da00bbSMatthew Dillon  *	If the page is paged in, it is placed in the inactive queue,
174992da00bbSMatthew Dillon  *	since it had its backing store ripped out from under it.
175092da00bbSMatthew Dillon  *	We also attempt to swap in all other pages in the swap block,
175192da00bbSMatthew Dillon  *	we only guarantee that the one at the specified index is
175292da00bbSMatthew Dillon  *	paged in.
175392da00bbSMatthew Dillon  *
175492da00bbSMatthew Dillon  *	XXX - The code to page the whole block in doesn't work, so we
175592da00bbSMatthew Dillon  *	      revert to the one-by-one behavior for now.  Sigh.
175692da00bbSMatthew Dillon  */
175762a59e8fSWarner Losh static inline void
1758b3fed13eSDavid Schultz swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex)
175992da00bbSMatthew Dillon {
176092da00bbSMatthew Dillon 	vm_page_t m;
176192da00bbSMatthew Dillon 
176292da00bbSMatthew Dillon 	vm_object_pip_add(object, 1);
17635944de8eSKonstantin Belousov 	m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL);
176492da00bbSMatthew Dillon 	if (m->valid == VM_PAGE_BITS_ALL) {
17650d8243ccSAttilio Rao 		vm_object_pip_wakeup(object);
176692da00bbSMatthew Dillon 		vm_page_dirty(m);
1767d061cdd5SAlan Cox 		vm_page_lock(m);
1768d061cdd5SAlan Cox 		vm_page_activate(m);
17692965a453SKip Macy 		vm_page_unlock(m);
1770c7aebda8SAttilio Rao 		vm_page_xunbusy(m);
177192da00bbSMatthew Dillon 		vm_pager_page_unswapped(m);
177292da00bbSMatthew Dillon 		return;
177392da00bbSMatthew Dillon 	}
177492da00bbSMatthew Dillon 
1775b3fed13eSDavid Schultz 	if (swap_pager_getpages(object, &m, 1, 0) != VM_PAGER_OK)
177692da00bbSMatthew Dillon 		panic("swap_pager_force_pagein: read from swap failed");/*XXX*/
17770d8243ccSAttilio Rao 	vm_object_pip_wakeup(object);
177892da00bbSMatthew Dillon 	vm_page_dirty(m);
1779db1f085eSAlan Cox 	vm_page_lock(m);
1780db1f085eSAlan Cox 	vm_page_deactivate(m);
17812965a453SKip Macy 	vm_page_unlock(m);
1782c7aebda8SAttilio Rao 	vm_page_xunbusy(m);
178392da00bbSMatthew Dillon 	vm_pager_page_unswapped(m);
178492da00bbSMatthew Dillon }
178592da00bbSMatthew Dillon 
178692da00bbSMatthew Dillon /*
178792da00bbSMatthew Dillon  *	swap_pager_swapoff:
178892da00bbSMatthew Dillon  *
178992da00bbSMatthew Dillon  *	Page in all of the pages that have been paged out to the
179092da00bbSMatthew Dillon  *	given device.  The corresponding blocks in the bitmap must be
179192da00bbSMatthew Dillon  *	marked as allocated and the device must be flagged SW_CLOSING.
179292da00bbSMatthew Dillon  *	There may be no processes swapped out to the device.
179392da00bbSMatthew Dillon  *
179492da00bbSMatthew Dillon  *	This routine may block.
179592da00bbSMatthew Dillon  */
1796e9c0cc15SPoul-Henning Kamp static void
1797b3fed13eSDavid Schultz swap_pager_swapoff(struct swdevt *sp)
179892da00bbSMatthew Dillon {
179992da00bbSMatthew Dillon 	struct swblock *swap;
18008bc61209SDavid Schultz 	int i, j, retries;
180192da00bbSMatthew Dillon 
180292da00bbSMatthew Dillon 	GIANT_REQUIRED;
180392da00bbSMatthew Dillon 
18048bc61209SDavid Schultz 	retries = 0;
180592da00bbSMatthew Dillon full_rescan:
1806b3fed13eSDavid Schultz 	mtx_lock(&swhash_mtx);
180792da00bbSMatthew Dillon 	for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */
180892da00bbSMatthew Dillon restart:
1809b3fed13eSDavid Schultz 		for (swap = swhash[i]; swap != NULL; swap = swap->swb_hnext) {
1810b3fed13eSDavid Schultz 			vm_object_t object = swap->swb_object;
1811b3fed13eSDavid Schultz 			vm_pindex_t pindex = swap->swb_index;
181292da00bbSMatthew Dillon 			for (j = 0; j < SWAP_META_PAGES; ++j) {
1813b3fed13eSDavid Schultz 				if (swp_pager_isondev(swap->swb_pages[j], sp)) {
1814b3fed13eSDavid Schultz 					/* avoid deadlock */
181589f6b863SAttilio Rao 					if (!VM_OBJECT_TRYWLOCK(object)) {
181692da00bbSMatthew Dillon 						break;
1817b3fed13eSDavid Schultz 					} else {
1818b3fed13eSDavid Schultz 						mtx_unlock(&swhash_mtx);
1819b3fed13eSDavid Schultz 						swp_pager_force_pagein(object,
1820b3fed13eSDavid Schultz 						    pindex + j);
182189f6b863SAttilio Rao 						VM_OBJECT_WUNLOCK(object);
1822b3fed13eSDavid Schultz 						mtx_lock(&swhash_mtx);
182392da00bbSMatthew Dillon 						goto restart;
182492da00bbSMatthew Dillon 					}
1825b3fed13eSDavid Schultz 				}
1826b3fed13eSDavid Schultz 			}
1827b3fed13eSDavid Schultz 		}
182892da00bbSMatthew Dillon 	}
1829d536c58fSAlan Cox 	mtx_unlock(&swhash_mtx);
18308bc61209SDavid Schultz 	if (sp->sw_used) {
183192da00bbSMatthew Dillon 		/*
18328bc61209SDavid Schultz 		 * Objects may be locked or paging to the device being
18338bc61209SDavid Schultz 		 * removed, so we will miss their pages and need to
18348bc61209SDavid Schultz 		 * make another pass.  We have marked this device as
18358bc61209SDavid Schultz 		 * SW_CLOSING, so the activity should finish soon.
183692da00bbSMatthew Dillon 		 */
18378bc61209SDavid Schultz 		retries++;
18388bc61209SDavid Schultz 		if (retries > 100) {
18398bc61209SDavid Schultz 			panic("swapoff: failed to locate %d swap blocks",
18408bc61209SDavid Schultz 			    sp->sw_used);
18418bc61209SDavid Schultz 		}
18424d70511aSJohn Baldwin 		pause("swpoff", hz / 20);
184392da00bbSMatthew Dillon 		goto full_rescan;
184492da00bbSMatthew Dillon 	}
184592da00bbSMatthew Dillon }
184692da00bbSMatthew Dillon 
18471c7c3c6aSMatthew Dillon /************************************************************************
18481c7c3c6aSMatthew Dillon  *				SWAP META DATA 				*
18491c7c3c6aSMatthew Dillon  ************************************************************************
18501c7c3c6aSMatthew Dillon  *
18511c7c3c6aSMatthew Dillon  *	These routines manipulate the swap metadata stored in the
1852cec9f109SDavid E. O'Brien  *	OBJT_SWAP object.
18531c7c3c6aSMatthew Dillon  *
18544dcc5c2dSMatthew Dillon  *	Swap metadata is implemented with a global hash and not directly
18554dcc5c2dSMatthew Dillon  *	linked into the object.  Instead the object simply contains
18564dcc5c2dSMatthew Dillon  *	appropriate tracking counters.
18571c7c3c6aSMatthew Dillon  */
18581c7c3c6aSMatthew Dillon 
18591c7c3c6aSMatthew Dillon /*
18601c7c3c6aSMatthew Dillon  * SWP_PAGER_META_BUILD() -	add swap block to swap meta data for object
18611c7c3c6aSMatthew Dillon  *
18621c7c3c6aSMatthew Dillon  *	We first convert the object to a swap object if it is a default
18631c7c3c6aSMatthew Dillon  *	object.
18641c7c3c6aSMatthew Dillon  *
18651c7c3c6aSMatthew Dillon  *	The specified swapblk is added to the object's swap metadata.  If
18661c7c3c6aSMatthew Dillon  *	the swapblk is not valid, it is freed instead.  Any previously
18671c7c3c6aSMatthew Dillon  *	assigned swapblk is freed.
18681c7c3c6aSMatthew Dillon  */
18691c7c3c6aSMatthew Dillon static void
18702f249180SPoul-Henning Kamp swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
18712f249180SPoul-Henning Kamp {
18723ff863f1SDag-Erling Smørgrav 	static volatile int exhausted;
18731c7c3c6aSMatthew Dillon 	struct swblock *swap;
18741c7c3c6aSMatthew Dillon 	struct swblock **pswap;
187523f09d50SIan Dowse 	int idx;
18761c7c3c6aSMatthew Dillon 
187789f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
18781c7c3c6aSMatthew Dillon 	/*
18791c7c3c6aSMatthew Dillon 	 * Convert default object to swap object if necessary
18801c7c3c6aSMatthew Dillon 	 */
18811c7c3c6aSMatthew Dillon 	if (object->type != OBJT_SWAP) {
18821c7c3c6aSMatthew Dillon 		object->type = OBJT_SWAP;
18831c7c3c6aSMatthew Dillon 		object->un_pager.swp.swp_bcount = 0;
18841c7c3c6aSMatthew Dillon 
18851c7c3c6aSMatthew Dillon 		if (object->handle != NULL) {
1886bd228075SAlan Cox 			mtx_lock(&sw_alloc_mtx);
18871c7c3c6aSMatthew Dillon 			TAILQ_INSERT_TAIL(
18881c7c3c6aSMatthew Dillon 			    NOBJLIST(object->handle),
18891c7c3c6aSMatthew Dillon 			    object,
18901c7c3c6aSMatthew Dillon 			    pager_object_list
18911c7c3c6aSMatthew Dillon 			);
1892a9fa2c05SAlfred Perlstein 			mtx_unlock(&sw_alloc_mtx);
18931c7c3c6aSMatthew Dillon 		}
1894bd228075SAlan Cox 	}
18951c7c3c6aSMatthew Dillon 
18961c7c3c6aSMatthew Dillon 	/*
18971c7c3c6aSMatthew Dillon 	 * Locate hash entry.  If not found create, but if we aren't adding
18984dcc5c2dSMatthew Dillon 	 * anything just return.  If we run out of space in the map we wait
18994dcc5c2dSMatthew Dillon 	 * and, since the hash table may have changed, retry.
19001c7c3c6aSMatthew Dillon 	 */
19014dcc5c2dSMatthew Dillon retry:
19027827d9b0SAlan Cox 	mtx_lock(&swhash_mtx);
190323f09d50SIan Dowse 	pswap = swp_pager_hash(object, pindex);
19041c7c3c6aSMatthew Dillon 
19051c7c3c6aSMatthew Dillon 	if ((swap = *pswap) == NULL) {
19061c7c3c6aSMatthew Dillon 		int i;
19071c7c3c6aSMatthew Dillon 
19081c7c3c6aSMatthew Dillon 		if (swapblk == SWAPBLK_NONE)
19097827d9b0SAlan Cox 			goto done;
19101c7c3c6aSMatthew Dillon 
19115a3c920fSKonstantin Belousov 		swap = *pswap = uma_zalloc(swap_zone, M_NOWAIT |
19125a3c920fSKonstantin Belousov 		    (curproc == pageproc ? M_USE_RESERVE : 0));
19134dcc5c2dSMatthew Dillon 		if (swap == NULL) {
19147827d9b0SAlan Cox 			mtx_unlock(&swhash_mtx);
191589f6b863SAttilio Rao 			VM_OBJECT_WUNLOCK(object);
19162025d69bSKonstantin Belousov 			if (uma_zone_exhausted(swap_zone)) {
1917dc1b35b5SDag-Erling Smørgrav 				if (atomic_cmpset_int(&exhausted, 0, 1))
19183ff863f1SDag-Erling Smørgrav 					printf("swap zone exhausted, "
19193ff863f1SDag-Erling Smørgrav 					    "increase kern.maxswzone\n");
19202025d69bSKonstantin Belousov 				vm_pageout_oom(VM_OOM_SWAPZ);
19212025d69bSKonstantin Belousov 				pause("swzonex", 10);
19222025d69bSKonstantin Belousov 			} else
19234dcc5c2dSMatthew Dillon 				VM_WAIT;
192489f6b863SAttilio Rao 			VM_OBJECT_WLOCK(object);
19254dcc5c2dSMatthew Dillon 			goto retry;
19264dcc5c2dSMatthew Dillon 		}
1927670d17b5SJeff Roberson 
1928dc1b35b5SDag-Erling Smørgrav 		if (atomic_cmpset_int(&exhausted, 1, 0))
19293ff863f1SDag-Erling Smørgrav 			printf("swap zone ok\n");
19303ff863f1SDag-Erling Smørgrav 
19311c7c3c6aSMatthew Dillon 		swap->swb_hnext = NULL;
19321c7c3c6aSMatthew Dillon 		swap->swb_object = object;
193323f09d50SIan Dowse 		swap->swb_index = pindex & ~(vm_pindex_t)SWAP_META_MASK;
19341c7c3c6aSMatthew Dillon 		swap->swb_count = 0;
19351c7c3c6aSMatthew Dillon 
19361c7c3c6aSMatthew Dillon 		++object->un_pager.swp.swp_bcount;
19371c7c3c6aSMatthew Dillon 
19381c7c3c6aSMatthew Dillon 		for (i = 0; i < SWAP_META_PAGES; ++i)
19391c7c3c6aSMatthew Dillon 			swap->swb_pages[i] = SWAPBLK_NONE;
19401c7c3c6aSMatthew Dillon 	}
19411c7c3c6aSMatthew Dillon 
19421c7c3c6aSMatthew Dillon 	/*
19431c7c3c6aSMatthew Dillon 	 * Delete prior contents of metadata
19441c7c3c6aSMatthew Dillon 	 */
194523f09d50SIan Dowse 	idx = pindex & SWAP_META_MASK;
19461c7c3c6aSMatthew Dillon 
194723f09d50SIan Dowse 	if (swap->swb_pages[idx] != SWAPBLK_NONE) {
194823f09d50SIan Dowse 		swp_pager_freeswapspace(swap->swb_pages[idx], 1);
19491c7c3c6aSMatthew Dillon 		--swap->swb_count;
19501c7c3c6aSMatthew Dillon 	}
19511c7c3c6aSMatthew Dillon 
19521c7c3c6aSMatthew Dillon 	/*
19531c7c3c6aSMatthew Dillon 	 * Enter block into metadata
19541c7c3c6aSMatthew Dillon 	 */
195523f09d50SIan Dowse 	swap->swb_pages[idx] = swapblk;
19564dcc5c2dSMatthew Dillon 	if (swapblk != SWAPBLK_NONE)
19571c7c3c6aSMatthew Dillon 		++swap->swb_count;
19587827d9b0SAlan Cox done:
19597827d9b0SAlan Cox 	mtx_unlock(&swhash_mtx);
19601c7c3c6aSMatthew Dillon }
19611c7c3c6aSMatthew Dillon 
19621c7c3c6aSMatthew Dillon /*
19631c7c3c6aSMatthew Dillon  * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
19641c7c3c6aSMatthew Dillon  *
19651c7c3c6aSMatthew Dillon  *	The requested range of blocks is freed, with any associated swap
19661c7c3c6aSMatthew Dillon  *	returned to the swap bitmap.
19671c7c3c6aSMatthew Dillon  *
19681c7c3c6aSMatthew Dillon  *	This routine will free swap metadata structures as they are cleaned
19691c7c3c6aSMatthew Dillon  *	out.  This routine does *NOT* operate on swap metadata associated
19701c7c3c6aSMatthew Dillon  *	with resident pages.
19711c7c3c6aSMatthew Dillon  */
19721c7c3c6aSMatthew Dillon static void
19734dcc5c2dSMatthew Dillon swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count)
19741c7c3c6aSMatthew Dillon {
19752928cef7SAlan Cox 
1976c25673ffSAttilio Rao 	VM_OBJECT_ASSERT_LOCKED(object);
19771c7c3c6aSMatthew Dillon 	if (object->type != OBJT_SWAP)
19781c7c3c6aSMatthew Dillon 		return;
19791c7c3c6aSMatthew Dillon 
19801c7c3c6aSMatthew Dillon 	while (count > 0) {
19811c7c3c6aSMatthew Dillon 		struct swblock **pswap;
19821c7c3c6aSMatthew Dillon 		struct swblock *swap;
19831c7c3c6aSMatthew Dillon 
19847827d9b0SAlan Cox 		mtx_lock(&swhash_mtx);
19851c7c3c6aSMatthew Dillon 		pswap = swp_pager_hash(object, index);
19861c7c3c6aSMatthew Dillon 
19871c7c3c6aSMatthew Dillon 		if ((swap = *pswap) != NULL) {
19881c7c3c6aSMatthew Dillon 			daddr_t v = swap->swb_pages[index & SWAP_META_MASK];
19891c7c3c6aSMatthew Dillon 
19901c7c3c6aSMatthew Dillon 			if (v != SWAPBLK_NONE) {
19911c7c3c6aSMatthew Dillon 				swp_pager_freeswapspace(v, 1);
19921c7c3c6aSMatthew Dillon 				swap->swb_pages[index & SWAP_META_MASK] =
19931c7c3c6aSMatthew Dillon 					SWAPBLK_NONE;
19941c7c3c6aSMatthew Dillon 				if (--swap->swb_count == 0) {
19951c7c3c6aSMatthew Dillon 					*pswap = swap->swb_hnext;
1996670d17b5SJeff Roberson 					uma_zfree(swap_zone, swap);
19971c7c3c6aSMatthew Dillon 					--object->un_pager.swp.swp_bcount;
19981c7c3c6aSMatthew Dillon 				}
19991c7c3c6aSMatthew Dillon 			}
20001c7c3c6aSMatthew Dillon 			--count;
20011c7c3c6aSMatthew Dillon 			++index;
20021c7c3c6aSMatthew Dillon 		} else {
20034dcc5c2dSMatthew Dillon 			int n = SWAP_META_PAGES - (index & SWAP_META_MASK);
20041c7c3c6aSMatthew Dillon 			count -= n;
20051c7c3c6aSMatthew Dillon 			index += n;
20061c7c3c6aSMatthew Dillon 		}
20077827d9b0SAlan Cox 		mtx_unlock(&swhash_mtx);
20081c7c3c6aSMatthew Dillon 	}
20091c7c3c6aSMatthew Dillon }
20101c7c3c6aSMatthew Dillon 
20111c7c3c6aSMatthew Dillon /*
20121c7c3c6aSMatthew Dillon  * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
20131c7c3c6aSMatthew Dillon  *
20141c7c3c6aSMatthew Dillon  *	This routine locates and destroys all swap metadata associated with
20151c7c3c6aSMatthew Dillon  *	an object.
20161c7c3c6aSMatthew Dillon  */
20171c7c3c6aSMatthew Dillon static void
20181c7c3c6aSMatthew Dillon swp_pager_meta_free_all(vm_object_t object)
20191c7c3c6aSMatthew Dillon {
20201c7c3c6aSMatthew Dillon 	daddr_t index = 0;
20211c7c3c6aSMatthew Dillon 
202289f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
20231c7c3c6aSMatthew Dillon 	if (object->type != OBJT_SWAP)
20241c7c3c6aSMatthew Dillon 		return;
20251c7c3c6aSMatthew Dillon 
20261c7c3c6aSMatthew Dillon 	while (object->un_pager.swp.swp_bcount) {
20271c7c3c6aSMatthew Dillon 		struct swblock **pswap;
20281c7c3c6aSMatthew Dillon 		struct swblock *swap;
20291c7c3c6aSMatthew Dillon 
20307827d9b0SAlan Cox 		mtx_lock(&swhash_mtx);
20311c7c3c6aSMatthew Dillon 		pswap = swp_pager_hash(object, index);
20321c7c3c6aSMatthew Dillon 		if ((swap = *pswap) != NULL) {
20331c7c3c6aSMatthew Dillon 			int i;
20341c7c3c6aSMatthew Dillon 
20351c7c3c6aSMatthew Dillon 			for (i = 0; i < SWAP_META_PAGES; ++i) {
20361c7c3c6aSMatthew Dillon 				daddr_t v = swap->swb_pages[i];
20371c7c3c6aSMatthew Dillon 				if (v != SWAPBLK_NONE) {
20381c7c3c6aSMatthew Dillon 					--swap->swb_count;
20394dcc5c2dSMatthew Dillon 					swp_pager_freeswapspace(v, 1);
20401c7c3c6aSMatthew Dillon 				}
20411c7c3c6aSMatthew Dillon 			}
20421c7c3c6aSMatthew Dillon 			if (swap->swb_count != 0)
20431c7c3c6aSMatthew Dillon 				panic("swap_pager_meta_free_all: swb_count != 0");
20441c7c3c6aSMatthew Dillon 			*pswap = swap->swb_hnext;
2045670d17b5SJeff Roberson 			uma_zfree(swap_zone, swap);
20461c7c3c6aSMatthew Dillon 			--object->un_pager.swp.swp_bcount;
20471c7c3c6aSMatthew Dillon 		}
20487827d9b0SAlan Cox 		mtx_unlock(&swhash_mtx);
20491c7c3c6aSMatthew Dillon 		index += SWAP_META_PAGES;
20501c7c3c6aSMatthew Dillon 	}
20511c7c3c6aSMatthew Dillon }
20521c7c3c6aSMatthew Dillon 
20531c7c3c6aSMatthew Dillon /*
20541c7c3c6aSMatthew Dillon  * SWP_PAGER_METACTL() -  misc control of swap and vm_page_t meta data.
20551c7c3c6aSMatthew Dillon  *
20561c7c3c6aSMatthew Dillon  *	This routine is capable of looking up, popping, or freeing
20571c7c3c6aSMatthew Dillon  *	swapblk assignments in the swap meta data or in the vm_page_t.
20581c7c3c6aSMatthew Dillon  *	The routine typically returns the swapblk being looked-up, or popped,
20591c7c3c6aSMatthew Dillon  *	or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block
20601c7c3c6aSMatthew Dillon  *	was invalid.  This routine will automatically free any invalid
20611c7c3c6aSMatthew Dillon  *	meta-data swapblks.
20621c7c3c6aSMatthew Dillon  *
20631c7c3c6aSMatthew Dillon  *	It is not possible to store invalid swapblks in the swap meta data
20641c7c3c6aSMatthew Dillon  *	(other then a literal 'SWAPBLK_NONE'), so we don't bother checking.
20651c7c3c6aSMatthew Dillon  *
20661c7c3c6aSMatthew Dillon  *	When acting on a busy resident page and paging is in progress, we
20671c7c3c6aSMatthew Dillon  *	have to wait until paging is complete but otherwise can act on the
20681c7c3c6aSMatthew Dillon  *	busy page.
20691c7c3c6aSMatthew Dillon  *
20704dcc5c2dSMatthew Dillon  *	SWM_FREE	remove and free swap block from metadata
20711c7c3c6aSMatthew Dillon  *	SWM_POP		remove from meta data but do not free.. pop it out
20721c7c3c6aSMatthew Dillon  */
20731c7c3c6aSMatthew Dillon static daddr_t
20742f249180SPoul-Henning Kamp swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
20752f249180SPoul-Henning Kamp {
20764dcc5c2dSMatthew Dillon 	struct swblock **pswap;
20774dcc5c2dSMatthew Dillon 	struct swblock *swap;
20784dcc5c2dSMatthew Dillon 	daddr_t r1;
207923f09d50SIan Dowse 	int idx;
20804dcc5c2dSMatthew Dillon 
2081c25673ffSAttilio Rao 	VM_OBJECT_ASSERT_LOCKED(object);
20821c7c3c6aSMatthew Dillon 	/*
20831c7c3c6aSMatthew Dillon 	 * The meta data only exists of the object is OBJT_SWAP
20841c7c3c6aSMatthew Dillon 	 * and even then might not be allocated yet.
20851c7c3c6aSMatthew Dillon 	 */
20864dcc5c2dSMatthew Dillon 	if (object->type != OBJT_SWAP)
20871c7c3c6aSMatthew Dillon 		return (SWAPBLK_NONE);
20881c7c3c6aSMatthew Dillon 
20894dcc5c2dSMatthew Dillon 	r1 = SWAPBLK_NONE;
20907827d9b0SAlan Cox 	mtx_lock(&swhash_mtx);
209123f09d50SIan Dowse 	pswap = swp_pager_hash(object, pindex);
20921c7c3c6aSMatthew Dillon 
20931c7c3c6aSMatthew Dillon 	if ((swap = *pswap) != NULL) {
209423f09d50SIan Dowse 		idx = pindex & SWAP_META_MASK;
209523f09d50SIan Dowse 		r1 = swap->swb_pages[idx];
20961c7c3c6aSMatthew Dillon 
20971c7c3c6aSMatthew Dillon 		if (r1 != SWAPBLK_NONE) {
20981c7c3c6aSMatthew Dillon 			if (flags & SWM_FREE) {
20994dcc5c2dSMatthew Dillon 				swp_pager_freeswapspace(r1, 1);
21001c7c3c6aSMatthew Dillon 				r1 = SWAPBLK_NONE;
21011c7c3c6aSMatthew Dillon 			}
21021c7c3c6aSMatthew Dillon 			if (flags & (SWM_FREE|SWM_POP)) {
210323f09d50SIan Dowse 				swap->swb_pages[idx] = SWAPBLK_NONE;
21041c7c3c6aSMatthew Dillon 				if (--swap->swb_count == 0) {
21051c7c3c6aSMatthew Dillon 					*pswap = swap->swb_hnext;
2106670d17b5SJeff Roberson 					uma_zfree(swap_zone, swap);
21071c7c3c6aSMatthew Dillon 					--object->un_pager.swp.swp_bcount;
21081c7c3c6aSMatthew Dillon 				}
21091c7c3c6aSMatthew Dillon 			}
21101c7c3c6aSMatthew Dillon 		}
21111c7c3c6aSMatthew Dillon 	}
21127827d9b0SAlan Cox 	mtx_unlock(&swhash_mtx);
21131c7c3c6aSMatthew Dillon 	return (r1);
21141c7c3c6aSMatthew Dillon }
21151c7c3c6aSMatthew Dillon 
2116e9c0cc15SPoul-Henning Kamp /*
2117e9c0cc15SPoul-Henning Kamp  * System call swapon(name) enables swapping on device name,
2118e9c0cc15SPoul-Henning Kamp  * which must be in the swdevsw.  Return EBUSY
2119e9c0cc15SPoul-Henning Kamp  * if already swapping on this device.
2120e9c0cc15SPoul-Henning Kamp  */
2121e9c0cc15SPoul-Henning Kamp #ifndef _SYS_SYSPROTO_H_
2122e9c0cc15SPoul-Henning Kamp struct swapon_args {
2123e9c0cc15SPoul-Henning Kamp 	char *name;
2124e9c0cc15SPoul-Henning Kamp };
2125e9c0cc15SPoul-Henning Kamp #endif
2126e9c0cc15SPoul-Henning Kamp 
2127e9c0cc15SPoul-Henning Kamp /*
2128e9c0cc15SPoul-Henning Kamp  * MPSAFE
2129e9c0cc15SPoul-Henning Kamp  */
2130e9c0cc15SPoul-Henning Kamp /* ARGSUSED */
2131e9c0cc15SPoul-Henning Kamp int
21328451d0ddSKip Macy sys_swapon(struct thread *td, struct swapon_args *uap)
2133e9c0cc15SPoul-Henning Kamp {
2134e9c0cc15SPoul-Henning Kamp 	struct vattr attr;
2135e9c0cc15SPoul-Henning Kamp 	struct vnode *vp;
2136e9c0cc15SPoul-Henning Kamp 	struct nameidata nd;
2137e9c0cc15SPoul-Henning Kamp 	int error;
2138e9c0cc15SPoul-Henning Kamp 
2139acd3428bSRobert Watson 	error = priv_check(td, PRIV_SWAPON);
2140e9c0cc15SPoul-Henning Kamp 	if (error)
2141acd3428bSRobert Watson 		return (error);
2142e9c0cc15SPoul-Henning Kamp 
2143acd3428bSRobert Watson 	mtx_lock(&Giant);
2144e9c0cc15SPoul-Henning Kamp 	while (swdev_syscall_active)
2145e9c0cc15SPoul-Henning Kamp 	    tsleep(&swdev_syscall_active, PUSER - 1, "swpon", 0);
2146e9c0cc15SPoul-Henning Kamp 	swdev_syscall_active = 1;
2147e9c0cc15SPoul-Henning Kamp 
2148e9c0cc15SPoul-Henning Kamp 	/*
2149e9c0cc15SPoul-Henning Kamp 	 * Swap metadata may not fit in the KVM if we have physical
2150e9c0cc15SPoul-Henning Kamp 	 * memory of >1GB.
2151e9c0cc15SPoul-Henning Kamp 	 */
2152e9c0cc15SPoul-Henning Kamp 	if (swap_zone == NULL) {
2153e9c0cc15SPoul-Henning Kamp 		error = ENOMEM;
2154e9c0cc15SPoul-Henning Kamp 		goto done;
2155e9c0cc15SPoul-Henning Kamp 	}
2156e9c0cc15SPoul-Henning Kamp 
2157d9135e72SRobert Watson 	NDINIT(&nd, LOOKUP, ISOPEN | FOLLOW | AUDITVNODE1, UIO_USERSPACE,
2158d9135e72SRobert Watson 	    uap->name, td);
2159e9c0cc15SPoul-Henning Kamp 	error = namei(&nd);
2160e9c0cc15SPoul-Henning Kamp 	if (error)
2161e9c0cc15SPoul-Henning Kamp 		goto done;
2162e9c0cc15SPoul-Henning Kamp 
2163e9c0cc15SPoul-Henning Kamp 	NDFREE(&nd, NDF_ONLY_PNBUF);
2164e9c0cc15SPoul-Henning Kamp 	vp = nd.ni_vp;
2165e9c0cc15SPoul-Henning Kamp 
216620da9c2eSPoul-Henning Kamp 	if (vn_isdisk(vp, &error)) {
2167dee34ca4SPoul-Henning Kamp 		error = swapongeom(td, vp);
216820da9c2eSPoul-Henning Kamp 	} else if (vp->v_type == VREG &&
2169e9c0cc15SPoul-Henning Kamp 	    (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
21700359a12eSAttilio Rao 	    (error = VOP_GETATTR(vp, &attr, td->td_ucred)) == 0) {
2171e9c0cc15SPoul-Henning Kamp 		/*
2172e9c0cc15SPoul-Henning Kamp 		 * Allow direct swapping to NFS regular files in the same
2173e9c0cc15SPoul-Henning Kamp 		 * way that nfs_mountroot() sets up diskless swapping.
2174e9c0cc15SPoul-Henning Kamp 		 */
217559efee01SPoul-Henning Kamp 		error = swaponvp(td, vp, attr.va_size / DEV_BSIZE);
2176e9c0cc15SPoul-Henning Kamp 	}
2177e9c0cc15SPoul-Henning Kamp 
2178e9c0cc15SPoul-Henning Kamp 	if (error)
2179e9c0cc15SPoul-Henning Kamp 		vrele(vp);
2180e9c0cc15SPoul-Henning Kamp done:
2181e9c0cc15SPoul-Henning Kamp 	swdev_syscall_active = 0;
2182e9c0cc15SPoul-Henning Kamp 	wakeup_one(&swdev_syscall_active);
2183e9c0cc15SPoul-Henning Kamp 	mtx_unlock(&Giant);
2184e9c0cc15SPoul-Henning Kamp 	return (error);
2185e9c0cc15SPoul-Henning Kamp }
2186e9c0cc15SPoul-Henning Kamp 
21873ff863f1SDag-Erling Smørgrav /*
21883ff863f1SDag-Erling Smørgrav  * Check that the total amount of swap currently configured does not
21893ff863f1SDag-Erling Smørgrav  * exceed half the theoretical maximum.  If it does, print a warning
21903ff863f1SDag-Erling Smørgrav  * message and return -1; otherwise, return 0.
21913ff863f1SDag-Erling Smørgrav  */
21923ff863f1SDag-Erling Smørgrav static int
21933ff863f1SDag-Erling Smørgrav swapon_check_swzone(unsigned long npages)
21943ff863f1SDag-Erling Smørgrav {
21953ff863f1SDag-Erling Smørgrav 	unsigned long maxpages;
21963ff863f1SDag-Erling Smørgrav 
21973ff863f1SDag-Erling Smørgrav 	/* absolute maximum we can handle assuming 100% efficiency */
21983ff863f1SDag-Erling Smørgrav 	maxpages = uma_zone_get_max(swap_zone) * SWAP_META_PAGES;
21993ff863f1SDag-Erling Smørgrav 
22003ff863f1SDag-Erling Smørgrav 	/* recommend using no more than half that amount */
22013ff863f1SDag-Erling Smørgrav 	if (npages > maxpages / 2) {
22023ff863f1SDag-Erling Smørgrav 		printf("warning: total configured swap (%lu pages) "
22033ff863f1SDag-Erling Smørgrav 		    "exceeds maximum recommended amount (%lu pages).\n",
22049462305cSSergey Kandaurov 		    npages, maxpages / 2);
22053ff863f1SDag-Erling Smørgrav 		printf("warning: increase kern.maxswzone "
22063ff863f1SDag-Erling Smørgrav 		    "or reduce amount of swap.\n");
22073ff863f1SDag-Erling Smørgrav 		return (-1);
22083ff863f1SDag-Erling Smørgrav 	}
22093ff863f1SDag-Erling Smørgrav 	return (0);
22103ff863f1SDag-Erling Smørgrav }
22113ff863f1SDag-Erling Smørgrav 
221259efee01SPoul-Henning Kamp static void
22132cc718a1SKonstantin Belousov swaponsomething(struct vnode *vp, void *id, u_long nblks,
22142cc718a1SKonstantin Belousov     sw_strategy_t *strategy, sw_close_t *close, dev_t dev, int flags)
2215e9c0cc15SPoul-Henning Kamp {
22162d9974c1SAlan Cox 	struct swdevt *sp, *tsp;
2217e9c0cc15SPoul-Henning Kamp 	swblk_t dvbase;
22188f60c087SPoul-Henning Kamp 	u_long mblocks;
2219e9c0cc15SPoul-Henning Kamp 
2220e9c0cc15SPoul-Henning Kamp 	/*
2221e9c0cc15SPoul-Henning Kamp 	 * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks.
2222e9c0cc15SPoul-Henning Kamp 	 * First chop nblks off to page-align it, then convert.
2223e9c0cc15SPoul-Henning Kamp 	 *
2224e9c0cc15SPoul-Henning Kamp 	 * sw->sw_nblks is in page-sized chunks now too.
2225e9c0cc15SPoul-Henning Kamp 	 */
2226e9c0cc15SPoul-Henning Kamp 	nblks &= ~(ctodb(1) - 1);
2227e9c0cc15SPoul-Henning Kamp 	nblks = dbtoc(nblks);
2228e9c0cc15SPoul-Henning Kamp 
22296e903bd0SKonstantin Belousov 	/*
22306e903bd0SKonstantin Belousov 	 * If we go beyond this, we get overflows in the radix
22316e903bd0SKonstantin Belousov 	 * tree bitmap code.
22326e903bd0SKonstantin Belousov 	 */
22336e903bd0SKonstantin Belousov 	mblocks = 0x40000000 / BLIST_META_RADIX;
22346e903bd0SKonstantin Belousov 	if (nblks > mblocks) {
22356e903bd0SKonstantin Belousov 		printf(
22366e903bd0SKonstantin Belousov     "WARNING: reducing swap size to maximum of %luMB per unit\n",
22376e903bd0SKonstantin Belousov 		    mblocks / 1024 / 1024 * PAGE_SIZE);
22386e903bd0SKonstantin Belousov 		nblks = mblocks;
22396e903bd0SKonstantin Belousov 	}
22406e903bd0SKonstantin Belousov 
22418f60c087SPoul-Henning Kamp 	sp = malloc(sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO);
2242dee34ca4SPoul-Henning Kamp 	sp->sw_vp = vp;
2243dee34ca4SPoul-Henning Kamp 	sp->sw_id = id;
2244f3732fd1SPoul-Henning Kamp 	sp->sw_dev = dev;
22458d677ef9SPoul-Henning Kamp 	sp->sw_flags = 0;
2246e9c0cc15SPoul-Henning Kamp 	sp->sw_nblks = nblks;
2247e9c0cc15SPoul-Henning Kamp 	sp->sw_used = 0;
224859efee01SPoul-Henning Kamp 	sp->sw_strategy = strategy;
2249dee34ca4SPoul-Henning Kamp 	sp->sw_close = close;
22502cc718a1SKonstantin Belousov 	sp->sw_flags = flags;
2251e9c0cc15SPoul-Henning Kamp 
2252c8c7ad92SKip Macy 	sp->sw_blist = blist_create(nblks, M_WAITOK);
2253e9c0cc15SPoul-Henning Kamp 	/*
2254ef3c5abdSPoul-Henning Kamp 	 * Do not free the first two block in order to avoid overwriting
22558f60c087SPoul-Henning Kamp 	 * any bsd label at the front of the partition
2256e9c0cc15SPoul-Henning Kamp 	 */
2257ef3c5abdSPoul-Henning Kamp 	blist_free(sp->sw_blist, 2, nblks - 2);
2258e9c0cc15SPoul-Henning Kamp 
22592d9974c1SAlan Cox 	dvbase = 0;
226020da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
22612d9974c1SAlan Cox 	TAILQ_FOREACH(tsp, &swtailq, sw_list) {
22622d9974c1SAlan Cox 		if (tsp->sw_end >= dvbase) {
22632d9974c1SAlan Cox 			/*
22642d9974c1SAlan Cox 			 * We put one uncovered page between the devices
22652d9974c1SAlan Cox 			 * in order to definitively prevent any cross-device
22662d9974c1SAlan Cox 			 * I/O requests
22672d9974c1SAlan Cox 			 */
22682d9974c1SAlan Cox 			dvbase = tsp->sw_end + 1;
22692d9974c1SAlan Cox 		}
22702d9974c1SAlan Cox 	}
22712d9974c1SAlan Cox 	sp->sw_first = dvbase;
22722d9974c1SAlan Cox 	sp->sw_end = dvbase + nblks;
22738f60c087SPoul-Henning Kamp 	TAILQ_INSERT_TAIL(&swtailq, sp, sw_list);
22748f60c087SPoul-Henning Kamp 	nswapdev++;
22758f60c087SPoul-Henning Kamp 	swap_pager_avail += nblks;
22763364c323SKonstantin Belousov 	swap_total += (vm_ooffset_t)nblks * PAGE_SIZE;
22773ff863f1SDag-Erling Smørgrav 	swapon_check_swzone(swap_total / PAGE_SIZE);
2278d05bc129SAlan Cox 	swp_sizecheck();
2279d05bc129SAlan Cox 	mtx_unlock(&sw_dev_mtx);
228059efee01SPoul-Henning Kamp }
2281e9c0cc15SPoul-Henning Kamp 
2282e9c0cc15SPoul-Henning Kamp /*
2283e9c0cc15SPoul-Henning Kamp  * SYSCALL: swapoff(devname)
2284e9c0cc15SPoul-Henning Kamp  *
2285e9c0cc15SPoul-Henning Kamp  * Disable swapping on the given device.
2286dee34ca4SPoul-Henning Kamp  *
2287dee34ca4SPoul-Henning Kamp  * XXX: Badly designed system call: it should use a device index
2288dee34ca4SPoul-Henning Kamp  * rather than filename as specification.  We keep sw_vp around
2289dee34ca4SPoul-Henning Kamp  * only to make this work.
2290e9c0cc15SPoul-Henning Kamp  */
2291e9c0cc15SPoul-Henning Kamp #ifndef _SYS_SYSPROTO_H_
2292e9c0cc15SPoul-Henning Kamp struct swapoff_args {
2293e9c0cc15SPoul-Henning Kamp 	char *name;
2294e9c0cc15SPoul-Henning Kamp };
2295e9c0cc15SPoul-Henning Kamp #endif
2296e9c0cc15SPoul-Henning Kamp 
2297e9c0cc15SPoul-Henning Kamp /*
2298e9c0cc15SPoul-Henning Kamp  * MPSAFE
2299e9c0cc15SPoul-Henning Kamp  */
2300e9c0cc15SPoul-Henning Kamp /* ARGSUSED */
2301e9c0cc15SPoul-Henning Kamp int
23028451d0ddSKip Macy sys_swapoff(struct thread *td, struct swapoff_args *uap)
2303e9c0cc15SPoul-Henning Kamp {
2304e9c0cc15SPoul-Henning Kamp 	struct vnode *vp;
2305e9c0cc15SPoul-Henning Kamp 	struct nameidata nd;
2306e9c0cc15SPoul-Henning Kamp 	struct swdevt *sp;
23078f60c087SPoul-Henning Kamp 	int error;
2308e9c0cc15SPoul-Henning Kamp 
2309acd3428bSRobert Watson 	error = priv_check(td, PRIV_SWAPOFF);
2310e9c0cc15SPoul-Henning Kamp 	if (error)
23110909f38aSPawel Jakub Dawidek 		return (error);
2312e9c0cc15SPoul-Henning Kamp 
23130909f38aSPawel Jakub Dawidek 	mtx_lock(&Giant);
2314e9c0cc15SPoul-Henning Kamp 	while (swdev_syscall_active)
2315e9c0cc15SPoul-Henning Kamp 	    tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
2316e9c0cc15SPoul-Henning Kamp 	swdev_syscall_active = 1;
2317e9c0cc15SPoul-Henning Kamp 
2318d9135e72SRobert Watson 	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, UIO_USERSPACE, uap->name,
2319d9135e72SRobert Watson 	    td);
2320e9c0cc15SPoul-Henning Kamp 	error = namei(&nd);
2321e9c0cc15SPoul-Henning Kamp 	if (error)
2322e9c0cc15SPoul-Henning Kamp 		goto done;
2323e9c0cc15SPoul-Henning Kamp 	NDFREE(&nd, NDF_ONLY_PNBUF);
2324e9c0cc15SPoul-Henning Kamp 	vp = nd.ni_vp;
2325e9c0cc15SPoul-Henning Kamp 
232620da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
23278f60c087SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2328dee34ca4SPoul-Henning Kamp 		if (sp->sw_vp == vp)
23290909f38aSPawel Jakub Dawidek 			break;
2330e9c0cc15SPoul-Henning Kamp 	}
233120da9c2eSPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
23320909f38aSPawel Jakub Dawidek 	if (sp == NULL) {
2333e9c0cc15SPoul-Henning Kamp 		error = EINVAL;
2334e9c0cc15SPoul-Henning Kamp 		goto done;
23350909f38aSPawel Jakub Dawidek 	}
233635918c55SChristian S.J. Peron 	error = swapoff_one(sp, td->td_ucred);
23370909f38aSPawel Jakub Dawidek done:
23380909f38aSPawel Jakub Dawidek 	swdev_syscall_active = 0;
23390909f38aSPawel Jakub Dawidek 	wakeup_one(&swdev_syscall_active);
23400909f38aSPawel Jakub Dawidek 	mtx_unlock(&Giant);
23410909f38aSPawel Jakub Dawidek 	return (error);
23420909f38aSPawel Jakub Dawidek }
23430909f38aSPawel Jakub Dawidek 
23440909f38aSPawel Jakub Dawidek static int
234535918c55SChristian S.J. Peron swapoff_one(struct swdevt *sp, struct ucred *cred)
23460909f38aSPawel Jakub Dawidek {
23470909f38aSPawel Jakub Dawidek 	u_long nblks, dvbase;
2348e9c0cc15SPoul-Henning Kamp #ifdef MAC
23490909f38aSPawel Jakub Dawidek 	int error;
2350e9c0cc15SPoul-Henning Kamp #endif
2351e9c0cc15SPoul-Henning Kamp 
23520909f38aSPawel Jakub Dawidek 	mtx_assert(&Giant, MA_OWNED);
23530909f38aSPawel Jakub Dawidek #ifdef MAC
2354cb05b60aSAttilio Rao 	(void) vn_lock(sp->sw_vp, LK_EXCLUSIVE | LK_RETRY);
235535918c55SChristian S.J. Peron 	error = mac_system_check_swapoff(cred, sp->sw_vp);
235622db15c0SAttilio Rao 	(void) VOP_UNLOCK(sp->sw_vp, 0);
23570909f38aSPawel Jakub Dawidek 	if (error != 0)
23580909f38aSPawel Jakub Dawidek 		return (error);
23590909f38aSPawel Jakub Dawidek #endif
2360e9c0cc15SPoul-Henning Kamp 	nblks = sp->sw_nblks;
2361e9c0cc15SPoul-Henning Kamp 
2362e9c0cc15SPoul-Henning Kamp 	/*
2363e9c0cc15SPoul-Henning Kamp 	 * We can turn off this swap device safely only if the
2364e9c0cc15SPoul-Henning Kamp 	 * available virtual memory in the system will fit the amount
2365e9c0cc15SPoul-Henning Kamp 	 * of data we will have to page back in, plus an epsilon so
2366e9c0cc15SPoul-Henning Kamp 	 * the system doesn't become critically low on swap space.
2367e9c0cc15SPoul-Henning Kamp 	 */
236844f1c916SBryan Drewery 	if (vm_cnt.v_free_count + vm_cnt.v_cache_count + swap_pager_avail <
23692feb50bfSAttilio Rao 	    nblks + nswap_lowat) {
23700909f38aSPawel Jakub Dawidek 		return (ENOMEM);
2371e9c0cc15SPoul-Henning Kamp 	}
2372e9c0cc15SPoul-Henning Kamp 
2373e9c0cc15SPoul-Henning Kamp 	/*
2374e9c0cc15SPoul-Henning Kamp 	 * Prevent further allocations on this device.
2375e9c0cc15SPoul-Henning Kamp 	 */
23762928cef7SAlan Cox 	mtx_lock(&sw_dev_mtx);
2377e9c0cc15SPoul-Henning Kamp 	sp->sw_flags |= SW_CLOSING;
23788f60c087SPoul-Henning Kamp 	for (dvbase = 0; dvbase < sp->sw_end; dvbase += dmmax) {
23798f60c087SPoul-Henning Kamp 		swap_pager_avail -= blist_fill(sp->sw_blist,
23808f60c087SPoul-Henning Kamp 		     dvbase, dmmax);
2381e9c0cc15SPoul-Henning Kamp 	}
23823364c323SKonstantin Belousov 	swap_total -= (vm_ooffset_t)nblks * PAGE_SIZE;
23832928cef7SAlan Cox 	mtx_unlock(&sw_dev_mtx);
2384e9c0cc15SPoul-Henning Kamp 
2385e9c0cc15SPoul-Henning Kamp 	/*
2386e9c0cc15SPoul-Henning Kamp 	 * Page in the contents of the device and close it.
2387e9c0cc15SPoul-Henning Kamp 	 */
2388b3fed13eSDavid Schultz 	swap_pager_swapoff(sp);
2389e9c0cc15SPoul-Henning Kamp 
239035918c55SChristian S.J. Peron 	sp->sw_close(curthread, sp);
239159efee01SPoul-Henning Kamp 	sp->sw_id = NULL;
239220da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
23938f60c087SPoul-Henning Kamp 	TAILQ_REMOVE(&swtailq, sp, sw_list);
23940676a140SAlan Cox 	nswapdev--;
23957dea2c2eSAlan Cox 	if (nswapdev == 0) {
23967dea2c2eSAlan Cox 		swap_pager_full = 2;
23977dea2c2eSAlan Cox 		swap_pager_almost_full = 1;
23987dea2c2eSAlan Cox 	}
23998f60c087SPoul-Henning Kamp 	if (swdevhd == sp)
24008f60c087SPoul-Henning Kamp 		swdevhd = NULL;
2401d05bc129SAlan Cox 	mtx_unlock(&sw_dev_mtx);
24028f60c087SPoul-Henning Kamp 	blist_destroy(sp->sw_blist);
24038f60c087SPoul-Henning Kamp 	free(sp, M_VMPGDATA);
24040909f38aSPawel Jakub Dawidek 	return (0);
24050909f38aSPawel Jakub Dawidek }
2406e9c0cc15SPoul-Henning Kamp 
24070909f38aSPawel Jakub Dawidek void
24080909f38aSPawel Jakub Dawidek swapoff_all(void)
24090909f38aSPawel Jakub Dawidek {
24100909f38aSPawel Jakub Dawidek 	struct swdevt *sp, *spt;
24110909f38aSPawel Jakub Dawidek 	const char *devname;
24120909f38aSPawel Jakub Dawidek 	int error;
24130909f38aSPawel Jakub Dawidek 
24140909f38aSPawel Jakub Dawidek 	mtx_lock(&Giant);
24150909f38aSPawel Jakub Dawidek 	while (swdev_syscall_active)
24160909f38aSPawel Jakub Dawidek 		tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
24170909f38aSPawel Jakub Dawidek 	swdev_syscall_active = 1;
24180909f38aSPawel Jakub Dawidek 
24190909f38aSPawel Jakub Dawidek 	mtx_lock(&sw_dev_mtx);
24200909f38aSPawel Jakub Dawidek 	TAILQ_FOREACH_SAFE(sp, &swtailq, sw_list, spt) {
24210909f38aSPawel Jakub Dawidek 		mtx_unlock(&sw_dev_mtx);
24220909f38aSPawel Jakub Dawidek 		if (vn_isdisk(sp->sw_vp, NULL))
24237870adb6SEd Schouten 			devname = devtoname(sp->sw_vp->v_rdev);
24240909f38aSPawel Jakub Dawidek 		else
24250909f38aSPawel Jakub Dawidek 			devname = "[file]";
242635918c55SChristian S.J. Peron 		error = swapoff_one(sp, thread0.td_ucred);
24270909f38aSPawel Jakub Dawidek 		if (error != 0) {
24280909f38aSPawel Jakub Dawidek 			printf("Cannot remove swap device %s (error=%d), "
24290909f38aSPawel Jakub Dawidek 			    "skipping.\n", devname, error);
24300909f38aSPawel Jakub Dawidek 		} else if (bootverbose) {
24310909f38aSPawel Jakub Dawidek 			printf("Swap device %s removed.\n", devname);
24320909f38aSPawel Jakub Dawidek 		}
24330909f38aSPawel Jakub Dawidek 		mtx_lock(&sw_dev_mtx);
24340909f38aSPawel Jakub Dawidek 	}
24350909f38aSPawel Jakub Dawidek 	mtx_unlock(&sw_dev_mtx);
24360909f38aSPawel Jakub Dawidek 
2437e9c0cc15SPoul-Henning Kamp 	swdev_syscall_active = 0;
2438e9c0cc15SPoul-Henning Kamp 	wakeup_one(&swdev_syscall_active);
2439e9c0cc15SPoul-Henning Kamp 	mtx_unlock(&Giant);
2440e9c0cc15SPoul-Henning Kamp }
2441e9c0cc15SPoul-Henning Kamp 
2442567104a1SPoul-Henning Kamp void
2443567104a1SPoul-Henning Kamp swap_pager_status(int *total, int *used)
2444567104a1SPoul-Henning Kamp {
2445567104a1SPoul-Henning Kamp 	struct swdevt *sp;
2446567104a1SPoul-Henning Kamp 
2447567104a1SPoul-Henning Kamp 	*total = 0;
2448567104a1SPoul-Henning Kamp 	*used = 0;
244920da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
24508f60c087SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2451567104a1SPoul-Henning Kamp 		*total += sp->sw_nblks;
2452567104a1SPoul-Henning Kamp 		*used += sp->sw_used;
2453567104a1SPoul-Henning Kamp 	}
245420da9c2eSPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
2455567104a1SPoul-Henning Kamp }
2456567104a1SPoul-Henning Kamp 
2457dda4f960SKonstantin Belousov int
2458dda4f960SKonstantin Belousov swap_dev_info(int name, struct xswdev *xs, char *devname, size_t len)
2459dda4f960SKonstantin Belousov {
2460dda4f960SKonstantin Belousov 	struct swdevt *sp;
24617870adb6SEd Schouten 	const char *tmp_devname;
2462dda4f960SKonstantin Belousov 	int error, n;
2463dda4f960SKonstantin Belousov 
2464dda4f960SKonstantin Belousov 	n = 0;
2465dda4f960SKonstantin Belousov 	error = ENOENT;
2466dda4f960SKonstantin Belousov 	mtx_lock(&sw_dev_mtx);
2467dda4f960SKonstantin Belousov 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2468dda4f960SKonstantin Belousov 		if (n != name) {
2469dda4f960SKonstantin Belousov 			n++;
2470dda4f960SKonstantin Belousov 			continue;
2471dda4f960SKonstantin Belousov 		}
2472dda4f960SKonstantin Belousov 		xs->xsw_version = XSWDEV_VERSION;
2473dda4f960SKonstantin Belousov 		xs->xsw_dev = sp->sw_dev;
2474dda4f960SKonstantin Belousov 		xs->xsw_flags = sp->sw_flags;
2475dda4f960SKonstantin Belousov 		xs->xsw_nblks = sp->sw_nblks;
2476dda4f960SKonstantin Belousov 		xs->xsw_used = sp->sw_used;
2477dda4f960SKonstantin Belousov 		if (devname != NULL) {
2478dda4f960SKonstantin Belousov 			if (vn_isdisk(sp->sw_vp, NULL))
24797870adb6SEd Schouten 				tmp_devname = devtoname(sp->sw_vp->v_rdev);
2480dda4f960SKonstantin Belousov 			else
2481dda4f960SKonstantin Belousov 				tmp_devname = "[file]";
2482dda4f960SKonstantin Belousov 			strncpy(devname, tmp_devname, len);
2483dda4f960SKonstantin Belousov 		}
2484dda4f960SKonstantin Belousov 		error = 0;
2485dda4f960SKonstantin Belousov 		break;
2486dda4f960SKonstantin Belousov 	}
2487dda4f960SKonstantin Belousov 	mtx_unlock(&sw_dev_mtx);
2488dda4f960SKonstantin Belousov 	return (error);
2489dda4f960SKonstantin Belousov }
2490dda4f960SKonstantin Belousov 
2491e9c0cc15SPoul-Henning Kamp static int
2492e9c0cc15SPoul-Henning Kamp sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS)
2493e9c0cc15SPoul-Henning Kamp {
2494e9c0cc15SPoul-Henning Kamp 	struct xswdev xs;
2495dda4f960SKonstantin Belousov 	int error;
2496e9c0cc15SPoul-Henning Kamp 
2497e9c0cc15SPoul-Henning Kamp 	if (arg2 != 1)			/* name length */
2498e9c0cc15SPoul-Henning Kamp 		return (EINVAL);
2499dda4f960SKonstantin Belousov 	error = swap_dev_info(*(int *)arg1, &xs, NULL, 0);
2500dda4f960SKonstantin Belousov 	if (error != 0)
2501dda4f960SKonstantin Belousov 		return (error);
2502e9c0cc15SPoul-Henning Kamp 	error = SYSCTL_OUT(req, &xs, sizeof(xs));
2503e9c0cc15SPoul-Henning Kamp 	return (error);
2504e9c0cc15SPoul-Henning Kamp }
2505e9c0cc15SPoul-Henning Kamp 
25068f60c087SPoul-Henning Kamp SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, &nswapdev, 0,
2507e9c0cc15SPoul-Henning Kamp     "Number of swap devices");
2508e9c0cc15SPoul-Henning Kamp SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD, sysctl_vm_swap_info,
2509e9c0cc15SPoul-Henning Kamp     "Swap statistics by device");
2510ec38b344SPoul-Henning Kamp 
2511ec38b344SPoul-Henning Kamp /*
25128eb5a1cdSKonstantin Belousov  * vmspace_swap_count() - count the approximate swap usage in pages for a
2513ec38b344SPoul-Henning Kamp  *			  vmspace.
2514ec38b344SPoul-Henning Kamp  *
2515ec38b344SPoul-Henning Kamp  *	The map must be locked.
2516ec38b344SPoul-Henning Kamp  *
25178eb5a1cdSKonstantin Belousov  *	Swap usage is determined by taking the proportional swap used by
2518ec38b344SPoul-Henning Kamp  *	VM objects backing the VM map.  To make up for fractional losses,
2519ec38b344SPoul-Henning Kamp  *	if the VM object has any swap use at all the associated map entries
2520ec38b344SPoul-Henning Kamp  *	count for at least 1 swap page.
2521ec38b344SPoul-Henning Kamp  */
25222860553aSRebecca Cran long
2523ec38b344SPoul-Henning Kamp vmspace_swap_count(struct vmspace *vmspace)
2524ec38b344SPoul-Henning Kamp {
252565d8409cSRebecca Cran 	vm_map_t map;
2526ec38b344SPoul-Henning Kamp 	vm_map_entry_t cur;
252765d8409cSRebecca Cran 	vm_object_t object;
25282860553aSRebecca Cran 	long count, n;
252965d8409cSRebecca Cran 
253065d8409cSRebecca Cran 	map = &vmspace->vm_map;
253165d8409cSRebecca Cran 	count = 0;
2532ec38b344SPoul-Henning Kamp 
2533ec38b344SPoul-Henning Kamp 	for (cur = map->header.next; cur != &map->header; cur = cur->next) {
2534ec38b344SPoul-Henning Kamp 		if ((cur->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
2535ec38b344SPoul-Henning Kamp 		    (object = cur->object.vm_object) != NULL) {
253689f6b863SAttilio Rao 			VM_OBJECT_WLOCK(object);
2537ec38b344SPoul-Henning Kamp 			if (object->type == OBJT_SWAP &&
2538ec38b344SPoul-Henning Kamp 			    object->un_pager.swp.swp_bcount != 0) {
253965d8409cSRebecca Cran 				n = (cur->end - cur->start) / PAGE_SIZE;
2540ec38b344SPoul-Henning Kamp 				count += object->un_pager.swp.swp_bcount *
2541ec38b344SPoul-Henning Kamp 				    SWAP_META_PAGES * n / object->size + 1;
2542ec38b344SPoul-Henning Kamp 			}
254389f6b863SAttilio Rao 			VM_OBJECT_WUNLOCK(object);
2544ec38b344SPoul-Henning Kamp 		}
2545ec38b344SPoul-Henning Kamp 	}
2546ec38b344SPoul-Henning Kamp 	return (count);
2547ec38b344SPoul-Henning Kamp }
2548dee34ca4SPoul-Henning Kamp 
2549dee34ca4SPoul-Henning Kamp /*
2550dee34ca4SPoul-Henning Kamp  * GEOM backend
2551dee34ca4SPoul-Henning Kamp  *
2552dee34ca4SPoul-Henning Kamp  * Swapping onto disk devices.
2553dee34ca4SPoul-Henning Kamp  *
2554dee34ca4SPoul-Henning Kamp  */
2555dee34ca4SPoul-Henning Kamp 
25565721c9c7SPoul-Henning Kamp static g_orphan_t swapgeom_orphan;
25575721c9c7SPoul-Henning Kamp 
2558dee34ca4SPoul-Henning Kamp static struct g_class g_swap_class = {
2559dee34ca4SPoul-Henning Kamp 	.name = "SWAP",
25605721c9c7SPoul-Henning Kamp 	.version = G_VERSION,
25615721c9c7SPoul-Henning Kamp 	.orphan = swapgeom_orphan,
2562dee34ca4SPoul-Henning Kamp };
2563dee34ca4SPoul-Henning Kamp 
2564dee34ca4SPoul-Henning Kamp DECLARE_GEOM_CLASS(g_swap_class, g_class);
2565dee34ca4SPoul-Henning Kamp 
2566dee34ca4SPoul-Henning Kamp 
2567dee34ca4SPoul-Henning Kamp static void
25683398491bSAlexander Motin swapgeom_close_ev(void *arg, int flags)
25693398491bSAlexander Motin {
25703398491bSAlexander Motin 	struct g_consumer *cp;
25713398491bSAlexander Motin 
25723398491bSAlexander Motin 	cp = arg;
25733398491bSAlexander Motin 	g_access(cp, -1, -1, 0);
25743398491bSAlexander Motin 	g_detach(cp);
25753398491bSAlexander Motin 	g_destroy_consumer(cp);
25763398491bSAlexander Motin }
25773398491bSAlexander Motin 
25783398491bSAlexander Motin static void
2579dee34ca4SPoul-Henning Kamp swapgeom_done(struct bio *bp2)
2580dee34ca4SPoul-Henning Kamp {
25813398491bSAlexander Motin 	struct swdevt *sp;
2582dee34ca4SPoul-Henning Kamp 	struct buf *bp;
25833398491bSAlexander Motin 	struct g_consumer *cp;
2584dee34ca4SPoul-Henning Kamp 
2585dee34ca4SPoul-Henning Kamp 	bp = bp2->bio_caller2;
25863398491bSAlexander Motin 	cp = bp2->bio_from;
2587c5d3d25eSPoul-Henning Kamp 	bp->b_ioflags = bp2->bio_flags;
2588dee34ca4SPoul-Henning Kamp 	if (bp2->bio_error)
2589dee34ca4SPoul-Henning Kamp 		bp->b_ioflags |= BIO_ERROR;
2590c5d3d25eSPoul-Henning Kamp 	bp->b_resid = bp->b_bcount - bp2->bio_completed;
2591c5d3d25eSPoul-Henning Kamp 	bp->b_error = bp2->bio_error;
2592dee34ca4SPoul-Henning Kamp 	bufdone(bp);
25933398491bSAlexander Motin 	mtx_lock(&sw_dev_mtx);
25940ada3afcSAlexander Motin 	if ((--cp->index) == 0 && cp->private) {
25950ada3afcSAlexander Motin 		if (g_post_event(swapgeom_close_ev, cp, M_NOWAIT, NULL) == 0) {
25963398491bSAlexander Motin 			sp = bp2->bio_caller1;
25973398491bSAlexander Motin 			sp->sw_id = NULL;
25983398491bSAlexander Motin 		}
25990ada3afcSAlexander Motin 	}
26003398491bSAlexander Motin 	mtx_unlock(&sw_dev_mtx);
2601dee34ca4SPoul-Henning Kamp 	g_destroy_bio(bp2);
2602dee34ca4SPoul-Henning Kamp }
2603dee34ca4SPoul-Henning Kamp 
2604dee34ca4SPoul-Henning Kamp static void
2605dee34ca4SPoul-Henning Kamp swapgeom_strategy(struct buf *bp, struct swdevt *sp)
2606dee34ca4SPoul-Henning Kamp {
2607dee34ca4SPoul-Henning Kamp 	struct bio *bio;
2608dee34ca4SPoul-Henning Kamp 	struct g_consumer *cp;
2609dee34ca4SPoul-Henning Kamp 
26103398491bSAlexander Motin 	mtx_lock(&sw_dev_mtx);
2611dee34ca4SPoul-Henning Kamp 	cp = sp->sw_id;
2612dee34ca4SPoul-Henning Kamp 	if (cp == NULL) {
26133398491bSAlexander Motin 		mtx_unlock(&sw_dev_mtx);
2614dee34ca4SPoul-Henning Kamp 		bp->b_error = ENXIO;
2615dee34ca4SPoul-Henning Kamp 		bp->b_ioflags |= BIO_ERROR;
2616dee34ca4SPoul-Henning Kamp 		bufdone(bp);
2617dee34ca4SPoul-Henning Kamp 		return;
2618dee34ca4SPoul-Henning Kamp 	}
26193398491bSAlexander Motin 	cp->index++;
26203398491bSAlexander Motin 	mtx_unlock(&sw_dev_mtx);
262111041003SKonstantin Belousov 	if (bp->b_iocmd == BIO_WRITE)
262211041003SKonstantin Belousov 		bio = g_new_bio();
262311041003SKonstantin Belousov 	else
26244f8205e5SPoul-Henning Kamp 		bio = g_alloc_bio();
26254f8205e5SPoul-Henning Kamp 	if (bio == NULL) {
26263e5b6861SPoul-Henning Kamp 		bp->b_error = ENOMEM;
26273e5b6861SPoul-Henning Kamp 		bp->b_ioflags |= BIO_ERROR;
26283e5b6861SPoul-Henning Kamp 		bufdone(bp);
26293e5b6861SPoul-Henning Kamp 		return;
26303e5b6861SPoul-Henning Kamp 	}
263111041003SKonstantin Belousov 
26323398491bSAlexander Motin 	bio->bio_caller1 = sp;
2633dee34ca4SPoul-Henning Kamp 	bio->bio_caller2 = bp;
2634c5d3d25eSPoul-Henning Kamp 	bio->bio_cmd = bp->b_iocmd;
2635dee34ca4SPoul-Henning Kamp 	bio->bio_offset = (bp->b_blkno - sp->sw_first) * PAGE_SIZE;
2636dee34ca4SPoul-Henning Kamp 	bio->bio_length = bp->b_bcount;
2637dee34ca4SPoul-Henning Kamp 	bio->bio_done = swapgeom_done;
26382cc718a1SKonstantin Belousov 	if ((bp->b_flags & B_UNMAPPED) != 0) {
26392cc718a1SKonstantin Belousov 		bio->bio_ma = bp->b_pages;
26402cc718a1SKonstantin Belousov 		bio->bio_data = unmapped_buf;
26412cc718a1SKonstantin Belousov 		bio->bio_ma_offset = (vm_offset_t)bp->b_offset & PAGE_MASK;
26422cc718a1SKonstantin Belousov 		bio->bio_ma_n = bp->b_npages;
26432cc718a1SKonstantin Belousov 		bio->bio_flags |= BIO_UNMAPPED;
26442cc718a1SKonstantin Belousov 	} else {
26452cc718a1SKonstantin Belousov 		bio->bio_data = bp->b_data;
26462cc718a1SKonstantin Belousov 		bio->bio_ma = NULL;
26472cc718a1SKonstantin Belousov 	}
2648dee34ca4SPoul-Henning Kamp 	g_io_request(bio, cp);
2649dee34ca4SPoul-Henning Kamp 	return;
2650dee34ca4SPoul-Henning Kamp }
2651dee34ca4SPoul-Henning Kamp 
2652dee34ca4SPoul-Henning Kamp static void
2653dee34ca4SPoul-Henning Kamp swapgeom_orphan(struct g_consumer *cp)
2654dee34ca4SPoul-Henning Kamp {
2655dee34ca4SPoul-Henning Kamp 	struct swdevt *sp;
26563398491bSAlexander Motin 	int destroy;
2657dee34ca4SPoul-Henning Kamp 
2658dee34ca4SPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
26593398491bSAlexander Motin 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
26603398491bSAlexander Motin 		if (sp->sw_id == cp) {
26618f12d83aSAlexander Motin 			sp->sw_flags |= SW_CLOSING;
26623398491bSAlexander Motin 			break;
2663dee34ca4SPoul-Henning Kamp 		}
26643398491bSAlexander Motin 	}
26653398491bSAlexander Motin 	cp->private = (void *)(uintptr_t)1;
26663398491bSAlexander Motin 	destroy = ((sp != NULL) && (cp->index == 0));
26673398491bSAlexander Motin 	if (destroy)
26683398491bSAlexander Motin 		sp->sw_id = NULL;
26693398491bSAlexander Motin 	mtx_unlock(&sw_dev_mtx);
26703398491bSAlexander Motin 	if (destroy)
26713398491bSAlexander Motin 		swapgeom_close_ev(cp, 0);
2672dee34ca4SPoul-Henning Kamp }
2673dee34ca4SPoul-Henning Kamp 
2674dee34ca4SPoul-Henning Kamp static void
2675dee34ca4SPoul-Henning Kamp swapgeom_close(struct thread *td, struct swdevt *sw)
2676dee34ca4SPoul-Henning Kamp {
26773398491bSAlexander Motin 	struct g_consumer *cp;
2678dee34ca4SPoul-Henning Kamp 
26793398491bSAlexander Motin 	mtx_lock(&sw_dev_mtx);
26803398491bSAlexander Motin 	cp = sw->sw_id;
26813398491bSAlexander Motin 	sw->sw_id = NULL;
26823398491bSAlexander Motin 	mtx_unlock(&sw_dev_mtx);
2683dee34ca4SPoul-Henning Kamp 	/* XXX: direct call when Giant untangled */
26843398491bSAlexander Motin 	if (cp != NULL)
26853398491bSAlexander Motin 		g_waitfor_event(swapgeom_close_ev, cp, M_WAITOK, NULL);
2686dee34ca4SPoul-Henning Kamp }
2687dee34ca4SPoul-Henning Kamp 
2688dee34ca4SPoul-Henning Kamp 
2689dee34ca4SPoul-Henning Kamp struct swh0h0 {
269089c9c53dSPoul-Henning Kamp 	struct cdev *dev;
2691dee34ca4SPoul-Henning Kamp 	struct vnode *vp;
2692dee34ca4SPoul-Henning Kamp 	int	error;
2693dee34ca4SPoul-Henning Kamp };
2694dee34ca4SPoul-Henning Kamp 
2695dee34ca4SPoul-Henning Kamp static void
2696dee34ca4SPoul-Henning Kamp swapongeom_ev(void *arg, int flags)
2697dee34ca4SPoul-Henning Kamp {
2698dee34ca4SPoul-Henning Kamp 	struct swh0h0 *swh;
2699dee34ca4SPoul-Henning Kamp 	struct g_provider *pp;
2700dee34ca4SPoul-Henning Kamp 	struct g_consumer *cp;
2701dee34ca4SPoul-Henning Kamp 	static struct g_geom *gp;
2702dee34ca4SPoul-Henning Kamp 	struct swdevt *sp;
2703dee34ca4SPoul-Henning Kamp 	u_long nblks;
2704dee34ca4SPoul-Henning Kamp 	int error;
2705dee34ca4SPoul-Henning Kamp 
2706dee34ca4SPoul-Henning Kamp 	swh = arg;
2707dee34ca4SPoul-Henning Kamp 	swh->error = 0;
2708dee34ca4SPoul-Henning Kamp 	pp = g_dev_getprovider(swh->dev);
2709dee34ca4SPoul-Henning Kamp 	if (pp == NULL) {
2710dee34ca4SPoul-Henning Kamp 		swh->error = ENODEV;
2711dee34ca4SPoul-Henning Kamp 		return;
2712dee34ca4SPoul-Henning Kamp 	}
2713dee34ca4SPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
2714dee34ca4SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2715dee34ca4SPoul-Henning Kamp 		cp = sp->sw_id;
2716dee34ca4SPoul-Henning Kamp 		if (cp != NULL && cp->provider == pp) {
2717dee34ca4SPoul-Henning Kamp 			mtx_unlock(&sw_dev_mtx);
2718dee34ca4SPoul-Henning Kamp 			swh->error = EBUSY;
2719dee34ca4SPoul-Henning Kamp 			return;
2720dee34ca4SPoul-Henning Kamp 		}
2721dee34ca4SPoul-Henning Kamp 	}
2722dee34ca4SPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
27235721c9c7SPoul-Henning Kamp 	if (gp == NULL)
272402c62349SJaakko Heinonen 		gp = g_new_geomf(&g_swap_class, "swap");
2725dee34ca4SPoul-Henning Kamp 	cp = g_new_consumer(gp);
27263398491bSAlexander Motin 	cp->index = 0;		/* Number of active I/Os. */
27273398491bSAlexander Motin 	cp->private = NULL;	/* Orphanization flag */
2728dee34ca4SPoul-Henning Kamp 	g_attach(cp, pp);
2729afeb65e6SPoul-Henning Kamp 	/*
2730afeb65e6SPoul-Henning Kamp 	 * XXX: Everytime you think you can improve the margin for
2731afeb65e6SPoul-Henning Kamp 	 * footshooting, somebody depends on the ability to do so:
2732afeb65e6SPoul-Henning Kamp 	 * savecore(8) wants to write to our swapdev so we cannot
2733afeb65e6SPoul-Henning Kamp 	 * set an exclusive count :-(
2734afeb65e6SPoul-Henning Kamp 	 */
2735d2bae332SPoul-Henning Kamp 	error = g_access(cp, 1, 1, 0);
2736dee34ca4SPoul-Henning Kamp 	if (error) {
2737dee34ca4SPoul-Henning Kamp 		g_detach(cp);
2738dee34ca4SPoul-Henning Kamp 		g_destroy_consumer(cp);
2739dee34ca4SPoul-Henning Kamp 		swh->error = error;
2740dee34ca4SPoul-Henning Kamp 		return;
2741dee34ca4SPoul-Henning Kamp 	}
2742dee34ca4SPoul-Henning Kamp 	nblks = pp->mediasize / DEV_BSIZE;
2743dee34ca4SPoul-Henning Kamp 	swaponsomething(swh->vp, cp, nblks, swapgeom_strategy,
27442cc718a1SKonstantin Belousov 	    swapgeom_close, dev2udev(swh->dev),
27452cc718a1SKonstantin Belousov 	    (pp->flags & G_PF_ACCEPT_UNMAPPED) != 0 ? SW_UNMAPPED : 0);
2746dee34ca4SPoul-Henning Kamp 	swh->error = 0;
2747dee34ca4SPoul-Henning Kamp }
2748dee34ca4SPoul-Henning Kamp 
2749dee34ca4SPoul-Henning Kamp static int
2750dee34ca4SPoul-Henning Kamp swapongeom(struct thread *td, struct vnode *vp)
2751dee34ca4SPoul-Henning Kamp {
2752dee34ca4SPoul-Henning Kamp 	int error;
2753dee34ca4SPoul-Henning Kamp 	struct swh0h0 swh;
2754dee34ca4SPoul-Henning Kamp 
2755cb05b60aSAttilio Rao 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2756dee34ca4SPoul-Henning Kamp 
2757dee34ca4SPoul-Henning Kamp 	swh.dev = vp->v_rdev;
2758dee34ca4SPoul-Henning Kamp 	swh.vp = vp;
2759dee34ca4SPoul-Henning Kamp 	swh.error = 0;
2760dee34ca4SPoul-Henning Kamp 	/* XXX: direct call when Giant untangled */
2761dee34ca4SPoul-Henning Kamp 	error = g_waitfor_event(swapongeom_ev, &swh, M_WAITOK, NULL);
2762dee34ca4SPoul-Henning Kamp 	if (!error)
2763dee34ca4SPoul-Henning Kamp 		error = swh.error;
276422db15c0SAttilio Rao 	VOP_UNLOCK(vp, 0);
2765dee34ca4SPoul-Henning Kamp 	return (error);
2766dee34ca4SPoul-Henning Kamp }
2767dee34ca4SPoul-Henning Kamp 
2768dee34ca4SPoul-Henning Kamp /*
2769dee34ca4SPoul-Henning Kamp  * VNODE backend
2770dee34ca4SPoul-Henning Kamp  *
2771dee34ca4SPoul-Henning Kamp  * This is used mainly for network filesystem (read: probably only tested
2772dee34ca4SPoul-Henning Kamp  * with NFS) swapfiles.
2773dee34ca4SPoul-Henning Kamp  *
2774dee34ca4SPoul-Henning Kamp  */
2775dee34ca4SPoul-Henning Kamp 
2776dee34ca4SPoul-Henning Kamp static void
2777dee34ca4SPoul-Henning Kamp swapdev_strategy(struct buf *bp, struct swdevt *sp)
2778dee34ca4SPoul-Henning Kamp {
2779494eb176SPoul-Henning Kamp 	struct vnode *vp2;
2780dee34ca4SPoul-Henning Kamp 
2781dee34ca4SPoul-Henning Kamp 	bp->b_blkno = ctodb(bp->b_blkno - sp->sw_first);
2782dee34ca4SPoul-Henning Kamp 
2783dee34ca4SPoul-Henning Kamp 	vp2 = sp->sw_id;
2784dee34ca4SPoul-Henning Kamp 	vhold(vp2);
2785dee34ca4SPoul-Henning Kamp 	if (bp->b_iocmd == BIO_WRITE) {
27863cfc7651SOlivier Houchard 		if (bp->b_bufobj)
2787494eb176SPoul-Henning Kamp 			bufobj_wdrop(bp->b_bufobj);
2788a76d8f4eSPoul-Henning Kamp 		bufobj_wref(&vp2->v_bufobj);
2789dee34ca4SPoul-Henning Kamp 	}
27903cfc7651SOlivier Houchard 	if (bp->b_bufobj != &vp2->v_bufobj)
27913cfc7651SOlivier Houchard 		bp->b_bufobj = &vp2->v_bufobj;
2792dee34ca4SPoul-Henning Kamp 	bp->b_vp = vp2;
27932c18019fSPoul-Henning Kamp 	bp->b_iooffset = dbtob(bp->b_blkno);
2794b792bebeSPoul-Henning Kamp 	bstrategy(bp);
2795dee34ca4SPoul-Henning Kamp 	return;
2796dee34ca4SPoul-Henning Kamp }
2797dee34ca4SPoul-Henning Kamp 
2798dee34ca4SPoul-Henning Kamp static void
2799dee34ca4SPoul-Henning Kamp swapdev_close(struct thread *td, struct swdevt *sp)
2800dee34ca4SPoul-Henning Kamp {
2801dee34ca4SPoul-Henning Kamp 
2802dee34ca4SPoul-Henning Kamp 	VOP_CLOSE(sp->sw_vp, FREAD | FWRITE, td->td_ucred, td);
2803dee34ca4SPoul-Henning Kamp 	vrele(sp->sw_vp);
2804dee34ca4SPoul-Henning Kamp }
2805dee34ca4SPoul-Henning Kamp 
2806dee34ca4SPoul-Henning Kamp 
2807dee34ca4SPoul-Henning Kamp static int
2808dee34ca4SPoul-Henning Kamp swaponvp(struct thread *td, struct vnode *vp, u_long nblks)
2809dee34ca4SPoul-Henning Kamp {
2810dee34ca4SPoul-Henning Kamp 	struct swdevt *sp;
2811dee34ca4SPoul-Henning Kamp 	int error;
2812dee34ca4SPoul-Henning Kamp 
2813dee34ca4SPoul-Henning Kamp 	if (nblks == 0)
2814dee34ca4SPoul-Henning Kamp 		return (ENXIO);
2815dee34ca4SPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
2816dee34ca4SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2817dee34ca4SPoul-Henning Kamp 		if (sp->sw_id == vp) {
2818dee34ca4SPoul-Henning Kamp 			mtx_unlock(&sw_dev_mtx);
2819dee34ca4SPoul-Henning Kamp 			return (EBUSY);
2820dee34ca4SPoul-Henning Kamp 		}
2821dee34ca4SPoul-Henning Kamp 	}
2822dee34ca4SPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
2823dee34ca4SPoul-Henning Kamp 
2824cb05b60aSAttilio Rao 	(void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2825dee34ca4SPoul-Henning Kamp #ifdef MAC
282630d239bcSRobert Watson 	error = mac_system_check_swapon(td->td_ucred, vp);
2827dee34ca4SPoul-Henning Kamp 	if (error == 0)
2828dee34ca4SPoul-Henning Kamp #endif
28299e223287SKonstantin Belousov 		error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, NULL);
283022db15c0SAttilio Rao 	(void) VOP_UNLOCK(vp, 0);
2831dee34ca4SPoul-Henning Kamp 	if (error)
2832dee34ca4SPoul-Henning Kamp 		return (error);
2833dee34ca4SPoul-Henning Kamp 
2834dee34ca4SPoul-Henning Kamp 	swaponsomething(vp, vp, nblks, swapdev_strategy, swapdev_close,
28352cc718a1SKonstantin Belousov 	    NODEV, 0);
2836dee34ca4SPoul-Henning Kamp 	return (0);
2837dee34ca4SPoul-Henning Kamp }
2838