xref: /freebsd/sys/vm/swap_pager.c (revision 69921123490b99c2588b0c743bc4af32bbe6601c)
160727d8bSWarner Losh /*-
21c7c3c6aSMatthew Dillon  * Copyright (c) 1998 Matthew Dillon,
326f9a767SRodney W. Grimes  * Copyright (c) 1994 John S. Dyson
4df8bae1dSRodney W. Grimes  * Copyright (c) 1990 University of Utah.
5e9c0cc15SPoul-Henning Kamp  * Copyright (c) 1982, 1986, 1989, 1993
6df8bae1dSRodney W. Grimes  *	The Regents of the University of California.  All rights reserved.
7df8bae1dSRodney W. Grimes  *
8df8bae1dSRodney W. Grimes  * This code is derived from software contributed to Berkeley by
9df8bae1dSRodney W. Grimes  * the Systems Programming Group of the University of Utah Computer
10df8bae1dSRodney W. Grimes  * Science Department.
11df8bae1dSRodney W. Grimes  *
12df8bae1dSRodney W. Grimes  * Redistribution and use in source and binary forms, with or without
13df8bae1dSRodney W. Grimes  * modification, are permitted provided that the following conditions
14df8bae1dSRodney W. Grimes  * are met:
15df8bae1dSRodney W. Grimes  * 1. Redistributions of source code must retain the above copyright
16df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer.
17df8bae1dSRodney W. Grimes  * 2. Redistributions in binary form must reproduce the above copyright
18df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer in the
19df8bae1dSRodney W. Grimes  *    documentation and/or other materials provided with the distribution.
20df8bae1dSRodney W. Grimes  * 3. All advertising materials mentioning features or use of this software
215929bcfaSPhilippe Charnier  *    must display the following acknowledgement:
22df8bae1dSRodney W. Grimes  *	This product includes software developed by the University of
23df8bae1dSRodney W. Grimes  *	California, Berkeley and its contributors.
24df8bae1dSRodney W. Grimes  * 4. Neither the name of the University nor the names of its contributors
25df8bae1dSRodney W. Grimes  *    may be used to endorse or promote products derived from this software
26df8bae1dSRodney W. Grimes  *    without specific prior written permission.
27df8bae1dSRodney W. Grimes  *
28df8bae1dSRodney W. Grimes  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
29df8bae1dSRodney W. Grimes  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30df8bae1dSRodney W. Grimes  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31df8bae1dSRodney W. Grimes  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
32df8bae1dSRodney W. Grimes  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33df8bae1dSRodney W. Grimes  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34df8bae1dSRodney W. Grimes  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35df8bae1dSRodney W. Grimes  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36df8bae1dSRodney W. Grimes  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37df8bae1dSRodney W. Grimes  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38df8bae1dSRodney W. Grimes  * SUCH DAMAGE.
39df8bae1dSRodney W. Grimes  *
401c7c3c6aSMatthew Dillon  *				New Swap System
411c7c3c6aSMatthew Dillon  *				Matthew Dillon
421c7c3c6aSMatthew Dillon  *
431c7c3c6aSMatthew Dillon  * Radix Bitmap 'blists'.
441c7c3c6aSMatthew Dillon  *
451c7c3c6aSMatthew Dillon  *	- The new swapper uses the new radix bitmap code.  This should scale
461c7c3c6aSMatthew Dillon  *	  to arbitrarily small or arbitrarily large swap spaces and an almost
471c7c3c6aSMatthew Dillon  *	  arbitrary degree of fragmentation.
481c7c3c6aSMatthew Dillon  *
491c7c3c6aSMatthew Dillon  * Features:
501c7c3c6aSMatthew Dillon  *
511c7c3c6aSMatthew Dillon  *	- on the fly reallocation of swap during putpages.  The new system
521c7c3c6aSMatthew Dillon  *	  does not try to keep previously allocated swap blocks for dirty
531c7c3c6aSMatthew Dillon  *	  pages.
541c7c3c6aSMatthew Dillon  *
551c7c3c6aSMatthew Dillon  *	- on the fly deallocation of swap
561c7c3c6aSMatthew Dillon  *
571c7c3c6aSMatthew Dillon  *	- No more garbage collection required.  Unnecessarily allocated swap
581c7c3c6aSMatthew Dillon  *	  blocks only exist for dirty vm_page_t's now and these are already
591c7c3c6aSMatthew Dillon  *	  cycled (in a high-load system) by the pager.  We also do on-the-fly
601c7c3c6aSMatthew Dillon  *	  removal of invalidated swap blocks when a page is destroyed
611c7c3c6aSMatthew Dillon  *	  or renamed.
621c7c3c6aSMatthew Dillon  *
63df8bae1dSRodney W. Grimes  * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
64df8bae1dSRodney W. Grimes  *
65df8bae1dSRodney W. Grimes  *	@(#)swap_pager.c	8.9 (Berkeley) 3/21/94
66e9c0cc15SPoul-Henning Kamp  *	@(#)vm_swap.c	8.5 (Berkeley) 2/17/94
67df8bae1dSRodney W. Grimes  */
68df8bae1dSRodney W. Grimes 
69874651b1SDavid E. O'Brien #include <sys/cdefs.h>
70874651b1SDavid E. O'Brien __FBSDID("$FreeBSD$");
71874651b1SDavid E. O'Brien 
72*69921123SKonstantin Belousov #include "opt_compat.h"
73e9c0cc15SPoul-Henning Kamp #include "opt_swap.h"
74e9c0cc15SPoul-Henning Kamp #include "opt_vm.h"
75e9c0cc15SPoul-Henning Kamp 
76df8bae1dSRodney W. Grimes #include <sys/param.h>
77df8bae1dSRodney W. Grimes #include <sys/systm.h>
78af647ddeSBruce Evans #include <sys/conf.h>
7964abb5a5SDavid Greenman #include <sys/kernel.h>
80acd3428bSRobert Watson #include <sys/priv.h>
81df8bae1dSRodney W. Grimes #include <sys/proc.h>
829626b608SPoul-Henning Kamp #include <sys/bio.h>
83df8bae1dSRodney W. Grimes #include <sys/buf.h>
84e9c0cc15SPoul-Henning Kamp #include <sys/disk.h>
85e9c0cc15SPoul-Henning Kamp #include <sys/fcntl.h>
86e9c0cc15SPoul-Henning Kamp #include <sys/mount.h>
87e9c0cc15SPoul-Henning Kamp #include <sys/namei.h>
88df8bae1dSRodney W. Grimes #include <sys/vnode.h>
89df8bae1dSRodney W. Grimes #include <sys/malloc.h>
901ba5ad42SEdward Tomasz Napierala #include <sys/racct.h>
913364c323SKonstantin Belousov #include <sys/resource.h>
923364c323SKonstantin Belousov #include <sys/resourcevar.h>
9389f6b863SAttilio Rao #include <sys/rwlock.h>
94327f4e83SMatthew Dillon #include <sys/sysctl.h>
95e9c0cc15SPoul-Henning Kamp #include <sys/sysproto.h>
961c7c3c6aSMatthew Dillon #include <sys/blist.h>
971c7c3c6aSMatthew Dillon #include <sys/lock.h>
980cddd8f0SMatthew Dillon #include <sys/sx.h>
99936524aaSMatthew Dillon #include <sys/vmmeter.h>
100df8bae1dSRodney W. Grimes 
101aed55708SRobert Watson #include <security/mac/mac_framework.h>
102aed55708SRobert Watson 
103df8bae1dSRodney W. Grimes #include <vm/vm.h>
10421cd6e62SSeigo Tanimura #include <vm/pmap.h>
10521cd6e62SSeigo Tanimura #include <vm/vm_map.h>
10621cd6e62SSeigo Tanimura #include <vm/vm_kern.h>
107efeaf95aSDavid Greenman #include <vm/vm_object.h>
108df8bae1dSRodney W. Grimes #include <vm/vm_page.h>
109efeaf95aSDavid Greenman #include <vm/vm_pager.h>
110df8bae1dSRodney W. Grimes #include <vm/vm_pageout.h>
111e9c0cc15SPoul-Henning Kamp #include <vm/vm_param.h>
112df8bae1dSRodney W. Grimes #include <vm/swap_pager.h>
113efeaf95aSDavid Greenman #include <vm/vm_extern.h>
114670d17b5SJeff Roberson #include <vm/uma.h>
115df8bae1dSRodney W. Grimes 
116dee34ca4SPoul-Henning Kamp #include <geom/geom.h>
117dee34ca4SPoul-Henning Kamp 
118ec38b344SPoul-Henning Kamp /*
11915523cf7SKonstantin Belousov  * SWB_NPAGES must be a power of 2.  It may be set to 1, 2, 4, 8, 16
12015523cf7SKonstantin Belousov  * or 32 pages per allocation.
12115523cf7SKonstantin Belousov  * The 32-page limit is due to the radix code (kern/subr_blist.c).
122ec38b344SPoul-Henning Kamp  */
123ec38b344SPoul-Henning Kamp #ifndef MAX_PAGEOUT_CLUSTER
124ec38b344SPoul-Henning Kamp #define MAX_PAGEOUT_CLUSTER 16
125ec38b344SPoul-Henning Kamp #endif
126ec38b344SPoul-Henning Kamp 
127ec38b344SPoul-Henning Kamp #if !defined(SWB_NPAGES)
128ec38b344SPoul-Henning Kamp #define SWB_NPAGES	MAX_PAGEOUT_CLUSTER
129ec38b344SPoul-Henning Kamp #endif
130ec38b344SPoul-Henning Kamp 
131ec38b344SPoul-Henning Kamp /*
13215523cf7SKonstantin Belousov  * The swblock structure maps an object and a small, fixed-size range
13315523cf7SKonstantin Belousov  * of page indices to disk addresses within a swap area.
13415523cf7SKonstantin Belousov  * The collection of these mappings is implemented as a hash table.
13515523cf7SKonstantin Belousov  * Unused disk addresses within a swap area are allocated and managed
13615523cf7SKonstantin Belousov  * using a blist.
137ec38b344SPoul-Henning Kamp  */
138ec38b344SPoul-Henning Kamp #define SWCORRECT(n) (sizeof(void *) * (n) / sizeof(daddr_t))
139ec38b344SPoul-Henning Kamp #define SWAP_META_PAGES		(SWB_NPAGES * 2)
140ec38b344SPoul-Henning Kamp #define SWAP_META_MASK		(SWAP_META_PAGES - 1)
141ec38b344SPoul-Henning Kamp 
142e9c0cc15SPoul-Henning Kamp struct swblock {
143e9c0cc15SPoul-Henning Kamp 	struct swblock	*swb_hnext;
144e9c0cc15SPoul-Henning Kamp 	vm_object_t	swb_object;
145e9c0cc15SPoul-Henning Kamp 	vm_pindex_t	swb_index;
146e9c0cc15SPoul-Henning Kamp 	int		swb_count;
147e9c0cc15SPoul-Henning Kamp 	daddr_t		swb_pages[SWAP_META_PAGES];
148e9c0cc15SPoul-Henning Kamp };
149e9c0cc15SPoul-Henning Kamp 
1502c4992dbSAlan Cox static MALLOC_DEFINE(M_VMPGDATA, "vm_pgdata", "swap pager private data");
15120da9c2eSPoul-Henning Kamp static struct mtx sw_dev_mtx;
1528f60c087SPoul-Henning Kamp static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq);
1538f60c087SPoul-Henning Kamp static struct swdevt *swdevhd;	/* Allocate from here next */
1548f60c087SPoul-Henning Kamp static int nswapdev;		/* Number of swap devices */
1558f60c087SPoul-Henning Kamp int swap_pager_avail;
15604533e1eSKonstantin Belousov static struct sx swdev_syscall_lock;	/* serialize swap(on|off) */
157e9c0cc15SPoul-Henning Kamp 
1583364c323SKonstantin Belousov static vm_ooffset_t swap_total;
1598a9c731fSIvan Voras SYSCTL_QUAD(_vm, OID_AUTO, swap_total, CTLFLAG_RD, &swap_total, 0,
1608a9c731fSIvan Voras     "Total amount of available swap storage.");
1613364c323SKonstantin Belousov static vm_ooffset_t swap_reserved;
1628a9c731fSIvan Voras SYSCTL_QUAD(_vm, OID_AUTO, swap_reserved, CTLFLAG_RD, &swap_reserved, 0,
1638a9c731fSIvan Voras     "Amount of swap storage needed to back all allocated anonymous memory.");
1643364c323SKonstantin Belousov static int overcommit = 0;
1658a9c731fSIvan Voras SYSCTL_INT(_vm, OID_AUTO, overcommit, CTLFLAG_RW, &overcommit, 0,
1668a9c731fSIvan Voras     "Configure virtual memory overcommit behavior. See tuning(7) "
1678a9c731fSIvan Voras     "for details.");
16861203277SDag-Erling Smørgrav static unsigned long swzone;
16961203277SDag-Erling Smørgrav SYSCTL_ULONG(_vm, OID_AUTO, swzone, CTLFLAG_RD, &swzone, 0,
17061203277SDag-Erling Smørgrav     "Actual size of swap metadata zone");
17161203277SDag-Erling Smørgrav static unsigned long swap_maxpages;
17261203277SDag-Erling Smørgrav SYSCTL_ULONG(_vm, OID_AUTO, swap_maxpages, CTLFLAG_RD, &swap_maxpages, 0,
17361203277SDag-Erling Smørgrav     "Maximum amount of swap supported");
1743364c323SKonstantin Belousov 
1753364c323SKonstantin Belousov /* bits from overcommit */
1763364c323SKonstantin Belousov #define	SWAP_RESERVE_FORCE_ON		(1 << 0)
1773364c323SKonstantin Belousov #define	SWAP_RESERVE_RLIMIT_ON		(1 << 1)
1783364c323SKonstantin Belousov #define	SWAP_RESERVE_ALLOW_NONWIRED	(1 << 2)
1793364c323SKonstantin Belousov 
1803364c323SKonstantin Belousov int
1813364c323SKonstantin Belousov swap_reserve(vm_ooffset_t incr)
1823364c323SKonstantin Belousov {
1833364c323SKonstantin Belousov 
184ef694c1aSEdward Tomasz Napierala 	return (swap_reserve_by_cred(incr, curthread->td_ucred));
1853364c323SKonstantin Belousov }
1863364c323SKonstantin Belousov 
1873364c323SKonstantin Belousov int
188ef694c1aSEdward Tomasz Napierala swap_reserve_by_cred(vm_ooffset_t incr, struct ucred *cred)
1893364c323SKonstantin Belousov {
1905c0e1c11SKonstantin Belousov 	vm_ooffset_t r, s;
1913364c323SKonstantin Belousov 	int res, error;
1923364c323SKonstantin Belousov 	static int curfail;
1933364c323SKonstantin Belousov 	static struct timeval lastfail;
194ef694c1aSEdward Tomasz Napierala 	struct uidinfo *uip;
195ef694c1aSEdward Tomasz Napierala 
196ef694c1aSEdward Tomasz Napierala 	uip = cred->cr_ruidinfo;
1973364c323SKonstantin Belousov 
1983364c323SKonstantin Belousov 	if (incr & PAGE_MASK)
1993364c323SKonstantin Belousov 		panic("swap_reserve: & PAGE_MASK");
2003364c323SKonstantin Belousov 
201afcc55f3SEdward Tomasz Napierala #ifdef RACCT
2024b5c9cf6SEdward Tomasz Napierala 	if (racct_enable) {
2031ba5ad42SEdward Tomasz Napierala 		PROC_LOCK(curproc);
2041ba5ad42SEdward Tomasz Napierala 		error = racct_add(curproc, RACCT_SWAP, incr);
2051ba5ad42SEdward Tomasz Napierala 		PROC_UNLOCK(curproc);
2061ba5ad42SEdward Tomasz Napierala 		if (error != 0)
2071ba5ad42SEdward Tomasz Napierala 			return (0);
2084b5c9cf6SEdward Tomasz Napierala 	}
209afcc55f3SEdward Tomasz Napierala #endif
2101ba5ad42SEdward Tomasz Napierala 
2113364c323SKonstantin Belousov 	res = 0;
2123364c323SKonstantin Belousov 	mtx_lock(&sw_dev_mtx);
2133364c323SKonstantin Belousov 	r = swap_reserved + incr;
2143364c323SKonstantin Belousov 	if (overcommit & SWAP_RESERVE_ALLOW_NONWIRED) {
21544f1c916SBryan Drewery 		s = vm_cnt.v_page_count - vm_cnt.v_free_reserved - vm_cnt.v_wire_count;
2163364c323SKonstantin Belousov 		s *= PAGE_SIZE;
2173364c323SKonstantin Belousov 	} else
2183364c323SKonstantin Belousov 		s = 0;
2193364c323SKonstantin Belousov 	s += swap_total;
2203364c323SKonstantin Belousov 	if ((overcommit & SWAP_RESERVE_FORCE_ON) == 0 || r <= s ||
2213364c323SKonstantin Belousov 	    (error = priv_check(curthread, PRIV_VM_SWAP_NOQUOTA)) == 0) {
2223364c323SKonstantin Belousov 		res = 1;
2233364c323SKonstantin Belousov 		swap_reserved = r;
2243364c323SKonstantin Belousov 	}
2253364c323SKonstantin Belousov 	mtx_unlock(&sw_dev_mtx);
2263364c323SKonstantin Belousov 
2273364c323SKonstantin Belousov 	if (res) {
2283364c323SKonstantin Belousov 		UIDINFO_VMSIZE_LOCK(uip);
2295c0e1c11SKonstantin Belousov 		if ((overcommit & SWAP_RESERVE_RLIMIT_ON) != 0 &&
230f6f6d240SMateusz Guzik 		    uip->ui_vmsize + incr > lim_cur(curthread, RLIMIT_SWAP) &&
2315c0e1c11SKonstantin Belousov 		    priv_check(curthread, PRIV_VM_SWAP_NORLIMIT))
2323364c323SKonstantin Belousov 			res = 0;
2333364c323SKonstantin Belousov 		else
2343364c323SKonstantin Belousov 			uip->ui_vmsize += incr;
2353364c323SKonstantin Belousov 		UIDINFO_VMSIZE_UNLOCK(uip);
2363364c323SKonstantin Belousov 		if (!res) {
2373364c323SKonstantin Belousov 			mtx_lock(&sw_dev_mtx);
2383364c323SKonstantin Belousov 			swap_reserved -= incr;
2393364c323SKonstantin Belousov 			mtx_unlock(&sw_dev_mtx);
2403364c323SKonstantin Belousov 		}
2413364c323SKonstantin Belousov 	}
2423364c323SKonstantin Belousov 	if (!res && ppsratecheck(&lastfail, &curfail, 1)) {
2433364c323SKonstantin Belousov 		printf("uid %d, pid %d: swap reservation for %jd bytes failed\n",
244134465d7SKonstantin Belousov 		    uip->ui_uid, curproc->p_pid, incr);
2453364c323SKonstantin Belousov 	}
2463364c323SKonstantin Belousov 
247afcc55f3SEdward Tomasz Napierala #ifdef RACCT
2481ba5ad42SEdward Tomasz Napierala 	if (!res) {
2491ba5ad42SEdward Tomasz Napierala 		PROC_LOCK(curproc);
2501ba5ad42SEdward Tomasz Napierala 		racct_sub(curproc, RACCT_SWAP, incr);
2511ba5ad42SEdward Tomasz Napierala 		PROC_UNLOCK(curproc);
2521ba5ad42SEdward Tomasz Napierala 	}
253afcc55f3SEdward Tomasz Napierala #endif
2541ba5ad42SEdward Tomasz Napierala 
2553364c323SKonstantin Belousov 	return (res);
2563364c323SKonstantin Belousov }
2573364c323SKonstantin Belousov 
2583364c323SKonstantin Belousov void
2593364c323SKonstantin Belousov swap_reserve_force(vm_ooffset_t incr)
2603364c323SKonstantin Belousov {
2613364c323SKonstantin Belousov 	struct uidinfo *uip;
2623364c323SKonstantin Belousov 
2633364c323SKonstantin Belousov 	mtx_lock(&sw_dev_mtx);
2643364c323SKonstantin Belousov 	swap_reserved += incr;
2653364c323SKonstantin Belousov 	mtx_unlock(&sw_dev_mtx);
2663364c323SKonstantin Belousov 
267afcc55f3SEdward Tomasz Napierala #ifdef RACCT
2681ba5ad42SEdward Tomasz Napierala 	PROC_LOCK(curproc);
2691ba5ad42SEdward Tomasz Napierala 	racct_add_force(curproc, RACCT_SWAP, incr);
2701ba5ad42SEdward Tomasz Napierala 	PROC_UNLOCK(curproc);
271afcc55f3SEdward Tomasz Napierala #endif
2721ba5ad42SEdward Tomasz Napierala 
2733364c323SKonstantin Belousov 	uip = curthread->td_ucred->cr_ruidinfo;
2743364c323SKonstantin Belousov 	PROC_LOCK(curproc);
2753364c323SKonstantin Belousov 	UIDINFO_VMSIZE_LOCK(uip);
2763364c323SKonstantin Belousov 	uip->ui_vmsize += incr;
2773364c323SKonstantin Belousov 	UIDINFO_VMSIZE_UNLOCK(uip);
2783364c323SKonstantin Belousov 	PROC_UNLOCK(curproc);
2793364c323SKonstantin Belousov }
2803364c323SKonstantin Belousov 
2813364c323SKonstantin Belousov void
2823364c323SKonstantin Belousov swap_release(vm_ooffset_t decr)
2833364c323SKonstantin Belousov {
284ef694c1aSEdward Tomasz Napierala 	struct ucred *cred;
2853364c323SKonstantin Belousov 
2863364c323SKonstantin Belousov 	PROC_LOCK(curproc);
287ef694c1aSEdward Tomasz Napierala 	cred = curthread->td_ucred;
288ef694c1aSEdward Tomasz Napierala 	swap_release_by_cred(decr, cred);
2893364c323SKonstantin Belousov 	PROC_UNLOCK(curproc);
2903364c323SKonstantin Belousov }
2913364c323SKonstantin Belousov 
2923364c323SKonstantin Belousov void
293ef694c1aSEdward Tomasz Napierala swap_release_by_cred(vm_ooffset_t decr, struct ucred *cred)
2943364c323SKonstantin Belousov {
295ef694c1aSEdward Tomasz Napierala  	struct uidinfo *uip;
296ef694c1aSEdward Tomasz Napierala 
297ef694c1aSEdward Tomasz Napierala 	uip = cred->cr_ruidinfo;
2983364c323SKonstantin Belousov 
2993364c323SKonstantin Belousov 	if (decr & PAGE_MASK)
3003364c323SKonstantin Belousov 		panic("swap_release: & PAGE_MASK");
3013364c323SKonstantin Belousov 
3023364c323SKonstantin Belousov 	mtx_lock(&sw_dev_mtx);
3033364c323SKonstantin Belousov 	if (swap_reserved < decr)
3043364c323SKonstantin Belousov 		panic("swap_reserved < decr");
3053364c323SKonstantin Belousov 	swap_reserved -= decr;
3063364c323SKonstantin Belousov 	mtx_unlock(&sw_dev_mtx);
3073364c323SKonstantin Belousov 
3083364c323SKonstantin Belousov 	UIDINFO_VMSIZE_LOCK(uip);
3093364c323SKonstantin Belousov 	if (uip->ui_vmsize < decr)
3103364c323SKonstantin Belousov 		printf("negative vmsize for uid = %d\n", uip->ui_uid);
3113364c323SKonstantin Belousov 	uip->ui_vmsize -= decr;
3123364c323SKonstantin Belousov 	UIDINFO_VMSIZE_UNLOCK(uip);
3131ba5ad42SEdward Tomasz Napierala 
3141ba5ad42SEdward Tomasz Napierala 	racct_sub_cred(cred, RACCT_SWAP, decr);
3153364c323SKonstantin Belousov }
3163364c323SKonstantin Belousov 
3171c7c3c6aSMatthew Dillon #define SWM_FREE	0x02	/* free, period			*/
3181c7c3c6aSMatthew Dillon #define SWM_POP		0x04	/* pop out			*/
31926f9a767SRodney W. Grimes 
3207dea2c2eSAlan Cox int swap_pager_full = 2;	/* swap space exhaustion (task killing) */
3217dea2c2eSAlan Cox static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/
3221c7c3c6aSMatthew Dillon static int nsw_rcount;		/* free read buffers			*/
323327f4e83SMatthew Dillon static int nsw_wcount_sync;	/* limit write buffers / synchronous	*/
324327f4e83SMatthew Dillon static int nsw_wcount_async;	/* limit write buffers / asynchronous	*/
325327f4e83SMatthew Dillon static int nsw_wcount_async_max;/* assigned maximum			*/
326327f4e83SMatthew Dillon static int nsw_cluster_max;	/* maximum VOP I/O allowed		*/
3271c7c3c6aSMatthew Dillon 
32889c241d1SGleb Smirnoff static int sysctl_swap_async_max(SYSCTL_HANDLER_ARGS);
3294c36e917SKonstantin Belousov SYSCTL_PROC(_vm, OID_AUTO, swap_async_max, CTLTYPE_INT | CTLFLAG_RW |
3304c36e917SKonstantin Belousov     CTLFLAG_MPSAFE, NULL, 0, sysctl_swap_async_max, "I",
3314c36e917SKonstantin Belousov     "Maximum running async swap ops");
33289c241d1SGleb Smirnoff 
3331c7c3c6aSMatthew Dillon static struct swblock **swhash;
3341c7c3c6aSMatthew Dillon static int swhash_mask;
3357827d9b0SAlan Cox static struct mtx swhash_mtx;
3367827d9b0SAlan Cox 
3370cddd8f0SMatthew Dillon static struct sx sw_alloc_sx;
338327f4e83SMatthew Dillon 
3391c7c3c6aSMatthew Dillon /*
3401c7c3c6aSMatthew Dillon  * "named" and "unnamed" anon region objects.  Try to reduce the overhead
3411c7c3c6aSMatthew Dillon  * of searching a named list by hashing it just a little.
3421c7c3c6aSMatthew Dillon  */
3431c7c3c6aSMatthew Dillon 
3441c7c3c6aSMatthew Dillon #define NOBJLISTS		8
3451c7c3c6aSMatthew Dillon 
3461c7c3c6aSMatthew Dillon #define NOBJLIST(handle)	\
347af647ddeSBruce Evans 	(&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)])
3481c7c3c6aSMatthew Dillon 
3491c7c3c6aSMatthew Dillon static struct pagerlst	swap_pager_object_list[NOBJLISTS];
350e9c0cc15SPoul-Henning Kamp static uma_zone_t	swap_zone;
3511c7c3c6aSMatthew Dillon 
3521c7c3c6aSMatthew Dillon /*
3531c7c3c6aSMatthew Dillon  * pagerops for OBJT_SWAP - "swap pager".  Some ops are also global procedure
3541c7c3c6aSMatthew Dillon  * calls hooked from other parts of the VM system and do not appear here.
3551c7c3c6aSMatthew Dillon  * (see vm/swap_pager.h).
3561c7c3c6aSMatthew Dillon  */
357ff98689dSBruce Evans static vm_object_t
35811caded3SAlfred Perlstein 		swap_pager_alloc(void *handle, vm_ooffset_t size,
3593364c323SKonstantin Belousov 		    vm_prot_t prot, vm_ooffset_t offset, struct ucred *);
36011caded3SAlfred Perlstein static void	swap_pager_dealloc(vm_object_t object);
361b0cd2017SGleb Smirnoff static int	swap_pager_getpages(vm_object_t, vm_page_t *, int, int *,
362b0cd2017SGleb Smirnoff     int *);
363b0cd2017SGleb Smirnoff static int	swap_pager_getpages_async(vm_object_t, vm_page_t *, int, int *,
364b0cd2017SGleb Smirnoff     int *, pgo_getpages_iodone_t, void *);
365751221fdSPoul-Henning Kamp static void	swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *);
3665ea4972cSAlan Cox static boolean_t
3675ea4972cSAlan Cox 		swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
36811caded3SAlfred Perlstein static void	swap_pager_init(void);
36911caded3SAlfred Perlstein static void	swap_pager_unswapped(vm_page_t);
370b3fed13eSDavid Schultz static void	swap_pager_swapoff(struct swdevt *sp);
371f708ef1bSPoul-Henning Kamp 
372df8bae1dSRodney W. Grimes struct pagerops swappagerops = {
373e04e4bacSPoul-Henning Kamp 	.pgo_init =	swap_pager_init,	/* early system initialization of pager	*/
374e04e4bacSPoul-Henning Kamp 	.pgo_alloc =	swap_pager_alloc,	/* allocate an OBJT_SWAP object		*/
375e04e4bacSPoul-Henning Kamp 	.pgo_dealloc =	swap_pager_dealloc,	/* deallocate an OBJT_SWAP object	*/
376e04e4bacSPoul-Henning Kamp 	.pgo_getpages =	swap_pager_getpages,	/* pagein				*/
37790effb23SGleb Smirnoff 	.pgo_getpages_async = swap_pager_getpages_async, /* pagein (async)		*/
378e04e4bacSPoul-Henning Kamp 	.pgo_putpages =	swap_pager_putpages,	/* pageout				*/
379e04e4bacSPoul-Henning Kamp 	.pgo_haspage =	swap_pager_haspage,	/* get backing store status for page	*/
380e04e4bacSPoul-Henning Kamp 	.pgo_pageunswapped = swap_pager_unswapped,	/* remove swap related to page		*/
381df8bae1dSRodney W. Grimes };
382df8bae1dSRodney W. Grimes 
3831c7c3c6aSMatthew Dillon /*
3841c7c3c6aSMatthew Dillon  * dmmax is in page-sized chunks with the new swap system.  It was
38564bcb9c8SMatthew Dillon  * dev-bsized chunks in the old.  dmmax is always a power of 2.
3861c7c3c6aSMatthew Dillon  *
3871c7c3c6aSMatthew Dillon  * swap_*() routines are externally accessible.  swp_*() routines are
3881c7c3c6aSMatthew Dillon  * internal.
3891c7c3c6aSMatthew Dillon  */
390751221fdSPoul-Henning Kamp static int dmmax;
391e9c0cc15SPoul-Henning Kamp static int nswap_lowat = 128;	/* in pages, swap_pager_almost_full warn */
392e9c0cc15SPoul-Henning Kamp static int nswap_hiwat = 512;	/* in pages, swap_pager_almost_full warn */
39326f9a767SRodney W. Grimes 
3942174a0c6SKonstantin Belousov SYSCTL_INT(_vm, OID_AUTO, dmmax, CTLFLAG_RD, &dmmax, 0,
3952174a0c6SKonstantin Belousov     "Maximum size of a swap block");
396cee313c4SRobert Watson 
397a5edd34aSPoul-Henning Kamp static void	swp_sizecheck(void);
39811caded3SAlfred Perlstein static void	swp_pager_async_iodone(struct buf *bp);
39988ad2d7bSKonstantin Belousov static int	swapongeom(struct vnode *);
40059efee01SPoul-Henning Kamp static int	swaponvp(struct thread *, struct vnode *, u_long);
40135918c55SChristian S.J. Peron static int	swapoff_one(struct swdevt *sp, struct ucred *cred);
40224a1cce3SDavid Greenman 
4031c7c3c6aSMatthew Dillon /*
4041c7c3c6aSMatthew Dillon  * Swap bitmap functions
4051c7c3c6aSMatthew Dillon  */
406a5edd34aSPoul-Henning Kamp static void	swp_pager_freeswapspace(daddr_t blk, int npages);
407a5edd34aSPoul-Henning Kamp static daddr_t	swp_pager_getswapspace(int npages);
4081c7c3c6aSMatthew Dillon 
4091c7c3c6aSMatthew Dillon /*
4101c7c3c6aSMatthew Dillon  * Metadata functions
4111c7c3c6aSMatthew Dillon  */
412a5edd34aSPoul-Henning Kamp static struct swblock **swp_pager_hash(vm_object_t object, vm_pindex_t index);
41311caded3SAlfred Perlstein static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
4142e56b64fSKonstantin Belousov static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t);
41511caded3SAlfred Perlstein static void swp_pager_meta_free_all(vm_object_t);
41611caded3SAlfred Perlstein static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);
4171c7c3c6aSMatthew Dillon 
4181c7c3c6aSMatthew Dillon /*
4191c7c3c6aSMatthew Dillon  * SWP_SIZECHECK() -	update swap_pager_full indication
4201c7c3c6aSMatthew Dillon  *
42120d3034fSMatthew Dillon  *	update the swap_pager_almost_full indication and warn when we are
42220d3034fSMatthew Dillon  *	about to run out of swap space, using lowat/hiwat hysteresis.
42320d3034fSMatthew Dillon  *
42420d3034fSMatthew Dillon  *	Clear swap_pager_full ( task killing ) indication when lowat is met.
4251c7c3c6aSMatthew Dillon  *
4261c7c3c6aSMatthew Dillon  *	No restrictions on call
4271c7c3c6aSMatthew Dillon  *	This routine may not block.
4281c7c3c6aSMatthew Dillon  */
429a5edd34aSPoul-Henning Kamp static void
4302f249180SPoul-Henning Kamp swp_sizecheck(void)
4310d94caffSDavid Greenman {
43223955314SAlfred Perlstein 
4338f60c087SPoul-Henning Kamp 	if (swap_pager_avail < nswap_lowat) {
43420d3034fSMatthew Dillon 		if (swap_pager_almost_full == 0) {
4351af87c92SDavid Greenman 			printf("swap_pager: out of swap space\n");
43620d3034fSMatthew Dillon 			swap_pager_almost_full = 1;
4372b0d37a4SMatthew Dillon 		}
43820d3034fSMatthew Dillon 	} else {
43926f9a767SRodney W. Grimes 		swap_pager_full = 0;
4408f60c087SPoul-Henning Kamp 		if (swap_pager_avail > nswap_hiwat)
44120d3034fSMatthew Dillon 			swap_pager_almost_full = 0;
44226f9a767SRodney W. Grimes 	}
4431c7c3c6aSMatthew Dillon }
4441c7c3c6aSMatthew Dillon 
4451c7c3c6aSMatthew Dillon /*
446da5fd145SPeter Wemm  * SWP_PAGER_HASH() -	hash swap meta data
447da5fd145SPeter Wemm  *
448a5edd34aSPoul-Henning Kamp  *	This is an helper function which hashes the swapblk given
449da5fd145SPeter Wemm  *	the object and page index.  It returns a pointer to a pointer
450da5fd145SPeter Wemm  *	to the object, or a pointer to a NULL pointer if it could not
451da5fd145SPeter Wemm  *	find a swapblk.
452da5fd145SPeter Wemm  */
453a5edd34aSPoul-Henning Kamp static struct swblock **
454da5fd145SPeter Wemm swp_pager_hash(vm_object_t object, vm_pindex_t index)
455da5fd145SPeter Wemm {
456da5fd145SPeter Wemm 	struct swblock **pswap;
457da5fd145SPeter Wemm 	struct swblock *swap;
458da5fd145SPeter Wemm 
459da5fd145SPeter Wemm 	index &= ~(vm_pindex_t)SWAP_META_MASK;
460da5fd145SPeter Wemm 	pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask];
461da5fd145SPeter Wemm 	while ((swap = *pswap) != NULL) {
462da5fd145SPeter Wemm 		if (swap->swb_object == object &&
463da5fd145SPeter Wemm 		    swap->swb_index == index
464da5fd145SPeter Wemm 		) {
465da5fd145SPeter Wemm 			break;
466da5fd145SPeter Wemm 		}
467da5fd145SPeter Wemm 		pswap = &swap->swb_hnext;
468da5fd145SPeter Wemm 	}
469da5fd145SPeter Wemm 	return (pswap);
470da5fd145SPeter Wemm }
471da5fd145SPeter Wemm 
472da5fd145SPeter Wemm /*
4731c7c3c6aSMatthew Dillon  * SWAP_PAGER_INIT() -	initialize the swap pager!
4741c7c3c6aSMatthew Dillon  *
4751c7c3c6aSMatthew Dillon  *	Expected to be started from system init.  NOTE:  This code is run
4761c7c3c6aSMatthew Dillon  *	before much else so be careful what you depend on.  Most of the VM
4771c7c3c6aSMatthew Dillon  *	system has yet to be initialized at this point.
4781c7c3c6aSMatthew Dillon  */
479f5a12711SPoul-Henning Kamp static void
4802f249180SPoul-Henning Kamp swap_pager_init(void)
481df8bae1dSRodney W. Grimes {
4821c7c3c6aSMatthew Dillon 	/*
4831c7c3c6aSMatthew Dillon 	 * Initialize object lists
4841c7c3c6aSMatthew Dillon 	 */
4851c7c3c6aSMatthew Dillon 	int i;
4861c7c3c6aSMatthew Dillon 
4871c7c3c6aSMatthew Dillon 	for (i = 0; i < NOBJLISTS; ++i)
4881c7c3c6aSMatthew Dillon 		TAILQ_INIT(&swap_pager_object_list[i]);
48920da9c2eSPoul-Henning Kamp 	mtx_init(&sw_dev_mtx, "swapdev", NULL, MTX_DEF);
49015719273SKonstantin Belousov 	sx_init(&sw_alloc_sx, "swspsx");
49104533e1eSKonstantin Belousov 	sx_init(&swdev_syscall_lock, "swsysc");
492df8bae1dSRodney W. Grimes 
493df8bae1dSRodney W. Grimes 	/*
4941c7c3c6aSMatthew Dillon 	 * Device Stripe, in PAGE_SIZE'd blocks
495df8bae1dSRodney W. Grimes 	 */
4961c7c3c6aSMatthew Dillon 	dmmax = SWB_NPAGES * 2;
4971c7c3c6aSMatthew Dillon }
49826f9a767SRodney W. Grimes 
499df8bae1dSRodney W. Grimes /*
5001c7c3c6aSMatthew Dillon  * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process
5011c7c3c6aSMatthew Dillon  *
5021c7c3c6aSMatthew Dillon  *	Expected to be started from pageout process once, prior to entering
5031c7c3c6aSMatthew Dillon  *	its main loop.
504df8bae1dSRodney W. Grimes  */
50524a1cce3SDavid Greenman void
5062f249180SPoul-Henning Kamp swap_pager_swap_init(void)
507df8bae1dSRodney W. Grimes {
50861203277SDag-Erling Smørgrav 	unsigned long n, n2;
5090d94caffSDavid Greenman 
51026f9a767SRodney W. Grimes 	/*
5111c7c3c6aSMatthew Dillon 	 * Number of in-transit swap bp operations.  Don't
5121c7c3c6aSMatthew Dillon 	 * exhaust the pbufs completely.  Make sure we
5131c7c3c6aSMatthew Dillon 	 * initialize workable values (0 will work for hysteresis
5141c7c3c6aSMatthew Dillon 	 * but it isn't very efficient).
5151c7c3c6aSMatthew Dillon 	 *
516327f4e83SMatthew Dillon 	 * The nsw_cluster_max is constrained by the bp->b_pages[]
5171c7c3c6aSMatthew Dillon 	 * array (MAXPHYS/PAGE_SIZE) and our locally defined
5181c7c3c6aSMatthew Dillon 	 * MAX_PAGEOUT_CLUSTER.   Also be aware that swap ops are
5191c7c3c6aSMatthew Dillon 	 * constrained by the swap device interleave stripe size.
520327f4e83SMatthew Dillon 	 *
521327f4e83SMatthew Dillon 	 * Currently we hardwire nsw_wcount_async to 4.  This limit is
522327f4e83SMatthew Dillon 	 * designed to prevent other I/O from having high latencies due to
523327f4e83SMatthew Dillon 	 * our pageout I/O.  The value 4 works well for one or two active swap
524327f4e83SMatthew Dillon 	 * devices but is probably a little low if you have more.  Even so,
525327f4e83SMatthew Dillon 	 * a higher value would probably generate only a limited improvement
526327f4e83SMatthew Dillon 	 * with three or four active swap devices since the system does not
527327f4e83SMatthew Dillon 	 * typically have to pageout at extreme bandwidths.   We will want
528327f4e83SMatthew Dillon 	 * at least 2 per swap devices, and 4 is a pretty good value if you
529327f4e83SMatthew Dillon 	 * have one NFS swap device due to the command/ack latency over NFS.
530327f4e83SMatthew Dillon 	 * So it all works out pretty well.
53126f9a767SRodney W. Grimes 	 */
532ad3cce20SMatthew Dillon 	nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER);
533327f4e83SMatthew Dillon 
5346d541bf1SJohn Baldwin 	mtx_lock(&pbuf_mtx);
5351c7c3c6aSMatthew Dillon 	nsw_rcount = (nswbuf + 1) / 2;
536327f4e83SMatthew Dillon 	nsw_wcount_sync = (nswbuf + 3) / 4;
537327f4e83SMatthew Dillon 	nsw_wcount_async = 4;
538327f4e83SMatthew Dillon 	nsw_wcount_async_max = nsw_wcount_async;
5396d541bf1SJohn Baldwin 	mtx_unlock(&pbuf_mtx);
54024a1cce3SDavid Greenman 
5411c7c3c6aSMatthew Dillon 	/*
5421c7c3c6aSMatthew Dillon 	 * Initialize our zone.  Right now I'm just guessing on the number
5431c7c3c6aSMatthew Dillon 	 * we need based on the number of pages in the system.  Each swblock
54461203277SDag-Erling Smørgrav 	 * can hold 32 pages, so this is probably overkill.  This reservation
545ec61f55dSMatthew Dillon 	 * is typically limited to around 32MB by default.
5461c7c3c6aSMatthew Dillon 	 */
54744f1c916SBryan Drewery 	n = vm_cnt.v_page_count / 2;
5482f9e4e80SMatthew Dillon 	if (maxswzone && n > maxswzone / sizeof(struct swblock))
5492f9e4e80SMatthew Dillon 		n = maxswzone / sizeof(struct swblock);
55022a5e6b9SDag-Erling Smørgrav 	n2 = n;
551670d17b5SJeff Roberson 	swap_zone = uma_zcreate("SWAPMETA", sizeof(struct swblock), NULL, NULL,
5527827d9b0SAlan Cox 	    NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM);
553010b1ca1SDavid Schultz 	if (swap_zone == NULL)
554010b1ca1SDavid Schultz 		panic("failed to create swap_zone.");
5558355f576SJeff Roberson 	do {
556a4915c21SAttilio Rao 		if (uma_zone_reserve_kva(swap_zone, n))
55761ce6eeeSAlfred Perlstein 			break;
55861ce6eeeSAlfred Perlstein 		/*
55961ce6eeeSAlfred Perlstein 		 * if the allocation failed, try a zone two thirds the
56061ce6eeeSAlfred Perlstein 		 * size of the previous attempt.
56161ce6eeeSAlfred Perlstein 		 */
56261ce6eeeSAlfred Perlstein 		n -= ((n + 2) / 3);
56361ce6eeeSAlfred Perlstein 	} while (n > 0);
56421cd6e62SSeigo Tanimura 	if (n2 != n)
56561203277SDag-Erling Smørgrav 		printf("Swap zone entries reduced from %lu to %lu.\n", n2, n);
56661203277SDag-Erling Smørgrav 	swap_maxpages = n * SWAP_META_PAGES;
56761203277SDag-Erling Smørgrav 	swzone = n * sizeof(struct swblock);
56822a5e6b9SDag-Erling Smørgrav 	n2 = n;
56924a1cce3SDavid Greenman 
5701c7c3c6aSMatthew Dillon 	/*
5711c7c3c6aSMatthew Dillon 	 * Initialize our meta-data hash table.  The swapper does not need to
5721c7c3c6aSMatthew Dillon 	 * be quite as efficient as the VM system, so we do not use an
5731c7c3c6aSMatthew Dillon 	 * oversized hash table.
5741c7c3c6aSMatthew Dillon 	 *
5751c7c3c6aSMatthew Dillon 	 * 	n: 		size of hash table, must be power of 2
5761c7c3c6aSMatthew Dillon 	 *	swhash_mask:	hash table index mask
5771c7c3c6aSMatthew Dillon 	 */
57861ce6eeeSAlfred Perlstein 	for (n = 1; n < n2 / 8; n *= 2)
5791c7c3c6aSMatthew Dillon 		;
580a163d034SWarner Losh 	swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO);
5811c7c3c6aSMatthew Dillon 	swhash_mask = n - 1;
5827827d9b0SAlan Cox 	mtx_init(&swhash_mtx, "swap_pager swhash", NULL, MTX_DEF);
58324a1cce3SDavid Greenman }
58424a1cce3SDavid Greenman 
585eb4d6a1bSKonstantin Belousov static vm_object_t
586eb4d6a1bSKonstantin Belousov swap_pager_alloc_init(void *handle, struct ucred *cred, vm_ooffset_t size,
587eb4d6a1bSKonstantin Belousov     vm_ooffset_t offset)
588eb4d6a1bSKonstantin Belousov {
589eb4d6a1bSKonstantin Belousov 	vm_object_t object;
590eb4d6a1bSKonstantin Belousov 
591eb4d6a1bSKonstantin Belousov 	if (cred != NULL) {
592eb4d6a1bSKonstantin Belousov 		if (!swap_reserve_by_cred(size, cred))
593eb4d6a1bSKonstantin Belousov 			return (NULL);
594eb4d6a1bSKonstantin Belousov 		crhold(cred);
595eb4d6a1bSKonstantin Belousov 	}
596eb4d6a1bSKonstantin Belousov 	object = vm_object_allocate(OBJT_SWAP, OFF_TO_IDX(offset +
597eb4d6a1bSKonstantin Belousov 	    PAGE_MASK + size));
598eb4d6a1bSKonstantin Belousov 	object->handle = handle;
599eb4d6a1bSKonstantin Belousov 	if (cred != NULL) {
600eb4d6a1bSKonstantin Belousov 		object->cred = cred;
601eb4d6a1bSKonstantin Belousov 		object->charge = size;
602eb4d6a1bSKonstantin Belousov 	}
603eb4d6a1bSKonstantin Belousov 	object->un_pager.swp.swp_bcount = 0;
604eb4d6a1bSKonstantin Belousov 	return (object);
605eb4d6a1bSKonstantin Belousov }
606eb4d6a1bSKonstantin Belousov 
60724a1cce3SDavid Greenman /*
6081c7c3c6aSMatthew Dillon  * SWAP_PAGER_ALLOC() -	allocate a new OBJT_SWAP VM object and instantiate
6091c7c3c6aSMatthew Dillon  *			its metadata structures.
6101c7c3c6aSMatthew Dillon  *
6111c7c3c6aSMatthew Dillon  *	This routine is called from the mmap and fork code to create a new
612eb4d6a1bSKonstantin Belousov  *	OBJT_SWAP object.
6131c7c3c6aSMatthew Dillon  *
614eb4d6a1bSKonstantin Belousov  *	This routine must ensure that no live duplicate is created for
615eb4d6a1bSKonstantin Belousov  *	the named object request, which is protected against by
616eb4d6a1bSKonstantin Belousov  *	holding the sw_alloc_sx lock in case handle != NULL.
61724a1cce3SDavid Greenman  */
618f5a12711SPoul-Henning Kamp static vm_object_t
6196cde7a16SDavid Greenman swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
6203364c323SKonstantin Belousov     vm_ooffset_t offset, struct ucred *cred)
62124a1cce3SDavid Greenman {
62224a1cce3SDavid Greenman 	vm_object_t object;
6232f7af3dbSAlan Cox 
624eb4d6a1bSKonstantin Belousov 	if (handle != NULL) {
6251c7c3c6aSMatthew Dillon 		/*
6261c7c3c6aSMatthew Dillon 		 * Reference existing named region or allocate new one.  There
6271c7c3c6aSMatthew Dillon 		 * should not be a race here against swp_pager_meta_build()
6281c7c3c6aSMatthew Dillon 		 * as called from vm_page_remove() in regards to the lookup
6291c7c3c6aSMatthew Dillon 		 * of the handle.
6301c7c3c6aSMatthew Dillon 		 */
6310cddd8f0SMatthew Dillon 		sx_xlock(&sw_alloc_sx);
6321c7c3c6aSMatthew Dillon 		object = vm_pager_object_lookup(NOBJLIST(handle), handle);
633b5e8f167SAlan Cox 		if (object == NULL) {
634eb4d6a1bSKonstantin Belousov 			object = swap_pager_alloc_init(handle, cred, size,
635eb4d6a1bSKonstantin Belousov 			    offset);
636eb4d6a1bSKonstantin Belousov 			if (object != NULL) {
637eb4d6a1bSKonstantin Belousov 				TAILQ_INSERT_TAIL(NOBJLIST(object->handle),
638eb4d6a1bSKonstantin Belousov 				    object, pager_object_list);
6393364c323SKonstantin Belousov 			}
64024a1cce3SDavid Greenman 		}
6410cddd8f0SMatthew Dillon 		sx_xunlock(&sw_alloc_sx);
64224a1cce3SDavid Greenman 	} else {
643eb4d6a1bSKonstantin Belousov 		object = swap_pager_alloc_init(handle, cred, size, offset);
64424a1cce3SDavid Greenman 	}
64524a1cce3SDavid Greenman 	return (object);
646df8bae1dSRodney W. Grimes }
647df8bae1dSRodney W. Grimes 
64826f9a767SRodney W. Grimes /*
6491c7c3c6aSMatthew Dillon  * SWAP_PAGER_DEALLOC() -	remove swap metadata from object
6501c7c3c6aSMatthew Dillon  *
6511c7c3c6aSMatthew Dillon  *	The swap backing for the object is destroyed.  The code is
6521c7c3c6aSMatthew Dillon  *	designed such that we can reinstantiate it later, but this
6531c7c3c6aSMatthew Dillon  *	routine is typically called only when the entire object is
6541c7c3c6aSMatthew Dillon  *	about to be destroyed.
6551c7c3c6aSMatthew Dillon  *
65615523cf7SKonstantin Belousov  *	The object must be locked.
65726f9a767SRodney W. Grimes  */
658df8bae1dSRodney W. Grimes static void
6592f249180SPoul-Henning Kamp swap_pager_dealloc(vm_object_t object)
66026f9a767SRodney W. Grimes {
6614dcc5c2dSMatthew Dillon 
662eb4d6a1bSKonstantin Belousov 	VM_OBJECT_ASSERT_WLOCKED(object);
663eb4d6a1bSKonstantin Belousov 	KASSERT((object->flags & OBJ_DEAD) != 0, ("dealloc of reachable obj"));
664eb4d6a1bSKonstantin Belousov 
66526f9a767SRodney W. Grimes 	/*
6661c7c3c6aSMatthew Dillon 	 * Remove from list right away so lookups will fail if we block for
6671c7c3c6aSMatthew Dillon 	 * pageout completion.
66826f9a767SRodney W. Grimes 	 */
669bd228075SAlan Cox 	if (object->handle != NULL) {
670eb4d6a1bSKonstantin Belousov 		VM_OBJECT_WUNLOCK(object);
671eb4d6a1bSKonstantin Belousov 		sx_xlock(&sw_alloc_sx);
672eb4d6a1bSKonstantin Belousov 		TAILQ_REMOVE(NOBJLIST(object->handle), object,
673eb4d6a1bSKonstantin Belousov 		    pager_object_list);
674eb4d6a1bSKonstantin Belousov 		sx_xunlock(&sw_alloc_sx);
675eb4d6a1bSKonstantin Belousov 		VM_OBJECT_WLOCK(object);
676bd228075SAlan Cox 	}
6771c7c3c6aSMatthew Dillon 
6781c7c3c6aSMatthew Dillon 	vm_object_pip_wait(object, "swpdea");
6791c7c3c6aSMatthew Dillon 
6801c7c3c6aSMatthew Dillon 	/*
6811c7c3c6aSMatthew Dillon 	 * Free all remaining metadata.  We only bother to free it from
6821c7c3c6aSMatthew Dillon 	 * the swap meta data.  We do not attempt to free swapblk's still
6831c7c3c6aSMatthew Dillon 	 * associated with vm_page_t's for this object.  We do not care
6841c7c3c6aSMatthew Dillon 	 * if paging is still in progress on some objects.
6851c7c3c6aSMatthew Dillon 	 */
6861c7c3c6aSMatthew Dillon 	swp_pager_meta_free_all(object);
687e735691bSJohn Baldwin 	object->handle = NULL;
688e735691bSJohn Baldwin 	object->type = OBJT_DEAD;
6891c7c3c6aSMatthew Dillon }
6901c7c3c6aSMatthew Dillon 
6911c7c3c6aSMatthew Dillon /************************************************************************
6921c7c3c6aSMatthew Dillon  *			SWAP PAGER BITMAP ROUTINES			*
6931c7c3c6aSMatthew Dillon  ************************************************************************/
6941c7c3c6aSMatthew Dillon 
6951c7c3c6aSMatthew Dillon /*
6961c7c3c6aSMatthew Dillon  * SWP_PAGER_GETSWAPSPACE() -	allocate raw swap space
6971c7c3c6aSMatthew Dillon  *
6981c7c3c6aSMatthew Dillon  *	Allocate swap for the requested number of pages.  The starting
6991c7c3c6aSMatthew Dillon  *	swap block number (a page index) is returned or SWAPBLK_NONE
7001c7c3c6aSMatthew Dillon  *	if the allocation failed.
7011c7c3c6aSMatthew Dillon  *
7021c7c3c6aSMatthew Dillon  *	Also has the side effect of advising that somebody made a mistake
7031c7c3c6aSMatthew Dillon  *	when they configured swap and didn't configure enough.
7041c7c3c6aSMatthew Dillon  *
70515523cf7SKonstantin Belousov  *	This routine may not sleep.
7068f60c087SPoul-Henning Kamp  *
7078f60c087SPoul-Henning Kamp  *	We allocate in round-robin fashion from the configured devices.
7081c7c3c6aSMatthew Dillon  */
709a5edd34aSPoul-Henning Kamp static daddr_t
7102f249180SPoul-Henning Kamp swp_pager_getswapspace(int npages)
7111c7c3c6aSMatthew Dillon {
7121c7c3c6aSMatthew Dillon 	daddr_t blk;
7138f60c087SPoul-Henning Kamp 	struct swdevt *sp;
7148f60c087SPoul-Henning Kamp 	int i;
7151c7c3c6aSMatthew Dillon 
7168f60c087SPoul-Henning Kamp 	blk = SWAPBLK_NONE;
71720da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
7188f60c087SPoul-Henning Kamp 	sp = swdevhd;
7198f60c087SPoul-Henning Kamp 	for (i = 0; i < nswapdev; i++) {
7208f60c087SPoul-Henning Kamp 		if (sp == NULL)
7218f60c087SPoul-Henning Kamp 			sp = TAILQ_FIRST(&swtailq);
7228f60c087SPoul-Henning Kamp 		if (!(sp->sw_flags & SW_CLOSING)) {
7238f60c087SPoul-Henning Kamp 			blk = blist_alloc(sp->sw_blist, npages);
7248f60c087SPoul-Henning Kamp 			if (blk != SWAPBLK_NONE) {
7258f60c087SPoul-Henning Kamp 				blk += sp->sw_first;
7268f60c087SPoul-Henning Kamp 				sp->sw_used += npages;
727d05bc129SAlan Cox 				swap_pager_avail -= npages;
7288f60c087SPoul-Henning Kamp 				swp_sizecheck();
7298f60c087SPoul-Henning Kamp 				swdevhd = TAILQ_NEXT(sp, sw_list);
730d05bc129SAlan Cox 				goto done;
7318f60c087SPoul-Henning Kamp 			}
7328f60c087SPoul-Henning Kamp 		}
7338f60c087SPoul-Henning Kamp 		sp = TAILQ_NEXT(sp, sw_list);
7348f60c087SPoul-Henning Kamp 	}
7352b0d37a4SMatthew Dillon 	if (swap_pager_full != 2) {
73620da9c2eSPoul-Henning Kamp 		printf("swap_pager_getswapspace(%d): failed\n", npages);
7372b0d37a4SMatthew Dillon 		swap_pager_full = 2;
73820d3034fSMatthew Dillon 		swap_pager_almost_full = 1;
7392b0d37a4SMatthew Dillon 	}
7408f60c087SPoul-Henning Kamp 	swdevhd = NULL;
741d05bc129SAlan Cox done:
742d05bc129SAlan Cox 	mtx_unlock(&sw_dev_mtx);
7431c7c3c6aSMatthew Dillon 	return (blk);
74426f9a767SRodney W. Grimes }
74526f9a767SRodney W. Grimes 
746b3fed13eSDavid Schultz static int
747b3fed13eSDavid Schultz swp_pager_isondev(daddr_t blk, struct swdevt *sp)
7488f60c087SPoul-Henning Kamp {
7498f60c087SPoul-Henning Kamp 
750b3fed13eSDavid Schultz 	return (blk >= sp->sw_first && blk < sp->sw_end);
7518f60c087SPoul-Henning Kamp }
7528f60c087SPoul-Henning Kamp 
7534b03903aSPoul-Henning Kamp static void
7544b03903aSPoul-Henning Kamp swp_pager_strategy(struct buf *bp)
7554b03903aSPoul-Henning Kamp {
7564b03903aSPoul-Henning Kamp 	struct swdevt *sp;
7574b03903aSPoul-Henning Kamp 
75820da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
7594b03903aSPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
7604b03903aSPoul-Henning Kamp 		if (bp->b_blkno >= sp->sw_first && bp->b_blkno < sp->sw_end) {
76120da9c2eSPoul-Henning Kamp 			mtx_unlock(&sw_dev_mtx);
7622cc718a1SKonstantin Belousov 			if ((sp->sw_flags & SW_UNMAPPED) != 0 &&
7632cc718a1SKonstantin Belousov 			    unmapped_buf_allowed) {
7642cc718a1SKonstantin Belousov 				bp->b_data = unmapped_buf;
7652cc718a1SKonstantin Belousov 				bp->b_offset = 0;
7662cc718a1SKonstantin Belousov 			} else {
7672cc718a1SKonstantin Belousov 				pmap_qenter((vm_offset_t)bp->b_data,
7682cc718a1SKonstantin Belousov 				    &bp->b_pages[0], bp->b_bcount / PAGE_SIZE);
7692cc718a1SKonstantin Belousov 			}
7704b03903aSPoul-Henning Kamp 			sp->sw_strategy(bp, sp);
7714b03903aSPoul-Henning Kamp 			return;
7724b03903aSPoul-Henning Kamp 		}
7734b03903aSPoul-Henning Kamp 	}
77420da9c2eSPoul-Henning Kamp 	panic("Swapdev not found");
7754b03903aSPoul-Henning Kamp }
7764b03903aSPoul-Henning Kamp 
7778f60c087SPoul-Henning Kamp 
77826f9a767SRodney W. Grimes /*
7791c7c3c6aSMatthew Dillon  * SWP_PAGER_FREESWAPSPACE() -	free raw swap space
7801c7c3c6aSMatthew Dillon  *
7811c7c3c6aSMatthew Dillon  *	This routine returns the specified swap blocks back to the bitmap.
7821c7c3c6aSMatthew Dillon  *
78315523cf7SKonstantin Belousov  *	This routine may not sleep.
78426f9a767SRodney W. Grimes  */
785a5edd34aSPoul-Henning Kamp static void
7868f60c087SPoul-Henning Kamp swp_pager_freeswapspace(daddr_t blk, int npages)
7870d94caffSDavid Greenman {
7888f60c087SPoul-Henning Kamp 	struct swdevt *sp;
78992da00bbSMatthew Dillon 
7907645e885SAlan Cox 	mtx_lock(&sw_dev_mtx);
7917645e885SAlan Cox 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
7927645e885SAlan Cox 		if (blk >= sp->sw_first && blk < sp->sw_end) {
79392da00bbSMatthew Dillon 			sp->sw_used -= npages;
79492da00bbSMatthew Dillon 			/*
7957645e885SAlan Cox 			 * If we are attempting to stop swapping on
7967645e885SAlan Cox 			 * this device, we don't want to mark any
7977645e885SAlan Cox 			 * blocks free lest they be reused.
79892da00bbSMatthew Dillon 			 */
7997645e885SAlan Cox 			if ((sp->sw_flags & SW_CLOSING) == 0) {
8007645e885SAlan Cox 				blist_free(sp->sw_blist, blk - sp->sw_first,
8017645e885SAlan Cox 				    npages);
8028f60c087SPoul-Henning Kamp 				swap_pager_avail += npages;
8031c7c3c6aSMatthew Dillon 				swp_sizecheck();
80426f9a767SRodney W. Grimes 			}
8057645e885SAlan Cox 			mtx_unlock(&sw_dev_mtx);
8067645e885SAlan Cox 			return;
8077645e885SAlan Cox 		}
8087645e885SAlan Cox 	}
8097645e885SAlan Cox 	panic("Swapdev not found");
8107645e885SAlan Cox }
8111c7c3c6aSMatthew Dillon 
81226f9a767SRodney W. Grimes /*
8131c7c3c6aSMatthew Dillon  * SWAP_PAGER_FREESPACE() -	frees swap blocks associated with a page
8141c7c3c6aSMatthew Dillon  *				range within an object.
8151c7c3c6aSMatthew Dillon  *
8161c7c3c6aSMatthew Dillon  *	This is a globally accessible routine.
8171c7c3c6aSMatthew Dillon  *
8181c7c3c6aSMatthew Dillon  *	This routine removes swapblk assignments from swap metadata.
8191c7c3c6aSMatthew Dillon  *
8201c7c3c6aSMatthew Dillon  *	The external callers of this routine typically have already destroyed
8211c7c3c6aSMatthew Dillon  *	or renamed vm_page_t's associated with this range in the object so
8221c7c3c6aSMatthew Dillon  *	we should be ok.
823c25673ffSAttilio Rao  *
824c25673ffSAttilio Rao  *	The object must be locked.
82526f9a767SRodney W. Grimes  */
82626f9a767SRodney W. Grimes void
8272f249180SPoul-Henning Kamp swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size)
82826f9a767SRodney W. Grimes {
82923955314SAlfred Perlstein 
8301c7c3c6aSMatthew Dillon 	swp_pager_meta_free(object, start, size);
8314dcc5c2dSMatthew Dillon }
8324dcc5c2dSMatthew Dillon 
8334dcc5c2dSMatthew Dillon /*
8344dcc5c2dSMatthew Dillon  * SWAP_PAGER_RESERVE() - reserve swap blocks in object
8354dcc5c2dSMatthew Dillon  *
8364dcc5c2dSMatthew Dillon  *	Assigns swap blocks to the specified range within the object.  The
83756ce850bSKonstantin Belousov  *	swap blocks are not zeroed.  Any previous swap assignment is destroyed.
8384dcc5c2dSMatthew Dillon  *
8394dcc5c2dSMatthew Dillon  *	Returns 0 on success, -1 on failure.
8404dcc5c2dSMatthew Dillon  */
8414dcc5c2dSMatthew Dillon int
8424dcc5c2dSMatthew Dillon swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
8434dcc5c2dSMatthew Dillon {
8444dcc5c2dSMatthew Dillon 	int n = 0;
8454dcc5c2dSMatthew Dillon 	daddr_t blk = SWAPBLK_NONE;
8464dcc5c2dSMatthew Dillon 	vm_pindex_t beg = start;	/* save start index */
8474dcc5c2dSMatthew Dillon 
84889f6b863SAttilio Rao 	VM_OBJECT_WLOCK(object);
8494dcc5c2dSMatthew Dillon 	while (size) {
8504dcc5c2dSMatthew Dillon 		if (n == 0) {
8514dcc5c2dSMatthew Dillon 			n = BLIST_MAX_ALLOC;
8524dcc5c2dSMatthew Dillon 			while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
8534dcc5c2dSMatthew Dillon 				n >>= 1;
8544dcc5c2dSMatthew Dillon 				if (n == 0) {
8554dcc5c2dSMatthew Dillon 					swp_pager_meta_free(object, beg, start - beg);
85689f6b863SAttilio Rao 					VM_OBJECT_WUNLOCK(object);
8574dcc5c2dSMatthew Dillon 					return (-1);
8584dcc5c2dSMatthew Dillon 				}
8594dcc5c2dSMatthew Dillon 			}
8604dcc5c2dSMatthew Dillon 		}
8614dcc5c2dSMatthew Dillon 		swp_pager_meta_build(object, start, blk);
8624dcc5c2dSMatthew Dillon 		--size;
8634dcc5c2dSMatthew Dillon 		++start;
8644dcc5c2dSMatthew Dillon 		++blk;
8654dcc5c2dSMatthew Dillon 		--n;
8664dcc5c2dSMatthew Dillon 	}
8674dcc5c2dSMatthew Dillon 	swp_pager_meta_free(object, start, n);
86889f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(object);
8694dcc5c2dSMatthew Dillon 	return (0);
87026f9a767SRodney W. Grimes }
87126f9a767SRodney W. Grimes 
8720a47b48bSJohn Dyson /*
8731c7c3c6aSMatthew Dillon  * SWAP_PAGER_COPY() -  copy blocks from source pager to destination pager
8741c7c3c6aSMatthew Dillon  *			and destroy the source.
8751c7c3c6aSMatthew Dillon  *
8761c7c3c6aSMatthew Dillon  *	Copy any valid swapblks from the source to the destination.  In
8771c7c3c6aSMatthew Dillon  *	cases where both the source and destination have a valid swapblk,
8781c7c3c6aSMatthew Dillon  *	we keep the destination's.
8791c7c3c6aSMatthew Dillon  *
88015523cf7SKonstantin Belousov  *	This routine is allowed to sleep.  It may sleep allocating metadata
8811c7c3c6aSMatthew Dillon  *	indirectly through swp_pager_meta_build() or if paging is still in
8821c7c3c6aSMatthew Dillon  *	progress on the source.
8831c7c3c6aSMatthew Dillon  *
8841c7c3c6aSMatthew Dillon  *	The source object contains no vm_page_t's (which is just as well)
8851c7c3c6aSMatthew Dillon  *
8861c7c3c6aSMatthew Dillon  *	The source object is of type OBJT_SWAP.
8871c7c3c6aSMatthew Dillon  *
88815523cf7SKonstantin Belousov  *	The source and destination objects must be locked.
88915523cf7SKonstantin Belousov  *	Both object locks may temporarily be released.
89026f9a767SRodney W. Grimes  */
89126f9a767SRodney W. Grimes void
8922f249180SPoul-Henning Kamp swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
8932f249180SPoul-Henning Kamp     vm_pindex_t offset, int destroysource)
89426f9a767SRodney W. Grimes {
895a316d390SJohn Dyson 	vm_pindex_t i;
8964dcc5c2dSMatthew Dillon 
89789f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(srcobject);
89889f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(dstobject);
8990cddd8f0SMatthew Dillon 
90026f9a767SRodney W. Grimes 	/*
9011c7c3c6aSMatthew Dillon 	 * If destroysource is set, we remove the source object from the
9021c7c3c6aSMatthew Dillon 	 * swap_pager internal queue now.
90326f9a767SRodney W. Grimes 	 */
904eb4d6a1bSKonstantin Belousov 	if (destroysource && srcobject->handle != NULL) {
905eb4d6a1bSKonstantin Belousov 		vm_object_pip_add(srcobject, 1);
906eb4d6a1bSKonstantin Belousov 		VM_OBJECT_WUNLOCK(srcobject);
907eb4d6a1bSKonstantin Belousov 		vm_object_pip_add(dstobject, 1);
908eb4d6a1bSKonstantin Belousov 		VM_OBJECT_WUNLOCK(dstobject);
909eb4d6a1bSKonstantin Belousov 		sx_xlock(&sw_alloc_sx);
910eb4d6a1bSKonstantin Belousov 		TAILQ_REMOVE(NOBJLIST(srcobject->handle), srcobject,
911eb4d6a1bSKonstantin Belousov 		    pager_object_list);
912eb4d6a1bSKonstantin Belousov 		sx_xunlock(&sw_alloc_sx);
913eb4d6a1bSKonstantin Belousov 		VM_OBJECT_WLOCK(dstobject);
914eb4d6a1bSKonstantin Belousov 		vm_object_pip_wakeup(dstobject);
915eb4d6a1bSKonstantin Belousov 		VM_OBJECT_WLOCK(srcobject);
916eb4d6a1bSKonstantin Belousov 		vm_object_pip_wakeup(srcobject);
917bd228075SAlan Cox 	}
91826f9a767SRodney W. Grimes 
9191c7c3c6aSMatthew Dillon 	/*
9201c7c3c6aSMatthew Dillon 	 * transfer source to destination.
9211c7c3c6aSMatthew Dillon 	 */
9221c7c3c6aSMatthew Dillon 	for (i = 0; i < dstobject->size; ++i) {
9231c7c3c6aSMatthew Dillon 		daddr_t dstaddr;
9241c7c3c6aSMatthew Dillon 
9251c7c3c6aSMatthew Dillon 		/*
9261c7c3c6aSMatthew Dillon 		 * Locate (without changing) the swapblk on the destination,
9271c7c3c6aSMatthew Dillon 		 * unless it is invalid in which case free it silently, or
9281c7c3c6aSMatthew Dillon 		 * if the destination is a resident page, in which case the
9291c7c3c6aSMatthew Dillon 		 * source is thrown away.
9301c7c3c6aSMatthew Dillon 		 */
9311c7c3c6aSMatthew Dillon 		dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
9321c7c3c6aSMatthew Dillon 
9331c7c3c6aSMatthew Dillon 		if (dstaddr == SWAPBLK_NONE) {
9341c7c3c6aSMatthew Dillon 			/*
9351c7c3c6aSMatthew Dillon 			 * Destination has no swapblk and is not resident,
9361c7c3c6aSMatthew Dillon 			 * copy source.
9371c7c3c6aSMatthew Dillon 			 */
9381c7c3c6aSMatthew Dillon 			daddr_t srcaddr;
9391c7c3c6aSMatthew Dillon 
9401c7c3c6aSMatthew Dillon 			srcaddr = swp_pager_meta_ctl(
9411c7c3c6aSMatthew Dillon 			    srcobject,
9421c7c3c6aSMatthew Dillon 			    i + offset,
9431c7c3c6aSMatthew Dillon 			    SWM_POP
9441c7c3c6aSMatthew Dillon 			);
9451c7c3c6aSMatthew Dillon 
946ee3dc7d7SAlan Cox 			if (srcaddr != SWAPBLK_NONE) {
947c7c8dd7eSAlan Cox 				/*
948c7c8dd7eSAlan Cox 				 * swp_pager_meta_build() can sleep.
949c7c8dd7eSAlan Cox 				 */
950c7c8dd7eSAlan Cox 				vm_object_pip_add(srcobject, 1);
95189f6b863SAttilio Rao 				VM_OBJECT_WUNLOCK(srcobject);
952c7c8dd7eSAlan Cox 				vm_object_pip_add(dstobject, 1);
9534dcc5c2dSMatthew Dillon 				swp_pager_meta_build(dstobject, i, srcaddr);
954c7c8dd7eSAlan Cox 				vm_object_pip_wakeup(dstobject);
95589f6b863SAttilio Rao 				VM_OBJECT_WLOCK(srcobject);
956c7c8dd7eSAlan Cox 				vm_object_pip_wakeup(srcobject);
957ee3dc7d7SAlan Cox 			}
9581c7c3c6aSMatthew Dillon 		} else {
9591c7c3c6aSMatthew Dillon 			/*
9601c7c3c6aSMatthew Dillon 			 * Destination has valid swapblk or it is represented
9611c7c3c6aSMatthew Dillon 			 * by a resident page.  We destroy the sourceblock.
9621c7c3c6aSMatthew Dillon 			 */
9631c7c3c6aSMatthew Dillon 
9641c7c3c6aSMatthew Dillon 			swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE);
9651c7c3c6aSMatthew Dillon 		}
96626f9a767SRodney W. Grimes 	}
96726f9a767SRodney W. Grimes 
96826f9a767SRodney W. Grimes 	/*
9691c7c3c6aSMatthew Dillon 	 * Free left over swap blocks in source.
9701c7c3c6aSMatthew Dillon 	 *
971763df3ecSPedro F. Giffuni 	 * We have to revert the type to OBJT_DEFAULT so we do not accidentally
9721c7c3c6aSMatthew Dillon 	 * double-remove the object from the swap queues.
97326f9a767SRodney W. Grimes 	 */
974c0877f10SJohn Dyson 	if (destroysource) {
9751c7c3c6aSMatthew Dillon 		swp_pager_meta_free_all(srcobject);
9761c7c3c6aSMatthew Dillon 		/*
9771c7c3c6aSMatthew Dillon 		 * Reverting the type is not necessary, the caller is going
9781c7c3c6aSMatthew Dillon 		 * to destroy srcobject directly, but I'm doing it here
979956f3135SPhilippe Charnier 		 * for consistency since we've removed the object from its
9801c7c3c6aSMatthew Dillon 		 * queues.
9811c7c3c6aSMatthew Dillon 		 */
9821c7c3c6aSMatthew Dillon 		srcobject->type = OBJT_DEFAULT;
983c0877f10SJohn Dyson 	}
98426f9a767SRodney W. Grimes }
98526f9a767SRodney W. Grimes 
986df8bae1dSRodney W. Grimes /*
9871c7c3c6aSMatthew Dillon  * SWAP_PAGER_HASPAGE() -	determine if we have good backing store for
9881c7c3c6aSMatthew Dillon  *				the requested page.
9891c7c3c6aSMatthew Dillon  *
9901c7c3c6aSMatthew Dillon  *	We determine whether good backing store exists for the requested
9911c7c3c6aSMatthew Dillon  *	page and return TRUE if it does, FALSE if it doesn't.
9921c7c3c6aSMatthew Dillon  *
9931c7c3c6aSMatthew Dillon  *	If TRUE, we also try to determine how much valid, contiguous backing
994915d1b71SMark Johnston  *	store exists before and after the requested page.
995df8bae1dSRodney W. Grimes  */
9965ea4972cSAlan Cox static boolean_t
997915d1b71SMark Johnston swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,
998915d1b71SMark Johnston     int *after)
99926f9a767SRodney W. Grimes {
1000915d1b71SMark Johnston 	daddr_t blk, blk0;
1001915d1b71SMark Johnston 	int i;
100226f9a767SRodney W. Grimes 
1003c25673ffSAttilio Rao 	VM_OBJECT_ASSERT_LOCKED(object);
1004915d1b71SMark Johnston 
10051c7c3c6aSMatthew Dillon 	/*
10061c7c3c6aSMatthew Dillon 	 * do we have good backing store at the requested index ?
10071c7c3c6aSMatthew Dillon 	 */
10081c7c3c6aSMatthew Dillon 	blk0 = swp_pager_meta_ctl(object, pindex, 0);
10094dcc5c2dSMatthew Dillon 	if (blk0 == SWAPBLK_NONE) {
10101c7c3c6aSMatthew Dillon 		if (before)
101124a1cce3SDavid Greenman 			*before = 0;
10121c7c3c6aSMatthew Dillon 		if (after)
101324a1cce3SDavid Greenman 			*after = 0;
101426f9a767SRodney W. Grimes 		return (FALSE);
101526f9a767SRodney W. Grimes 	}
101626f9a767SRodney W. Grimes 
101726f9a767SRodney W. Grimes 	/*
10181c7c3c6aSMatthew Dillon 	 * find backwards-looking contiguous good backing store
1019e47ed70bSJohn Dyson 	 */
10201c7c3c6aSMatthew Dillon 	if (before != NULL) {
1021915d1b71SMark Johnston 		for (i = 1; i < SWB_NPAGES; i++) {
10221c7c3c6aSMatthew Dillon 			if (i > pindex)
10231c7c3c6aSMatthew Dillon 				break;
10241c7c3c6aSMatthew Dillon 			blk = swp_pager_meta_ctl(object, pindex - i, 0);
10251c7c3c6aSMatthew Dillon 			if (blk != blk0 - i)
10261c7c3c6aSMatthew Dillon 				break;
1027ffc82b0aSJohn Dyson 		}
1028915d1b71SMark Johnston 		*before = i - 1;
102926f9a767SRodney W. Grimes 	}
103026f9a767SRodney W. Grimes 
103126f9a767SRodney W. Grimes 	/*
10321c7c3c6aSMatthew Dillon 	 * find forward-looking contiguous good backing store
103326f9a767SRodney W. Grimes 	 */
10341c7c3c6aSMatthew Dillon 	if (after != NULL) {
1035915d1b71SMark Johnston 		for (i = 1; i < SWB_NPAGES; i++) {
10361c7c3c6aSMatthew Dillon 			blk = swp_pager_meta_ctl(object, pindex + i, 0);
10371c7c3c6aSMatthew Dillon 			if (blk != blk0 + i)
10381c7c3c6aSMatthew Dillon 				break;
103926f9a767SRodney W. Grimes 		}
1040915d1b71SMark Johnston 		*after = i - 1;
10411c7c3c6aSMatthew Dillon 	}
10421c7c3c6aSMatthew Dillon 	return (TRUE);
10431c7c3c6aSMatthew Dillon }
10441c7c3c6aSMatthew Dillon 
10451c7c3c6aSMatthew Dillon /*
10461c7c3c6aSMatthew Dillon  * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page
10471c7c3c6aSMatthew Dillon  *
10481c7c3c6aSMatthew Dillon  *	This removes any associated swap backing store, whether valid or
10491c7c3c6aSMatthew Dillon  *	not, from the page.
10501c7c3c6aSMatthew Dillon  *
10511c7c3c6aSMatthew Dillon  *	This routine is typically called when a page is made dirty, at
10521c7c3c6aSMatthew Dillon  *	which point any associated swap can be freed.  MADV_FREE also
10531c7c3c6aSMatthew Dillon  *	calls us in a special-case situation
10541c7c3c6aSMatthew Dillon  *
10551c7c3c6aSMatthew Dillon  *	NOTE!!!  If the page is clean and the swap was valid, the caller
10561c7c3c6aSMatthew Dillon  *	should make the page dirty before calling this routine.  This routine
10571c7c3c6aSMatthew Dillon  *	does NOT change the m->dirty status of the page.  Also: MADV_FREE
10581c7c3c6aSMatthew Dillon  *	depends on it.
10591c7c3c6aSMatthew Dillon  *
106015523cf7SKonstantin Belousov  *	This routine may not sleep.
1061c25673ffSAttilio Rao  *
1062c25673ffSAttilio Rao  *	The object containing the page must be locked.
10631c7c3c6aSMatthew Dillon  */
10641c7c3c6aSMatthew Dillon static void
10652f249180SPoul-Henning Kamp swap_pager_unswapped(vm_page_t m)
10661c7c3c6aSMatthew Dillon {
10672f249180SPoul-Henning Kamp 
10681c7c3c6aSMatthew Dillon 	swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE);
10691c7c3c6aSMatthew Dillon }
10701c7c3c6aSMatthew Dillon 
10711c7c3c6aSMatthew Dillon /*
1072915d1b71SMark Johnston  * swap_pager_getpages() - bring pages in from swap
10731c7c3c6aSMatthew Dillon  *
1074915d1b71SMark Johnston  *	Attempt to page in the pages in array "m" of length "count".  The caller
1075915d1b71SMark Johnston  *	may optionally specify that additional pages preceding and succeeding
1076915d1b71SMark Johnston  *	the specified range be paged in.  The number of such pages is returned
1077915d1b71SMark Johnston  *	in the "rbehind" and "rahead" parameters, and they will be in the
1078915d1b71SMark Johnston  *	inactive queue upon return.
10791c7c3c6aSMatthew Dillon  *
1080915d1b71SMark Johnston  *	The pages in "m" must be busied and will remain busied upon return.
10811c7c3c6aSMatthew Dillon  */
1082f708ef1bSPoul-Henning Kamp static int
1083b0cd2017SGleb Smirnoff swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind,
1084b0cd2017SGleb Smirnoff     int *rahead)
1085df8bae1dSRodney W. Grimes {
10861c7c3c6aSMatthew Dillon 	struct buf *bp;
1087915d1b71SMark Johnston 	vm_page_t mpred, msucc, p;
1088915d1b71SMark Johnston 	vm_pindex_t pindex;
10891c7c3c6aSMatthew Dillon 	daddr_t blk;
1090dd9cb6daSMark Johnston 	int i, j, maxahead, maxbehind, reqcount, shift;
10910d94caffSDavid Greenman 
1092915d1b71SMark Johnston 	reqcount = count;
10931c7c3c6aSMatthew Dillon 
109489f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(object);
10951c7c3c6aSMatthew Dillon 	bp = getpbuf(&nsw_rcount);
1096915d1b71SMark Johnston 	VM_OBJECT_WLOCK(object);
109726f9a767SRodney W. Grimes 
1098dd9cb6daSMark Johnston 	if (!swap_pager_haspage(object, m[0]->pindex, &maxbehind, &maxahead)) {
1099915d1b71SMark Johnston 		relpbuf(bp, &nsw_rcount);
1100915d1b71SMark Johnston 		return (VM_PAGER_FAIL);
1101915d1b71SMark Johnston 	}
1102915d1b71SMark Johnston 
1103915d1b71SMark Johnston 	/*
1104915d1b71SMark Johnston 	 * Clip the readahead and readbehind ranges to exclude resident pages.
1105915d1b71SMark Johnston 	 */
1106915d1b71SMark Johnston 	if (rahead != NULL) {
1107dd9cb6daSMark Johnston 		KASSERT(reqcount - 1 <= maxahead,
1108915d1b71SMark Johnston 		    ("page count %d extends beyond swap block", reqcount));
1109dd9cb6daSMark Johnston 		*rahead = imin(*rahead, maxahead - (reqcount - 1));
1110915d1b71SMark Johnston 		pindex = m[reqcount - 1]->pindex;
1111915d1b71SMark Johnston 		msucc = TAILQ_NEXT(m[reqcount - 1], listq);
1112915d1b71SMark Johnston 		if (msucc != NULL && msucc->pindex - pindex - 1 < *rahead)
1113915d1b71SMark Johnston 			*rahead = msucc->pindex - pindex - 1;
1114915d1b71SMark Johnston 	}
1115915d1b71SMark Johnston 	if (rbehind != NULL) {
1116dd9cb6daSMark Johnston 		*rbehind = imin(*rbehind, maxbehind);
1117915d1b71SMark Johnston 		pindex = m[0]->pindex;
1118915d1b71SMark Johnston 		mpred = TAILQ_PREV(m[0], pglist, listq);
1119915d1b71SMark Johnston 		if (mpred != NULL && pindex - mpred->pindex - 1 < *rbehind)
1120915d1b71SMark Johnston 			*rbehind = pindex - mpred->pindex - 1;
1121915d1b71SMark Johnston 	}
1122915d1b71SMark Johnston 
1123915d1b71SMark Johnston 	/*
1124915d1b71SMark Johnston 	 * Allocate readahead and readbehind pages.
1125915d1b71SMark Johnston 	 */
1126915d1b71SMark Johnston 	shift = rbehind != NULL ? *rbehind : 0;
1127915d1b71SMark Johnston 	if (shift != 0) {
1128915d1b71SMark Johnston 		for (i = 1; i <= shift; i++) {
1129915d1b71SMark Johnston 			p = vm_page_alloc(object, m[0]->pindex - i,
11307667839aSAlan Cox 			    VM_ALLOC_NORMAL);
1131915d1b71SMark Johnston 			if (p == NULL) {
1132915d1b71SMark Johnston 				/* Shift allocated pages to the left. */
1133915d1b71SMark Johnston 				for (j = 0; j < i - 1; j++)
1134915d1b71SMark Johnston 					bp->b_pages[j] =
1135915d1b71SMark Johnston 					    bp->b_pages[j + shift - i + 1];
1136915d1b71SMark Johnston 				break;
1137915d1b71SMark Johnston 			}
1138915d1b71SMark Johnston 			bp->b_pages[shift - i] = p;
1139915d1b71SMark Johnston 		}
1140915d1b71SMark Johnston 		shift = i - 1;
1141915d1b71SMark Johnston 		*rbehind = shift;
1142915d1b71SMark Johnston 	}
1143915d1b71SMark Johnston 	for (i = 0; i < reqcount; i++)
1144915d1b71SMark Johnston 		bp->b_pages[i + shift] = m[i];
1145915d1b71SMark Johnston 	if (rahead != NULL) {
1146915d1b71SMark Johnston 		for (i = 0; i < *rahead; i++) {
1147915d1b71SMark Johnston 			p = vm_page_alloc(object,
11487667839aSAlan Cox 			    m[reqcount - 1]->pindex + i + 1, VM_ALLOC_NORMAL);
1149915d1b71SMark Johnston 			if (p == NULL)
1150915d1b71SMark Johnston 				break;
1151915d1b71SMark Johnston 			bp->b_pages[shift + reqcount + i] = p;
1152915d1b71SMark Johnston 		}
1153915d1b71SMark Johnston 		*rahead = i;
1154915d1b71SMark Johnston 	}
1155915d1b71SMark Johnston 	if (rbehind != NULL)
1156915d1b71SMark Johnston 		count += *rbehind;
1157915d1b71SMark Johnston 	if (rahead != NULL)
1158915d1b71SMark Johnston 		count += *rahead;
1159915d1b71SMark Johnston 
1160915d1b71SMark Johnston 	vm_object_pip_add(object, count);
1161915d1b71SMark Johnston 
1162915d1b71SMark Johnston 	for (i = 0; i < count; i++)
1163915d1b71SMark Johnston 		bp->b_pages[i]->oflags |= VPO_SWAPINPROG;
1164915d1b71SMark Johnston 
1165915d1b71SMark Johnston 	pindex = bp->b_pages[0]->pindex;
1166915d1b71SMark Johnston 	blk = swp_pager_meta_ctl(object, pindex, 0);
1167915d1b71SMark Johnston 	KASSERT(blk != SWAPBLK_NONE,
1168915d1b71SMark Johnston 	    ("no swap blocking containing %p(%jx)", object, (uintmax_t)pindex));
1169915d1b71SMark Johnston 
1170915d1b71SMark Johnston 	VM_OBJECT_WUNLOCK(object);
1171915d1b71SMark Johnston 
1172915d1b71SMark Johnston 	bp->b_flags |= B_PAGING;
117321144e3bSPoul-Henning Kamp 	bp->b_iocmd = BIO_READ;
11741c7c3c6aSMatthew Dillon 	bp->b_iodone = swp_pager_async_iodone;
1175fdcc1cc0SJohn Baldwin 	bp->b_rcred = crhold(thread0.td_ucred);
1176fdcc1cc0SJohn Baldwin 	bp->b_wcred = crhold(thread0.td_ucred);
1177b0cd2017SGleb Smirnoff 	bp->b_blkno = blk;
1178b0cd2017SGleb Smirnoff 	bp->b_bcount = PAGE_SIZE * count;
1179b0cd2017SGleb Smirnoff 	bp->b_bufsize = PAGE_SIZE * count;
1180b0cd2017SGleb Smirnoff 	bp->b_npages = count;
1181915d1b71SMark Johnston 	bp->b_pgbefore = rbehind != NULL ? *rbehind : 0;
1182915d1b71SMark Johnston 	bp->b_pgafter = rahead != NULL ? *rahead : 0;
118326f9a767SRodney W. Grimes 
118483c9dea1SGleb Smirnoff 	VM_CNT_INC(v_swapin);
118583c9dea1SGleb Smirnoff 	VM_CNT_ADD(v_swappgsin, count);
11861c7c3c6aSMatthew Dillon 
11871c7c3c6aSMatthew Dillon 	/*
11881c7c3c6aSMatthew Dillon 	 * perform the I/O.  NOTE!!!  bp cannot be considered valid after
11891c7c3c6aSMatthew Dillon 	 * this point because we automatically release it on completion.
11901c7c3c6aSMatthew Dillon 	 * Instead, we look at the one page we are interested in which we
11911c7c3c6aSMatthew Dillon 	 * still hold a lock on even through the I/O completion.
11921c7c3c6aSMatthew Dillon 	 *
11931c7c3c6aSMatthew Dillon 	 * The other pages in our m[] array are also released on completion,
11941c7c3c6aSMatthew Dillon 	 * so we cannot assume they are valid anymore either.
11951c7c3c6aSMatthew Dillon 	 *
1196c37a77eeSPoul-Henning Kamp 	 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
11971c7c3c6aSMatthew Dillon 	 */
1198b890cb2cSPeter Wemm 	BUF_KERNPROC(bp);
11994b03903aSPoul-Henning Kamp 	swp_pager_strategy(bp);
120026f9a767SRodney W. Grimes 
120126f9a767SRodney W. Grimes 	/*
1202915d1b71SMark Johnston 	 * Wait for the pages we want to complete.  VPO_SWAPINPROG is always
12031c7c3c6aSMatthew Dillon 	 * cleared on completion.  If an I/O error occurs, SWAPBLK_NONE
1204915d1b71SMark Johnston 	 * is set in the metadata for each page in the request.
120526f9a767SRodney W. Grimes 	 */
120689f6b863SAttilio Rao 	VM_OBJECT_WLOCK(object);
1207b0cd2017SGleb Smirnoff 	while ((m[0]->oflags & VPO_SWAPINPROG) != 0) {
1208b0cd2017SGleb Smirnoff 		m[0]->oflags |= VPO_SWAPSLEEP;
120983c9dea1SGleb Smirnoff 		VM_CNT_INC(v_intrans);
1210c7aebda8SAttilio Rao 		if (VM_OBJECT_SLEEP(object, &object->paging_in_progress, PSWP,
1211c7aebda8SAttilio Rao 		    "swread", hz * 20)) {
12129bd86a98SBruce M Simpson 			printf(
1213c5690651SPoul-Henning Kamp "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n",
1214c5690651SPoul-Henning Kamp 			    bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount);
12151c7c3c6aSMatthew Dillon 		}
12161b119d9dSDavid Greenman 	}
121726f9a767SRodney W. Grimes 
121826f9a767SRodney W. Grimes 	/*
1219b0cd2017SGleb Smirnoff 	 * If we had an unrecoverable read error pages will not be valid.
122026f9a767SRodney W. Grimes 	 */
1221915d1b71SMark Johnston 	for (i = 0; i < reqcount; i++)
1222b0cd2017SGleb Smirnoff 		if (m[i]->valid != VM_PAGE_BITS_ALL)
12231c7c3c6aSMatthew Dillon 			return (VM_PAGER_ERROR);
1224b0cd2017SGleb Smirnoff 
12251c7c3c6aSMatthew Dillon 	return (VM_PAGER_OK);
12261c7c3c6aSMatthew Dillon 
12271c7c3c6aSMatthew Dillon 	/*
12281c7c3c6aSMatthew Dillon 	 * A final note: in a low swap situation, we cannot deallocate swap
12291c7c3c6aSMatthew Dillon 	 * and mark a page dirty here because the caller is likely to mark
12301c7c3c6aSMatthew Dillon 	 * the page clean when we return, causing the page to possibly revert
12311c7c3c6aSMatthew Dillon 	 * to all-zero's later.
12321c7c3c6aSMatthew Dillon 	 */
1233df8bae1dSRodney W. Grimes }
1234df8bae1dSRodney W. Grimes 
12351c7c3c6aSMatthew Dillon /*
123690effb23SGleb Smirnoff  * 	swap_pager_getpages_async():
123790effb23SGleb Smirnoff  *
123890effb23SGleb Smirnoff  *	Right now this is emulation of asynchronous operation on top of
123990effb23SGleb Smirnoff  *	swap_pager_getpages().
124090effb23SGleb Smirnoff  */
124190effb23SGleb Smirnoff static int
124290effb23SGleb Smirnoff swap_pager_getpages_async(vm_object_t object, vm_page_t *m, int count,
1243b0cd2017SGleb Smirnoff     int *rbehind, int *rahead, pgo_getpages_iodone_t iodone, void *arg)
124490effb23SGleb Smirnoff {
124590effb23SGleb Smirnoff 	int r, error;
124690effb23SGleb Smirnoff 
1247b0cd2017SGleb Smirnoff 	r = swap_pager_getpages(object, m, count, rbehind, rahead);
124890effb23SGleb Smirnoff 	VM_OBJECT_WUNLOCK(object);
124990effb23SGleb Smirnoff 	switch (r) {
125090effb23SGleb Smirnoff 	case VM_PAGER_OK:
125190effb23SGleb Smirnoff 		error = 0;
125290effb23SGleb Smirnoff 		break;
125390effb23SGleb Smirnoff 	case VM_PAGER_ERROR:
125490effb23SGleb Smirnoff 		error = EIO;
125590effb23SGleb Smirnoff 		break;
125690effb23SGleb Smirnoff 	case VM_PAGER_FAIL:
125790effb23SGleb Smirnoff 		error = EINVAL;
125890effb23SGleb Smirnoff 		break;
125990effb23SGleb Smirnoff 	default:
1260d9328101SGleb Smirnoff 		panic("unhandled swap_pager_getpages() error %d", r);
126190effb23SGleb Smirnoff 	}
126290effb23SGleb Smirnoff 	(iodone)(arg, m, count, error);
126390effb23SGleb Smirnoff 	VM_OBJECT_WLOCK(object);
126490effb23SGleb Smirnoff 
126590effb23SGleb Smirnoff 	return (r);
126690effb23SGleb Smirnoff }
126790effb23SGleb Smirnoff 
126890effb23SGleb Smirnoff /*
12691c7c3c6aSMatthew Dillon  *	swap_pager_putpages:
12701c7c3c6aSMatthew Dillon  *
12711c7c3c6aSMatthew Dillon  *	Assign swap (if necessary) and initiate I/O on the specified pages.
12721c7c3c6aSMatthew Dillon  *
12731c7c3c6aSMatthew Dillon  *	We support both OBJT_DEFAULT and OBJT_SWAP objects.  DEFAULT objects
12741c7c3c6aSMatthew Dillon  *	are automatically converted to SWAP objects.
12751c7c3c6aSMatthew Dillon  *
1276ea3aecf5SPeter Wemm  *	In a low memory situation we may block in VOP_STRATEGY(), but the new
12771c7c3c6aSMatthew Dillon  *	vm_page reservation system coupled with properly written VFS devices
12781c7c3c6aSMatthew Dillon  *	should ensure that no low-memory deadlock occurs.  This is an area
12791c7c3c6aSMatthew Dillon  *	which needs work.
12801c7c3c6aSMatthew Dillon  *
12811c7c3c6aSMatthew Dillon  *	The parent has N vm_object_pip_add() references prior to
12821c7c3c6aSMatthew Dillon  *	calling us and will remove references for rtvals[] that are
12831c7c3c6aSMatthew Dillon  *	not set to VM_PAGER_PEND.  We need to remove the rest on I/O
12841c7c3c6aSMatthew Dillon  *	completion.
12851c7c3c6aSMatthew Dillon  *
12861c7c3c6aSMatthew Dillon  *	The parent has soft-busy'd the pages it passes us and will unbusy
12871c7c3c6aSMatthew Dillon  *	those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
12881c7c3c6aSMatthew Dillon  *	We need to unbusy the rest on I/O completion.
12891c7c3c6aSMatthew Dillon  */
1290d635a37fSWarner Losh static void
12912f249180SPoul-Henning Kamp swap_pager_putpages(vm_object_t object, vm_page_t *m, int count,
1292e065e87cSKonstantin Belousov     int flags, int *rtvals)
1293df8bae1dSRodney W. Grimes {
1294e065e87cSKonstantin Belousov 	int i, n;
1295e065e87cSKonstantin Belousov 	boolean_t sync;
1296df8bae1dSRodney W. Grimes 
12971c7c3c6aSMatthew Dillon 	if (count && m[0]->object != object) {
12987036145bSMaxim Konovalov 		panic("swap_pager_putpages: object mismatch %p/%p",
12991c7c3c6aSMatthew Dillon 		    object,
13001c7c3c6aSMatthew Dillon 		    m[0]->object
13011c7c3c6aSMatthew Dillon 		);
13021c7c3c6aSMatthew Dillon 	}
1303ee3dc7d7SAlan Cox 
13041c7c3c6aSMatthew Dillon 	/*
13051c7c3c6aSMatthew Dillon 	 * Step 1
13061c7c3c6aSMatthew Dillon 	 *
13071c7c3c6aSMatthew Dillon 	 * Turn object into OBJT_SWAP
13081c7c3c6aSMatthew Dillon 	 * check for bogus sysops
13091c7c3c6aSMatthew Dillon 	 * force sync if not pageout process
13101c7c3c6aSMatthew Dillon 	 */
13114dcc5c2dSMatthew Dillon 	if (object->type != OBJT_SWAP)
13124dcc5c2dSMatthew Dillon 		swp_pager_meta_build(object, 0, SWAPBLK_NONE);
131389f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(object);
1314e47ed70bSJohn Dyson 
1315e065e87cSKonstantin Belousov 	n = 0;
1316e47ed70bSJohn Dyson 	if (curproc != pageproc)
1317e47ed70bSJohn Dyson 		sync = TRUE;
1318e065e87cSKonstantin Belousov 	else
1319e065e87cSKonstantin Belousov 		sync = (flags & VM_PAGER_PUT_SYNC) != 0;
132026f9a767SRodney W. Grimes 
13211c7c3c6aSMatthew Dillon 	/*
13221c7c3c6aSMatthew Dillon 	 * Step 2
13231c7c3c6aSMatthew Dillon 	 *
13241c7c3c6aSMatthew Dillon 	 * Assign swap blocks and issue I/O.  We reallocate swap on the fly.
13251c7c3c6aSMatthew Dillon 	 * The page is left dirty until the pageout operation completes
13261c7c3c6aSMatthew Dillon 	 * successfully.
13271c7c3c6aSMatthew Dillon 	 */
13281c7c3c6aSMatthew Dillon 	for (i = 0; i < count; i += n) {
13291c7c3c6aSMatthew Dillon 		int j;
13301c7c3c6aSMatthew Dillon 		struct buf *bp;
1331a316d390SJohn Dyson 		daddr_t blk;
133226f9a767SRodney W. Grimes 
1333df8bae1dSRodney W. Grimes 		/*
13341c7c3c6aSMatthew Dillon 		 * Maximum I/O size is limited by a number of factors.
1335df8bae1dSRodney W. Grimes 		 */
13361c7c3c6aSMatthew Dillon 		n = min(BLIST_MAX_ALLOC, count - i);
1337327f4e83SMatthew Dillon 		n = min(n, nsw_cluster_max);
13381c7c3c6aSMatthew Dillon 
133926f9a767SRodney W. Grimes 		/*
13401c7c3c6aSMatthew Dillon 		 * Get biggest block of swap we can.  If we fail, fall
13411c7c3c6aSMatthew Dillon 		 * back and try to allocate a smaller block.  Don't go
13421c7c3c6aSMatthew Dillon 		 * overboard trying to allocate space if it would overly
13431c7c3c6aSMatthew Dillon 		 * fragment swap.
134426f9a767SRodney W. Grimes 		 */
13451c7c3c6aSMatthew Dillon 		while (
13461c7c3c6aSMatthew Dillon 		    (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE &&
13471c7c3c6aSMatthew Dillon 		    n > 4
13481c7c3c6aSMatthew Dillon 		) {
13491c7c3c6aSMatthew Dillon 			n >>= 1;
135026f9a767SRodney W. Grimes 		}
13511c7c3c6aSMatthew Dillon 		if (blk == SWAPBLK_NONE) {
13524dcc5c2dSMatthew Dillon 			for (j = 0; j < n; ++j)
13531c7c3c6aSMatthew Dillon 				rtvals[i+j] = VM_PAGER_FAIL;
13541c7c3c6aSMatthew Dillon 			continue;
135526f9a767SRodney W. Grimes 		}
135626f9a767SRodney W. Grimes 
135726f9a767SRodney W. Grimes 		/*
13581c7c3c6aSMatthew Dillon 		 * All I/O parameters have been satisfied, build the I/O
13591c7c3c6aSMatthew Dillon 		 * request and assign the swap space.
136026f9a767SRodney W. Grimes 		 */
1361327f4e83SMatthew Dillon 		if (sync == TRUE) {
1362327f4e83SMatthew Dillon 			bp = getpbuf(&nsw_wcount_sync);
1363327f4e83SMatthew Dillon 		} else {
1364327f4e83SMatthew Dillon 			bp = getpbuf(&nsw_wcount_async);
136521144e3bSPoul-Henning Kamp 			bp->b_flags = B_ASYNC;
1366327f4e83SMatthew Dillon 		}
13675e04322aSPoul-Henning Kamp 		bp->b_flags |= B_PAGING;
1368912e4ae9SPoul-Henning Kamp 		bp->b_iocmd = BIO_WRITE;
136926f9a767SRodney W. Grimes 
1370fdcc1cc0SJohn Baldwin 		bp->b_rcred = crhold(thread0.td_ucred);
1371fdcc1cc0SJohn Baldwin 		bp->b_wcred = crhold(thread0.td_ucred);
13721c7c3c6aSMatthew Dillon 		bp->b_bcount = PAGE_SIZE * n;
13731c7c3c6aSMatthew Dillon 		bp->b_bufsize = PAGE_SIZE * n;
13741c7c3c6aSMatthew Dillon 		bp->b_blkno = blk;
1375e47ed70bSJohn Dyson 
137689f6b863SAttilio Rao 		VM_OBJECT_WLOCK(object);
13771c7c3c6aSMatthew Dillon 		for (j = 0; j < n; ++j) {
13781c7c3c6aSMatthew Dillon 			vm_page_t mreq = m[i+j];
13791c7c3c6aSMatthew Dillon 
13801c7c3c6aSMatthew Dillon 			swp_pager_meta_build(
13811c7c3c6aSMatthew Dillon 			    mreq->object,
13821c7c3c6aSMatthew Dillon 			    mreq->pindex,
13834dcc5c2dSMatthew Dillon 			    blk + j
13841c7c3c6aSMatthew Dillon 			);
13857dbf82dcSMatthew Dillon 			vm_page_dirty(mreq);
13865786be7cSAlan Cox 			mreq->oflags |= VPO_SWAPINPROG;
13871c7c3c6aSMatthew Dillon 			bp->b_pages[j] = mreq;
13881c7c3c6aSMatthew Dillon 		}
138989f6b863SAttilio Rao 		VM_OBJECT_WUNLOCK(object);
13901c7c3c6aSMatthew Dillon 		bp->b_npages = n;
1391a5296b05SJulian Elischer 		/*
1392a5296b05SJulian Elischer 		 * Must set dirty range for NFS to work.
1393a5296b05SJulian Elischer 		 */
1394a5296b05SJulian Elischer 		bp->b_dirtyoff = 0;
1395a5296b05SJulian Elischer 		bp->b_dirtyend = bp->b_bcount;
13961c7c3c6aSMatthew Dillon 
139783c9dea1SGleb Smirnoff 		VM_CNT_INC(v_swapout);
139883c9dea1SGleb Smirnoff 		VM_CNT_ADD(v_swappgsout, bp->b_npages);
139926f9a767SRodney W. Grimes 
140026f9a767SRodney W. Grimes 		/*
140177923df2SAlan Cox 		 * We unconditionally set rtvals[] to VM_PAGER_PEND so that we
140277923df2SAlan Cox 		 * can call the async completion routine at the end of a
140377923df2SAlan Cox 		 * synchronous I/O operation.  Otherwise, our caller would
140477923df2SAlan Cox 		 * perform duplicate unbusy and wakeup operations on the page
140577923df2SAlan Cox 		 * and object, respectively.
140677923df2SAlan Cox 		 */
140777923df2SAlan Cox 		for (j = 0; j < n; j++)
140877923df2SAlan Cox 			rtvals[i + j] = VM_PAGER_PEND;
140977923df2SAlan Cox 
141077923df2SAlan Cox 		/*
14111c7c3c6aSMatthew Dillon 		 * asynchronous
14121c7c3c6aSMatthew Dillon 		 *
1413c37a77eeSPoul-Henning Kamp 		 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
141426f9a767SRodney W. Grimes 		 */
14151c7c3c6aSMatthew Dillon 		if (sync == FALSE) {
14161c7c3c6aSMatthew Dillon 			bp->b_iodone = swp_pager_async_iodone;
141767812eacSKirk McKusick 			BUF_KERNPROC(bp);
14184b03903aSPoul-Henning Kamp 			swp_pager_strategy(bp);
14191c7c3c6aSMatthew Dillon 			continue;
142026f9a767SRodney W. Grimes 		}
1421e47ed70bSJohn Dyson 
142226f9a767SRodney W. Grimes 		/*
14231c7c3c6aSMatthew Dillon 		 * synchronous
14241c7c3c6aSMatthew Dillon 		 *
1425c37a77eeSPoul-Henning Kamp 		 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
14261c7c3c6aSMatthew Dillon 		 */
14272c840b1fSAlan Cox 		bp->b_iodone = bdone;
14284b03903aSPoul-Henning Kamp 		swp_pager_strategy(bp);
14291c7c3c6aSMatthew Dillon 
14301c7c3c6aSMatthew Dillon 		/*
143177923df2SAlan Cox 		 * Wait for the sync I/O to complete.
143226f9a767SRodney W. Grimes 		 */
14332c840b1fSAlan Cox 		bwait(bp, PVM, "swwrt");
143477923df2SAlan Cox 
14351c7c3c6aSMatthew Dillon 		/*
14361c7c3c6aSMatthew Dillon 		 * Now that we are through with the bp, we can call the
14371c7c3c6aSMatthew Dillon 		 * normal async completion, which frees everything up.
14381c7c3c6aSMatthew Dillon 		 */
14391c7c3c6aSMatthew Dillon 		swp_pager_async_iodone(bp);
14401c7c3c6aSMatthew Dillon 	}
144189f6b863SAttilio Rao 	VM_OBJECT_WLOCK(object);
14421c7c3c6aSMatthew Dillon }
14431c7c3c6aSMatthew Dillon 
14441c7c3c6aSMatthew Dillon /*
14451c7c3c6aSMatthew Dillon  *	swp_pager_async_iodone:
14461c7c3c6aSMatthew Dillon  *
14471c7c3c6aSMatthew Dillon  *	Completion routine for asynchronous reads and writes from/to swap.
14481c7c3c6aSMatthew Dillon  *	Also called manually by synchronous code to finish up a bp.
14491c7c3c6aSMatthew Dillon  *
145015523cf7SKonstantin Belousov  *	This routine may not sleep.
14511c7c3c6aSMatthew Dillon  */
14521c7c3c6aSMatthew Dillon static void
14532f249180SPoul-Henning Kamp swp_pager_async_iodone(struct buf *bp)
14541c7c3c6aSMatthew Dillon {
14551c7c3c6aSMatthew Dillon 	int i;
14561c7c3c6aSMatthew Dillon 	vm_object_t object = NULL;
14571c7c3c6aSMatthew Dillon 
14581c7c3c6aSMatthew Dillon 	/*
14591c7c3c6aSMatthew Dillon 	 * report error
14601c7c3c6aSMatthew Dillon 	 */
1461c244d2deSPoul-Henning Kamp 	if (bp->b_ioflags & BIO_ERROR) {
14621c7c3c6aSMatthew Dillon 		printf(
14631c7c3c6aSMatthew Dillon 		    "swap_pager: I/O error - %s failed; blkno %ld,"
14641c7c3c6aSMatthew Dillon 			"size %ld, error %d\n",
146521144e3bSPoul-Henning Kamp 		    ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
14661c7c3c6aSMatthew Dillon 		    (long)bp->b_blkno,
14671c7c3c6aSMatthew Dillon 		    (long)bp->b_bcount,
14681c7c3c6aSMatthew Dillon 		    bp->b_error
14691c7c3c6aSMatthew Dillon 		);
14701c7c3c6aSMatthew Dillon 	}
14711c7c3c6aSMatthew Dillon 
14721c7c3c6aSMatthew Dillon 	/*
147326f9a767SRodney W. Grimes 	 * remove the mapping for kernel virtual
147426f9a767SRodney W. Grimes 	 */
1475fade8dd7SJeff Roberson 	if (buf_mapped(bp))
14761c7c3c6aSMatthew Dillon 		pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
1477fade8dd7SJeff Roberson 	else
1478fade8dd7SJeff Roberson 		bp->b_data = bp->b_kvabase;
147926f9a767SRodney W. Grimes 
148033a609ecSAlan Cox 	if (bp->b_npages) {
148133a609ecSAlan Cox 		object = bp->b_pages[0]->object;
148289f6b863SAttilio Rao 		VM_OBJECT_WLOCK(object);
148333a609ecSAlan Cox 	}
14842965a453SKip Macy 
148526f9a767SRodney W. Grimes 	/*
14861c7c3c6aSMatthew Dillon 	 * cleanup pages.  If an error occurs writing to swap, we are in
14871c7c3c6aSMatthew Dillon 	 * very serious trouble.  If it happens to be a disk error, though,
14881c7c3c6aSMatthew Dillon 	 * we may be able to recover by reassigning the swap later on.  So
14891c7c3c6aSMatthew Dillon 	 * in this case we remove the m->swapblk assignment for the page
14901c7c3c6aSMatthew Dillon 	 * but do not free it in the rlist.  The errornous block(s) are thus
14911c7c3c6aSMatthew Dillon 	 * never reallocated as swap.  Redirty the page and continue.
149226f9a767SRodney W. Grimes 	 */
14931c7c3c6aSMatthew Dillon 	for (i = 0; i < bp->b_npages; ++i) {
14941c7c3c6aSMatthew Dillon 		vm_page_t m = bp->b_pages[i];
1495e47ed70bSJohn Dyson 
14965786be7cSAlan Cox 		m->oflags &= ~VPO_SWAPINPROG;
1497c7aebda8SAttilio Rao 		if (m->oflags & VPO_SWAPSLEEP) {
1498c7aebda8SAttilio Rao 			m->oflags &= ~VPO_SWAPSLEEP;
1499c7aebda8SAttilio Rao 			wakeup(&object->paging_in_progress);
1500c7aebda8SAttilio Rao 		}
1501e47ed70bSJohn Dyson 
1502c244d2deSPoul-Henning Kamp 		if (bp->b_ioflags & BIO_ERROR) {
1503ffc82b0aSJohn Dyson 			/*
15041c7c3c6aSMatthew Dillon 			 * If an error occurs I'd love to throw the swapblk
15051c7c3c6aSMatthew Dillon 			 * away without freeing it back to swapspace, so it
15061c7c3c6aSMatthew Dillon 			 * can never be used again.  But I can't from an
15071c7c3c6aSMatthew Dillon 			 * interrupt.
1508ffc82b0aSJohn Dyson 			 */
150921144e3bSPoul-Henning Kamp 			if (bp->b_iocmd == BIO_READ) {
15101c7c3c6aSMatthew Dillon 				/*
15111c7c3c6aSMatthew Dillon 				 * NOTE: for reads, m->dirty will probably
1512956f3135SPhilippe Charnier 				 * be overridden by the original caller of
15131c7c3c6aSMatthew Dillon 				 * getpages so don't play cute tricks here.
15141c7c3c6aSMatthew Dillon 				 */
15151c7c3c6aSMatthew Dillon 				m->valid = 0;
15161c7c3c6aSMatthew Dillon 			} else {
15171c7c3c6aSMatthew Dillon 				/*
15181c7c3c6aSMatthew Dillon 				 * If a write error occurs, reactivate page
15191c7c3c6aSMatthew Dillon 				 * so it doesn't clog the inactive list,
15201c7c3c6aSMatthew Dillon 				 * then finish the I/O.
15211c7c3c6aSMatthew Dillon 				 */
15227dbf82dcSMatthew Dillon 				vm_page_dirty(m);
15233c4a2440SAlan Cox 				vm_page_lock(m);
15241c7c3c6aSMatthew Dillon 				vm_page_activate(m);
15253c4a2440SAlan Cox 				vm_page_unlock(m);
1526c7aebda8SAttilio Rao 				vm_page_sunbusy(m);
15271c7c3c6aSMatthew Dillon 			}
152821144e3bSPoul-Henning Kamp 		} else if (bp->b_iocmd == BIO_READ) {
15291c7c3c6aSMatthew Dillon 			/*
15301c7c3c6aSMatthew Dillon 			 * NOTE: for reads, m->dirty will probably be
1531956f3135SPhilippe Charnier 			 * overridden by the original caller of getpages so
15321c7c3c6aSMatthew Dillon 			 * we cannot set them in order to free the underlying
15331c7c3c6aSMatthew Dillon 			 * swap in a low-swap situation.  I don't think we'd
15341c7c3c6aSMatthew Dillon 			 * want to do that anyway, but it was an optimization
15351c7c3c6aSMatthew Dillon 			 * that existed in the old swapper for a time before
15361c7c3c6aSMatthew Dillon 			 * it got ripped out due to precisely this problem.
15371c7c3c6aSMatthew Dillon 			 */
1538016a3c93SAlan Cox 			KASSERT(!pmap_page_is_mapped(m),
1539016a3c93SAlan Cox 			    ("swp_pager_async_iodone: page %p is mapped", m));
1540016a3c93SAlan Cox 			KASSERT(m->dirty == 0,
1541016a3c93SAlan Cox 			    ("swp_pager_async_iodone: page %p is dirty", m));
1542915d1b71SMark Johnston 
1543b0cd2017SGleb Smirnoff 			m->valid = VM_PAGE_BITS_ALL;
1544915d1b71SMark Johnston 			if (i < bp->b_pgbefore ||
1545915d1b71SMark Johnston 			    i >= bp->b_npages - bp->b_pgafter)
1546915d1b71SMark Johnston 				vm_page_readahead_finish(m);
15471c7c3c6aSMatthew Dillon 		} else {
15481c7c3c6aSMatthew Dillon 			/*
1549016a3c93SAlan Cox 			 * For write success, clear the dirty
15501c7c3c6aSMatthew Dillon 			 * status, then finish the I/O ( which decrements the
15511c7c3c6aSMatthew Dillon 			 * busy count and possibly wakes waiter's up ).
1552ebcddc72SAlan Cox 			 * A page is only written to swap after a period of
1553ebcddc72SAlan Cox 			 * inactivity.  Therefore, we do not expect it to be
1554ebcddc72SAlan Cox 			 * reused.
15551c7c3c6aSMatthew Dillon 			 */
15566031c68dSAlan Cox 			KASSERT(!pmap_page_is_write_mapped(m),
1557016a3c93SAlan Cox 			    ("swp_pager_async_iodone: page %p is not write"
1558016a3c93SAlan Cox 			    " protected", m));
1559c52e7044SAlan Cox 			vm_page_undirty(m);
15603c4a2440SAlan Cox 			vm_page_lock(m);
1561ebcddc72SAlan Cox 			vm_page_deactivate_noreuse(m);
15622965a453SKip Macy 			vm_page_unlock(m);
1563ebcddc72SAlan Cox 			vm_page_sunbusy(m);
15643c4a2440SAlan Cox 		}
15653c4a2440SAlan Cox 	}
156626f9a767SRodney W. Grimes 
15671c7c3c6aSMatthew Dillon 	/*
15681c7c3c6aSMatthew Dillon 	 * adjust pip.  NOTE: the original parent may still have its own
15691c7c3c6aSMatthew Dillon 	 * pip refs on the object.
15701c7c3c6aSMatthew Dillon 	 */
15710d420ad3SAlan Cox 	if (object != NULL) {
15721c7c3c6aSMatthew Dillon 		vm_object_pip_wakeupn(object, bp->b_npages);
157389f6b863SAttilio Rao 		VM_OBJECT_WUNLOCK(object);
15740d420ad3SAlan Cox 	}
157526f9a767SRodney W. Grimes 
15761c7c3c6aSMatthew Dillon 	/*
1577100650deSOlivier Houchard 	 * swapdev_strategy() manually sets b_vp and b_bufobj before calling
1578100650deSOlivier Houchard 	 * bstrategy(). Set them back to NULL now we're done with it, or we'll
1579100650deSOlivier Houchard 	 * trigger a KASSERT in relpbuf().
1580100650deSOlivier Houchard 	 */
1581100650deSOlivier Houchard 	if (bp->b_vp) {
1582100650deSOlivier Houchard 		    bp->b_vp = NULL;
1583100650deSOlivier Houchard 		    bp->b_bufobj = NULL;
1584100650deSOlivier Houchard 	}
1585100650deSOlivier Houchard 	/*
15861c7c3c6aSMatthew Dillon 	 * release the physical I/O buffer
15871c7c3c6aSMatthew Dillon 	 */
1588327f4e83SMatthew Dillon 	relpbuf(
1589327f4e83SMatthew Dillon 	    bp,
159021144e3bSPoul-Henning Kamp 	    ((bp->b_iocmd == BIO_READ) ? &nsw_rcount :
1591327f4e83SMatthew Dillon 		((bp->b_flags & B_ASYNC) ?
1592327f4e83SMatthew Dillon 		    &nsw_wcount_async :
1593327f4e83SMatthew Dillon 		    &nsw_wcount_sync
1594327f4e83SMatthew Dillon 		)
1595327f4e83SMatthew Dillon 	    )
1596327f4e83SMatthew Dillon 	);
159726f9a767SRodney W. Grimes }
15981c7c3c6aSMatthew Dillon 
159992da00bbSMatthew Dillon /*
160092da00bbSMatthew Dillon  *	swap_pager_isswapped:
160192da00bbSMatthew Dillon  *
160292da00bbSMatthew Dillon  *	Return 1 if at least one page in the given object is paged
160392da00bbSMatthew Dillon  *	out to the given swap device.
160492da00bbSMatthew Dillon  *
160515523cf7SKonstantin Belousov  *	This routine may not sleep.
160692da00bbSMatthew Dillon  */
16078f60c087SPoul-Henning Kamp int
16088f60c087SPoul-Henning Kamp swap_pager_isswapped(vm_object_t object, struct swdevt *sp)
16098f60c087SPoul-Henning Kamp {
161092da00bbSMatthew Dillon 	daddr_t index = 0;
161192da00bbSMatthew Dillon 	int bcount;
161292da00bbSMatthew Dillon 	int i;
161392da00bbSMatthew Dillon 
161489f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
1615f6bcadc4SDavid Schultz 	if (object->type != OBJT_SWAP)
1616f6bcadc4SDavid Schultz 		return (0);
1617f6bcadc4SDavid Schultz 
1618b3fed13eSDavid Schultz 	mtx_lock(&swhash_mtx);
161992da00bbSMatthew Dillon 	for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) {
162092da00bbSMatthew Dillon 		struct swblock *swap;
162192da00bbSMatthew Dillon 
162292da00bbSMatthew Dillon 		if ((swap = *swp_pager_hash(object, index)) != NULL) {
162392da00bbSMatthew Dillon 			for (i = 0; i < SWAP_META_PAGES; ++i) {
1624b3fed13eSDavid Schultz 				if (swp_pager_isondev(swap->swb_pages[i], sp)) {
16257827d9b0SAlan Cox 					mtx_unlock(&swhash_mtx);
1626b3fed13eSDavid Schultz 					return (1);
162792da00bbSMatthew Dillon 				}
162892da00bbSMatthew Dillon 			}
16297827d9b0SAlan Cox 		}
163092da00bbSMatthew Dillon 		index += SWAP_META_PAGES;
163192da00bbSMatthew Dillon 	}
1632b3fed13eSDavid Schultz 	mtx_unlock(&swhash_mtx);
1633b3fed13eSDavid Schultz 	return (0);
163492da00bbSMatthew Dillon }
163592da00bbSMatthew Dillon 
1636b1fd102eSMark Johnston int
1637b1fd102eSMark Johnston swap_pager_nswapdev(void)
1638b1fd102eSMark Johnston {
1639b1fd102eSMark Johnston 
1640b1fd102eSMark Johnston 	return (nswapdev);
1641b1fd102eSMark Johnston }
1642b1fd102eSMark Johnston 
164392da00bbSMatthew Dillon /*
164492da00bbSMatthew Dillon  * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in
164592da00bbSMatthew Dillon  *
1646ebcddc72SAlan Cox  *	This routine dissociates the page at the given index within an object
1647ebcddc72SAlan Cox  *	from its backing store, paging it in if it does not reside in memory.
1648ebcddc72SAlan Cox  *	If the page is paged in, it is marked dirty and placed in the laundry
1649ebcddc72SAlan Cox  *	queue.  The page is marked dirty because it no longer has backing
1650ebcddc72SAlan Cox  *	store.  It is placed in the laundry queue because it has not been
1651ebcddc72SAlan Cox  *	accessed recently.  Otherwise, it would already reside in memory.
1652ebcddc72SAlan Cox  *
1653ebcddc72SAlan Cox  *	We also attempt to swap in all other pages in the swap block.
1654ebcddc72SAlan Cox  *	However, we only guarantee that the one at the specified index is
165592da00bbSMatthew Dillon  *	paged in.
165692da00bbSMatthew Dillon  *
165792da00bbSMatthew Dillon  *	XXX - The code to page the whole block in doesn't work, so we
165892da00bbSMatthew Dillon  *	      revert to the one-by-one behavior for now.  Sigh.
165992da00bbSMatthew Dillon  */
166062a59e8fSWarner Losh static inline void
1661b3fed13eSDavid Schultz swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex)
166292da00bbSMatthew Dillon {
166392da00bbSMatthew Dillon 	vm_page_t m;
166492da00bbSMatthew Dillon 
166592da00bbSMatthew Dillon 	vm_object_pip_add(object, 1);
16665944de8eSKonstantin Belousov 	m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL);
166792da00bbSMatthew Dillon 	if (m->valid == VM_PAGE_BITS_ALL) {
16680d8243ccSAttilio Rao 		vm_object_pip_wakeup(object);
166992da00bbSMatthew Dillon 		vm_page_dirty(m);
1670d061cdd5SAlan Cox 		vm_page_lock(m);
1671d061cdd5SAlan Cox 		vm_page_activate(m);
16722965a453SKip Macy 		vm_page_unlock(m);
1673c7aebda8SAttilio Rao 		vm_page_xunbusy(m);
167492da00bbSMatthew Dillon 		vm_pager_page_unswapped(m);
167592da00bbSMatthew Dillon 		return;
167692da00bbSMatthew Dillon 	}
167792da00bbSMatthew Dillon 
1678b0cd2017SGleb Smirnoff 	if (swap_pager_getpages(object, &m, 1, NULL, NULL) != VM_PAGER_OK)
167992da00bbSMatthew Dillon 		panic("swap_pager_force_pagein: read from swap failed");/*XXX*/
16800d8243ccSAttilio Rao 	vm_object_pip_wakeup(object);
168192da00bbSMatthew Dillon 	vm_page_dirty(m);
1682db1f085eSAlan Cox 	vm_page_lock(m);
1683ebcddc72SAlan Cox 	vm_page_launder(m);
16842965a453SKip Macy 	vm_page_unlock(m);
1685c7aebda8SAttilio Rao 	vm_page_xunbusy(m);
168692da00bbSMatthew Dillon 	vm_pager_page_unswapped(m);
168792da00bbSMatthew Dillon }
168892da00bbSMatthew Dillon 
168992da00bbSMatthew Dillon /*
169092da00bbSMatthew Dillon  *	swap_pager_swapoff:
169192da00bbSMatthew Dillon  *
169292da00bbSMatthew Dillon  *	Page in all of the pages that have been paged out to the
169392da00bbSMatthew Dillon  *	given device.  The corresponding blocks in the bitmap must be
169492da00bbSMatthew Dillon  *	marked as allocated and the device must be flagged SW_CLOSING.
169592da00bbSMatthew Dillon  *	There may be no processes swapped out to the device.
169692da00bbSMatthew Dillon  *
169792da00bbSMatthew Dillon  *	This routine may block.
169892da00bbSMatthew Dillon  */
1699e9c0cc15SPoul-Henning Kamp static void
1700b3fed13eSDavid Schultz swap_pager_swapoff(struct swdevt *sp)
170192da00bbSMatthew Dillon {
170292da00bbSMatthew Dillon 	struct swblock *swap;
170398150664SKonstantin Belousov 	vm_object_t locked_obj, object;
170498150664SKonstantin Belousov 	vm_pindex_t pindex;
17058bc61209SDavid Schultz 	int i, j, retries;
170692da00bbSMatthew Dillon 
170704533e1eSKonstantin Belousov 	sx_assert(&swdev_syscall_lock, SA_XLOCKED);
170892da00bbSMatthew Dillon 
17098bc61209SDavid Schultz 	retries = 0;
171098150664SKonstantin Belousov 	locked_obj = NULL;
171192da00bbSMatthew Dillon full_rescan:
1712b3fed13eSDavid Schultz 	mtx_lock(&swhash_mtx);
171392da00bbSMatthew Dillon 	for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */
171492da00bbSMatthew Dillon restart:
1715b3fed13eSDavid Schultz 		for (swap = swhash[i]; swap != NULL; swap = swap->swb_hnext) {
171698150664SKonstantin Belousov 			object = swap->swb_object;
171798150664SKonstantin Belousov 			pindex = swap->swb_index;
171892da00bbSMatthew Dillon 			for (j = 0; j < SWAP_META_PAGES; ++j) {
171998150664SKonstantin Belousov 				if (!swp_pager_isondev(swap->swb_pages[j], sp))
172098150664SKonstantin Belousov 					continue;
172198150664SKonstantin Belousov 				if (locked_obj != object) {
172298150664SKonstantin Belousov 					if (locked_obj != NULL)
172398150664SKonstantin Belousov 						VM_OBJECT_WUNLOCK(locked_obj);
172498150664SKonstantin Belousov 					locked_obj = object;
172589f6b863SAttilio Rao 					if (!VM_OBJECT_TRYWLOCK(object)) {
1726b3fed13eSDavid Schultz 						mtx_unlock(&swhash_mtx);
172798150664SKonstantin Belousov 						/* Depends on type-stability. */
172898150664SKonstantin Belousov 						VM_OBJECT_WLOCK(object);
172998150664SKonstantin Belousov 						mtx_lock(&swhash_mtx);
173098150664SKonstantin Belousov 						goto restart;
173198150664SKonstantin Belousov 					}
173298150664SKonstantin Belousov 				}
173398150664SKonstantin Belousov 				MPASS(locked_obj == object);
173498150664SKonstantin Belousov 				mtx_unlock(&swhash_mtx);
173598150664SKonstantin Belousov 				swp_pager_force_pagein(object, pindex + j);
1736b3fed13eSDavid Schultz 				mtx_lock(&swhash_mtx);
173792da00bbSMatthew Dillon 				goto restart;
173892da00bbSMatthew Dillon 			}
1739b3fed13eSDavid Schultz 		}
1740b3fed13eSDavid Schultz 	}
1741d536c58fSAlan Cox 	mtx_unlock(&swhash_mtx);
174298150664SKonstantin Belousov 	if (locked_obj != NULL) {
174398150664SKonstantin Belousov 		VM_OBJECT_WUNLOCK(locked_obj);
174498150664SKonstantin Belousov 		locked_obj = NULL;
174598150664SKonstantin Belousov 	}
17468bc61209SDavid Schultz 	if (sp->sw_used) {
174792da00bbSMatthew Dillon 		/*
17488bc61209SDavid Schultz 		 * Objects may be locked or paging to the device being
17498bc61209SDavid Schultz 		 * removed, so we will miss their pages and need to
17508bc61209SDavid Schultz 		 * make another pass.  We have marked this device as
17518bc61209SDavid Schultz 		 * SW_CLOSING, so the activity should finish soon.
175292da00bbSMatthew Dillon 		 */
17538bc61209SDavid Schultz 		retries++;
17548bc61209SDavid Schultz 		if (retries > 100) {
17558bc61209SDavid Schultz 			panic("swapoff: failed to locate %d swap blocks",
17568bc61209SDavid Schultz 			    sp->sw_used);
17578bc61209SDavid Schultz 		}
17584d70511aSJohn Baldwin 		pause("swpoff", hz / 20);
175992da00bbSMatthew Dillon 		goto full_rescan;
176092da00bbSMatthew Dillon 	}
1761b1fd102eSMark Johnston 	EVENTHANDLER_INVOKE(swapoff, sp);
176292da00bbSMatthew Dillon }
176392da00bbSMatthew Dillon 
17641c7c3c6aSMatthew Dillon /************************************************************************
17651c7c3c6aSMatthew Dillon  *				SWAP META DATA 				*
17661c7c3c6aSMatthew Dillon  ************************************************************************
17671c7c3c6aSMatthew Dillon  *
17681c7c3c6aSMatthew Dillon  *	These routines manipulate the swap metadata stored in the
1769cec9f109SDavid E. O'Brien  *	OBJT_SWAP object.
17701c7c3c6aSMatthew Dillon  *
17714dcc5c2dSMatthew Dillon  *	Swap metadata is implemented with a global hash and not directly
17724dcc5c2dSMatthew Dillon  *	linked into the object.  Instead the object simply contains
17734dcc5c2dSMatthew Dillon  *	appropriate tracking counters.
17741c7c3c6aSMatthew Dillon  */
17751c7c3c6aSMatthew Dillon 
17761c7c3c6aSMatthew Dillon /*
17771c7c3c6aSMatthew Dillon  * SWP_PAGER_META_BUILD() -	add swap block to swap meta data for object
17781c7c3c6aSMatthew Dillon  *
17791c7c3c6aSMatthew Dillon  *	We first convert the object to a swap object if it is a default
17801c7c3c6aSMatthew Dillon  *	object.
17811c7c3c6aSMatthew Dillon  *
17821c7c3c6aSMatthew Dillon  *	The specified swapblk is added to the object's swap metadata.  If
17831c7c3c6aSMatthew Dillon  *	the swapblk is not valid, it is freed instead.  Any previously
17841c7c3c6aSMatthew Dillon  *	assigned swapblk is freed.
17851c7c3c6aSMatthew Dillon  */
17861c7c3c6aSMatthew Dillon static void
17872f249180SPoul-Henning Kamp swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
17882f249180SPoul-Henning Kamp {
17893ff863f1SDag-Erling Smørgrav 	static volatile int exhausted;
17901c7c3c6aSMatthew Dillon 	struct swblock *swap;
17911c7c3c6aSMatthew Dillon 	struct swblock **pswap;
179223f09d50SIan Dowse 	int idx;
17931c7c3c6aSMatthew Dillon 
179489f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
17951c7c3c6aSMatthew Dillon 	/*
17961c7c3c6aSMatthew Dillon 	 * Convert default object to swap object if necessary
17971c7c3c6aSMatthew Dillon 	 */
17981c7c3c6aSMatthew Dillon 	if (object->type != OBJT_SWAP) {
17991c7c3c6aSMatthew Dillon 		object->type = OBJT_SWAP;
18001c7c3c6aSMatthew Dillon 		object->un_pager.swp.swp_bcount = 0;
1801eb4d6a1bSKonstantin Belousov 		KASSERT(object->handle == NULL, ("default pager with handle"));
1802bd228075SAlan Cox 	}
18031c7c3c6aSMatthew Dillon 
18041c7c3c6aSMatthew Dillon 	/*
18051c7c3c6aSMatthew Dillon 	 * Locate hash entry.  If not found create, but if we aren't adding
18064dcc5c2dSMatthew Dillon 	 * anything just return.  If we run out of space in the map we wait
18074dcc5c2dSMatthew Dillon 	 * and, since the hash table may have changed, retry.
18081c7c3c6aSMatthew Dillon 	 */
18094dcc5c2dSMatthew Dillon retry:
18107827d9b0SAlan Cox 	mtx_lock(&swhash_mtx);
181123f09d50SIan Dowse 	pswap = swp_pager_hash(object, pindex);
18121c7c3c6aSMatthew Dillon 
18131c7c3c6aSMatthew Dillon 	if ((swap = *pswap) == NULL) {
18141c7c3c6aSMatthew Dillon 		int i;
18151c7c3c6aSMatthew Dillon 
18161c7c3c6aSMatthew Dillon 		if (swapblk == SWAPBLK_NONE)
18177827d9b0SAlan Cox 			goto done;
18181c7c3c6aSMatthew Dillon 
18195a3c920fSKonstantin Belousov 		swap = *pswap = uma_zalloc(swap_zone, M_NOWAIT |
18205a3c920fSKonstantin Belousov 		    (curproc == pageproc ? M_USE_RESERVE : 0));
18214dcc5c2dSMatthew Dillon 		if (swap == NULL) {
18227827d9b0SAlan Cox 			mtx_unlock(&swhash_mtx);
182389f6b863SAttilio Rao 			VM_OBJECT_WUNLOCK(object);
18242025d69bSKonstantin Belousov 			if (uma_zone_exhausted(swap_zone)) {
1825dc1b35b5SDag-Erling Smørgrav 				if (atomic_cmpset_int(&exhausted, 0, 1))
18263ff863f1SDag-Erling Smørgrav 					printf("swap zone exhausted, "
18273ff863f1SDag-Erling Smørgrav 					    "increase kern.maxswzone\n");
18282025d69bSKonstantin Belousov 				vm_pageout_oom(VM_OOM_SWAPZ);
18292025d69bSKonstantin Belousov 				pause("swzonex", 10);
18302025d69bSKonstantin Belousov 			} else
18314dcc5c2dSMatthew Dillon 				VM_WAIT;
183289f6b863SAttilio Rao 			VM_OBJECT_WLOCK(object);
18334dcc5c2dSMatthew Dillon 			goto retry;
18344dcc5c2dSMatthew Dillon 		}
1835670d17b5SJeff Roberson 
1836dc1b35b5SDag-Erling Smørgrav 		if (atomic_cmpset_int(&exhausted, 1, 0))
18373ff863f1SDag-Erling Smørgrav 			printf("swap zone ok\n");
18383ff863f1SDag-Erling Smørgrav 
18391c7c3c6aSMatthew Dillon 		swap->swb_hnext = NULL;
18401c7c3c6aSMatthew Dillon 		swap->swb_object = object;
184123f09d50SIan Dowse 		swap->swb_index = pindex & ~(vm_pindex_t)SWAP_META_MASK;
18421c7c3c6aSMatthew Dillon 		swap->swb_count = 0;
18431c7c3c6aSMatthew Dillon 
18441c7c3c6aSMatthew Dillon 		++object->un_pager.swp.swp_bcount;
18451c7c3c6aSMatthew Dillon 
18461c7c3c6aSMatthew Dillon 		for (i = 0; i < SWAP_META_PAGES; ++i)
18471c7c3c6aSMatthew Dillon 			swap->swb_pages[i] = SWAPBLK_NONE;
18481c7c3c6aSMatthew Dillon 	}
18491c7c3c6aSMatthew Dillon 
18501c7c3c6aSMatthew Dillon 	/*
18511c7c3c6aSMatthew Dillon 	 * Delete prior contents of metadata
18521c7c3c6aSMatthew Dillon 	 */
185323f09d50SIan Dowse 	idx = pindex & SWAP_META_MASK;
18541c7c3c6aSMatthew Dillon 
185523f09d50SIan Dowse 	if (swap->swb_pages[idx] != SWAPBLK_NONE) {
185623f09d50SIan Dowse 		swp_pager_freeswapspace(swap->swb_pages[idx], 1);
18571c7c3c6aSMatthew Dillon 		--swap->swb_count;
18581c7c3c6aSMatthew Dillon 	}
18591c7c3c6aSMatthew Dillon 
18601c7c3c6aSMatthew Dillon 	/*
18611c7c3c6aSMatthew Dillon 	 * Enter block into metadata
18621c7c3c6aSMatthew Dillon 	 */
186323f09d50SIan Dowse 	swap->swb_pages[idx] = swapblk;
18644dcc5c2dSMatthew Dillon 	if (swapblk != SWAPBLK_NONE)
18651c7c3c6aSMatthew Dillon 		++swap->swb_count;
18667827d9b0SAlan Cox done:
18677827d9b0SAlan Cox 	mtx_unlock(&swhash_mtx);
18681c7c3c6aSMatthew Dillon }
18691c7c3c6aSMatthew Dillon 
18701c7c3c6aSMatthew Dillon /*
18711c7c3c6aSMatthew Dillon  * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
18721c7c3c6aSMatthew Dillon  *
18731c7c3c6aSMatthew Dillon  *	The requested range of blocks is freed, with any associated swap
18741c7c3c6aSMatthew Dillon  *	returned to the swap bitmap.
18751c7c3c6aSMatthew Dillon  *
18761c7c3c6aSMatthew Dillon  *	This routine will free swap metadata structures as they are cleaned
18771c7c3c6aSMatthew Dillon  *	out.  This routine does *NOT* operate on swap metadata associated
18781c7c3c6aSMatthew Dillon  *	with resident pages.
18791c7c3c6aSMatthew Dillon  */
18801c7c3c6aSMatthew Dillon static void
18812e56b64fSKonstantin Belousov swp_pager_meta_free(vm_object_t object, vm_pindex_t index, vm_pindex_t count)
18821c7c3c6aSMatthew Dillon {
18832e56b64fSKonstantin Belousov 	struct swblock **pswap, *swap;
18842e56b64fSKonstantin Belousov 	vm_pindex_t c;
18852e56b64fSKonstantin Belousov 	daddr_t v;
18862e56b64fSKonstantin Belousov 	int n, sidx;
18872928cef7SAlan Cox 
1888c25673ffSAttilio Rao 	VM_OBJECT_ASSERT_LOCKED(object);
18892e56b64fSKonstantin Belousov 	if (object->type != OBJT_SWAP || count == 0)
18901c7c3c6aSMatthew Dillon 		return;
18911c7c3c6aSMatthew Dillon 
18927827d9b0SAlan Cox 	mtx_lock(&swhash_mtx);
18932e56b64fSKonstantin Belousov 	for (c = 0; c < count;) {
18941c7c3c6aSMatthew Dillon 		pswap = swp_pager_hash(object, index);
18952e56b64fSKonstantin Belousov 		sidx = index & SWAP_META_MASK;
18962e56b64fSKonstantin Belousov 		n = SWAP_META_PAGES - sidx;
18972e56b64fSKonstantin Belousov 		index += n;
18982e56b64fSKonstantin Belousov 		if ((swap = *pswap) == NULL) {
18992e56b64fSKonstantin Belousov 			c += n;
19002e56b64fSKonstantin Belousov 			continue;
19012e56b64fSKonstantin Belousov 		}
19022e56b64fSKonstantin Belousov 		for (; c < count && sidx < SWAP_META_PAGES; ++c, ++sidx) {
19032e56b64fSKonstantin Belousov 			if ((v = swap->swb_pages[sidx]) == SWAPBLK_NONE)
19042e56b64fSKonstantin Belousov 				continue;
19051c7c3c6aSMatthew Dillon 			swp_pager_freeswapspace(v, 1);
19062e56b64fSKonstantin Belousov 			swap->swb_pages[sidx] = SWAPBLK_NONE;
19071c7c3c6aSMatthew Dillon 			if (--swap->swb_count == 0) {
19081c7c3c6aSMatthew Dillon 				*pswap = swap->swb_hnext;
1909670d17b5SJeff Roberson 				uma_zfree(swap_zone, swap);
19101c7c3c6aSMatthew Dillon 				--object->un_pager.swp.swp_bcount;
19112e56b64fSKonstantin Belousov 				c += SWAP_META_PAGES - sidx;
19122e56b64fSKonstantin Belousov 				break;
19131c7c3c6aSMatthew Dillon 			}
19141c7c3c6aSMatthew Dillon 		}
19151c7c3c6aSMatthew Dillon 	}
19167827d9b0SAlan Cox 	mtx_unlock(&swhash_mtx);
19171c7c3c6aSMatthew Dillon }
19181c7c3c6aSMatthew Dillon 
19191c7c3c6aSMatthew Dillon /*
19201c7c3c6aSMatthew Dillon  * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
19211c7c3c6aSMatthew Dillon  *
19221c7c3c6aSMatthew Dillon  *	This routine locates and destroys all swap metadata associated with
19231c7c3c6aSMatthew Dillon  *	an object.
19241c7c3c6aSMatthew Dillon  */
19251c7c3c6aSMatthew Dillon static void
19261c7c3c6aSMatthew Dillon swp_pager_meta_free_all(vm_object_t object)
19271c7c3c6aSMatthew Dillon {
192871057cd2SKonstantin Belousov 	struct swblock **pswap, *swap;
192971057cd2SKonstantin Belousov 	vm_pindex_t index;
193071057cd2SKonstantin Belousov 	daddr_t v;
193171057cd2SKonstantin Belousov 	int i;
19321c7c3c6aSMatthew Dillon 
193389f6b863SAttilio Rao 	VM_OBJECT_ASSERT_WLOCKED(object);
19341c7c3c6aSMatthew Dillon 	if (object->type != OBJT_SWAP)
19351c7c3c6aSMatthew Dillon 		return;
19361c7c3c6aSMatthew Dillon 
193771057cd2SKonstantin Belousov 	index = 0;
193871057cd2SKonstantin Belousov 	while (object->un_pager.swp.swp_bcount != 0) {
19397827d9b0SAlan Cox 		mtx_lock(&swhash_mtx);
19401c7c3c6aSMatthew Dillon 		pswap = swp_pager_hash(object, index);
19411c7c3c6aSMatthew Dillon 		if ((swap = *pswap) != NULL) {
19421c7c3c6aSMatthew Dillon 			for (i = 0; i < SWAP_META_PAGES; ++i) {
194371057cd2SKonstantin Belousov 				v = swap->swb_pages[i];
19441c7c3c6aSMatthew Dillon 				if (v != SWAPBLK_NONE) {
19451c7c3c6aSMatthew Dillon 					--swap->swb_count;
19464dcc5c2dSMatthew Dillon 					swp_pager_freeswapspace(v, 1);
19471c7c3c6aSMatthew Dillon 				}
19481c7c3c6aSMatthew Dillon 			}
19491c7c3c6aSMatthew Dillon 			if (swap->swb_count != 0)
195071057cd2SKonstantin Belousov 				panic(
195171057cd2SKonstantin Belousov 				    "swap_pager_meta_free_all: swb_count != 0");
19521c7c3c6aSMatthew Dillon 			*pswap = swap->swb_hnext;
1953670d17b5SJeff Roberson 			uma_zfree(swap_zone, swap);
19541c7c3c6aSMatthew Dillon 			--object->un_pager.swp.swp_bcount;
19551c7c3c6aSMatthew Dillon 		}
19567827d9b0SAlan Cox 		mtx_unlock(&swhash_mtx);
19571c7c3c6aSMatthew Dillon 		index += SWAP_META_PAGES;
19581c7c3c6aSMatthew Dillon 	}
19591c7c3c6aSMatthew Dillon }
19601c7c3c6aSMatthew Dillon 
19611c7c3c6aSMatthew Dillon /*
19621c7c3c6aSMatthew Dillon  * SWP_PAGER_METACTL() -  misc control of swap and vm_page_t meta data.
19631c7c3c6aSMatthew Dillon  *
19641c7c3c6aSMatthew Dillon  *	This routine is capable of looking up, popping, or freeing
19651c7c3c6aSMatthew Dillon  *	swapblk assignments in the swap meta data or in the vm_page_t.
19661c7c3c6aSMatthew Dillon  *	The routine typically returns the swapblk being looked-up, or popped,
19671c7c3c6aSMatthew Dillon  *	or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block
19681c7c3c6aSMatthew Dillon  *	was invalid.  This routine will automatically free any invalid
19691c7c3c6aSMatthew Dillon  *	meta-data swapblks.
19701c7c3c6aSMatthew Dillon  *
19711c7c3c6aSMatthew Dillon  *	It is not possible to store invalid swapblks in the swap meta data
19721c7c3c6aSMatthew Dillon  *	(other then a literal 'SWAPBLK_NONE'), so we don't bother checking.
19731c7c3c6aSMatthew Dillon  *
19741c7c3c6aSMatthew Dillon  *	When acting on a busy resident page and paging is in progress, we
19751c7c3c6aSMatthew Dillon  *	have to wait until paging is complete but otherwise can act on the
19761c7c3c6aSMatthew Dillon  *	busy page.
19771c7c3c6aSMatthew Dillon  *
19784dcc5c2dSMatthew Dillon  *	SWM_FREE	remove and free swap block from metadata
19791c7c3c6aSMatthew Dillon  *	SWM_POP		remove from meta data but do not free.. pop it out
19801c7c3c6aSMatthew Dillon  */
19811c7c3c6aSMatthew Dillon static daddr_t
19822f249180SPoul-Henning Kamp swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
19832f249180SPoul-Henning Kamp {
19844dcc5c2dSMatthew Dillon 	struct swblock **pswap;
19854dcc5c2dSMatthew Dillon 	struct swblock *swap;
19864dcc5c2dSMatthew Dillon 	daddr_t r1;
198723f09d50SIan Dowse 	int idx;
19884dcc5c2dSMatthew Dillon 
1989c25673ffSAttilio Rao 	VM_OBJECT_ASSERT_LOCKED(object);
19901c7c3c6aSMatthew Dillon 	/*
19911c7c3c6aSMatthew Dillon 	 * The meta data only exists of the object is OBJT_SWAP
19921c7c3c6aSMatthew Dillon 	 * and even then might not be allocated yet.
19931c7c3c6aSMatthew Dillon 	 */
19944dcc5c2dSMatthew Dillon 	if (object->type != OBJT_SWAP)
19951c7c3c6aSMatthew Dillon 		return (SWAPBLK_NONE);
19961c7c3c6aSMatthew Dillon 
19974dcc5c2dSMatthew Dillon 	r1 = SWAPBLK_NONE;
19987827d9b0SAlan Cox 	mtx_lock(&swhash_mtx);
199923f09d50SIan Dowse 	pswap = swp_pager_hash(object, pindex);
20001c7c3c6aSMatthew Dillon 
20011c7c3c6aSMatthew Dillon 	if ((swap = *pswap) != NULL) {
200223f09d50SIan Dowse 		idx = pindex & SWAP_META_MASK;
200323f09d50SIan Dowse 		r1 = swap->swb_pages[idx];
20041c7c3c6aSMatthew Dillon 
20051c7c3c6aSMatthew Dillon 		if (r1 != SWAPBLK_NONE) {
20061c7c3c6aSMatthew Dillon 			if (flags & SWM_FREE) {
20074dcc5c2dSMatthew Dillon 				swp_pager_freeswapspace(r1, 1);
20081c7c3c6aSMatthew Dillon 				r1 = SWAPBLK_NONE;
20091c7c3c6aSMatthew Dillon 			}
20101c7c3c6aSMatthew Dillon 			if (flags & (SWM_FREE|SWM_POP)) {
201123f09d50SIan Dowse 				swap->swb_pages[idx] = SWAPBLK_NONE;
20121c7c3c6aSMatthew Dillon 				if (--swap->swb_count == 0) {
20131c7c3c6aSMatthew Dillon 					*pswap = swap->swb_hnext;
2014670d17b5SJeff Roberson 					uma_zfree(swap_zone, swap);
20151c7c3c6aSMatthew Dillon 					--object->un_pager.swp.swp_bcount;
20161c7c3c6aSMatthew Dillon 				}
20171c7c3c6aSMatthew Dillon 			}
20181c7c3c6aSMatthew Dillon 		}
20191c7c3c6aSMatthew Dillon 	}
20207827d9b0SAlan Cox 	mtx_unlock(&swhash_mtx);
20211c7c3c6aSMatthew Dillon 	return (r1);
20221c7c3c6aSMatthew Dillon }
20231c7c3c6aSMatthew Dillon 
2024e9c0cc15SPoul-Henning Kamp /*
202577d6fd97SKonstantin Belousov  * Returns the least page index which is greater than or equal to the
202677d6fd97SKonstantin Belousov  * parameter pindex and for which there is a swap block allocated.
202777d6fd97SKonstantin Belousov  * Returns object's size if the object's type is not swap or if there
202877d6fd97SKonstantin Belousov  * are no allocated swap blocks for the object after the requested
202977d6fd97SKonstantin Belousov  * pindex.
203077d6fd97SKonstantin Belousov  */
203177d6fd97SKonstantin Belousov vm_pindex_t
203277d6fd97SKonstantin Belousov swap_pager_find_least(vm_object_t object, vm_pindex_t pindex)
203377d6fd97SKonstantin Belousov {
203477d6fd97SKonstantin Belousov 	struct swblock **pswap, *swap;
203577d6fd97SKonstantin Belousov 	vm_pindex_t i, j, lim;
203677d6fd97SKonstantin Belousov 	int idx;
203777d6fd97SKonstantin Belousov 
203877d6fd97SKonstantin Belousov 	VM_OBJECT_ASSERT_LOCKED(object);
203977d6fd97SKonstantin Belousov 	if (object->type != OBJT_SWAP || object->un_pager.swp.swp_bcount == 0)
204077d6fd97SKonstantin Belousov 		return (object->size);
204177d6fd97SKonstantin Belousov 
204277d6fd97SKonstantin Belousov 	mtx_lock(&swhash_mtx);
204377d6fd97SKonstantin Belousov 	for (j = pindex; j < object->size; j = lim) {
204477d6fd97SKonstantin Belousov 		pswap = swp_pager_hash(object, j);
204577d6fd97SKonstantin Belousov 		lim = rounddown2(j + SWAP_META_PAGES, SWAP_META_PAGES);
204677d6fd97SKonstantin Belousov 		if (lim > object->size)
204777d6fd97SKonstantin Belousov 			lim = object->size;
204877d6fd97SKonstantin Belousov 		if ((swap = *pswap) != NULL) {
204977d6fd97SKonstantin Belousov 			for (idx = j & SWAP_META_MASK, i = j; i < lim;
205077d6fd97SKonstantin Belousov 			    i++, idx++) {
205177d6fd97SKonstantin Belousov 				if (swap->swb_pages[idx] != SWAPBLK_NONE)
205277d6fd97SKonstantin Belousov 					goto found;
205377d6fd97SKonstantin Belousov 			}
205477d6fd97SKonstantin Belousov 		}
205577d6fd97SKonstantin Belousov 	}
205677d6fd97SKonstantin Belousov 	i = object->size;
205777d6fd97SKonstantin Belousov found:
205877d6fd97SKonstantin Belousov 	mtx_unlock(&swhash_mtx);
205977d6fd97SKonstantin Belousov 	return (i);
206077d6fd97SKonstantin Belousov }
206177d6fd97SKonstantin Belousov 
206277d6fd97SKonstantin Belousov /*
2063e9c0cc15SPoul-Henning Kamp  * System call swapon(name) enables swapping on device name,
2064e9c0cc15SPoul-Henning Kamp  * which must be in the swdevsw.  Return EBUSY
2065e9c0cc15SPoul-Henning Kamp  * if already swapping on this device.
2066e9c0cc15SPoul-Henning Kamp  */
2067e9c0cc15SPoul-Henning Kamp #ifndef _SYS_SYSPROTO_H_
2068e9c0cc15SPoul-Henning Kamp struct swapon_args {
2069e9c0cc15SPoul-Henning Kamp 	char *name;
2070e9c0cc15SPoul-Henning Kamp };
2071e9c0cc15SPoul-Henning Kamp #endif
2072e9c0cc15SPoul-Henning Kamp 
2073e9c0cc15SPoul-Henning Kamp /*
2074e9c0cc15SPoul-Henning Kamp  * MPSAFE
2075e9c0cc15SPoul-Henning Kamp  */
2076e9c0cc15SPoul-Henning Kamp /* ARGSUSED */
2077e9c0cc15SPoul-Henning Kamp int
20788451d0ddSKip Macy sys_swapon(struct thread *td, struct swapon_args *uap)
2079e9c0cc15SPoul-Henning Kamp {
2080e9c0cc15SPoul-Henning Kamp 	struct vattr attr;
2081e9c0cc15SPoul-Henning Kamp 	struct vnode *vp;
2082e9c0cc15SPoul-Henning Kamp 	struct nameidata nd;
2083e9c0cc15SPoul-Henning Kamp 	int error;
2084e9c0cc15SPoul-Henning Kamp 
2085acd3428bSRobert Watson 	error = priv_check(td, PRIV_SWAPON);
2086e9c0cc15SPoul-Henning Kamp 	if (error)
2087acd3428bSRobert Watson 		return (error);
2088e9c0cc15SPoul-Henning Kamp 
208904533e1eSKonstantin Belousov 	sx_xlock(&swdev_syscall_lock);
2090e9c0cc15SPoul-Henning Kamp 
2091e9c0cc15SPoul-Henning Kamp 	/*
2092e9c0cc15SPoul-Henning Kamp 	 * Swap metadata may not fit in the KVM if we have physical
2093e9c0cc15SPoul-Henning Kamp 	 * memory of >1GB.
2094e9c0cc15SPoul-Henning Kamp 	 */
2095e9c0cc15SPoul-Henning Kamp 	if (swap_zone == NULL) {
2096e9c0cc15SPoul-Henning Kamp 		error = ENOMEM;
2097e9c0cc15SPoul-Henning Kamp 		goto done;
2098e9c0cc15SPoul-Henning Kamp 	}
2099e9c0cc15SPoul-Henning Kamp 
2100d9135e72SRobert Watson 	NDINIT(&nd, LOOKUP, ISOPEN | FOLLOW | AUDITVNODE1, UIO_USERSPACE,
2101d9135e72SRobert Watson 	    uap->name, td);
2102e9c0cc15SPoul-Henning Kamp 	error = namei(&nd);
2103e9c0cc15SPoul-Henning Kamp 	if (error)
2104e9c0cc15SPoul-Henning Kamp 		goto done;
2105e9c0cc15SPoul-Henning Kamp 
2106e9c0cc15SPoul-Henning Kamp 	NDFREE(&nd, NDF_ONLY_PNBUF);
2107e9c0cc15SPoul-Henning Kamp 	vp = nd.ni_vp;
2108e9c0cc15SPoul-Henning Kamp 
210920da9c2eSPoul-Henning Kamp 	if (vn_isdisk(vp, &error)) {
211088ad2d7bSKonstantin Belousov 		error = swapongeom(vp);
211120da9c2eSPoul-Henning Kamp 	} else if (vp->v_type == VREG &&
2112e9c0cc15SPoul-Henning Kamp 	    (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
21130359a12eSAttilio Rao 	    (error = VOP_GETATTR(vp, &attr, td->td_ucred)) == 0) {
2114e9c0cc15SPoul-Henning Kamp 		/*
2115e9c0cc15SPoul-Henning Kamp 		 * Allow direct swapping to NFS regular files in the same
2116e9c0cc15SPoul-Henning Kamp 		 * way that nfs_mountroot() sets up diskless swapping.
2117e9c0cc15SPoul-Henning Kamp 		 */
211859efee01SPoul-Henning Kamp 		error = swaponvp(td, vp, attr.va_size / DEV_BSIZE);
2119e9c0cc15SPoul-Henning Kamp 	}
2120e9c0cc15SPoul-Henning Kamp 
2121e9c0cc15SPoul-Henning Kamp 	if (error)
2122e9c0cc15SPoul-Henning Kamp 		vrele(vp);
2123e9c0cc15SPoul-Henning Kamp done:
212404533e1eSKonstantin Belousov 	sx_xunlock(&swdev_syscall_lock);
2125e9c0cc15SPoul-Henning Kamp 	return (error);
2126e9c0cc15SPoul-Henning Kamp }
2127e9c0cc15SPoul-Henning Kamp 
21283ff863f1SDag-Erling Smørgrav /*
21293ff863f1SDag-Erling Smørgrav  * Check that the total amount of swap currently configured does not
21303ff863f1SDag-Erling Smørgrav  * exceed half the theoretical maximum.  If it does, print a warning
21313ff863f1SDag-Erling Smørgrav  * message and return -1; otherwise, return 0.
21323ff863f1SDag-Erling Smørgrav  */
21333ff863f1SDag-Erling Smørgrav static int
21343ff863f1SDag-Erling Smørgrav swapon_check_swzone(unsigned long npages)
21353ff863f1SDag-Erling Smørgrav {
21363ff863f1SDag-Erling Smørgrav 	unsigned long maxpages;
21373ff863f1SDag-Erling Smørgrav 
21383ff863f1SDag-Erling Smørgrav 	/* absolute maximum we can handle assuming 100% efficiency */
21393ff863f1SDag-Erling Smørgrav 	maxpages = uma_zone_get_max(swap_zone) * SWAP_META_PAGES;
21403ff863f1SDag-Erling Smørgrav 
21413ff863f1SDag-Erling Smørgrav 	/* recommend using no more than half that amount */
21423ff863f1SDag-Erling Smørgrav 	if (npages > maxpages / 2) {
21433ff863f1SDag-Erling Smørgrav 		printf("warning: total configured swap (%lu pages) "
21443ff863f1SDag-Erling Smørgrav 		    "exceeds maximum recommended amount (%lu pages).\n",
21459462305cSSergey Kandaurov 		    npages, maxpages / 2);
21463ff863f1SDag-Erling Smørgrav 		printf("warning: increase kern.maxswzone "
21473ff863f1SDag-Erling Smørgrav 		    "or reduce amount of swap.\n");
21483ff863f1SDag-Erling Smørgrav 		return (-1);
21493ff863f1SDag-Erling Smørgrav 	}
21503ff863f1SDag-Erling Smørgrav 	return (0);
21513ff863f1SDag-Erling Smørgrav }
21523ff863f1SDag-Erling Smørgrav 
215359efee01SPoul-Henning Kamp static void
21542cc718a1SKonstantin Belousov swaponsomething(struct vnode *vp, void *id, u_long nblks,
21552cc718a1SKonstantin Belousov     sw_strategy_t *strategy, sw_close_t *close, dev_t dev, int flags)
2156e9c0cc15SPoul-Henning Kamp {
21572d9974c1SAlan Cox 	struct swdevt *sp, *tsp;
2158e9c0cc15SPoul-Henning Kamp 	swblk_t dvbase;
21598f60c087SPoul-Henning Kamp 	u_long mblocks;
2160e9c0cc15SPoul-Henning Kamp 
2161e9c0cc15SPoul-Henning Kamp 	/*
2162e9c0cc15SPoul-Henning Kamp 	 * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks.
2163e9c0cc15SPoul-Henning Kamp 	 * First chop nblks off to page-align it, then convert.
2164e9c0cc15SPoul-Henning Kamp 	 *
2165e9c0cc15SPoul-Henning Kamp 	 * sw->sw_nblks is in page-sized chunks now too.
2166e9c0cc15SPoul-Henning Kamp 	 */
2167e9c0cc15SPoul-Henning Kamp 	nblks &= ~(ctodb(1) - 1);
2168e9c0cc15SPoul-Henning Kamp 	nblks = dbtoc(nblks);
2169e9c0cc15SPoul-Henning Kamp 
21706e903bd0SKonstantin Belousov 	/*
21716e903bd0SKonstantin Belousov 	 * If we go beyond this, we get overflows in the radix
21726e903bd0SKonstantin Belousov 	 * tree bitmap code.
21736e903bd0SKonstantin Belousov 	 */
21746e903bd0SKonstantin Belousov 	mblocks = 0x40000000 / BLIST_META_RADIX;
21756e903bd0SKonstantin Belousov 	if (nblks > mblocks) {
21766e903bd0SKonstantin Belousov 		printf(
21776e903bd0SKonstantin Belousov     "WARNING: reducing swap size to maximum of %luMB per unit\n",
21786e903bd0SKonstantin Belousov 		    mblocks / 1024 / 1024 * PAGE_SIZE);
21796e903bd0SKonstantin Belousov 		nblks = mblocks;
21806e903bd0SKonstantin Belousov 	}
21816e903bd0SKonstantin Belousov 
21828f60c087SPoul-Henning Kamp 	sp = malloc(sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO);
2183dee34ca4SPoul-Henning Kamp 	sp->sw_vp = vp;
2184dee34ca4SPoul-Henning Kamp 	sp->sw_id = id;
2185f3732fd1SPoul-Henning Kamp 	sp->sw_dev = dev;
21868d677ef9SPoul-Henning Kamp 	sp->sw_flags = 0;
2187e9c0cc15SPoul-Henning Kamp 	sp->sw_nblks = nblks;
2188e9c0cc15SPoul-Henning Kamp 	sp->sw_used = 0;
218959efee01SPoul-Henning Kamp 	sp->sw_strategy = strategy;
2190dee34ca4SPoul-Henning Kamp 	sp->sw_close = close;
21912cc718a1SKonstantin Belousov 	sp->sw_flags = flags;
2192e9c0cc15SPoul-Henning Kamp 
2193c8c7ad92SKip Macy 	sp->sw_blist = blist_create(nblks, M_WAITOK);
2194e9c0cc15SPoul-Henning Kamp 	/*
2195ef3c5abdSPoul-Henning Kamp 	 * Do not free the first two block in order to avoid overwriting
21968f60c087SPoul-Henning Kamp 	 * any bsd label at the front of the partition
2197e9c0cc15SPoul-Henning Kamp 	 */
2198ef3c5abdSPoul-Henning Kamp 	blist_free(sp->sw_blist, 2, nblks - 2);
2199e9c0cc15SPoul-Henning Kamp 
22002d9974c1SAlan Cox 	dvbase = 0;
220120da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
22022d9974c1SAlan Cox 	TAILQ_FOREACH(tsp, &swtailq, sw_list) {
22032d9974c1SAlan Cox 		if (tsp->sw_end >= dvbase) {
22042d9974c1SAlan Cox 			/*
22052d9974c1SAlan Cox 			 * We put one uncovered page between the devices
22062d9974c1SAlan Cox 			 * in order to definitively prevent any cross-device
22072d9974c1SAlan Cox 			 * I/O requests
22082d9974c1SAlan Cox 			 */
22092d9974c1SAlan Cox 			dvbase = tsp->sw_end + 1;
22102d9974c1SAlan Cox 		}
22112d9974c1SAlan Cox 	}
22122d9974c1SAlan Cox 	sp->sw_first = dvbase;
22132d9974c1SAlan Cox 	sp->sw_end = dvbase + nblks;
22148f60c087SPoul-Henning Kamp 	TAILQ_INSERT_TAIL(&swtailq, sp, sw_list);
22158f60c087SPoul-Henning Kamp 	nswapdev++;
22168f60c087SPoul-Henning Kamp 	swap_pager_avail += nblks;
22173364c323SKonstantin Belousov 	swap_total += (vm_ooffset_t)nblks * PAGE_SIZE;
22183ff863f1SDag-Erling Smørgrav 	swapon_check_swzone(swap_total / PAGE_SIZE);
2219d05bc129SAlan Cox 	swp_sizecheck();
2220d05bc129SAlan Cox 	mtx_unlock(&sw_dev_mtx);
2221b1fd102eSMark Johnston 	EVENTHANDLER_INVOKE(swapon, sp);
222259efee01SPoul-Henning Kamp }
2223e9c0cc15SPoul-Henning Kamp 
2224e9c0cc15SPoul-Henning Kamp /*
2225e9c0cc15SPoul-Henning Kamp  * SYSCALL: swapoff(devname)
2226e9c0cc15SPoul-Henning Kamp  *
2227e9c0cc15SPoul-Henning Kamp  * Disable swapping on the given device.
2228dee34ca4SPoul-Henning Kamp  *
2229dee34ca4SPoul-Henning Kamp  * XXX: Badly designed system call: it should use a device index
2230dee34ca4SPoul-Henning Kamp  * rather than filename as specification.  We keep sw_vp around
2231dee34ca4SPoul-Henning Kamp  * only to make this work.
2232e9c0cc15SPoul-Henning Kamp  */
2233e9c0cc15SPoul-Henning Kamp #ifndef _SYS_SYSPROTO_H_
2234e9c0cc15SPoul-Henning Kamp struct swapoff_args {
2235e9c0cc15SPoul-Henning Kamp 	char *name;
2236e9c0cc15SPoul-Henning Kamp };
2237e9c0cc15SPoul-Henning Kamp #endif
2238e9c0cc15SPoul-Henning Kamp 
2239e9c0cc15SPoul-Henning Kamp /*
2240e9c0cc15SPoul-Henning Kamp  * MPSAFE
2241e9c0cc15SPoul-Henning Kamp  */
2242e9c0cc15SPoul-Henning Kamp /* ARGSUSED */
2243e9c0cc15SPoul-Henning Kamp int
22448451d0ddSKip Macy sys_swapoff(struct thread *td, struct swapoff_args *uap)
2245e9c0cc15SPoul-Henning Kamp {
2246e9c0cc15SPoul-Henning Kamp 	struct vnode *vp;
2247e9c0cc15SPoul-Henning Kamp 	struct nameidata nd;
2248e9c0cc15SPoul-Henning Kamp 	struct swdevt *sp;
22498f60c087SPoul-Henning Kamp 	int error;
2250e9c0cc15SPoul-Henning Kamp 
2251acd3428bSRobert Watson 	error = priv_check(td, PRIV_SWAPOFF);
2252e9c0cc15SPoul-Henning Kamp 	if (error)
22530909f38aSPawel Jakub Dawidek 		return (error);
2254e9c0cc15SPoul-Henning Kamp 
225504533e1eSKonstantin Belousov 	sx_xlock(&swdev_syscall_lock);
2256e9c0cc15SPoul-Henning Kamp 
2257d9135e72SRobert Watson 	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, UIO_USERSPACE, uap->name,
2258d9135e72SRobert Watson 	    td);
2259e9c0cc15SPoul-Henning Kamp 	error = namei(&nd);
2260e9c0cc15SPoul-Henning Kamp 	if (error)
2261e9c0cc15SPoul-Henning Kamp 		goto done;
2262e9c0cc15SPoul-Henning Kamp 	NDFREE(&nd, NDF_ONLY_PNBUF);
2263e9c0cc15SPoul-Henning Kamp 	vp = nd.ni_vp;
2264e9c0cc15SPoul-Henning Kamp 
226520da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
22668f60c087SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2267dee34ca4SPoul-Henning Kamp 		if (sp->sw_vp == vp)
22680909f38aSPawel Jakub Dawidek 			break;
2269e9c0cc15SPoul-Henning Kamp 	}
227020da9c2eSPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
22710909f38aSPawel Jakub Dawidek 	if (sp == NULL) {
2272e9c0cc15SPoul-Henning Kamp 		error = EINVAL;
2273e9c0cc15SPoul-Henning Kamp 		goto done;
22740909f38aSPawel Jakub Dawidek 	}
227535918c55SChristian S.J. Peron 	error = swapoff_one(sp, td->td_ucred);
22760909f38aSPawel Jakub Dawidek done:
227704533e1eSKonstantin Belousov 	sx_xunlock(&swdev_syscall_lock);
22780909f38aSPawel Jakub Dawidek 	return (error);
22790909f38aSPawel Jakub Dawidek }
22800909f38aSPawel Jakub Dawidek 
22810909f38aSPawel Jakub Dawidek static int
228235918c55SChristian S.J. Peron swapoff_one(struct swdevt *sp, struct ucred *cred)
22830909f38aSPawel Jakub Dawidek {
22840909f38aSPawel Jakub Dawidek 	u_long nblks, dvbase;
2285e9c0cc15SPoul-Henning Kamp #ifdef MAC
22860909f38aSPawel Jakub Dawidek 	int error;
2287e9c0cc15SPoul-Henning Kamp #endif
2288e9c0cc15SPoul-Henning Kamp 
228904533e1eSKonstantin Belousov 	sx_assert(&swdev_syscall_lock, SA_XLOCKED);
22900909f38aSPawel Jakub Dawidek #ifdef MAC
2291cb05b60aSAttilio Rao 	(void) vn_lock(sp->sw_vp, LK_EXCLUSIVE | LK_RETRY);
229235918c55SChristian S.J. Peron 	error = mac_system_check_swapoff(cred, sp->sw_vp);
229322db15c0SAttilio Rao 	(void) VOP_UNLOCK(sp->sw_vp, 0);
22940909f38aSPawel Jakub Dawidek 	if (error != 0)
22950909f38aSPawel Jakub Dawidek 		return (error);
22960909f38aSPawel Jakub Dawidek #endif
2297e9c0cc15SPoul-Henning Kamp 	nblks = sp->sw_nblks;
2298e9c0cc15SPoul-Henning Kamp 
2299e9c0cc15SPoul-Henning Kamp 	/*
2300e9c0cc15SPoul-Henning Kamp 	 * We can turn off this swap device safely only if the
2301e9c0cc15SPoul-Henning Kamp 	 * available virtual memory in the system will fit the amount
2302e9c0cc15SPoul-Henning Kamp 	 * of data we will have to page back in, plus an epsilon so
2303e9c0cc15SPoul-Henning Kamp 	 * the system doesn't become critically low on swap space.
2304e9c0cc15SPoul-Henning Kamp 	 */
2305bba39b9aSAlan Cox 	if (vm_cnt.v_free_count + swap_pager_avail < nblks + nswap_lowat)
23060909f38aSPawel Jakub Dawidek 		return (ENOMEM);
2307e9c0cc15SPoul-Henning Kamp 
2308e9c0cc15SPoul-Henning Kamp 	/*
2309e9c0cc15SPoul-Henning Kamp 	 * Prevent further allocations on this device.
2310e9c0cc15SPoul-Henning Kamp 	 */
23112928cef7SAlan Cox 	mtx_lock(&sw_dev_mtx);
2312e9c0cc15SPoul-Henning Kamp 	sp->sw_flags |= SW_CLOSING;
23138f60c087SPoul-Henning Kamp 	for (dvbase = 0; dvbase < sp->sw_end; dvbase += dmmax) {
23148f60c087SPoul-Henning Kamp 		swap_pager_avail -= blist_fill(sp->sw_blist,
23158f60c087SPoul-Henning Kamp 		     dvbase, dmmax);
2316e9c0cc15SPoul-Henning Kamp 	}
23173364c323SKonstantin Belousov 	swap_total -= (vm_ooffset_t)nblks * PAGE_SIZE;
23182928cef7SAlan Cox 	mtx_unlock(&sw_dev_mtx);
2319e9c0cc15SPoul-Henning Kamp 
2320e9c0cc15SPoul-Henning Kamp 	/*
2321e9c0cc15SPoul-Henning Kamp 	 * Page in the contents of the device and close it.
2322e9c0cc15SPoul-Henning Kamp 	 */
2323b3fed13eSDavid Schultz 	swap_pager_swapoff(sp);
2324e9c0cc15SPoul-Henning Kamp 
232535918c55SChristian S.J. Peron 	sp->sw_close(curthread, sp);
232620da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
23279e3e3fe5SWarner Losh 	sp->sw_id = NULL;
23288f60c087SPoul-Henning Kamp 	TAILQ_REMOVE(&swtailq, sp, sw_list);
23290676a140SAlan Cox 	nswapdev--;
23307dea2c2eSAlan Cox 	if (nswapdev == 0) {
23317dea2c2eSAlan Cox 		swap_pager_full = 2;
23327dea2c2eSAlan Cox 		swap_pager_almost_full = 1;
23337dea2c2eSAlan Cox 	}
23348f60c087SPoul-Henning Kamp 	if (swdevhd == sp)
23358f60c087SPoul-Henning Kamp 		swdevhd = NULL;
2336d05bc129SAlan Cox 	mtx_unlock(&sw_dev_mtx);
23378f60c087SPoul-Henning Kamp 	blist_destroy(sp->sw_blist);
23388f60c087SPoul-Henning Kamp 	free(sp, M_VMPGDATA);
23390909f38aSPawel Jakub Dawidek 	return (0);
23400909f38aSPawel Jakub Dawidek }
2341e9c0cc15SPoul-Henning Kamp 
23420909f38aSPawel Jakub Dawidek void
23430909f38aSPawel Jakub Dawidek swapoff_all(void)
23440909f38aSPawel Jakub Dawidek {
23450909f38aSPawel Jakub Dawidek 	struct swdevt *sp, *spt;
23460909f38aSPawel Jakub Dawidek 	const char *devname;
23470909f38aSPawel Jakub Dawidek 	int error;
23480909f38aSPawel Jakub Dawidek 
234904533e1eSKonstantin Belousov 	sx_xlock(&swdev_syscall_lock);
23500909f38aSPawel Jakub Dawidek 
23510909f38aSPawel Jakub Dawidek 	mtx_lock(&sw_dev_mtx);
23520909f38aSPawel Jakub Dawidek 	TAILQ_FOREACH_SAFE(sp, &swtailq, sw_list, spt) {
23530909f38aSPawel Jakub Dawidek 		mtx_unlock(&sw_dev_mtx);
23540909f38aSPawel Jakub Dawidek 		if (vn_isdisk(sp->sw_vp, NULL))
23557870adb6SEd Schouten 			devname = devtoname(sp->sw_vp->v_rdev);
23560909f38aSPawel Jakub Dawidek 		else
23570909f38aSPawel Jakub Dawidek 			devname = "[file]";
235835918c55SChristian S.J. Peron 		error = swapoff_one(sp, thread0.td_ucred);
23590909f38aSPawel Jakub Dawidek 		if (error != 0) {
23600909f38aSPawel Jakub Dawidek 			printf("Cannot remove swap device %s (error=%d), "
23610909f38aSPawel Jakub Dawidek 			    "skipping.\n", devname, error);
23620909f38aSPawel Jakub Dawidek 		} else if (bootverbose) {
23630909f38aSPawel Jakub Dawidek 			printf("Swap device %s removed.\n", devname);
23640909f38aSPawel Jakub Dawidek 		}
23650909f38aSPawel Jakub Dawidek 		mtx_lock(&sw_dev_mtx);
23660909f38aSPawel Jakub Dawidek 	}
23670909f38aSPawel Jakub Dawidek 	mtx_unlock(&sw_dev_mtx);
23680909f38aSPawel Jakub Dawidek 
236904533e1eSKonstantin Belousov 	sx_xunlock(&swdev_syscall_lock);
2370e9c0cc15SPoul-Henning Kamp }
2371e9c0cc15SPoul-Henning Kamp 
2372567104a1SPoul-Henning Kamp void
2373567104a1SPoul-Henning Kamp swap_pager_status(int *total, int *used)
2374567104a1SPoul-Henning Kamp {
2375567104a1SPoul-Henning Kamp 	struct swdevt *sp;
2376567104a1SPoul-Henning Kamp 
2377567104a1SPoul-Henning Kamp 	*total = 0;
2378567104a1SPoul-Henning Kamp 	*used = 0;
237920da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
23808f60c087SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2381567104a1SPoul-Henning Kamp 		*total += sp->sw_nblks;
2382567104a1SPoul-Henning Kamp 		*used += sp->sw_used;
2383567104a1SPoul-Henning Kamp 	}
238420da9c2eSPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
2385567104a1SPoul-Henning Kamp }
2386567104a1SPoul-Henning Kamp 
2387dda4f960SKonstantin Belousov int
2388dda4f960SKonstantin Belousov swap_dev_info(int name, struct xswdev *xs, char *devname, size_t len)
2389dda4f960SKonstantin Belousov {
2390dda4f960SKonstantin Belousov 	struct swdevt *sp;
23917870adb6SEd Schouten 	const char *tmp_devname;
2392dda4f960SKonstantin Belousov 	int error, n;
2393dda4f960SKonstantin Belousov 
2394dda4f960SKonstantin Belousov 	n = 0;
2395dda4f960SKonstantin Belousov 	error = ENOENT;
2396dda4f960SKonstantin Belousov 	mtx_lock(&sw_dev_mtx);
2397dda4f960SKonstantin Belousov 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2398dda4f960SKonstantin Belousov 		if (n != name) {
2399dda4f960SKonstantin Belousov 			n++;
2400dda4f960SKonstantin Belousov 			continue;
2401dda4f960SKonstantin Belousov 		}
2402dda4f960SKonstantin Belousov 		xs->xsw_version = XSWDEV_VERSION;
2403dda4f960SKonstantin Belousov 		xs->xsw_dev = sp->sw_dev;
2404dda4f960SKonstantin Belousov 		xs->xsw_flags = sp->sw_flags;
2405dda4f960SKonstantin Belousov 		xs->xsw_nblks = sp->sw_nblks;
2406dda4f960SKonstantin Belousov 		xs->xsw_used = sp->sw_used;
2407dda4f960SKonstantin Belousov 		if (devname != NULL) {
2408dda4f960SKonstantin Belousov 			if (vn_isdisk(sp->sw_vp, NULL))
24097870adb6SEd Schouten 				tmp_devname = devtoname(sp->sw_vp->v_rdev);
2410dda4f960SKonstantin Belousov 			else
2411dda4f960SKonstantin Belousov 				tmp_devname = "[file]";
2412dda4f960SKonstantin Belousov 			strncpy(devname, tmp_devname, len);
2413dda4f960SKonstantin Belousov 		}
2414dda4f960SKonstantin Belousov 		error = 0;
2415dda4f960SKonstantin Belousov 		break;
2416dda4f960SKonstantin Belousov 	}
2417dda4f960SKonstantin Belousov 	mtx_unlock(&sw_dev_mtx);
2418dda4f960SKonstantin Belousov 	return (error);
2419dda4f960SKonstantin Belousov }
2420dda4f960SKonstantin Belousov 
2421*69921123SKonstantin Belousov #if defined(COMPAT_FREEBSD11)
2422*69921123SKonstantin Belousov #define XSWDEV_VERSION_11	1
2423*69921123SKonstantin Belousov struct xswdev11 {
2424*69921123SKonstantin Belousov 	u_int	xsw_version;
2425*69921123SKonstantin Belousov 	uint32_t xsw_dev;
2426*69921123SKonstantin Belousov 	int	xsw_flags;
2427*69921123SKonstantin Belousov 	int	xsw_nblks;
2428*69921123SKonstantin Belousov 	int     xsw_used;
2429*69921123SKonstantin Belousov };
2430*69921123SKonstantin Belousov #endif
2431*69921123SKonstantin Belousov 
2432e9c0cc15SPoul-Henning Kamp static int
2433e9c0cc15SPoul-Henning Kamp sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS)
2434e9c0cc15SPoul-Henning Kamp {
2435e9c0cc15SPoul-Henning Kamp 	struct xswdev xs;
2436*69921123SKonstantin Belousov #if defined(COMPAT_FREEBSD11)
2437*69921123SKonstantin Belousov 	struct xswdev11 xs11;
2438*69921123SKonstantin Belousov #endif
2439dda4f960SKonstantin Belousov 	int error;
2440e9c0cc15SPoul-Henning Kamp 
2441e9c0cc15SPoul-Henning Kamp 	if (arg2 != 1)			/* name length */
2442e9c0cc15SPoul-Henning Kamp 		return (EINVAL);
2443dda4f960SKonstantin Belousov 	error = swap_dev_info(*(int *)arg1, &xs, NULL, 0);
2444dda4f960SKonstantin Belousov 	if (error != 0)
2445dda4f960SKonstantin Belousov 		return (error);
2446*69921123SKonstantin Belousov #if defined(COMPAT_FREEBSD11)
2447*69921123SKonstantin Belousov 	if (req->oldlen == sizeof(xs11)) {
2448*69921123SKonstantin Belousov 		xs11.xsw_version = XSWDEV_VERSION_11;
2449*69921123SKonstantin Belousov 		xs11.xsw_dev = xs.xsw_dev; /* truncation */
2450*69921123SKonstantin Belousov 		xs11.xsw_flags = xs.xsw_flags;
2451*69921123SKonstantin Belousov 		xs11.xsw_nblks = xs.xsw_nblks;
2452*69921123SKonstantin Belousov 		xs11.xsw_used = xs.xsw_used;
2453*69921123SKonstantin Belousov 		error = SYSCTL_OUT(req, &xs11, sizeof(xs11));
2454*69921123SKonstantin Belousov 	} else
2455*69921123SKonstantin Belousov #endif
2456e9c0cc15SPoul-Henning Kamp 		error = SYSCTL_OUT(req, &xs, sizeof(xs));
2457e9c0cc15SPoul-Henning Kamp 	return (error);
2458e9c0cc15SPoul-Henning Kamp }
2459e9c0cc15SPoul-Henning Kamp 
24608f60c087SPoul-Henning Kamp SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, &nswapdev, 0,
2461e9c0cc15SPoul-Henning Kamp     "Number of swap devices");
24624c36e917SKonstantin Belousov SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD | CTLFLAG_MPSAFE,
24634c36e917SKonstantin Belousov     sysctl_vm_swap_info,
2464e9c0cc15SPoul-Henning Kamp     "Swap statistics by device");
2465ec38b344SPoul-Henning Kamp 
2466ec38b344SPoul-Henning Kamp /*
24678eb5a1cdSKonstantin Belousov  * vmspace_swap_count() - count the approximate swap usage in pages for a
2468ec38b344SPoul-Henning Kamp  *			  vmspace.
2469ec38b344SPoul-Henning Kamp  *
2470ec38b344SPoul-Henning Kamp  *	The map must be locked.
2471ec38b344SPoul-Henning Kamp  *
24728eb5a1cdSKonstantin Belousov  *	Swap usage is determined by taking the proportional swap used by
2473ec38b344SPoul-Henning Kamp  *	VM objects backing the VM map.  To make up for fractional losses,
2474ec38b344SPoul-Henning Kamp  *	if the VM object has any swap use at all the associated map entries
2475ec38b344SPoul-Henning Kamp  *	count for at least 1 swap page.
2476ec38b344SPoul-Henning Kamp  */
24772860553aSRebecca Cran long
2478ec38b344SPoul-Henning Kamp vmspace_swap_count(struct vmspace *vmspace)
2479ec38b344SPoul-Henning Kamp {
248065d8409cSRebecca Cran 	vm_map_t map;
2481ec38b344SPoul-Henning Kamp 	vm_map_entry_t cur;
248265d8409cSRebecca Cran 	vm_object_t object;
24832860553aSRebecca Cran 	long count, n;
248465d8409cSRebecca Cran 
248565d8409cSRebecca Cran 	map = &vmspace->vm_map;
248665d8409cSRebecca Cran 	count = 0;
2487ec38b344SPoul-Henning Kamp 
2488ec38b344SPoul-Henning Kamp 	for (cur = map->header.next; cur != &map->header; cur = cur->next) {
2489ec38b344SPoul-Henning Kamp 		if ((cur->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
2490ec38b344SPoul-Henning Kamp 		    (object = cur->object.vm_object) != NULL) {
249189f6b863SAttilio Rao 			VM_OBJECT_WLOCK(object);
2492ec38b344SPoul-Henning Kamp 			if (object->type == OBJT_SWAP &&
2493ec38b344SPoul-Henning Kamp 			    object->un_pager.swp.swp_bcount != 0) {
249465d8409cSRebecca Cran 				n = (cur->end - cur->start) / PAGE_SIZE;
2495ec38b344SPoul-Henning Kamp 				count += object->un_pager.swp.swp_bcount *
2496ec38b344SPoul-Henning Kamp 				    SWAP_META_PAGES * n / object->size + 1;
2497ec38b344SPoul-Henning Kamp 			}
249889f6b863SAttilio Rao 			VM_OBJECT_WUNLOCK(object);
2499ec38b344SPoul-Henning Kamp 		}
2500ec38b344SPoul-Henning Kamp 	}
2501ec38b344SPoul-Henning Kamp 	return (count);
2502ec38b344SPoul-Henning Kamp }
2503dee34ca4SPoul-Henning Kamp 
2504dee34ca4SPoul-Henning Kamp /*
2505dee34ca4SPoul-Henning Kamp  * GEOM backend
2506dee34ca4SPoul-Henning Kamp  *
2507dee34ca4SPoul-Henning Kamp  * Swapping onto disk devices.
2508dee34ca4SPoul-Henning Kamp  *
2509dee34ca4SPoul-Henning Kamp  */
2510dee34ca4SPoul-Henning Kamp 
25115721c9c7SPoul-Henning Kamp static g_orphan_t swapgeom_orphan;
25125721c9c7SPoul-Henning Kamp 
2513dee34ca4SPoul-Henning Kamp static struct g_class g_swap_class = {
2514dee34ca4SPoul-Henning Kamp 	.name = "SWAP",
25155721c9c7SPoul-Henning Kamp 	.version = G_VERSION,
25165721c9c7SPoul-Henning Kamp 	.orphan = swapgeom_orphan,
2517dee34ca4SPoul-Henning Kamp };
2518dee34ca4SPoul-Henning Kamp 
2519dee34ca4SPoul-Henning Kamp DECLARE_GEOM_CLASS(g_swap_class, g_class);
2520dee34ca4SPoul-Henning Kamp 
2521dee34ca4SPoul-Henning Kamp 
2522dee34ca4SPoul-Henning Kamp static void
25233398491bSAlexander Motin swapgeom_close_ev(void *arg, int flags)
25243398491bSAlexander Motin {
25253398491bSAlexander Motin 	struct g_consumer *cp;
25263398491bSAlexander Motin 
25273398491bSAlexander Motin 	cp = arg;
25283398491bSAlexander Motin 	g_access(cp, -1, -1, 0);
25293398491bSAlexander Motin 	g_detach(cp);
25303398491bSAlexander Motin 	g_destroy_consumer(cp);
25313398491bSAlexander Motin }
25323398491bSAlexander Motin 
25339e3e3fe5SWarner Losh /*
25349e3e3fe5SWarner Losh  * Add a reference to the g_consumer for an inflight transaction.
25359e3e3fe5SWarner Losh  */
25369e3e3fe5SWarner Losh static void
25379e3e3fe5SWarner Losh swapgeom_acquire(struct g_consumer *cp)
25389e3e3fe5SWarner Losh {
25399e3e3fe5SWarner Losh 
25409e3e3fe5SWarner Losh 	mtx_assert(&sw_dev_mtx, MA_OWNED);
25419e3e3fe5SWarner Losh 	cp->index++;
25429e3e3fe5SWarner Losh }
25439e3e3fe5SWarner Losh 
25449e3e3fe5SWarner Losh /*
25450c657d22SKonstantin Belousov  * Remove a reference from the g_consumer.  Post a close event if all
25460c657d22SKonstantin Belousov  * references go away, since the function might be called from the
25470c657d22SKonstantin Belousov  * biodone context.
25489e3e3fe5SWarner Losh  */
25499e3e3fe5SWarner Losh static void
25509e3e3fe5SWarner Losh swapgeom_release(struct g_consumer *cp, struct swdevt *sp)
25519e3e3fe5SWarner Losh {
25529e3e3fe5SWarner Losh 
25539e3e3fe5SWarner Losh 	mtx_assert(&sw_dev_mtx, MA_OWNED);
25549e3e3fe5SWarner Losh 	cp->index--;
25559e3e3fe5SWarner Losh 	if (cp->index == 0) {
25569e3e3fe5SWarner Losh 		if (g_post_event(swapgeom_close_ev, cp, M_NOWAIT, NULL) == 0)
25579e3e3fe5SWarner Losh 			sp->sw_id = NULL;
25589e3e3fe5SWarner Losh 	}
25599e3e3fe5SWarner Losh }
25609e3e3fe5SWarner Losh 
25613398491bSAlexander Motin static void
2562dee34ca4SPoul-Henning Kamp swapgeom_done(struct bio *bp2)
2563dee34ca4SPoul-Henning Kamp {
25643398491bSAlexander Motin 	struct swdevt *sp;
2565dee34ca4SPoul-Henning Kamp 	struct buf *bp;
25663398491bSAlexander Motin 	struct g_consumer *cp;
2567dee34ca4SPoul-Henning Kamp 
2568dee34ca4SPoul-Henning Kamp 	bp = bp2->bio_caller2;
25693398491bSAlexander Motin 	cp = bp2->bio_from;
2570c5d3d25eSPoul-Henning Kamp 	bp->b_ioflags = bp2->bio_flags;
2571dee34ca4SPoul-Henning Kamp 	if (bp2->bio_error)
2572dee34ca4SPoul-Henning Kamp 		bp->b_ioflags |= BIO_ERROR;
2573c5d3d25eSPoul-Henning Kamp 	bp->b_resid = bp->b_bcount - bp2->bio_completed;
2574c5d3d25eSPoul-Henning Kamp 	bp->b_error = bp2->bio_error;
2575dee34ca4SPoul-Henning Kamp 	bufdone(bp);
25763398491bSAlexander Motin 	sp = bp2->bio_caller1;
25779e3e3fe5SWarner Losh 	mtx_lock(&sw_dev_mtx);
25789e3e3fe5SWarner Losh 	swapgeom_release(cp, sp);
25793398491bSAlexander Motin 	mtx_unlock(&sw_dev_mtx);
2580dee34ca4SPoul-Henning Kamp 	g_destroy_bio(bp2);
2581dee34ca4SPoul-Henning Kamp }
2582dee34ca4SPoul-Henning Kamp 
2583dee34ca4SPoul-Henning Kamp static void
2584dee34ca4SPoul-Henning Kamp swapgeom_strategy(struct buf *bp, struct swdevt *sp)
2585dee34ca4SPoul-Henning Kamp {
2586dee34ca4SPoul-Henning Kamp 	struct bio *bio;
2587dee34ca4SPoul-Henning Kamp 	struct g_consumer *cp;
2588dee34ca4SPoul-Henning Kamp 
25893398491bSAlexander Motin 	mtx_lock(&sw_dev_mtx);
2590dee34ca4SPoul-Henning Kamp 	cp = sp->sw_id;
2591dee34ca4SPoul-Henning Kamp 	if (cp == NULL) {
25923398491bSAlexander Motin 		mtx_unlock(&sw_dev_mtx);
2593dee34ca4SPoul-Henning Kamp 		bp->b_error = ENXIO;
2594dee34ca4SPoul-Henning Kamp 		bp->b_ioflags |= BIO_ERROR;
2595dee34ca4SPoul-Henning Kamp 		bufdone(bp);
2596dee34ca4SPoul-Henning Kamp 		return;
2597dee34ca4SPoul-Henning Kamp 	}
25989e3e3fe5SWarner Losh 	swapgeom_acquire(cp);
25993398491bSAlexander Motin 	mtx_unlock(&sw_dev_mtx);
260011041003SKonstantin Belousov 	if (bp->b_iocmd == BIO_WRITE)
260111041003SKonstantin Belousov 		bio = g_new_bio();
260211041003SKonstantin Belousov 	else
26034f8205e5SPoul-Henning Kamp 		bio = g_alloc_bio();
26044f8205e5SPoul-Henning Kamp 	if (bio == NULL) {
26059e3e3fe5SWarner Losh 		mtx_lock(&sw_dev_mtx);
26069e3e3fe5SWarner Losh 		swapgeom_release(cp, sp);
26079e3e3fe5SWarner Losh 		mtx_unlock(&sw_dev_mtx);
26083e5b6861SPoul-Henning Kamp 		bp->b_error = ENOMEM;
26093e5b6861SPoul-Henning Kamp 		bp->b_ioflags |= BIO_ERROR;
26103e5b6861SPoul-Henning Kamp 		bufdone(bp);
26113e5b6861SPoul-Henning Kamp 		return;
26123e5b6861SPoul-Henning Kamp 	}
261311041003SKonstantin Belousov 
26143398491bSAlexander Motin 	bio->bio_caller1 = sp;
2615dee34ca4SPoul-Henning Kamp 	bio->bio_caller2 = bp;
2616c5d3d25eSPoul-Henning Kamp 	bio->bio_cmd = bp->b_iocmd;
2617dee34ca4SPoul-Henning Kamp 	bio->bio_offset = (bp->b_blkno - sp->sw_first) * PAGE_SIZE;
2618dee34ca4SPoul-Henning Kamp 	bio->bio_length = bp->b_bcount;
2619dee34ca4SPoul-Henning Kamp 	bio->bio_done = swapgeom_done;
2620fade8dd7SJeff Roberson 	if (!buf_mapped(bp)) {
26212cc718a1SKonstantin Belousov 		bio->bio_ma = bp->b_pages;
26222cc718a1SKonstantin Belousov 		bio->bio_data = unmapped_buf;
26232cc718a1SKonstantin Belousov 		bio->bio_ma_offset = (vm_offset_t)bp->b_offset & PAGE_MASK;
26242cc718a1SKonstantin Belousov 		bio->bio_ma_n = bp->b_npages;
26252cc718a1SKonstantin Belousov 		bio->bio_flags |= BIO_UNMAPPED;
26262cc718a1SKonstantin Belousov 	} else {
26272cc718a1SKonstantin Belousov 		bio->bio_data = bp->b_data;
26282cc718a1SKonstantin Belousov 		bio->bio_ma = NULL;
26292cc718a1SKonstantin Belousov 	}
2630dee34ca4SPoul-Henning Kamp 	g_io_request(bio, cp);
2631dee34ca4SPoul-Henning Kamp 	return;
2632dee34ca4SPoul-Henning Kamp }
2633dee34ca4SPoul-Henning Kamp 
2634dee34ca4SPoul-Henning Kamp static void
2635dee34ca4SPoul-Henning Kamp swapgeom_orphan(struct g_consumer *cp)
2636dee34ca4SPoul-Henning Kamp {
2637dee34ca4SPoul-Henning Kamp 	struct swdevt *sp;
26383398491bSAlexander Motin 	int destroy;
2639dee34ca4SPoul-Henning Kamp 
2640dee34ca4SPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
26413398491bSAlexander Motin 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
26423398491bSAlexander Motin 		if (sp->sw_id == cp) {
26438f12d83aSAlexander Motin 			sp->sw_flags |= SW_CLOSING;
26443398491bSAlexander Motin 			break;
2645dee34ca4SPoul-Henning Kamp 		}
26463398491bSAlexander Motin 	}
26479e3e3fe5SWarner Losh 	/*
26489e3e3fe5SWarner Losh 	 * Drop reference we were created with. Do directly since we're in a
26499e3e3fe5SWarner Losh 	 * special context where we don't have to queue the call to
26509e3e3fe5SWarner Losh 	 * swapgeom_close_ev().
26519e3e3fe5SWarner Losh 	 */
26529e3e3fe5SWarner Losh 	cp->index--;
26533398491bSAlexander Motin 	destroy = ((sp != NULL) && (cp->index == 0));
26543398491bSAlexander Motin 	if (destroy)
26553398491bSAlexander Motin 		sp->sw_id = NULL;
26563398491bSAlexander Motin 	mtx_unlock(&sw_dev_mtx);
26573398491bSAlexander Motin 	if (destroy)
26583398491bSAlexander Motin 		swapgeom_close_ev(cp, 0);
2659dee34ca4SPoul-Henning Kamp }
2660dee34ca4SPoul-Henning Kamp 
2661dee34ca4SPoul-Henning Kamp static void
2662dee34ca4SPoul-Henning Kamp swapgeom_close(struct thread *td, struct swdevt *sw)
2663dee34ca4SPoul-Henning Kamp {
26643398491bSAlexander Motin 	struct g_consumer *cp;
2665dee34ca4SPoul-Henning Kamp 
26663398491bSAlexander Motin 	mtx_lock(&sw_dev_mtx);
26673398491bSAlexander Motin 	cp = sw->sw_id;
26683398491bSAlexander Motin 	sw->sw_id = NULL;
26693398491bSAlexander Motin 	mtx_unlock(&sw_dev_mtx);
26700c657d22SKonstantin Belousov 
26710c657d22SKonstantin Belousov 	/*
26720c657d22SKonstantin Belousov 	 * swapgeom_close() may be called from the biodone context,
26730c657d22SKonstantin Belousov 	 * where we cannot perform topology changes.  Delegate the
26740c657d22SKonstantin Belousov 	 * work to the events thread.
26750c657d22SKonstantin Belousov 	 */
26763398491bSAlexander Motin 	if (cp != NULL)
26773398491bSAlexander Motin 		g_waitfor_event(swapgeom_close_ev, cp, M_WAITOK, NULL);
2678dee34ca4SPoul-Henning Kamp }
2679dee34ca4SPoul-Henning Kamp 
268088ad2d7bSKonstantin Belousov static int
268188ad2d7bSKonstantin Belousov swapongeom_locked(struct cdev *dev, struct vnode *vp)
2682dee34ca4SPoul-Henning Kamp {
2683dee34ca4SPoul-Henning Kamp 	struct g_provider *pp;
2684dee34ca4SPoul-Henning Kamp 	struct g_consumer *cp;
2685dee34ca4SPoul-Henning Kamp 	static struct g_geom *gp;
2686dee34ca4SPoul-Henning Kamp 	struct swdevt *sp;
2687dee34ca4SPoul-Henning Kamp 	u_long nblks;
2688dee34ca4SPoul-Henning Kamp 	int error;
2689dee34ca4SPoul-Henning Kamp 
269088ad2d7bSKonstantin Belousov 	pp = g_dev_getprovider(dev);
269188ad2d7bSKonstantin Belousov 	if (pp == NULL)
269288ad2d7bSKonstantin Belousov 		return (ENODEV);
2693dee34ca4SPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
2694dee34ca4SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2695dee34ca4SPoul-Henning Kamp 		cp = sp->sw_id;
2696dee34ca4SPoul-Henning Kamp 		if (cp != NULL && cp->provider == pp) {
2697dee34ca4SPoul-Henning Kamp 			mtx_unlock(&sw_dev_mtx);
269888ad2d7bSKonstantin Belousov 			return (EBUSY);
2699dee34ca4SPoul-Henning Kamp 		}
2700dee34ca4SPoul-Henning Kamp 	}
2701dee34ca4SPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
27025721c9c7SPoul-Henning Kamp 	if (gp == NULL)
270302c62349SJaakko Heinonen 		gp = g_new_geomf(&g_swap_class, "swap");
2704dee34ca4SPoul-Henning Kamp 	cp = g_new_consumer(gp);
27059e3e3fe5SWarner Losh 	cp->index = 1;	/* Number of active I/Os, plus one for being active. */
27069e3e3fe5SWarner Losh 	cp->flags |=  G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
2707dee34ca4SPoul-Henning Kamp 	g_attach(cp, pp);
2708afeb65e6SPoul-Henning Kamp 	/*
2709afeb65e6SPoul-Henning Kamp 	 * XXX: Every time you think you can improve the margin for
2710afeb65e6SPoul-Henning Kamp 	 * footshooting, somebody depends on the ability to do so:
2711afeb65e6SPoul-Henning Kamp 	 * savecore(8) wants to write to our swapdev so we cannot
2712afeb65e6SPoul-Henning Kamp 	 * set an exclusive count :-(
2713afeb65e6SPoul-Henning Kamp 	 */
2714d2bae332SPoul-Henning Kamp 	error = g_access(cp, 1, 1, 0);
271588ad2d7bSKonstantin Belousov 	if (error != 0) {
2716dee34ca4SPoul-Henning Kamp 		g_detach(cp);
2717dee34ca4SPoul-Henning Kamp 		g_destroy_consumer(cp);
271888ad2d7bSKonstantin Belousov 		return (error);
2719dee34ca4SPoul-Henning Kamp 	}
2720dee34ca4SPoul-Henning Kamp 	nblks = pp->mediasize / DEV_BSIZE;
272188ad2d7bSKonstantin Belousov 	swaponsomething(vp, cp, nblks, swapgeom_strategy,
272288ad2d7bSKonstantin Belousov 	    swapgeom_close, dev2udev(dev),
27232cc718a1SKonstantin Belousov 	    (pp->flags & G_PF_ACCEPT_UNMAPPED) != 0 ? SW_UNMAPPED : 0);
272488ad2d7bSKonstantin Belousov 	return (0);
2725dee34ca4SPoul-Henning Kamp }
2726dee34ca4SPoul-Henning Kamp 
2727dee34ca4SPoul-Henning Kamp static int
272888ad2d7bSKonstantin Belousov swapongeom(struct vnode *vp)
2729dee34ca4SPoul-Henning Kamp {
2730dee34ca4SPoul-Henning Kamp 	int error;
2731dee34ca4SPoul-Henning Kamp 
2732cb05b60aSAttilio Rao 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
273388ad2d7bSKonstantin Belousov 	if (vp->v_type != VCHR || (vp->v_iflag & VI_DOOMED) != 0) {
273488ad2d7bSKonstantin Belousov 		error = ENOENT;
273588ad2d7bSKonstantin Belousov 	} else {
273688ad2d7bSKonstantin Belousov 		g_topology_lock();
273788ad2d7bSKonstantin Belousov 		error = swapongeom_locked(vp->v_rdev, vp);
273888ad2d7bSKonstantin Belousov 		g_topology_unlock();
273988ad2d7bSKonstantin Belousov 	}
274022db15c0SAttilio Rao 	VOP_UNLOCK(vp, 0);
2741dee34ca4SPoul-Henning Kamp 	return (error);
2742dee34ca4SPoul-Henning Kamp }
2743dee34ca4SPoul-Henning Kamp 
2744dee34ca4SPoul-Henning Kamp /*
2745dee34ca4SPoul-Henning Kamp  * VNODE backend
2746dee34ca4SPoul-Henning Kamp  *
2747dee34ca4SPoul-Henning Kamp  * This is used mainly for network filesystem (read: probably only tested
2748dee34ca4SPoul-Henning Kamp  * with NFS) swapfiles.
2749dee34ca4SPoul-Henning Kamp  *
2750dee34ca4SPoul-Henning Kamp  */
2751dee34ca4SPoul-Henning Kamp 
2752dee34ca4SPoul-Henning Kamp static void
2753dee34ca4SPoul-Henning Kamp swapdev_strategy(struct buf *bp, struct swdevt *sp)
2754dee34ca4SPoul-Henning Kamp {
2755494eb176SPoul-Henning Kamp 	struct vnode *vp2;
2756dee34ca4SPoul-Henning Kamp 
2757dee34ca4SPoul-Henning Kamp 	bp->b_blkno = ctodb(bp->b_blkno - sp->sw_first);
2758dee34ca4SPoul-Henning Kamp 
2759dee34ca4SPoul-Henning Kamp 	vp2 = sp->sw_id;
2760dee34ca4SPoul-Henning Kamp 	vhold(vp2);
2761dee34ca4SPoul-Henning Kamp 	if (bp->b_iocmd == BIO_WRITE) {
27623cfc7651SOlivier Houchard 		if (bp->b_bufobj)
2763494eb176SPoul-Henning Kamp 			bufobj_wdrop(bp->b_bufobj);
2764a76d8f4eSPoul-Henning Kamp 		bufobj_wref(&vp2->v_bufobj);
2765dee34ca4SPoul-Henning Kamp 	}
27663cfc7651SOlivier Houchard 	if (bp->b_bufobj != &vp2->v_bufobj)
27673cfc7651SOlivier Houchard 		bp->b_bufobj = &vp2->v_bufobj;
2768dee34ca4SPoul-Henning Kamp 	bp->b_vp = vp2;
27692c18019fSPoul-Henning Kamp 	bp->b_iooffset = dbtob(bp->b_blkno);
2770b792bebeSPoul-Henning Kamp 	bstrategy(bp);
2771dee34ca4SPoul-Henning Kamp 	return;
2772dee34ca4SPoul-Henning Kamp }
2773dee34ca4SPoul-Henning Kamp 
2774dee34ca4SPoul-Henning Kamp static void
2775dee34ca4SPoul-Henning Kamp swapdev_close(struct thread *td, struct swdevt *sp)
2776dee34ca4SPoul-Henning Kamp {
2777dee34ca4SPoul-Henning Kamp 
2778dee34ca4SPoul-Henning Kamp 	VOP_CLOSE(sp->sw_vp, FREAD | FWRITE, td->td_ucred, td);
2779dee34ca4SPoul-Henning Kamp 	vrele(sp->sw_vp);
2780dee34ca4SPoul-Henning Kamp }
2781dee34ca4SPoul-Henning Kamp 
2782dee34ca4SPoul-Henning Kamp 
2783dee34ca4SPoul-Henning Kamp static int
2784dee34ca4SPoul-Henning Kamp swaponvp(struct thread *td, struct vnode *vp, u_long nblks)
2785dee34ca4SPoul-Henning Kamp {
2786dee34ca4SPoul-Henning Kamp 	struct swdevt *sp;
2787dee34ca4SPoul-Henning Kamp 	int error;
2788dee34ca4SPoul-Henning Kamp 
2789dee34ca4SPoul-Henning Kamp 	if (nblks == 0)
2790dee34ca4SPoul-Henning Kamp 		return (ENXIO);
2791dee34ca4SPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
2792dee34ca4SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2793dee34ca4SPoul-Henning Kamp 		if (sp->sw_id == vp) {
2794dee34ca4SPoul-Henning Kamp 			mtx_unlock(&sw_dev_mtx);
2795dee34ca4SPoul-Henning Kamp 			return (EBUSY);
2796dee34ca4SPoul-Henning Kamp 		}
2797dee34ca4SPoul-Henning Kamp 	}
2798dee34ca4SPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
2799dee34ca4SPoul-Henning Kamp 
2800cb05b60aSAttilio Rao 	(void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2801dee34ca4SPoul-Henning Kamp #ifdef MAC
280230d239bcSRobert Watson 	error = mac_system_check_swapon(td->td_ucred, vp);
2803dee34ca4SPoul-Henning Kamp 	if (error == 0)
2804dee34ca4SPoul-Henning Kamp #endif
28059e223287SKonstantin Belousov 		error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, NULL);
280622db15c0SAttilio Rao 	(void) VOP_UNLOCK(vp, 0);
2807dee34ca4SPoul-Henning Kamp 	if (error)
2808dee34ca4SPoul-Henning Kamp 		return (error);
2809dee34ca4SPoul-Henning Kamp 
2810dee34ca4SPoul-Henning Kamp 	swaponsomething(vp, vp, nblks, swapdev_strategy, swapdev_close,
28112cc718a1SKonstantin Belousov 	    NODEV, 0);
2812dee34ca4SPoul-Henning Kamp 	return (0);
2813dee34ca4SPoul-Henning Kamp }
281489c241d1SGleb Smirnoff 
281589c241d1SGleb Smirnoff static int
281689c241d1SGleb Smirnoff sysctl_swap_async_max(SYSCTL_HANDLER_ARGS)
281789c241d1SGleb Smirnoff {
281889c241d1SGleb Smirnoff 	int error, new, n;
281989c241d1SGleb Smirnoff 
282089c241d1SGleb Smirnoff 	new = nsw_wcount_async_max;
282189c241d1SGleb Smirnoff 	error = sysctl_handle_int(oidp, &new, 0, req);
282289c241d1SGleb Smirnoff 	if (error != 0 || req->newptr == NULL)
282389c241d1SGleb Smirnoff 		return (error);
282489c241d1SGleb Smirnoff 
282589c241d1SGleb Smirnoff 	if (new > nswbuf / 2 || new < 1)
282689c241d1SGleb Smirnoff 		return (EINVAL);
282789c241d1SGleb Smirnoff 
282889c241d1SGleb Smirnoff 	mtx_lock(&pbuf_mtx);
282989c241d1SGleb Smirnoff 	while (nsw_wcount_async_max != new) {
283089c241d1SGleb Smirnoff 		/*
283189c241d1SGleb Smirnoff 		 * Adjust difference.  If the current async count is too low,
283289c241d1SGleb Smirnoff 		 * we will need to sqeeze our update slowly in.  Sleep with a
283389c241d1SGleb Smirnoff 		 * higher priority than getpbuf() to finish faster.
283489c241d1SGleb Smirnoff 		 */
283589c241d1SGleb Smirnoff 		n = new - nsw_wcount_async_max;
283689c241d1SGleb Smirnoff 		if (nsw_wcount_async + n >= 0) {
283789c241d1SGleb Smirnoff 			nsw_wcount_async += n;
283889c241d1SGleb Smirnoff 			nsw_wcount_async_max += n;
283989c241d1SGleb Smirnoff 			wakeup(&nsw_wcount_async);
284089c241d1SGleb Smirnoff 		} else {
284189c241d1SGleb Smirnoff 			nsw_wcount_async_max -= nsw_wcount_async;
284289c241d1SGleb Smirnoff 			nsw_wcount_async = 0;
284389c241d1SGleb Smirnoff 			msleep(&nsw_wcount_async, &pbuf_mtx, PSWP,
284489c241d1SGleb Smirnoff 			    "swpsysctl", 0);
284589c241d1SGleb Smirnoff 		}
284689c241d1SGleb Smirnoff 	}
284789c241d1SGleb Smirnoff 	mtx_unlock(&pbuf_mtx);
284889c241d1SGleb Smirnoff 
284989c241d1SGleb Smirnoff 	return (0);
285089c241d1SGleb Smirnoff }
2851