xref: /freebsd/sys/vm/swap_pager.c (revision 1ba5ad4210627e14e9b65653121b97820a812029)
160727d8bSWarner Losh /*-
21c7c3c6aSMatthew Dillon  * Copyright (c) 1998 Matthew Dillon,
326f9a767SRodney W. Grimes  * Copyright (c) 1994 John S. Dyson
4df8bae1dSRodney W. Grimes  * Copyright (c) 1990 University of Utah.
5e9c0cc15SPoul-Henning Kamp  * Copyright (c) 1982, 1986, 1989, 1993
6df8bae1dSRodney W. Grimes  *	The Regents of the University of California.  All rights reserved.
7df8bae1dSRodney W. Grimes  *
8df8bae1dSRodney W. Grimes  * This code is derived from software contributed to Berkeley by
9df8bae1dSRodney W. Grimes  * the Systems Programming Group of the University of Utah Computer
10df8bae1dSRodney W. Grimes  * Science Department.
11df8bae1dSRodney W. Grimes  *
12df8bae1dSRodney W. Grimes  * Redistribution and use in source and binary forms, with or without
13df8bae1dSRodney W. Grimes  * modification, are permitted provided that the following conditions
14df8bae1dSRodney W. Grimes  * are met:
15df8bae1dSRodney W. Grimes  * 1. Redistributions of source code must retain the above copyright
16df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer.
17df8bae1dSRodney W. Grimes  * 2. Redistributions in binary form must reproduce the above copyright
18df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer in the
19df8bae1dSRodney W. Grimes  *    documentation and/or other materials provided with the distribution.
20df8bae1dSRodney W. Grimes  * 3. All advertising materials mentioning features or use of this software
215929bcfaSPhilippe Charnier  *    must display the following acknowledgement:
22df8bae1dSRodney W. Grimes  *	This product includes software developed by the University of
23df8bae1dSRodney W. Grimes  *	California, Berkeley and its contributors.
24df8bae1dSRodney W. Grimes  * 4. Neither the name of the University nor the names of its contributors
25df8bae1dSRodney W. Grimes  *    may be used to endorse or promote products derived from this software
26df8bae1dSRodney W. Grimes  *    without specific prior written permission.
27df8bae1dSRodney W. Grimes  *
28df8bae1dSRodney W. Grimes  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
29df8bae1dSRodney W. Grimes  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30df8bae1dSRodney W. Grimes  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31df8bae1dSRodney W. Grimes  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
32df8bae1dSRodney W. Grimes  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33df8bae1dSRodney W. Grimes  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34df8bae1dSRodney W. Grimes  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35df8bae1dSRodney W. Grimes  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36df8bae1dSRodney W. Grimes  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37df8bae1dSRodney W. Grimes  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38df8bae1dSRodney W. Grimes  * SUCH DAMAGE.
39df8bae1dSRodney W. Grimes  *
401c7c3c6aSMatthew Dillon  *				New Swap System
411c7c3c6aSMatthew Dillon  *				Matthew Dillon
421c7c3c6aSMatthew Dillon  *
431c7c3c6aSMatthew Dillon  * Radix Bitmap 'blists'.
441c7c3c6aSMatthew Dillon  *
451c7c3c6aSMatthew Dillon  *	- The new swapper uses the new radix bitmap code.  This should scale
461c7c3c6aSMatthew Dillon  *	  to arbitrarily small or arbitrarily large swap spaces and an almost
471c7c3c6aSMatthew Dillon  *	  arbitrary degree of fragmentation.
481c7c3c6aSMatthew Dillon  *
491c7c3c6aSMatthew Dillon  * Features:
501c7c3c6aSMatthew Dillon  *
511c7c3c6aSMatthew Dillon  *	- on the fly reallocation of swap during putpages.  The new system
521c7c3c6aSMatthew Dillon  *	  does not try to keep previously allocated swap blocks for dirty
531c7c3c6aSMatthew Dillon  *	  pages.
541c7c3c6aSMatthew Dillon  *
551c7c3c6aSMatthew Dillon  *	- on the fly deallocation of swap
561c7c3c6aSMatthew Dillon  *
571c7c3c6aSMatthew Dillon  *	- No more garbage collection required.  Unnecessarily allocated swap
581c7c3c6aSMatthew Dillon  *	  blocks only exist for dirty vm_page_t's now and these are already
591c7c3c6aSMatthew Dillon  *	  cycled (in a high-load system) by the pager.  We also do on-the-fly
601c7c3c6aSMatthew Dillon  *	  removal of invalidated swap blocks when a page is destroyed
611c7c3c6aSMatthew Dillon  *	  or renamed.
621c7c3c6aSMatthew Dillon  *
63df8bae1dSRodney W. Grimes  * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
64df8bae1dSRodney W. Grimes  *
65df8bae1dSRodney W. Grimes  *	@(#)swap_pager.c	8.9 (Berkeley) 3/21/94
66e9c0cc15SPoul-Henning Kamp  *	@(#)vm_swap.c	8.5 (Berkeley) 2/17/94
67df8bae1dSRodney W. Grimes  */
68df8bae1dSRodney W. Grimes 
69874651b1SDavid E. O'Brien #include <sys/cdefs.h>
70874651b1SDavid E. O'Brien __FBSDID("$FreeBSD$");
71874651b1SDavid E. O'Brien 
72e9c0cc15SPoul-Henning Kamp #include "opt_swap.h"
73e9c0cc15SPoul-Henning Kamp #include "opt_vm.h"
74e9c0cc15SPoul-Henning Kamp 
75df8bae1dSRodney W. Grimes #include <sys/param.h>
76df8bae1dSRodney W. Grimes #include <sys/systm.h>
77af647ddeSBruce Evans #include <sys/conf.h>
7864abb5a5SDavid Greenman #include <sys/kernel.h>
79acd3428bSRobert Watson #include <sys/priv.h>
80df8bae1dSRodney W. Grimes #include <sys/proc.h>
819626b608SPoul-Henning Kamp #include <sys/bio.h>
82df8bae1dSRodney W. Grimes #include <sys/buf.h>
83e9c0cc15SPoul-Henning Kamp #include <sys/disk.h>
84e9c0cc15SPoul-Henning Kamp #include <sys/fcntl.h>
85e9c0cc15SPoul-Henning Kamp #include <sys/mount.h>
86e9c0cc15SPoul-Henning Kamp #include <sys/namei.h>
87df8bae1dSRodney W. Grimes #include <sys/vnode.h>
88df8bae1dSRodney W. Grimes #include <sys/malloc.h>
89*1ba5ad42SEdward Tomasz Napierala #include <sys/racct.h>
903364c323SKonstantin Belousov #include <sys/resource.h>
913364c323SKonstantin Belousov #include <sys/resourcevar.h>
92327f4e83SMatthew Dillon #include <sys/sysctl.h>
93e9c0cc15SPoul-Henning Kamp #include <sys/sysproto.h>
941c7c3c6aSMatthew Dillon #include <sys/blist.h>
951c7c3c6aSMatthew Dillon #include <sys/lock.h>
960cddd8f0SMatthew Dillon #include <sys/sx.h>
97936524aaSMatthew Dillon #include <sys/vmmeter.h>
98df8bae1dSRodney W. Grimes 
99aed55708SRobert Watson #include <security/mac/mac_framework.h>
100aed55708SRobert Watson 
101df8bae1dSRodney W. Grimes #include <vm/vm.h>
10221cd6e62SSeigo Tanimura #include <vm/pmap.h>
10321cd6e62SSeigo Tanimura #include <vm/vm_map.h>
10421cd6e62SSeigo Tanimura #include <vm/vm_kern.h>
105efeaf95aSDavid Greenman #include <vm/vm_object.h>
106df8bae1dSRodney W. Grimes #include <vm/vm_page.h>
107efeaf95aSDavid Greenman #include <vm/vm_pager.h>
108df8bae1dSRodney W. Grimes #include <vm/vm_pageout.h>
109e9c0cc15SPoul-Henning Kamp #include <vm/vm_param.h>
110df8bae1dSRodney W. Grimes #include <vm/swap_pager.h>
111efeaf95aSDavid Greenman #include <vm/vm_extern.h>
112670d17b5SJeff Roberson #include <vm/uma.h>
113df8bae1dSRodney W. Grimes 
114dee34ca4SPoul-Henning Kamp #include <geom/geom.h>
115dee34ca4SPoul-Henning Kamp 
116ec38b344SPoul-Henning Kamp /*
117ec38b344SPoul-Henning Kamp  * SWB_NPAGES must be a power of 2.  It may be set to 1, 2, 4, 8, or 16
118ec38b344SPoul-Henning Kamp  * pages per allocation.  We recommend you stick with the default of 8.
119ec38b344SPoul-Henning Kamp  * The 16-page limit is due to the radix code (kern/subr_blist.c).
120ec38b344SPoul-Henning Kamp  */
121ec38b344SPoul-Henning Kamp #ifndef MAX_PAGEOUT_CLUSTER
122ec38b344SPoul-Henning Kamp #define MAX_PAGEOUT_CLUSTER 16
123ec38b344SPoul-Henning Kamp #endif
124ec38b344SPoul-Henning Kamp 
125ec38b344SPoul-Henning Kamp #if !defined(SWB_NPAGES)
126ec38b344SPoul-Henning Kamp #define SWB_NPAGES	MAX_PAGEOUT_CLUSTER
127ec38b344SPoul-Henning Kamp #endif
128ec38b344SPoul-Henning Kamp 
129ec38b344SPoul-Henning Kamp /*
130ec38b344SPoul-Henning Kamp  * Piecemeal swap metadata structure.  Swap is stored in a radix tree.
131ec38b344SPoul-Henning Kamp  *
132ec38b344SPoul-Henning Kamp  * If SWB_NPAGES is 8 and sizeof(char *) == sizeof(daddr_t), our radix
133ec38b344SPoul-Henning Kamp  * is basically 8.  Assuming PAGE_SIZE == 4096, one tree level represents
134ec38b344SPoul-Henning Kamp  * 32K worth of data, two levels represent 256K, three levels represent
135ec38b344SPoul-Henning Kamp  * 2 MBytes.   This is acceptable.
136ec38b344SPoul-Henning Kamp  *
137ec38b344SPoul-Henning Kamp  * Overall memory utilization is about the same as the old swap structure.
138ec38b344SPoul-Henning Kamp  */
139ec38b344SPoul-Henning Kamp #define SWCORRECT(n) (sizeof(void *) * (n) / sizeof(daddr_t))
140ec38b344SPoul-Henning Kamp #define SWAP_META_PAGES		(SWB_NPAGES * 2)
141ec38b344SPoul-Henning Kamp #define SWAP_META_MASK		(SWAP_META_PAGES - 1)
142ec38b344SPoul-Henning Kamp 
143e9c0cc15SPoul-Henning Kamp struct swblock {
144e9c0cc15SPoul-Henning Kamp 	struct swblock	*swb_hnext;
145e9c0cc15SPoul-Henning Kamp 	vm_object_t	swb_object;
146e9c0cc15SPoul-Henning Kamp 	vm_pindex_t	swb_index;
147e9c0cc15SPoul-Henning Kamp 	int		swb_count;
148e9c0cc15SPoul-Henning Kamp 	daddr_t		swb_pages[SWAP_META_PAGES];
149e9c0cc15SPoul-Henning Kamp };
150e9c0cc15SPoul-Henning Kamp 
1512c4992dbSAlan Cox static MALLOC_DEFINE(M_VMPGDATA, "vm_pgdata", "swap pager private data");
15220da9c2eSPoul-Henning Kamp static struct mtx sw_dev_mtx;
1538f60c087SPoul-Henning Kamp static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq);
1548f60c087SPoul-Henning Kamp static struct swdevt *swdevhd;	/* Allocate from here next */
1558f60c087SPoul-Henning Kamp static int nswapdev;		/* Number of swap devices */
1568f60c087SPoul-Henning Kamp int swap_pager_avail;
157e9c0cc15SPoul-Henning Kamp static int swdev_syscall_active = 0; /* serialize swap(on|off) */
158e9c0cc15SPoul-Henning Kamp 
1593364c323SKonstantin Belousov static vm_ooffset_t swap_total;
1608a9c731fSIvan Voras SYSCTL_QUAD(_vm, OID_AUTO, swap_total, CTLFLAG_RD, &swap_total, 0,
1618a9c731fSIvan Voras     "Total amount of available swap storage.");
1623364c323SKonstantin Belousov static vm_ooffset_t swap_reserved;
1638a9c731fSIvan Voras SYSCTL_QUAD(_vm, OID_AUTO, swap_reserved, CTLFLAG_RD, &swap_reserved, 0,
1648a9c731fSIvan Voras     "Amount of swap storage needed to back all allocated anonymous memory.");
1653364c323SKonstantin Belousov static int overcommit = 0;
1668a9c731fSIvan Voras SYSCTL_INT(_vm, OID_AUTO, overcommit, CTLFLAG_RW, &overcommit, 0,
1678a9c731fSIvan Voras     "Configure virtual memory overcommit behavior. See tuning(7) "
1688a9c731fSIvan Voras     "for details.");
1693364c323SKonstantin Belousov 
1703364c323SKonstantin Belousov /* bits from overcommit */
1713364c323SKonstantin Belousov #define	SWAP_RESERVE_FORCE_ON		(1 << 0)
1723364c323SKonstantin Belousov #define	SWAP_RESERVE_RLIMIT_ON		(1 << 1)
1733364c323SKonstantin Belousov #define	SWAP_RESERVE_ALLOW_NONWIRED	(1 << 2)
1743364c323SKonstantin Belousov 
1753364c323SKonstantin Belousov int
1763364c323SKonstantin Belousov swap_reserve(vm_ooffset_t incr)
1773364c323SKonstantin Belousov {
1783364c323SKonstantin Belousov 
179ef694c1aSEdward Tomasz Napierala 	return (swap_reserve_by_cred(incr, curthread->td_ucred));
1803364c323SKonstantin Belousov }
1813364c323SKonstantin Belousov 
1823364c323SKonstantin Belousov int
183ef694c1aSEdward Tomasz Napierala swap_reserve_by_cred(vm_ooffset_t incr, struct ucred *cred)
1843364c323SKonstantin Belousov {
1855c0e1c11SKonstantin Belousov 	vm_ooffset_t r, s;
1863364c323SKonstantin Belousov 	int res, error;
1873364c323SKonstantin Belousov 	static int curfail;
1883364c323SKonstantin Belousov 	static struct timeval lastfail;
189ef694c1aSEdward Tomasz Napierala 	struct uidinfo *uip;
190ef694c1aSEdward Tomasz Napierala 
191ef694c1aSEdward Tomasz Napierala 	uip = cred->cr_ruidinfo;
1923364c323SKonstantin Belousov 
1933364c323SKonstantin Belousov 	if (incr & PAGE_MASK)
1943364c323SKonstantin Belousov 		panic("swap_reserve: & PAGE_MASK");
1953364c323SKonstantin Belousov 
196*1ba5ad42SEdward Tomasz Napierala 	PROC_LOCK(curproc);
197*1ba5ad42SEdward Tomasz Napierala 	error = racct_add(curproc, RACCT_SWAP, incr);
198*1ba5ad42SEdward Tomasz Napierala 	PROC_UNLOCK(curproc);
199*1ba5ad42SEdward Tomasz Napierala 	if (error != 0)
200*1ba5ad42SEdward Tomasz Napierala 		return (0);
201*1ba5ad42SEdward Tomasz Napierala 
2023364c323SKonstantin Belousov 	res = 0;
2033364c323SKonstantin Belousov 	mtx_lock(&sw_dev_mtx);
2043364c323SKonstantin Belousov 	r = swap_reserved + incr;
2053364c323SKonstantin Belousov 	if (overcommit & SWAP_RESERVE_ALLOW_NONWIRED) {
2063364c323SKonstantin Belousov 		s = cnt.v_page_count - cnt.v_free_reserved - cnt.v_wire_count;
2073364c323SKonstantin Belousov 		s *= PAGE_SIZE;
2083364c323SKonstantin Belousov 	} else
2093364c323SKonstantin Belousov 		s = 0;
2103364c323SKonstantin Belousov 	s += swap_total;
2113364c323SKonstantin Belousov 	if ((overcommit & SWAP_RESERVE_FORCE_ON) == 0 || r <= s ||
2123364c323SKonstantin Belousov 	    (error = priv_check(curthread, PRIV_VM_SWAP_NOQUOTA)) == 0) {
2133364c323SKonstantin Belousov 		res = 1;
2143364c323SKonstantin Belousov 		swap_reserved = r;
2153364c323SKonstantin Belousov 	}
2163364c323SKonstantin Belousov 	mtx_unlock(&sw_dev_mtx);
2173364c323SKonstantin Belousov 
2183364c323SKonstantin Belousov 	if (res) {
2193364c323SKonstantin Belousov 		PROC_LOCK(curproc);
2203364c323SKonstantin Belousov 		UIDINFO_VMSIZE_LOCK(uip);
2215c0e1c11SKonstantin Belousov 		if ((overcommit & SWAP_RESERVE_RLIMIT_ON) != 0 &&
2225c0e1c11SKonstantin Belousov 		    uip->ui_vmsize + incr > lim_cur(curproc, RLIMIT_SWAP) &&
2235c0e1c11SKonstantin Belousov 		    priv_check(curthread, PRIV_VM_SWAP_NORLIMIT))
2243364c323SKonstantin Belousov 			res = 0;
2253364c323SKonstantin Belousov 		else
2263364c323SKonstantin Belousov 			uip->ui_vmsize += incr;
2273364c323SKonstantin Belousov 		UIDINFO_VMSIZE_UNLOCK(uip);
2283364c323SKonstantin Belousov 		PROC_UNLOCK(curproc);
2293364c323SKonstantin Belousov 		if (!res) {
2303364c323SKonstantin Belousov 			mtx_lock(&sw_dev_mtx);
2313364c323SKonstantin Belousov 			swap_reserved -= incr;
2323364c323SKonstantin Belousov 			mtx_unlock(&sw_dev_mtx);
2333364c323SKonstantin Belousov 		}
2343364c323SKonstantin Belousov 	}
2353364c323SKonstantin Belousov 	if (!res && ppsratecheck(&lastfail, &curfail, 1)) {
2363364c323SKonstantin Belousov 		printf("uid %d, pid %d: swap reservation for %jd bytes failed\n",
2373364c323SKonstantin Belousov 		    curproc->p_pid, uip->ui_uid, incr);
2383364c323SKonstantin Belousov 	}
2393364c323SKonstantin Belousov 
240*1ba5ad42SEdward Tomasz Napierala 	if (!res) {
241*1ba5ad42SEdward Tomasz Napierala 		PROC_LOCK(curproc);
242*1ba5ad42SEdward Tomasz Napierala 		racct_sub(curproc, RACCT_SWAP, incr);
243*1ba5ad42SEdward Tomasz Napierala 		PROC_UNLOCK(curproc);
244*1ba5ad42SEdward Tomasz Napierala 	}
245*1ba5ad42SEdward Tomasz Napierala 
2463364c323SKonstantin Belousov 	return (res);
2473364c323SKonstantin Belousov }
2483364c323SKonstantin Belousov 
2493364c323SKonstantin Belousov void
2503364c323SKonstantin Belousov swap_reserve_force(vm_ooffset_t incr)
2513364c323SKonstantin Belousov {
2523364c323SKonstantin Belousov 	struct uidinfo *uip;
2533364c323SKonstantin Belousov 
2543364c323SKonstantin Belousov 	mtx_lock(&sw_dev_mtx);
2553364c323SKonstantin Belousov 	swap_reserved += incr;
2563364c323SKonstantin Belousov 	mtx_unlock(&sw_dev_mtx);
2573364c323SKonstantin Belousov 
258*1ba5ad42SEdward Tomasz Napierala 	PROC_LOCK(curproc);
259*1ba5ad42SEdward Tomasz Napierala 	racct_add_force(curproc, RACCT_SWAP, incr);
260*1ba5ad42SEdward Tomasz Napierala 	PROC_UNLOCK(curproc);
261*1ba5ad42SEdward Tomasz Napierala 
2623364c323SKonstantin Belousov 	uip = curthread->td_ucred->cr_ruidinfo;
2633364c323SKonstantin Belousov 	PROC_LOCK(curproc);
2643364c323SKonstantin Belousov 	UIDINFO_VMSIZE_LOCK(uip);
2653364c323SKonstantin Belousov 	uip->ui_vmsize += incr;
2663364c323SKonstantin Belousov 	UIDINFO_VMSIZE_UNLOCK(uip);
2673364c323SKonstantin Belousov 	PROC_UNLOCK(curproc);
2683364c323SKonstantin Belousov }
2693364c323SKonstantin Belousov 
2703364c323SKonstantin Belousov void
2713364c323SKonstantin Belousov swap_release(vm_ooffset_t decr)
2723364c323SKonstantin Belousov {
273ef694c1aSEdward Tomasz Napierala 	struct ucred *cred;
2743364c323SKonstantin Belousov 
2753364c323SKonstantin Belousov 	PROC_LOCK(curproc);
276ef694c1aSEdward Tomasz Napierala 	cred = curthread->td_ucred;
277ef694c1aSEdward Tomasz Napierala 	swap_release_by_cred(decr, cred);
2783364c323SKonstantin Belousov 	PROC_UNLOCK(curproc);
2793364c323SKonstantin Belousov }
2803364c323SKonstantin Belousov 
2813364c323SKonstantin Belousov void
282ef694c1aSEdward Tomasz Napierala swap_release_by_cred(vm_ooffset_t decr, struct ucred *cred)
2833364c323SKonstantin Belousov {
284ef694c1aSEdward Tomasz Napierala  	struct uidinfo *uip;
285ef694c1aSEdward Tomasz Napierala 
286ef694c1aSEdward Tomasz Napierala 	uip = cred->cr_ruidinfo;
2873364c323SKonstantin Belousov 
2883364c323SKonstantin Belousov 	if (decr & PAGE_MASK)
2893364c323SKonstantin Belousov 		panic("swap_release: & PAGE_MASK");
2903364c323SKonstantin Belousov 
2913364c323SKonstantin Belousov 	mtx_lock(&sw_dev_mtx);
2923364c323SKonstantin Belousov 	if (swap_reserved < decr)
2933364c323SKonstantin Belousov 		panic("swap_reserved < decr");
2943364c323SKonstantin Belousov 	swap_reserved -= decr;
2953364c323SKonstantin Belousov 	mtx_unlock(&sw_dev_mtx);
2963364c323SKonstantin Belousov 
2973364c323SKonstantin Belousov 	UIDINFO_VMSIZE_LOCK(uip);
2983364c323SKonstantin Belousov 	if (uip->ui_vmsize < decr)
2993364c323SKonstantin Belousov 		printf("negative vmsize for uid = %d\n", uip->ui_uid);
3003364c323SKonstantin Belousov 	uip->ui_vmsize -= decr;
3013364c323SKonstantin Belousov 	UIDINFO_VMSIZE_UNLOCK(uip);
302*1ba5ad42SEdward Tomasz Napierala 
303*1ba5ad42SEdward Tomasz Napierala 	racct_sub_cred(cred, RACCT_SWAP, decr);
3043364c323SKonstantin Belousov }
3053364c323SKonstantin Belousov 
3064b03903aSPoul-Henning Kamp static void swapdev_strategy(struct buf *, struct swdevt *sw);
307e9c0cc15SPoul-Henning Kamp 
3081c7c3c6aSMatthew Dillon #define SWM_FREE	0x02	/* free, period			*/
3091c7c3c6aSMatthew Dillon #define SWM_POP		0x04	/* pop out			*/
31026f9a767SRodney W. Grimes 
3117dea2c2eSAlan Cox int swap_pager_full = 2;	/* swap space exhaustion (task killing) */
3127dea2c2eSAlan Cox static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/
3131c7c3c6aSMatthew Dillon static int nsw_rcount;		/* free read buffers			*/
314327f4e83SMatthew Dillon static int nsw_wcount_sync;	/* limit write buffers / synchronous	*/
315327f4e83SMatthew Dillon static int nsw_wcount_async;	/* limit write buffers / asynchronous	*/
316327f4e83SMatthew Dillon static int nsw_wcount_async_max;/* assigned maximum			*/
317327f4e83SMatthew Dillon static int nsw_cluster_max;	/* maximum VOP I/O allowed		*/
3181c7c3c6aSMatthew Dillon 
3191c7c3c6aSMatthew Dillon static struct swblock **swhash;
3201c7c3c6aSMatthew Dillon static int swhash_mask;
3217827d9b0SAlan Cox static struct mtx swhash_mtx;
3227827d9b0SAlan Cox 
323327f4e83SMatthew Dillon static int swap_async_max = 4;	/* maximum in-progress async I/O's	*/
3240cddd8f0SMatthew Dillon static struct sx sw_alloc_sx;
325327f4e83SMatthew Dillon 
32624e7ab7cSPoul-Henning Kamp 
327327f4e83SMatthew Dillon SYSCTL_INT(_vm, OID_AUTO, swap_async_max,
328327f4e83SMatthew Dillon         CTLFLAG_RW, &swap_async_max, 0, "Maximum running async swap ops");
3291c7c3c6aSMatthew Dillon 
3301c7c3c6aSMatthew Dillon /*
3311c7c3c6aSMatthew Dillon  * "named" and "unnamed" anon region objects.  Try to reduce the overhead
3321c7c3c6aSMatthew Dillon  * of searching a named list by hashing it just a little.
3331c7c3c6aSMatthew Dillon  */
3341c7c3c6aSMatthew Dillon 
3351c7c3c6aSMatthew Dillon #define NOBJLISTS		8
3361c7c3c6aSMatthew Dillon 
3371c7c3c6aSMatthew Dillon #define NOBJLIST(handle)	\
338af647ddeSBruce Evans 	(&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)])
3391c7c3c6aSMatthew Dillon 
340a9fa2c05SAlfred Perlstein static struct mtx sw_alloc_mtx;	/* protect list manipulation */
3411c7c3c6aSMatthew Dillon static struct pagerlst	swap_pager_object_list[NOBJLISTS];
342e9c0cc15SPoul-Henning Kamp static uma_zone_t	swap_zone;
3435285558aSAlan Cox static struct vm_object	swap_zone_obj;
3441c7c3c6aSMatthew Dillon 
3451c7c3c6aSMatthew Dillon /*
3461c7c3c6aSMatthew Dillon  * pagerops for OBJT_SWAP - "swap pager".  Some ops are also global procedure
3471c7c3c6aSMatthew Dillon  * calls hooked from other parts of the VM system and do not appear here.
3481c7c3c6aSMatthew Dillon  * (see vm/swap_pager.h).
3491c7c3c6aSMatthew Dillon  */
350ff98689dSBruce Evans static vm_object_t
35111caded3SAlfred Perlstein 		swap_pager_alloc(void *handle, vm_ooffset_t size,
3523364c323SKonstantin Belousov 		    vm_prot_t prot, vm_ooffset_t offset, struct ucred *);
35311caded3SAlfred Perlstein static void	swap_pager_dealloc(vm_object_t object);
35411caded3SAlfred Perlstein static int	swap_pager_getpages(vm_object_t, vm_page_t *, int, int);
355751221fdSPoul-Henning Kamp static void	swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *);
3565ea4972cSAlan Cox static boolean_t
3575ea4972cSAlan Cox 		swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
35811caded3SAlfred Perlstein static void	swap_pager_init(void);
35911caded3SAlfred Perlstein static void	swap_pager_unswapped(vm_page_t);
360b3fed13eSDavid Schultz static void	swap_pager_swapoff(struct swdevt *sp);
361f708ef1bSPoul-Henning Kamp 
362df8bae1dSRodney W. Grimes struct pagerops swappagerops = {
363e04e4bacSPoul-Henning Kamp 	.pgo_init =	swap_pager_init,	/* early system initialization of pager	*/
364e04e4bacSPoul-Henning Kamp 	.pgo_alloc =	swap_pager_alloc,	/* allocate an OBJT_SWAP object		*/
365e04e4bacSPoul-Henning Kamp 	.pgo_dealloc =	swap_pager_dealloc,	/* deallocate an OBJT_SWAP object	*/
366e04e4bacSPoul-Henning Kamp 	.pgo_getpages =	swap_pager_getpages,	/* pagein				*/
367e04e4bacSPoul-Henning Kamp 	.pgo_putpages =	swap_pager_putpages,	/* pageout				*/
368e04e4bacSPoul-Henning Kamp 	.pgo_haspage =	swap_pager_haspage,	/* get backing store status for page	*/
369e04e4bacSPoul-Henning Kamp 	.pgo_pageunswapped = swap_pager_unswapped,	/* remove swap related to page		*/
370df8bae1dSRodney W. Grimes };
371df8bae1dSRodney W. Grimes 
3721c7c3c6aSMatthew Dillon /*
3731c7c3c6aSMatthew Dillon  * dmmax is in page-sized chunks with the new swap system.  It was
37464bcb9c8SMatthew Dillon  * dev-bsized chunks in the old.  dmmax is always a power of 2.
3751c7c3c6aSMatthew Dillon  *
3761c7c3c6aSMatthew Dillon  * swap_*() routines are externally accessible.  swp_*() routines are
3771c7c3c6aSMatthew Dillon  * internal.
3781c7c3c6aSMatthew Dillon  */
379751221fdSPoul-Henning Kamp static int dmmax;
380e9c0cc15SPoul-Henning Kamp static int nswap_lowat = 128;	/* in pages, swap_pager_almost_full warn */
381e9c0cc15SPoul-Henning Kamp static int nswap_hiwat = 512;	/* in pages, swap_pager_almost_full warn */
38226f9a767SRodney W. Grimes 
383cee313c4SRobert Watson SYSCTL_INT(_vm, OID_AUTO, dmmax,
384cee313c4SRobert Watson 	CTLFLAG_RD, &dmmax, 0, "Maximum size of a swap block");
385cee313c4SRobert Watson 
386a5edd34aSPoul-Henning Kamp static void	swp_sizecheck(void);
38711caded3SAlfred Perlstein static void	swp_pager_async_iodone(struct buf *bp);
388dee34ca4SPoul-Henning Kamp static int	swapongeom(struct thread *, struct vnode *);
38959efee01SPoul-Henning Kamp static int	swaponvp(struct thread *, struct vnode *, u_long);
39035918c55SChristian S.J. Peron static int	swapoff_one(struct swdevt *sp, struct ucred *cred);
39124a1cce3SDavid Greenman 
3921c7c3c6aSMatthew Dillon /*
3931c7c3c6aSMatthew Dillon  * Swap bitmap functions
3941c7c3c6aSMatthew Dillon  */
395a5edd34aSPoul-Henning Kamp static void	swp_pager_freeswapspace(daddr_t blk, int npages);
396a5edd34aSPoul-Henning Kamp static daddr_t	swp_pager_getswapspace(int npages);
3971c7c3c6aSMatthew Dillon 
3981c7c3c6aSMatthew Dillon /*
3991c7c3c6aSMatthew Dillon  * Metadata functions
4001c7c3c6aSMatthew Dillon  */
401a5edd34aSPoul-Henning Kamp static struct swblock **swp_pager_hash(vm_object_t object, vm_pindex_t index);
40211caded3SAlfred Perlstein static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
40311caded3SAlfred Perlstein static void swp_pager_meta_free(vm_object_t, vm_pindex_t, daddr_t);
40411caded3SAlfred Perlstein static void swp_pager_meta_free_all(vm_object_t);
40511caded3SAlfred Perlstein static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);
4061c7c3c6aSMatthew Dillon 
407e86a87e9SKonstantin Belousov static void
408e86a87e9SKonstantin Belousov swp_pager_free_nrpage(vm_page_t m)
409e86a87e9SKonstantin Belousov {
410e86a87e9SKonstantin Belousov 
4113c4a2440SAlan Cox 	vm_page_lock(m);
412e86a87e9SKonstantin Belousov 	if (m->wire_count == 0)
413e86a87e9SKonstantin Belousov 		vm_page_free(m);
4143c4a2440SAlan Cox 	vm_page_unlock(m);
415e86a87e9SKonstantin Belousov }
416e86a87e9SKonstantin Belousov 
4171c7c3c6aSMatthew Dillon /*
4181c7c3c6aSMatthew Dillon  * SWP_SIZECHECK() -	update swap_pager_full indication
4191c7c3c6aSMatthew Dillon  *
42020d3034fSMatthew Dillon  *	update the swap_pager_almost_full indication and warn when we are
42120d3034fSMatthew Dillon  *	about to run out of swap space, using lowat/hiwat hysteresis.
42220d3034fSMatthew Dillon  *
42320d3034fSMatthew Dillon  *	Clear swap_pager_full ( task killing ) indication when lowat is met.
4241c7c3c6aSMatthew Dillon  *
4251c7c3c6aSMatthew Dillon  *	No restrictions on call
4261c7c3c6aSMatthew Dillon  *	This routine may not block.
4271c7c3c6aSMatthew Dillon  *	This routine must be called at splvm()
4281c7c3c6aSMatthew Dillon  */
429a5edd34aSPoul-Henning Kamp static void
4302f249180SPoul-Henning Kamp swp_sizecheck(void)
4310d94caffSDavid Greenman {
43223955314SAlfred Perlstein 
4338f60c087SPoul-Henning Kamp 	if (swap_pager_avail < nswap_lowat) {
43420d3034fSMatthew Dillon 		if (swap_pager_almost_full == 0) {
4351af87c92SDavid Greenman 			printf("swap_pager: out of swap space\n");
43620d3034fSMatthew Dillon 			swap_pager_almost_full = 1;
4372b0d37a4SMatthew Dillon 		}
43820d3034fSMatthew Dillon 	} else {
43926f9a767SRodney W. Grimes 		swap_pager_full = 0;
4408f60c087SPoul-Henning Kamp 		if (swap_pager_avail > nswap_hiwat)
44120d3034fSMatthew Dillon 			swap_pager_almost_full = 0;
44226f9a767SRodney W. Grimes 	}
4431c7c3c6aSMatthew Dillon }
4441c7c3c6aSMatthew Dillon 
4451c7c3c6aSMatthew Dillon /*
446da5fd145SPeter Wemm  * SWP_PAGER_HASH() -	hash swap meta data
447da5fd145SPeter Wemm  *
448a5edd34aSPoul-Henning Kamp  *	This is an helper function which hashes the swapblk given
449da5fd145SPeter Wemm  *	the object and page index.  It returns a pointer to a pointer
450da5fd145SPeter Wemm  *	to the object, or a pointer to a NULL pointer if it could not
451da5fd145SPeter Wemm  *	find a swapblk.
452da5fd145SPeter Wemm  *
453da5fd145SPeter Wemm  *	This routine must be called at splvm().
454da5fd145SPeter Wemm  */
455a5edd34aSPoul-Henning Kamp static struct swblock **
456da5fd145SPeter Wemm swp_pager_hash(vm_object_t object, vm_pindex_t index)
457da5fd145SPeter Wemm {
458da5fd145SPeter Wemm 	struct swblock **pswap;
459da5fd145SPeter Wemm 	struct swblock *swap;
460da5fd145SPeter Wemm 
461da5fd145SPeter Wemm 	index &= ~(vm_pindex_t)SWAP_META_MASK;
462da5fd145SPeter Wemm 	pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask];
463da5fd145SPeter Wemm 	while ((swap = *pswap) != NULL) {
464da5fd145SPeter Wemm 		if (swap->swb_object == object &&
465da5fd145SPeter Wemm 		    swap->swb_index == index
466da5fd145SPeter Wemm 		) {
467da5fd145SPeter Wemm 			break;
468da5fd145SPeter Wemm 		}
469da5fd145SPeter Wemm 		pswap = &swap->swb_hnext;
470da5fd145SPeter Wemm 	}
471da5fd145SPeter Wemm 	return (pswap);
472da5fd145SPeter Wemm }
473da5fd145SPeter Wemm 
474da5fd145SPeter Wemm /*
4751c7c3c6aSMatthew Dillon  * SWAP_PAGER_INIT() -	initialize the swap pager!
4761c7c3c6aSMatthew Dillon  *
4771c7c3c6aSMatthew Dillon  *	Expected to be started from system init.  NOTE:  This code is run
4781c7c3c6aSMatthew Dillon  *	before much else so be careful what you depend on.  Most of the VM
4791c7c3c6aSMatthew Dillon  *	system has yet to be initialized at this point.
4801c7c3c6aSMatthew Dillon  */
481f5a12711SPoul-Henning Kamp static void
4822f249180SPoul-Henning Kamp swap_pager_init(void)
483df8bae1dSRodney W. Grimes {
4841c7c3c6aSMatthew Dillon 	/*
4851c7c3c6aSMatthew Dillon 	 * Initialize object lists
4861c7c3c6aSMatthew Dillon 	 */
4871c7c3c6aSMatthew Dillon 	int i;
4881c7c3c6aSMatthew Dillon 
4891c7c3c6aSMatthew Dillon 	for (i = 0; i < NOBJLISTS; ++i)
4901c7c3c6aSMatthew Dillon 		TAILQ_INIT(&swap_pager_object_list[i]);
4916008862bSJohn Baldwin 	mtx_init(&sw_alloc_mtx, "swap_pager list", NULL, MTX_DEF);
49220da9c2eSPoul-Henning Kamp 	mtx_init(&sw_dev_mtx, "swapdev", NULL, MTX_DEF);
493df8bae1dSRodney W. Grimes 
494df8bae1dSRodney W. Grimes 	/*
4951c7c3c6aSMatthew Dillon 	 * Device Stripe, in PAGE_SIZE'd blocks
496df8bae1dSRodney W. Grimes 	 */
4971c7c3c6aSMatthew Dillon 	dmmax = SWB_NPAGES * 2;
4981c7c3c6aSMatthew Dillon }
49926f9a767SRodney W. Grimes 
500df8bae1dSRodney W. Grimes /*
5011c7c3c6aSMatthew Dillon  * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process
5021c7c3c6aSMatthew Dillon  *
5031c7c3c6aSMatthew Dillon  *	Expected to be started from pageout process once, prior to entering
5041c7c3c6aSMatthew Dillon  *	its main loop.
505df8bae1dSRodney W. Grimes  */
50624a1cce3SDavid Greenman void
5072f249180SPoul-Henning Kamp swap_pager_swap_init(void)
508df8bae1dSRodney W. Grimes {
50921cd6e62SSeigo Tanimura 	int n, n2;
5100d94caffSDavid Greenman 
51126f9a767SRodney W. Grimes 	/*
5121c7c3c6aSMatthew Dillon 	 * Number of in-transit swap bp operations.  Don't
5131c7c3c6aSMatthew Dillon 	 * exhaust the pbufs completely.  Make sure we
5141c7c3c6aSMatthew Dillon 	 * initialize workable values (0 will work for hysteresis
5151c7c3c6aSMatthew Dillon 	 * but it isn't very efficient).
5161c7c3c6aSMatthew Dillon 	 *
517327f4e83SMatthew Dillon 	 * The nsw_cluster_max is constrained by the bp->b_pages[]
5181c7c3c6aSMatthew Dillon 	 * array (MAXPHYS/PAGE_SIZE) and our locally defined
5191c7c3c6aSMatthew Dillon 	 * MAX_PAGEOUT_CLUSTER.   Also be aware that swap ops are
5201c7c3c6aSMatthew Dillon 	 * constrained by the swap device interleave stripe size.
521327f4e83SMatthew Dillon 	 *
522327f4e83SMatthew Dillon 	 * Currently we hardwire nsw_wcount_async to 4.  This limit is
523327f4e83SMatthew Dillon 	 * designed to prevent other I/O from having high latencies due to
524327f4e83SMatthew Dillon 	 * our pageout I/O.  The value 4 works well for one or two active swap
525327f4e83SMatthew Dillon 	 * devices but is probably a little low if you have more.  Even so,
526327f4e83SMatthew Dillon 	 * a higher value would probably generate only a limited improvement
527327f4e83SMatthew Dillon 	 * with three or four active swap devices since the system does not
528327f4e83SMatthew Dillon 	 * typically have to pageout at extreme bandwidths.   We will want
529327f4e83SMatthew Dillon 	 * at least 2 per swap devices, and 4 is a pretty good value if you
530327f4e83SMatthew Dillon 	 * have one NFS swap device due to the command/ack latency over NFS.
531327f4e83SMatthew Dillon 	 * So it all works out pretty well.
53226f9a767SRodney W. Grimes 	 */
533ad3cce20SMatthew Dillon 	nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER);
534327f4e83SMatthew Dillon 
5356d541bf1SJohn Baldwin 	mtx_lock(&pbuf_mtx);
5361c7c3c6aSMatthew Dillon 	nsw_rcount = (nswbuf + 1) / 2;
537327f4e83SMatthew Dillon 	nsw_wcount_sync = (nswbuf + 3) / 4;
538327f4e83SMatthew Dillon 	nsw_wcount_async = 4;
539327f4e83SMatthew Dillon 	nsw_wcount_async_max = nsw_wcount_async;
5406d541bf1SJohn Baldwin 	mtx_unlock(&pbuf_mtx);
54124a1cce3SDavid Greenman 
5421c7c3c6aSMatthew Dillon 	/*
5431c7c3c6aSMatthew Dillon 	 * Initialize our zone.  Right now I'm just guessing on the number
5441c7c3c6aSMatthew Dillon 	 * we need based on the number of pages in the system.  Each swblock
5452f9e4e80SMatthew Dillon 	 * can hold 16 pages, so this is probably overkill.  This reservation
546ec61f55dSMatthew Dillon 	 * is typically limited to around 32MB by default.
5471c7c3c6aSMatthew Dillon 	 */
5482feb50bfSAttilio Rao 	n = cnt.v_page_count / 2;
5492f9e4e80SMatthew Dillon 	if (maxswzone && n > maxswzone / sizeof(struct swblock))
5502f9e4e80SMatthew Dillon 		n = maxswzone / sizeof(struct swblock);
55121cd6e62SSeigo Tanimura 	n2 = n;
552670d17b5SJeff Roberson 	swap_zone = uma_zcreate("SWAPMETA", sizeof(struct swblock), NULL, NULL,
5537827d9b0SAlan Cox 	    NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM);
554010b1ca1SDavid Schultz 	if (swap_zone == NULL)
555010b1ca1SDavid Schultz 		panic("failed to create swap_zone.");
5568355f576SJeff Roberson 	do {
5575285558aSAlan Cox 		if (uma_zone_set_obj(swap_zone, &swap_zone_obj, n))
55861ce6eeeSAlfred Perlstein 			break;
55961ce6eeeSAlfred Perlstein 		/*
56061ce6eeeSAlfred Perlstein 		 * if the allocation failed, try a zone two thirds the
56161ce6eeeSAlfred Perlstein 		 * size of the previous attempt.
56261ce6eeeSAlfred Perlstein 		 */
56361ce6eeeSAlfred Perlstein 		n -= ((n + 2) / 3);
56461ce6eeeSAlfred Perlstein 	} while (n > 0);
56521cd6e62SSeigo Tanimura 	if (n2 != n)
56661ce6eeeSAlfred Perlstein 		printf("Swap zone entries reduced from %d to %d.\n", n2, n);
56721cd6e62SSeigo Tanimura 	n2 = n;
56824a1cce3SDavid Greenman 
5691c7c3c6aSMatthew Dillon 	/*
5701c7c3c6aSMatthew Dillon 	 * Initialize our meta-data hash table.  The swapper does not need to
5711c7c3c6aSMatthew Dillon 	 * be quite as efficient as the VM system, so we do not use an
5721c7c3c6aSMatthew Dillon 	 * oversized hash table.
5731c7c3c6aSMatthew Dillon 	 *
5741c7c3c6aSMatthew Dillon 	 * 	n: 		size of hash table, must be power of 2
5751c7c3c6aSMatthew Dillon 	 *	swhash_mask:	hash table index mask
5761c7c3c6aSMatthew Dillon 	 */
57761ce6eeeSAlfred Perlstein 	for (n = 1; n < n2 / 8; n *= 2)
5781c7c3c6aSMatthew Dillon 		;
579a163d034SWarner Losh 	swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO);
5801c7c3c6aSMatthew Dillon 	swhash_mask = n - 1;
5817827d9b0SAlan Cox 	mtx_init(&swhash_mtx, "swap_pager swhash", NULL, MTX_DEF);
58224a1cce3SDavid Greenman }
58324a1cce3SDavid Greenman 
58424a1cce3SDavid Greenman /*
5851c7c3c6aSMatthew Dillon  * SWAP_PAGER_ALLOC() -	allocate a new OBJT_SWAP VM object and instantiate
5861c7c3c6aSMatthew Dillon  *			its metadata structures.
5871c7c3c6aSMatthew Dillon  *
5881c7c3c6aSMatthew Dillon  *	This routine is called from the mmap and fork code to create a new
5891c7c3c6aSMatthew Dillon  *	OBJT_SWAP object.  We do this by creating an OBJT_DEFAULT object
5901c7c3c6aSMatthew Dillon  *	and then converting it with swp_pager_meta_build().
5911c7c3c6aSMatthew Dillon  *
5921c7c3c6aSMatthew Dillon  *	This routine may block in vm_object_allocate() and create a named
5931c7c3c6aSMatthew Dillon  *	object lookup race, so we must interlock.   We must also run at
5941c7c3c6aSMatthew Dillon  *	splvm() for the object lookup to handle races with interrupts, but
5951c7c3c6aSMatthew Dillon  *	we do not have to maintain splvm() in between the lookup and the
5961c7c3c6aSMatthew Dillon  *	add because (I believe) it is not possible to attempt to create
5971c7c3c6aSMatthew Dillon  *	a new swap object w/handle when a default object with that handle
5981c7c3c6aSMatthew Dillon  *	already exists.
59924c46d03SAlan Cox  *
60024c46d03SAlan Cox  * MPSAFE
60124a1cce3SDavid Greenman  */
602f5a12711SPoul-Henning Kamp static vm_object_t
6036cde7a16SDavid Greenman swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
6043364c323SKonstantin Belousov     vm_ooffset_t offset, struct ucred *cred)
60524a1cce3SDavid Greenman {
60624a1cce3SDavid Greenman 	vm_object_t object;
6072f7af3dbSAlan Cox 	vm_pindex_t pindex;
6082f7af3dbSAlan Cox 
6092f7af3dbSAlan Cox 	pindex = OFF_TO_IDX(offset + PAGE_MASK + size);
61024a1cce3SDavid Greenman 	if (handle) {
611e793b779SAlan Cox 		mtx_lock(&Giant);
6121c7c3c6aSMatthew Dillon 		/*
6131c7c3c6aSMatthew Dillon 		 * Reference existing named region or allocate new one.  There
6141c7c3c6aSMatthew Dillon 		 * should not be a race here against swp_pager_meta_build()
6151c7c3c6aSMatthew Dillon 		 * as called from vm_page_remove() in regards to the lookup
6161c7c3c6aSMatthew Dillon 		 * of the handle.
6171c7c3c6aSMatthew Dillon 		 */
6180cddd8f0SMatthew Dillon 		sx_xlock(&sw_alloc_sx);
6191c7c3c6aSMatthew Dillon 		object = vm_pager_object_lookup(NOBJLIST(handle), handle);
620b5e8f167SAlan Cox 		if (object == NULL) {
6213364c323SKonstantin Belousov 			if (cred != NULL) {
622ef694c1aSEdward Tomasz Napierala 				if (!swap_reserve_by_cred(size, cred)) {
6233364c323SKonstantin Belousov 					sx_xunlock(&sw_alloc_sx);
6243364c323SKonstantin Belousov 					mtx_unlock(&Giant);
6253364c323SKonstantin Belousov 					return (NULL);
6263364c323SKonstantin Belousov 				}
627ef694c1aSEdward Tomasz Napierala 				crhold(cred);
6283364c323SKonstantin Belousov 			}
6292f7af3dbSAlan Cox 			object = vm_object_allocate(OBJT_DEFAULT, pindex);
630ee3dc7d7SAlan Cox 			VM_OBJECT_LOCK(object);
6313364c323SKonstantin Belousov 			object->handle = handle;
6323364c323SKonstantin Belousov 			if (cred != NULL) {
633ef694c1aSEdward Tomasz Napierala 				object->cred = cred;
6343364c323SKonstantin Belousov 				object->charge = size;
6353364c323SKonstantin Belousov 			}
6364dcc5c2dSMatthew Dillon 			swp_pager_meta_build(object, 0, SWAPBLK_NONE);
637ee3dc7d7SAlan Cox 			VM_OBJECT_UNLOCK(object);
63824a1cce3SDavid Greenman 		}
6390cddd8f0SMatthew Dillon 		sx_xunlock(&sw_alloc_sx);
640e793b779SAlan Cox 		mtx_unlock(&Giant);
64124a1cce3SDavid Greenman 	} else {
6423364c323SKonstantin Belousov 		if (cred != NULL) {
643ef694c1aSEdward Tomasz Napierala 			if (!swap_reserve_by_cred(size, cred))
6443364c323SKonstantin Belousov 				return (NULL);
645ef694c1aSEdward Tomasz Napierala 			crhold(cred);
6463364c323SKonstantin Belousov 		}
6472f7af3dbSAlan Cox 		object = vm_object_allocate(OBJT_DEFAULT, pindex);
648ee3dc7d7SAlan Cox 		VM_OBJECT_LOCK(object);
6493364c323SKonstantin Belousov 		if (cred != NULL) {
650ef694c1aSEdward Tomasz Napierala 			object->cred = cred;
6513364c323SKonstantin Belousov 			object->charge = size;
6523364c323SKonstantin Belousov 		}
6534dcc5c2dSMatthew Dillon 		swp_pager_meta_build(object, 0, SWAPBLK_NONE);
654ee3dc7d7SAlan Cox 		VM_OBJECT_UNLOCK(object);
65524a1cce3SDavid Greenman 	}
65624a1cce3SDavid Greenman 	return (object);
657df8bae1dSRodney W. Grimes }
658df8bae1dSRodney W. Grimes 
65926f9a767SRodney W. Grimes /*
6601c7c3c6aSMatthew Dillon  * SWAP_PAGER_DEALLOC() -	remove swap metadata from object
6611c7c3c6aSMatthew Dillon  *
6621c7c3c6aSMatthew Dillon  *	The swap backing for the object is destroyed.  The code is
6631c7c3c6aSMatthew Dillon  *	designed such that we can reinstantiate it later, but this
6641c7c3c6aSMatthew Dillon  *	routine is typically called only when the entire object is
6651c7c3c6aSMatthew Dillon  *	about to be destroyed.
6661c7c3c6aSMatthew Dillon  *
6671c7c3c6aSMatthew Dillon  *	This routine may block, but no longer does.
6681c7c3c6aSMatthew Dillon  *
6691c7c3c6aSMatthew Dillon  *	The object must be locked or unreferenceable.
67026f9a767SRodney W. Grimes  */
671df8bae1dSRodney W. Grimes static void
6722f249180SPoul-Henning Kamp swap_pager_dealloc(vm_object_t object)
67326f9a767SRodney W. Grimes {
6744dcc5c2dSMatthew Dillon 
67526f9a767SRodney W. Grimes 	/*
6761c7c3c6aSMatthew Dillon 	 * Remove from list right away so lookups will fail if we block for
6771c7c3c6aSMatthew Dillon 	 * pageout completion.
67826f9a767SRodney W. Grimes 	 */
679bd228075SAlan Cox 	if (object->handle != NULL) {
680a9fa2c05SAlfred Perlstein 		mtx_lock(&sw_alloc_mtx);
6811c7c3c6aSMatthew Dillon 		TAILQ_REMOVE(NOBJLIST(object->handle), object, pager_object_list);
682a9fa2c05SAlfred Perlstein 		mtx_unlock(&sw_alloc_mtx);
683bd228075SAlan Cox 	}
6841c7c3c6aSMatthew Dillon 
685658ad5ffSAlan Cox 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
6861c7c3c6aSMatthew Dillon 	vm_object_pip_wait(object, "swpdea");
6871c7c3c6aSMatthew Dillon 
6881c7c3c6aSMatthew Dillon 	/*
6891c7c3c6aSMatthew Dillon 	 * Free all remaining metadata.  We only bother to free it from
6901c7c3c6aSMatthew Dillon 	 * the swap meta data.  We do not attempt to free swapblk's still
6911c7c3c6aSMatthew Dillon 	 * associated with vm_page_t's for this object.  We do not care
6921c7c3c6aSMatthew Dillon 	 * if paging is still in progress on some objects.
6931c7c3c6aSMatthew Dillon 	 */
6941c7c3c6aSMatthew Dillon 	swp_pager_meta_free_all(object);
6951c7c3c6aSMatthew Dillon }
6961c7c3c6aSMatthew Dillon 
6971c7c3c6aSMatthew Dillon /************************************************************************
6981c7c3c6aSMatthew Dillon  *			SWAP PAGER BITMAP ROUTINES			*
6991c7c3c6aSMatthew Dillon  ************************************************************************/
7001c7c3c6aSMatthew Dillon 
7011c7c3c6aSMatthew Dillon /*
7021c7c3c6aSMatthew Dillon  * SWP_PAGER_GETSWAPSPACE() -	allocate raw swap space
7031c7c3c6aSMatthew Dillon  *
7041c7c3c6aSMatthew Dillon  *	Allocate swap for the requested number of pages.  The starting
7051c7c3c6aSMatthew Dillon  *	swap block number (a page index) is returned or SWAPBLK_NONE
7061c7c3c6aSMatthew Dillon  *	if the allocation failed.
7071c7c3c6aSMatthew Dillon  *
7081c7c3c6aSMatthew Dillon  *	Also has the side effect of advising that somebody made a mistake
7091c7c3c6aSMatthew Dillon  *	when they configured swap and didn't configure enough.
7101c7c3c6aSMatthew Dillon  *
7111c7c3c6aSMatthew Dillon  *	Must be called at splvm() to avoid races with bitmap frees from
7121c7c3c6aSMatthew Dillon  *	vm_page_remove() aka swap_pager_page_removed().
7131c7c3c6aSMatthew Dillon  *
7141c7c3c6aSMatthew Dillon  *	This routine may not block
7151c7c3c6aSMatthew Dillon  *	This routine must be called at splvm().
7168f60c087SPoul-Henning Kamp  *
7178f60c087SPoul-Henning Kamp  *	We allocate in round-robin fashion from the configured devices.
7181c7c3c6aSMatthew Dillon  */
719a5edd34aSPoul-Henning Kamp static daddr_t
7202f249180SPoul-Henning Kamp swp_pager_getswapspace(int npages)
7211c7c3c6aSMatthew Dillon {
7221c7c3c6aSMatthew Dillon 	daddr_t blk;
7238f60c087SPoul-Henning Kamp 	struct swdevt *sp;
7248f60c087SPoul-Henning Kamp 	int i;
7251c7c3c6aSMatthew Dillon 
7268f60c087SPoul-Henning Kamp 	blk = SWAPBLK_NONE;
72720da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
7288f60c087SPoul-Henning Kamp 	sp = swdevhd;
7298f60c087SPoul-Henning Kamp 	for (i = 0; i < nswapdev; i++) {
7308f60c087SPoul-Henning Kamp 		if (sp == NULL)
7318f60c087SPoul-Henning Kamp 			sp = TAILQ_FIRST(&swtailq);
7328f60c087SPoul-Henning Kamp 		if (!(sp->sw_flags & SW_CLOSING)) {
7338f60c087SPoul-Henning Kamp 			blk = blist_alloc(sp->sw_blist, npages);
7348f60c087SPoul-Henning Kamp 			if (blk != SWAPBLK_NONE) {
7358f60c087SPoul-Henning Kamp 				blk += sp->sw_first;
7368f60c087SPoul-Henning Kamp 				sp->sw_used += npages;
737d05bc129SAlan Cox 				swap_pager_avail -= npages;
7388f60c087SPoul-Henning Kamp 				swp_sizecheck();
7398f60c087SPoul-Henning Kamp 				swdevhd = TAILQ_NEXT(sp, sw_list);
740d05bc129SAlan Cox 				goto done;
7418f60c087SPoul-Henning Kamp 			}
7428f60c087SPoul-Henning Kamp 		}
7438f60c087SPoul-Henning Kamp 		sp = TAILQ_NEXT(sp, sw_list);
7448f60c087SPoul-Henning Kamp 	}
7452b0d37a4SMatthew Dillon 	if (swap_pager_full != 2) {
74620da9c2eSPoul-Henning Kamp 		printf("swap_pager_getswapspace(%d): failed\n", npages);
7472b0d37a4SMatthew Dillon 		swap_pager_full = 2;
74820d3034fSMatthew Dillon 		swap_pager_almost_full = 1;
7492b0d37a4SMatthew Dillon 	}
7508f60c087SPoul-Henning Kamp 	swdevhd = NULL;
751d05bc129SAlan Cox done:
752d05bc129SAlan Cox 	mtx_unlock(&sw_dev_mtx);
7531c7c3c6aSMatthew Dillon 	return (blk);
75426f9a767SRodney W. Grimes }
75526f9a767SRodney W. Grimes 
756b3fed13eSDavid Schultz static int
757b3fed13eSDavid Schultz swp_pager_isondev(daddr_t blk, struct swdevt *sp)
7588f60c087SPoul-Henning Kamp {
7598f60c087SPoul-Henning Kamp 
760b3fed13eSDavid Schultz 	return (blk >= sp->sw_first && blk < sp->sw_end);
7618f60c087SPoul-Henning Kamp }
7628f60c087SPoul-Henning Kamp 
7634b03903aSPoul-Henning Kamp static void
7644b03903aSPoul-Henning Kamp swp_pager_strategy(struct buf *bp)
7654b03903aSPoul-Henning Kamp {
7664b03903aSPoul-Henning Kamp 	struct swdevt *sp;
7674b03903aSPoul-Henning Kamp 
76820da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
7694b03903aSPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
7704b03903aSPoul-Henning Kamp 		if (bp->b_blkno >= sp->sw_first && bp->b_blkno < sp->sw_end) {
77120da9c2eSPoul-Henning Kamp 			mtx_unlock(&sw_dev_mtx);
7724b03903aSPoul-Henning Kamp 			sp->sw_strategy(bp, sp);
7734b03903aSPoul-Henning Kamp 			return;
7744b03903aSPoul-Henning Kamp 		}
7754b03903aSPoul-Henning Kamp 	}
77620da9c2eSPoul-Henning Kamp 	panic("Swapdev not found");
7774b03903aSPoul-Henning Kamp }
7784b03903aSPoul-Henning Kamp 
7798f60c087SPoul-Henning Kamp 
78026f9a767SRodney W. Grimes /*
7811c7c3c6aSMatthew Dillon  * SWP_PAGER_FREESWAPSPACE() -	free raw swap space
7821c7c3c6aSMatthew Dillon  *
7831c7c3c6aSMatthew Dillon  *	This routine returns the specified swap blocks back to the bitmap.
7841c7c3c6aSMatthew Dillon  *
7851c7c3c6aSMatthew Dillon  *	Note:  This routine may not block (it could in the old swap code),
7861c7c3c6aSMatthew Dillon  *	and through the use of the new blist routines it does not block.
7871c7c3c6aSMatthew Dillon  *
7881c7c3c6aSMatthew Dillon  *	We must be called at splvm() to avoid races with bitmap frees from
7891c7c3c6aSMatthew Dillon  *	vm_page_remove() aka swap_pager_page_removed().
7901c7c3c6aSMatthew Dillon  *
7911c7c3c6aSMatthew Dillon  *	This routine may not block
7921c7c3c6aSMatthew Dillon  *	This routine must be called at splvm().
79326f9a767SRodney W. Grimes  */
794a5edd34aSPoul-Henning Kamp static void
7958f60c087SPoul-Henning Kamp swp_pager_freeswapspace(daddr_t blk, int npages)
7960d94caffSDavid Greenman {
7978f60c087SPoul-Henning Kamp 	struct swdevt *sp;
79892da00bbSMatthew Dillon 
7997645e885SAlan Cox 	mtx_lock(&sw_dev_mtx);
8007645e885SAlan Cox 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
8017645e885SAlan Cox 		if (blk >= sp->sw_first && blk < sp->sw_end) {
80292da00bbSMatthew Dillon 			sp->sw_used -= npages;
80392da00bbSMatthew Dillon 			/*
8047645e885SAlan Cox 			 * If we are attempting to stop swapping on
8057645e885SAlan Cox 			 * this device, we don't want to mark any
8067645e885SAlan Cox 			 * blocks free lest they be reused.
80792da00bbSMatthew Dillon 			 */
8087645e885SAlan Cox 			if ((sp->sw_flags & SW_CLOSING) == 0) {
8097645e885SAlan Cox 				blist_free(sp->sw_blist, blk - sp->sw_first,
8107645e885SAlan Cox 				    npages);
8118f60c087SPoul-Henning Kamp 				swap_pager_avail += npages;
8121c7c3c6aSMatthew Dillon 				swp_sizecheck();
81326f9a767SRodney W. Grimes 			}
8147645e885SAlan Cox 			mtx_unlock(&sw_dev_mtx);
8157645e885SAlan Cox 			return;
8167645e885SAlan Cox 		}
8177645e885SAlan Cox 	}
8187645e885SAlan Cox 	panic("Swapdev not found");
8197645e885SAlan Cox }
8201c7c3c6aSMatthew Dillon 
82126f9a767SRodney W. Grimes /*
8221c7c3c6aSMatthew Dillon  * SWAP_PAGER_FREESPACE() -	frees swap blocks associated with a page
8231c7c3c6aSMatthew Dillon  *				range within an object.
8241c7c3c6aSMatthew Dillon  *
8251c7c3c6aSMatthew Dillon  *	This is a globally accessible routine.
8261c7c3c6aSMatthew Dillon  *
8271c7c3c6aSMatthew Dillon  *	This routine removes swapblk assignments from swap metadata.
8281c7c3c6aSMatthew Dillon  *
8291c7c3c6aSMatthew Dillon  *	The external callers of this routine typically have already destroyed
8301c7c3c6aSMatthew Dillon  *	or renamed vm_page_t's associated with this range in the object so
8311c7c3c6aSMatthew Dillon  *	we should be ok.
8324dcc5c2dSMatthew Dillon  *
8334dcc5c2dSMatthew Dillon  *	This routine may be called at any spl.  We up our spl to splvm temporarily
8344dcc5c2dSMatthew Dillon  *	in order to perform the metadata removal.
83526f9a767SRodney W. Grimes  */
83626f9a767SRodney W. Grimes void
8372f249180SPoul-Henning Kamp swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size)
83826f9a767SRodney W. Grimes {
83923955314SAlfred Perlstein 
84019ba4c8eSAlan Cox 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
8411c7c3c6aSMatthew Dillon 	swp_pager_meta_free(object, start, size);
8424dcc5c2dSMatthew Dillon }
8434dcc5c2dSMatthew Dillon 
8444dcc5c2dSMatthew Dillon /*
8454dcc5c2dSMatthew Dillon  * SWAP_PAGER_RESERVE() - reserve swap blocks in object
8464dcc5c2dSMatthew Dillon  *
8474dcc5c2dSMatthew Dillon  *	Assigns swap blocks to the specified range within the object.  The
8484dcc5c2dSMatthew Dillon  *	swap blocks are not zerod.  Any previous swap assignment is destroyed.
8494dcc5c2dSMatthew Dillon  *
8504dcc5c2dSMatthew Dillon  *	Returns 0 on success, -1 on failure.
8514dcc5c2dSMatthew Dillon  */
8524dcc5c2dSMatthew Dillon int
8534dcc5c2dSMatthew Dillon swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
8544dcc5c2dSMatthew Dillon {
8554dcc5c2dSMatthew Dillon 	int n = 0;
8564dcc5c2dSMatthew Dillon 	daddr_t blk = SWAPBLK_NONE;
8574dcc5c2dSMatthew Dillon 	vm_pindex_t beg = start;	/* save start index */
8584dcc5c2dSMatthew Dillon 
859ee3dc7d7SAlan Cox 	VM_OBJECT_LOCK(object);
8604dcc5c2dSMatthew Dillon 	while (size) {
8614dcc5c2dSMatthew Dillon 		if (n == 0) {
8624dcc5c2dSMatthew Dillon 			n = BLIST_MAX_ALLOC;
8634dcc5c2dSMatthew Dillon 			while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
8644dcc5c2dSMatthew Dillon 				n >>= 1;
8654dcc5c2dSMatthew Dillon 				if (n == 0) {
8664dcc5c2dSMatthew Dillon 					swp_pager_meta_free(object, beg, start - beg);
867ee3dc7d7SAlan Cox 					VM_OBJECT_UNLOCK(object);
8684dcc5c2dSMatthew Dillon 					return (-1);
8694dcc5c2dSMatthew Dillon 				}
8704dcc5c2dSMatthew Dillon 			}
8714dcc5c2dSMatthew Dillon 		}
8724dcc5c2dSMatthew Dillon 		swp_pager_meta_build(object, start, blk);
8734dcc5c2dSMatthew Dillon 		--size;
8744dcc5c2dSMatthew Dillon 		++start;
8754dcc5c2dSMatthew Dillon 		++blk;
8764dcc5c2dSMatthew Dillon 		--n;
8774dcc5c2dSMatthew Dillon 	}
8784dcc5c2dSMatthew Dillon 	swp_pager_meta_free(object, start, n);
879ee3dc7d7SAlan Cox 	VM_OBJECT_UNLOCK(object);
8804dcc5c2dSMatthew Dillon 	return (0);
88126f9a767SRodney W. Grimes }
88226f9a767SRodney W. Grimes 
8830a47b48bSJohn Dyson /*
8841c7c3c6aSMatthew Dillon  * SWAP_PAGER_COPY() -  copy blocks from source pager to destination pager
8851c7c3c6aSMatthew Dillon  *			and destroy the source.
8861c7c3c6aSMatthew Dillon  *
8871c7c3c6aSMatthew Dillon  *	Copy any valid swapblks from the source to the destination.  In
8881c7c3c6aSMatthew Dillon  *	cases where both the source and destination have a valid swapblk,
8891c7c3c6aSMatthew Dillon  *	we keep the destination's.
8901c7c3c6aSMatthew Dillon  *
8911c7c3c6aSMatthew Dillon  *	This routine is allowed to block.  It may block allocating metadata
8921c7c3c6aSMatthew Dillon  *	indirectly through swp_pager_meta_build() or if paging is still in
8931c7c3c6aSMatthew Dillon  *	progress on the source.
8941c7c3c6aSMatthew Dillon  *
8954dcc5c2dSMatthew Dillon  *	This routine can be called at any spl
8964dcc5c2dSMatthew Dillon  *
8971c7c3c6aSMatthew Dillon  *	XXX vm_page_collapse() kinda expects us not to block because we
8981c7c3c6aSMatthew Dillon  *	supposedly do not need to allocate memory, but for the moment we
8991c7c3c6aSMatthew Dillon  *	*may* have to get a little memory from the zone allocator, but
9001c7c3c6aSMatthew Dillon  *	it is taken from the interrupt memory.  We should be ok.
9011c7c3c6aSMatthew Dillon  *
9021c7c3c6aSMatthew Dillon  *	The source object contains no vm_page_t's (which is just as well)
9031c7c3c6aSMatthew Dillon  *
9041c7c3c6aSMatthew Dillon  *	The source object is of type OBJT_SWAP.
9051c7c3c6aSMatthew Dillon  *
9064dcc5c2dSMatthew Dillon  *	The source and destination objects must be locked or
9074dcc5c2dSMatthew Dillon  *	inaccessible (XXX are they ?)
90826f9a767SRodney W. Grimes  */
90926f9a767SRodney W. Grimes void
9102f249180SPoul-Henning Kamp swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
9112f249180SPoul-Henning Kamp     vm_pindex_t offset, int destroysource)
91226f9a767SRodney W. Grimes {
913a316d390SJohn Dyson 	vm_pindex_t i;
9144dcc5c2dSMatthew Dillon 
915c7c8dd7eSAlan Cox 	VM_OBJECT_LOCK_ASSERT(srcobject, MA_OWNED);
916c7c8dd7eSAlan Cox 	VM_OBJECT_LOCK_ASSERT(dstobject, MA_OWNED);
9170cddd8f0SMatthew Dillon 
91826f9a767SRodney W. Grimes 	/*
9191c7c3c6aSMatthew Dillon 	 * If destroysource is set, we remove the source object from the
9201c7c3c6aSMatthew Dillon 	 * swap_pager internal queue now.
92126f9a767SRodney W. Grimes 	 */
922cbd8ec09SJohn Dyson 	if (destroysource) {
923bd228075SAlan Cox 		if (srcobject->handle != NULL) {
924a9fa2c05SAlfred Perlstein 			mtx_lock(&sw_alloc_mtx);
9251c7c3c6aSMatthew Dillon 			TAILQ_REMOVE(
9261c7c3c6aSMatthew Dillon 			    NOBJLIST(srcobject->handle),
9271c7c3c6aSMatthew Dillon 			    srcobject,
9281c7c3c6aSMatthew Dillon 			    pager_object_list
9291c7c3c6aSMatthew Dillon 			);
930a9fa2c05SAlfred Perlstein 			mtx_unlock(&sw_alloc_mtx);
931cbd8ec09SJohn Dyson 		}
932bd228075SAlan Cox 	}
93326f9a767SRodney W. Grimes 
9341c7c3c6aSMatthew Dillon 	/*
9351c7c3c6aSMatthew Dillon 	 * transfer source to destination.
9361c7c3c6aSMatthew Dillon 	 */
9371c7c3c6aSMatthew Dillon 	for (i = 0; i < dstobject->size; ++i) {
9381c7c3c6aSMatthew Dillon 		daddr_t dstaddr;
9391c7c3c6aSMatthew Dillon 
9401c7c3c6aSMatthew Dillon 		/*
9411c7c3c6aSMatthew Dillon 		 * Locate (without changing) the swapblk on the destination,
9421c7c3c6aSMatthew Dillon 		 * unless it is invalid in which case free it silently, or
9431c7c3c6aSMatthew Dillon 		 * if the destination is a resident page, in which case the
9441c7c3c6aSMatthew Dillon 		 * source is thrown away.
9451c7c3c6aSMatthew Dillon 		 */
9461c7c3c6aSMatthew Dillon 		dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
9471c7c3c6aSMatthew Dillon 
9481c7c3c6aSMatthew Dillon 		if (dstaddr == SWAPBLK_NONE) {
9491c7c3c6aSMatthew Dillon 			/*
9501c7c3c6aSMatthew Dillon 			 * Destination has no swapblk and is not resident,
9511c7c3c6aSMatthew Dillon 			 * copy source.
9521c7c3c6aSMatthew Dillon 			 */
9531c7c3c6aSMatthew Dillon 			daddr_t srcaddr;
9541c7c3c6aSMatthew Dillon 
9551c7c3c6aSMatthew Dillon 			srcaddr = swp_pager_meta_ctl(
9561c7c3c6aSMatthew Dillon 			    srcobject,
9571c7c3c6aSMatthew Dillon 			    i + offset,
9581c7c3c6aSMatthew Dillon 			    SWM_POP
9591c7c3c6aSMatthew Dillon 			);
9601c7c3c6aSMatthew Dillon 
961ee3dc7d7SAlan Cox 			if (srcaddr != SWAPBLK_NONE) {
962c7c8dd7eSAlan Cox 				/*
963c7c8dd7eSAlan Cox 				 * swp_pager_meta_build() can sleep.
964c7c8dd7eSAlan Cox 				 */
965c7c8dd7eSAlan Cox 				vm_object_pip_add(srcobject, 1);
966c7c8dd7eSAlan Cox 				VM_OBJECT_UNLOCK(srcobject);
967c7c8dd7eSAlan Cox 				vm_object_pip_add(dstobject, 1);
9684dcc5c2dSMatthew Dillon 				swp_pager_meta_build(dstobject, i, srcaddr);
969c7c8dd7eSAlan Cox 				vm_object_pip_wakeup(dstobject);
970c7c8dd7eSAlan Cox 				VM_OBJECT_LOCK(srcobject);
971c7c8dd7eSAlan Cox 				vm_object_pip_wakeup(srcobject);
972ee3dc7d7SAlan Cox 			}
9731c7c3c6aSMatthew Dillon 		} else {
9741c7c3c6aSMatthew Dillon 			/*
9751c7c3c6aSMatthew Dillon 			 * Destination has valid swapblk or it is represented
9761c7c3c6aSMatthew Dillon 			 * by a resident page.  We destroy the sourceblock.
9771c7c3c6aSMatthew Dillon 			 */
9781c7c3c6aSMatthew Dillon 
9791c7c3c6aSMatthew Dillon 			swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE);
9801c7c3c6aSMatthew Dillon 		}
98126f9a767SRodney W. Grimes 	}
98226f9a767SRodney W. Grimes 
98326f9a767SRodney W. Grimes 	/*
9841c7c3c6aSMatthew Dillon 	 * Free left over swap blocks in source.
9851c7c3c6aSMatthew Dillon 	 *
9861c7c3c6aSMatthew Dillon 	 * We have to revert the type to OBJT_DEFAULT so we do not accidently
9871c7c3c6aSMatthew Dillon 	 * double-remove the object from the swap queues.
98826f9a767SRodney W. Grimes 	 */
989c0877f10SJohn Dyson 	if (destroysource) {
9901c7c3c6aSMatthew Dillon 		swp_pager_meta_free_all(srcobject);
9911c7c3c6aSMatthew Dillon 		/*
9921c7c3c6aSMatthew Dillon 		 * Reverting the type is not necessary, the caller is going
9931c7c3c6aSMatthew Dillon 		 * to destroy srcobject directly, but I'm doing it here
994956f3135SPhilippe Charnier 		 * for consistency since we've removed the object from its
9951c7c3c6aSMatthew Dillon 		 * queues.
9961c7c3c6aSMatthew Dillon 		 */
9971c7c3c6aSMatthew Dillon 		srcobject->type = OBJT_DEFAULT;
998c0877f10SJohn Dyson 	}
99926f9a767SRodney W. Grimes }
100026f9a767SRodney W. Grimes 
1001df8bae1dSRodney W. Grimes /*
10021c7c3c6aSMatthew Dillon  * SWAP_PAGER_HASPAGE() -	determine if we have good backing store for
10031c7c3c6aSMatthew Dillon  *				the requested page.
10041c7c3c6aSMatthew Dillon  *
10051c7c3c6aSMatthew Dillon  *	We determine whether good backing store exists for the requested
10061c7c3c6aSMatthew Dillon  *	page and return TRUE if it does, FALSE if it doesn't.
10071c7c3c6aSMatthew Dillon  *
10081c7c3c6aSMatthew Dillon  *	If TRUE, we also try to determine how much valid, contiguous backing
10091c7c3c6aSMatthew Dillon  *	store exists before and after the requested page within a reasonable
10101c7c3c6aSMatthew Dillon  *	distance.  We do not try to restrict it to the swap device stripe
10111c7c3c6aSMatthew Dillon  *	(that is handled in getpages/putpages).  It probably isn't worth
10121c7c3c6aSMatthew Dillon  *	doing here.
1013df8bae1dSRodney W. Grimes  */
10145ea4972cSAlan Cox static boolean_t
10152f249180SPoul-Henning Kamp swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after)
101626f9a767SRodney W. Grimes {
10171c7c3c6aSMatthew Dillon 	daddr_t blk0;
101826f9a767SRodney W. Grimes 
1019ee3dc7d7SAlan Cox 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
10201c7c3c6aSMatthew Dillon 	/*
10211c7c3c6aSMatthew Dillon 	 * do we have good backing store at the requested index ?
10221c7c3c6aSMatthew Dillon 	 */
10231c7c3c6aSMatthew Dillon 	blk0 = swp_pager_meta_ctl(object, pindex, 0);
10241c7c3c6aSMatthew Dillon 
10254dcc5c2dSMatthew Dillon 	if (blk0 == SWAPBLK_NONE) {
10261c7c3c6aSMatthew Dillon 		if (before)
102724a1cce3SDavid Greenman 			*before = 0;
10281c7c3c6aSMatthew Dillon 		if (after)
102924a1cce3SDavid Greenman 			*after = 0;
103026f9a767SRodney W. Grimes 		return (FALSE);
103126f9a767SRodney W. Grimes 	}
103226f9a767SRodney W. Grimes 
103326f9a767SRodney W. Grimes 	/*
10341c7c3c6aSMatthew Dillon 	 * find backwards-looking contiguous good backing store
1035e47ed70bSJohn Dyson 	 */
10361c7c3c6aSMatthew Dillon 	if (before != NULL) {
103726f9a767SRodney W. Grimes 		int i;
10380d94caffSDavid Greenman 
10391c7c3c6aSMatthew Dillon 		for (i = 1; i < (SWB_NPAGES/2); ++i) {
10401c7c3c6aSMatthew Dillon 			daddr_t blk;
10411c7c3c6aSMatthew Dillon 
10421c7c3c6aSMatthew Dillon 			if (i > pindex)
10431c7c3c6aSMatthew Dillon 				break;
10441c7c3c6aSMatthew Dillon 			blk = swp_pager_meta_ctl(object, pindex - i, 0);
10451c7c3c6aSMatthew Dillon 			if (blk != blk0 - i)
10461c7c3c6aSMatthew Dillon 				break;
1047ffc82b0aSJohn Dyson 		}
10481c7c3c6aSMatthew Dillon 		*before = (i - 1);
104926f9a767SRodney W. Grimes 	}
105026f9a767SRodney W. Grimes 
105126f9a767SRodney W. Grimes 	/*
10521c7c3c6aSMatthew Dillon 	 * find forward-looking contiguous good backing store
105326f9a767SRodney W. Grimes 	 */
10541c7c3c6aSMatthew Dillon 	if (after != NULL) {
10551c7c3c6aSMatthew Dillon 		int i;
10561c7c3c6aSMatthew Dillon 
10571c7c3c6aSMatthew Dillon 		for (i = 1; i < (SWB_NPAGES/2); ++i) {
10581c7c3c6aSMatthew Dillon 			daddr_t blk;
10591c7c3c6aSMatthew Dillon 
10601c7c3c6aSMatthew Dillon 			blk = swp_pager_meta_ctl(object, pindex + i, 0);
10611c7c3c6aSMatthew Dillon 			if (blk != blk0 + i)
10621c7c3c6aSMatthew Dillon 				break;
106326f9a767SRodney W. Grimes 		}
10641c7c3c6aSMatthew Dillon 		*after = (i - 1);
10651c7c3c6aSMatthew Dillon 	}
10661c7c3c6aSMatthew Dillon 	return (TRUE);
10671c7c3c6aSMatthew Dillon }
10681c7c3c6aSMatthew Dillon 
10691c7c3c6aSMatthew Dillon /*
10701c7c3c6aSMatthew Dillon  * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page
10711c7c3c6aSMatthew Dillon  *
10721c7c3c6aSMatthew Dillon  *	This removes any associated swap backing store, whether valid or
10731c7c3c6aSMatthew Dillon  *	not, from the page.
10741c7c3c6aSMatthew Dillon  *
10751c7c3c6aSMatthew Dillon  *	This routine is typically called when a page is made dirty, at
10761c7c3c6aSMatthew Dillon  *	which point any associated swap can be freed.  MADV_FREE also
10771c7c3c6aSMatthew Dillon  *	calls us in a special-case situation
10781c7c3c6aSMatthew Dillon  *
10791c7c3c6aSMatthew Dillon  *	NOTE!!!  If the page is clean and the swap was valid, the caller
10801c7c3c6aSMatthew Dillon  *	should make the page dirty before calling this routine.  This routine
10811c7c3c6aSMatthew Dillon  *	does NOT change the m->dirty status of the page.  Also: MADV_FREE
10821c7c3c6aSMatthew Dillon  *	depends on it.
10831c7c3c6aSMatthew Dillon  *
10841c7c3c6aSMatthew Dillon  *	This routine may not block
10854dcc5c2dSMatthew Dillon  *	This routine must be called at splvm()
10861c7c3c6aSMatthew Dillon  */
10871c7c3c6aSMatthew Dillon static void
10882f249180SPoul-Henning Kamp swap_pager_unswapped(vm_page_t m)
10891c7c3c6aSMatthew Dillon {
10902f249180SPoul-Henning Kamp 
1091ee3dc7d7SAlan Cox 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
10921c7c3c6aSMatthew Dillon 	swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE);
10931c7c3c6aSMatthew Dillon }
10941c7c3c6aSMatthew Dillon 
10951c7c3c6aSMatthew Dillon /*
10961c7c3c6aSMatthew Dillon  * SWAP_PAGER_GETPAGES() - bring pages in from swap
10971c7c3c6aSMatthew Dillon  *
10981c7c3c6aSMatthew Dillon  *	Attempt to retrieve (m, count) pages from backing store, but make
10991c7c3c6aSMatthew Dillon  *	sure we retrieve at least m[reqpage].  We try to load in as large
11001c7c3c6aSMatthew Dillon  *	a chunk surrounding m[reqpage] as is contiguous in swap and which
11011c7c3c6aSMatthew Dillon  *	belongs to the same object.
11021c7c3c6aSMatthew Dillon  *
11031c7c3c6aSMatthew Dillon  *	The code is designed for asynchronous operation and
11041c7c3c6aSMatthew Dillon  *	immediate-notification of 'reqpage' but tends not to be
11051c7c3c6aSMatthew Dillon  *	used that way.  Please do not optimize-out this algorithmic
11061c7c3c6aSMatthew Dillon  *	feature, I intend to improve on it in the future.
11071c7c3c6aSMatthew Dillon  *
11081c7c3c6aSMatthew Dillon  *	The parent has a single vm_object_pip_add() reference prior to
11091c7c3c6aSMatthew Dillon  *	calling us and we should return with the same.
11101c7c3c6aSMatthew Dillon  *
11111c7c3c6aSMatthew Dillon  *	The parent has BUSY'd the pages.  We should return with 'm'
11121c7c3c6aSMatthew Dillon  *	left busy, but the others adjusted.
11131c7c3c6aSMatthew Dillon  */
1114f708ef1bSPoul-Henning Kamp static int
11152f249180SPoul-Henning Kamp swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage)
1116df8bae1dSRodney W. Grimes {
11171c7c3c6aSMatthew Dillon 	struct buf *bp;
11181c7c3c6aSMatthew Dillon 	vm_page_t mreq;
111926f9a767SRodney W. Grimes 	int i;
112026f9a767SRodney W. Grimes 	int j;
11211c7c3c6aSMatthew Dillon 	daddr_t blk;
11220d94caffSDavid Greenman 
11231c7c3c6aSMatthew Dillon 	mreq = m[reqpage];
11241c7c3c6aSMatthew Dillon 
1125e04e4bacSPoul-Henning Kamp 	KASSERT(mreq->object == object,
1126e04e4bacSPoul-Henning Kamp 	    ("swap_pager_getpages: object mismatch %p/%p",
1127e04e4bacSPoul-Henning Kamp 	    object, mreq->object));
1128e04e4bacSPoul-Henning Kamp 
11291c7c3c6aSMatthew Dillon 	/*
11301c7c3c6aSMatthew Dillon 	 * Calculate range to retrieve.  The pages have already been assigned
1131751221fdSPoul-Henning Kamp 	 * their swapblks.  We require a *contiguous* range but we know it to
1132751221fdSPoul-Henning Kamp 	 * not span devices.   If we do not supply it, bad things
11334dcc5c2dSMatthew Dillon 	 * happen.  Note that blk, iblk & jblk can be SWAPBLK_NONE, but the
11344dcc5c2dSMatthew Dillon 	 * loops are set up such that the case(s) are handled implicitly.
11354dcc5c2dSMatthew Dillon 	 *
11360b6ace47SAlan Cox 	 * The swp_*() calls must be made with the object locked.
11371c7c3c6aSMatthew Dillon 	 */
11381c7c3c6aSMatthew Dillon 	blk = swp_pager_meta_ctl(mreq->object, mreq->pindex, 0);
11391c7c3c6aSMatthew Dillon 
11401c7c3c6aSMatthew Dillon 	for (i = reqpage - 1; i >= 0; --i) {
11411c7c3c6aSMatthew Dillon 		daddr_t iblk;
11421c7c3c6aSMatthew Dillon 
11431c7c3c6aSMatthew Dillon 		iblk = swp_pager_meta_ctl(m[i]->object, m[i]->pindex, 0);
11441c7c3c6aSMatthew Dillon 		if (blk != iblk + (reqpage - i))
114526f9a767SRodney W. Grimes 			break;
114626f9a767SRodney W. Grimes 	}
11471c7c3c6aSMatthew Dillon 	++i;
11481c7c3c6aSMatthew Dillon 
11491c7c3c6aSMatthew Dillon 	for (j = reqpage + 1; j < count; ++j) {
11501c7c3c6aSMatthew Dillon 		daddr_t jblk;
11511c7c3c6aSMatthew Dillon 
11521c7c3c6aSMatthew Dillon 		jblk = swp_pager_meta_ctl(m[j]->object, m[j]->pindex, 0);
11531c7c3c6aSMatthew Dillon 		if (blk != jblk - (j - reqpage))
11541c7c3c6aSMatthew Dillon 			break;
11551c7c3c6aSMatthew Dillon 	}
11561c7c3c6aSMatthew Dillon 
11571c7c3c6aSMatthew Dillon 	/*
11581c7c3c6aSMatthew Dillon 	 * free pages outside our collection range.   Note: we never free
11591c7c3c6aSMatthew Dillon 	 * mreq, it must remain busy throughout.
11601c7c3c6aSMatthew Dillon 	 */
1161071a1710SAlan Cox 	if (0 < i || j < count) {
11621c7c3c6aSMatthew Dillon 		int k;
11631c7c3c6aSMatthew Dillon 
11643c4a2440SAlan Cox 		for (k = 0; k < i; ++k)
1165e86a87e9SKonstantin Belousov 			swp_pager_free_nrpage(m[k]);
11663c4a2440SAlan Cox 		for (k = j; k < count; ++k)
11672965a453SKip Macy 			swp_pager_free_nrpage(m[k]);
1168071a1710SAlan Cox 	}
11691c7c3c6aSMatthew Dillon 
11701c7c3c6aSMatthew Dillon 	/*
11714dcc5c2dSMatthew Dillon 	 * Return VM_PAGER_FAIL if we have nothing to do.  Return mreq
11724dcc5c2dSMatthew Dillon 	 * still busy, but the others unbusied.
11731c7c3c6aSMatthew Dillon 	 */
11744dcc5c2dSMatthew Dillon 	if (blk == SWAPBLK_NONE)
117526f9a767SRodney W. Grimes 		return (VM_PAGER_FAIL);
1176df8bae1dSRodney W. Grimes 
117716f62314SDavid Greenman 	/*
11788630c117SAlan Cox 	 * Getpbuf() can sleep.
11798630c117SAlan Cox 	 */
11808630c117SAlan Cox 	VM_OBJECT_UNLOCK(object);
11818630c117SAlan Cox 	/*
118216f62314SDavid Greenman 	 * Get a swap buffer header to perform the IO
118316f62314SDavid Greenman 	 */
11841c7c3c6aSMatthew Dillon 	bp = getpbuf(&nsw_rcount);
11855e04322aSPoul-Henning Kamp 	bp->b_flags |= B_PAGING;
118626f9a767SRodney W. Grimes 
118716f62314SDavid Greenman 	/*
118816f62314SDavid Greenman 	 * map our page(s) into kva for input
118916f62314SDavid Greenman 	 */
1190d68d828bSAlan Cox 	pmap_qenter((vm_offset_t)bp->b_data, m + i, j - i);
11911c7c3c6aSMatthew Dillon 
119221144e3bSPoul-Henning Kamp 	bp->b_iocmd = BIO_READ;
11931c7c3c6aSMatthew Dillon 	bp->b_iodone = swp_pager_async_iodone;
1194fdcc1cc0SJohn Baldwin 	bp->b_rcred = crhold(thread0.td_ucred);
1195fdcc1cc0SJohn Baldwin 	bp->b_wcred = crhold(thread0.td_ucred);
11961c7c3c6aSMatthew Dillon 	bp->b_blkno = blk - (reqpage - i);
11971c7c3c6aSMatthew Dillon 	bp->b_bcount = PAGE_SIZE * (j - i);
11981c7c3c6aSMatthew Dillon 	bp->b_bufsize = PAGE_SIZE * (j - i);
11991c7c3c6aSMatthew Dillon 	bp->b_pager.pg_reqpage = reqpage - i;
12001c7c3c6aSMatthew Dillon 
12018630c117SAlan Cox 	VM_OBJECT_LOCK(object);
12021c7c3c6aSMatthew Dillon 	{
12031c7c3c6aSMatthew Dillon 		int k;
12041c7c3c6aSMatthew Dillon 
12051c7c3c6aSMatthew Dillon 		for (k = i; k < j; ++k) {
12061c7c3c6aSMatthew Dillon 			bp->b_pages[k - i] = m[k];
12075786be7cSAlan Cox 			m[k]->oflags |= VPO_SWAPINPROG;
12081c7c3c6aSMatthew Dillon 		}
12091c7c3c6aSMatthew Dillon 	}
12101c7c3c6aSMatthew Dillon 	bp->b_npages = j - i;
121126f9a767SRodney W. Grimes 
1212b4b70819SAttilio Rao 	PCPU_INC(cnt.v_swapin);
1213b4b70819SAttilio Rao 	PCPU_ADD(cnt.v_swappgsin, bp->b_npages);
12141c7c3c6aSMatthew Dillon 
1215df8bae1dSRodney W. Grimes 	/*
12161c7c3c6aSMatthew Dillon 	 * We still hold the lock on mreq, and our automatic completion routine
12171c7c3c6aSMatthew Dillon 	 * does not remove it.
1218df8bae1dSRodney W. Grimes 	 */
1219071a1710SAlan Cox 	vm_object_pip_add(object, bp->b_npages);
1220071a1710SAlan Cox 	VM_OBJECT_UNLOCK(object);
12211c7c3c6aSMatthew Dillon 
12221c7c3c6aSMatthew Dillon 	/*
12231c7c3c6aSMatthew Dillon 	 * perform the I/O.  NOTE!!!  bp cannot be considered valid after
12241c7c3c6aSMatthew Dillon 	 * this point because we automatically release it on completion.
12251c7c3c6aSMatthew Dillon 	 * Instead, we look at the one page we are interested in which we
12261c7c3c6aSMatthew Dillon 	 * still hold a lock on even through the I/O completion.
12271c7c3c6aSMatthew Dillon 	 *
12281c7c3c6aSMatthew Dillon 	 * The other pages in our m[] array are also released on completion,
12291c7c3c6aSMatthew Dillon 	 * so we cannot assume they are valid anymore either.
12301c7c3c6aSMatthew Dillon 	 *
1231c37a77eeSPoul-Henning Kamp 	 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
12321c7c3c6aSMatthew Dillon 	 */
1233b890cb2cSPeter Wemm 	BUF_KERNPROC(bp);
12344b03903aSPoul-Henning Kamp 	swp_pager_strategy(bp);
123526f9a767SRodney W. Grimes 
123626f9a767SRodney W. Grimes 	/*
12375786be7cSAlan Cox 	 * wait for the page we want to complete.  VPO_SWAPINPROG is always
12381c7c3c6aSMatthew Dillon 	 * cleared on completion.  If an I/O error occurs, SWAPBLK_NONE
12391c7c3c6aSMatthew Dillon 	 * is set in the meta-data.
124026f9a767SRodney W. Grimes 	 */
124191449ce9SAlan Cox 	VM_OBJECT_LOCK(object);
12425786be7cSAlan Cox 	while ((mreq->oflags & VPO_SWAPINPROG) != 0) {
12435786be7cSAlan Cox 		mreq->oflags |= VPO_WANTED;
1244b4b70819SAttilio Rao 		PCPU_INC(cnt.v_intrans);
124591449ce9SAlan Cox 		if (msleep(mreq, VM_OBJECT_MTX(object), PSWP, "swread", hz*20)) {
12469bd86a98SBruce M Simpson 			printf(
1247c5690651SPoul-Henning Kamp "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n",
1248c5690651SPoul-Henning Kamp 			    bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount);
12491c7c3c6aSMatthew Dillon 		}
12501b119d9dSDavid Greenman 	}
125126f9a767SRodney W. Grimes 
125226f9a767SRodney W. Grimes 	/*
1253a1287949SEivind Eklund 	 * mreq is left busied after completion, but all the other pages
12541c7c3c6aSMatthew Dillon 	 * are freed.  If we had an unrecoverable read error the page will
12551c7c3c6aSMatthew Dillon 	 * not be valid.
125626f9a767SRodney W. Grimes 	 */
12571c7c3c6aSMatthew Dillon 	if (mreq->valid != VM_PAGE_BITS_ALL) {
12581c7c3c6aSMatthew Dillon 		return (VM_PAGER_ERROR);
125926f9a767SRodney W. Grimes 	} else {
12601c7c3c6aSMatthew Dillon 		return (VM_PAGER_OK);
126126f9a767SRodney W. Grimes 	}
12621c7c3c6aSMatthew Dillon 
12631c7c3c6aSMatthew Dillon 	/*
12641c7c3c6aSMatthew Dillon 	 * A final note: in a low swap situation, we cannot deallocate swap
12651c7c3c6aSMatthew Dillon 	 * and mark a page dirty here because the caller is likely to mark
12661c7c3c6aSMatthew Dillon 	 * the page clean when we return, causing the page to possibly revert
12671c7c3c6aSMatthew Dillon 	 * to all-zero's later.
12681c7c3c6aSMatthew Dillon 	 */
1269df8bae1dSRodney W. Grimes }
1270df8bae1dSRodney W. Grimes 
12711c7c3c6aSMatthew Dillon /*
12721c7c3c6aSMatthew Dillon  *	swap_pager_putpages:
12731c7c3c6aSMatthew Dillon  *
12741c7c3c6aSMatthew Dillon  *	Assign swap (if necessary) and initiate I/O on the specified pages.
12751c7c3c6aSMatthew Dillon  *
12761c7c3c6aSMatthew Dillon  *	We support both OBJT_DEFAULT and OBJT_SWAP objects.  DEFAULT objects
12771c7c3c6aSMatthew Dillon  *	are automatically converted to SWAP objects.
12781c7c3c6aSMatthew Dillon  *
1279ea3aecf5SPeter Wemm  *	In a low memory situation we may block in VOP_STRATEGY(), but the new
12801c7c3c6aSMatthew Dillon  *	vm_page reservation system coupled with properly written VFS devices
12811c7c3c6aSMatthew Dillon  *	should ensure that no low-memory deadlock occurs.  This is an area
12821c7c3c6aSMatthew Dillon  *	which needs work.
12831c7c3c6aSMatthew Dillon  *
12841c7c3c6aSMatthew Dillon  *	The parent has N vm_object_pip_add() references prior to
12851c7c3c6aSMatthew Dillon  *	calling us and will remove references for rtvals[] that are
12861c7c3c6aSMatthew Dillon  *	not set to VM_PAGER_PEND.  We need to remove the rest on I/O
12871c7c3c6aSMatthew Dillon  *	completion.
12881c7c3c6aSMatthew Dillon  *
12891c7c3c6aSMatthew Dillon  *	The parent has soft-busy'd the pages it passes us and will unbusy
12901c7c3c6aSMatthew Dillon  *	those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
12911c7c3c6aSMatthew Dillon  *	We need to unbusy the rest on I/O completion.
12921c7c3c6aSMatthew Dillon  */
1293e4542174SMatthew Dillon void
12942f249180SPoul-Henning Kamp swap_pager_putpages(vm_object_t object, vm_page_t *m, int count,
12952f249180SPoul-Henning Kamp     boolean_t sync, int *rtvals)
1296df8bae1dSRodney W. Grimes {
12971c7c3c6aSMatthew Dillon 	int i;
12981c7c3c6aSMatthew Dillon 	int n = 0;
1299df8bae1dSRodney W. Grimes 
13001c7c3c6aSMatthew Dillon 	if (count && m[0]->object != object) {
13017036145bSMaxim Konovalov 		panic("swap_pager_putpages: object mismatch %p/%p",
13021c7c3c6aSMatthew Dillon 		    object,
13031c7c3c6aSMatthew Dillon 		    m[0]->object
13041c7c3c6aSMatthew Dillon 		);
13051c7c3c6aSMatthew Dillon 	}
1306ee3dc7d7SAlan Cox 
13071c7c3c6aSMatthew Dillon 	/*
13081c7c3c6aSMatthew Dillon 	 * Step 1
13091c7c3c6aSMatthew Dillon 	 *
13101c7c3c6aSMatthew Dillon 	 * Turn object into OBJT_SWAP
13111c7c3c6aSMatthew Dillon 	 * check for bogus sysops
13121c7c3c6aSMatthew Dillon 	 * force sync if not pageout process
13131c7c3c6aSMatthew Dillon 	 */
13144dcc5c2dSMatthew Dillon 	if (object->type != OBJT_SWAP)
13154dcc5c2dSMatthew Dillon 		swp_pager_meta_build(object, 0, SWAPBLK_NONE);
1316ee3dc7d7SAlan Cox 	VM_OBJECT_UNLOCK(object);
1317e47ed70bSJohn Dyson 
1318e47ed70bSJohn Dyson 	if (curproc != pageproc)
1319e47ed70bSJohn Dyson 		sync = TRUE;
132026f9a767SRodney W. Grimes 
13211c7c3c6aSMatthew Dillon 	/*
13221c7c3c6aSMatthew Dillon 	 * Step 2
13231c7c3c6aSMatthew Dillon 	 *
1324ad3cce20SMatthew Dillon 	 * Update nsw parameters from swap_async_max sysctl values.
1325ad3cce20SMatthew Dillon 	 * Do not let the sysop crash the machine with bogus numbers.
1326327f4e83SMatthew Dillon 	 */
13276d541bf1SJohn Baldwin 	mtx_lock(&pbuf_mtx);
1328327f4e83SMatthew Dillon 	if (swap_async_max != nsw_wcount_async_max) {
1329327f4e83SMatthew Dillon 		int n;
1330327f4e83SMatthew Dillon 
1331327f4e83SMatthew Dillon 		/*
1332327f4e83SMatthew Dillon 		 * limit range
1333327f4e83SMatthew Dillon 		 */
1334327f4e83SMatthew Dillon 		if ((n = swap_async_max) > nswbuf / 2)
1335327f4e83SMatthew Dillon 			n = nswbuf / 2;
1336327f4e83SMatthew Dillon 		if (n < 1)
1337327f4e83SMatthew Dillon 			n = 1;
1338327f4e83SMatthew Dillon 		swap_async_max = n;
1339327f4e83SMatthew Dillon 
1340327f4e83SMatthew Dillon 		/*
1341327f4e83SMatthew Dillon 		 * Adjust difference ( if possible ).  If the current async
1342327f4e83SMatthew Dillon 		 * count is too low, we may not be able to make the adjustment
1343327f4e83SMatthew Dillon 		 * at this time.
1344327f4e83SMatthew Dillon 		 */
1345327f4e83SMatthew Dillon 		n -= nsw_wcount_async_max;
1346327f4e83SMatthew Dillon 		if (nsw_wcount_async + n >= 0) {
1347327f4e83SMatthew Dillon 			nsw_wcount_async += n;
1348327f4e83SMatthew Dillon 			nsw_wcount_async_max += n;
1349327f4e83SMatthew Dillon 			wakeup(&nsw_wcount_async);
1350327f4e83SMatthew Dillon 		}
1351327f4e83SMatthew Dillon 	}
13526d541bf1SJohn Baldwin 	mtx_unlock(&pbuf_mtx);
1353327f4e83SMatthew Dillon 
1354327f4e83SMatthew Dillon 	/*
1355327f4e83SMatthew Dillon 	 * Step 3
1356327f4e83SMatthew Dillon 	 *
13571c7c3c6aSMatthew Dillon 	 * Assign swap blocks and issue I/O.  We reallocate swap on the fly.
13581c7c3c6aSMatthew Dillon 	 * The page is left dirty until the pageout operation completes
13591c7c3c6aSMatthew Dillon 	 * successfully.
13601c7c3c6aSMatthew Dillon 	 */
13611c7c3c6aSMatthew Dillon 	for (i = 0; i < count; i += n) {
13621c7c3c6aSMatthew Dillon 		int j;
13631c7c3c6aSMatthew Dillon 		struct buf *bp;
1364a316d390SJohn Dyson 		daddr_t blk;
136526f9a767SRodney W. Grimes 
1366df8bae1dSRodney W. Grimes 		/*
13671c7c3c6aSMatthew Dillon 		 * Maximum I/O size is limited by a number of factors.
1368df8bae1dSRodney W. Grimes 		 */
13691c7c3c6aSMatthew Dillon 		n = min(BLIST_MAX_ALLOC, count - i);
1370327f4e83SMatthew Dillon 		n = min(n, nsw_cluster_max);
13711c7c3c6aSMatthew Dillon 
137226f9a767SRodney W. Grimes 		/*
13731c7c3c6aSMatthew Dillon 		 * Get biggest block of swap we can.  If we fail, fall
13741c7c3c6aSMatthew Dillon 		 * back and try to allocate a smaller block.  Don't go
13751c7c3c6aSMatthew Dillon 		 * overboard trying to allocate space if it would overly
13761c7c3c6aSMatthew Dillon 		 * fragment swap.
137726f9a767SRodney W. Grimes 		 */
13781c7c3c6aSMatthew Dillon 		while (
13791c7c3c6aSMatthew Dillon 		    (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE &&
13801c7c3c6aSMatthew Dillon 		    n > 4
13811c7c3c6aSMatthew Dillon 		) {
13821c7c3c6aSMatthew Dillon 			n >>= 1;
138326f9a767SRodney W. Grimes 		}
13841c7c3c6aSMatthew Dillon 		if (blk == SWAPBLK_NONE) {
13854dcc5c2dSMatthew Dillon 			for (j = 0; j < n; ++j)
13861c7c3c6aSMatthew Dillon 				rtvals[i+j] = VM_PAGER_FAIL;
13871c7c3c6aSMatthew Dillon 			continue;
138826f9a767SRodney W. Grimes 		}
138926f9a767SRodney W. Grimes 
139026f9a767SRodney W. Grimes 		/*
13911c7c3c6aSMatthew Dillon 		 * All I/O parameters have been satisfied, build the I/O
13921c7c3c6aSMatthew Dillon 		 * request and assign the swap space.
139326f9a767SRodney W. Grimes 		 */
1394327f4e83SMatthew Dillon 		if (sync == TRUE) {
1395327f4e83SMatthew Dillon 			bp = getpbuf(&nsw_wcount_sync);
1396327f4e83SMatthew Dillon 		} else {
1397327f4e83SMatthew Dillon 			bp = getpbuf(&nsw_wcount_async);
139821144e3bSPoul-Henning Kamp 			bp->b_flags = B_ASYNC;
1399327f4e83SMatthew Dillon 		}
14005e04322aSPoul-Henning Kamp 		bp->b_flags |= B_PAGING;
1401912e4ae9SPoul-Henning Kamp 		bp->b_iocmd = BIO_WRITE;
140226f9a767SRodney W. Grimes 
14031c7c3c6aSMatthew Dillon 		pmap_qenter((vm_offset_t)bp->b_data, &m[i], n);
14041c7c3c6aSMatthew Dillon 
1405fdcc1cc0SJohn Baldwin 		bp->b_rcred = crhold(thread0.td_ucred);
1406fdcc1cc0SJohn Baldwin 		bp->b_wcred = crhold(thread0.td_ucred);
14071c7c3c6aSMatthew Dillon 		bp->b_bcount = PAGE_SIZE * n;
14081c7c3c6aSMatthew Dillon 		bp->b_bufsize = PAGE_SIZE * n;
14091c7c3c6aSMatthew Dillon 		bp->b_blkno = blk;
1410e47ed70bSJohn Dyson 
1411ee3dc7d7SAlan Cox 		VM_OBJECT_LOCK(object);
14121c7c3c6aSMatthew Dillon 		for (j = 0; j < n; ++j) {
14131c7c3c6aSMatthew Dillon 			vm_page_t mreq = m[i+j];
14141c7c3c6aSMatthew Dillon 
14151c7c3c6aSMatthew Dillon 			swp_pager_meta_build(
14161c7c3c6aSMatthew Dillon 			    mreq->object,
14171c7c3c6aSMatthew Dillon 			    mreq->pindex,
14184dcc5c2dSMatthew Dillon 			    blk + j
14191c7c3c6aSMatthew Dillon 			);
14207dbf82dcSMatthew Dillon 			vm_page_dirty(mreq);
14211c7c3c6aSMatthew Dillon 			rtvals[i+j] = VM_PAGER_OK;
14221c7c3c6aSMatthew Dillon 
14235786be7cSAlan Cox 			mreq->oflags |= VPO_SWAPINPROG;
14241c7c3c6aSMatthew Dillon 			bp->b_pages[j] = mreq;
14251c7c3c6aSMatthew Dillon 		}
1426ee3dc7d7SAlan Cox 		VM_OBJECT_UNLOCK(object);
14271c7c3c6aSMatthew Dillon 		bp->b_npages = n;
1428a5296b05SJulian Elischer 		/*
1429a5296b05SJulian Elischer 		 * Must set dirty range for NFS to work.
1430a5296b05SJulian Elischer 		 */
1431a5296b05SJulian Elischer 		bp->b_dirtyoff = 0;
1432a5296b05SJulian Elischer 		bp->b_dirtyend = bp->b_bcount;
14331c7c3c6aSMatthew Dillon 
1434b4b70819SAttilio Rao 		PCPU_INC(cnt.v_swapout);
1435b4b70819SAttilio Rao 		PCPU_ADD(cnt.v_swappgsout, bp->b_npages);
143626f9a767SRodney W. Grimes 
143726f9a767SRodney W. Grimes 		/*
14381c7c3c6aSMatthew Dillon 		 * asynchronous
14391c7c3c6aSMatthew Dillon 		 *
1440c37a77eeSPoul-Henning Kamp 		 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
144126f9a767SRodney W. Grimes 		 */
14421c7c3c6aSMatthew Dillon 		if (sync == FALSE) {
14431c7c3c6aSMatthew Dillon 			bp->b_iodone = swp_pager_async_iodone;
144467812eacSKirk McKusick 			BUF_KERNPROC(bp);
14454b03903aSPoul-Henning Kamp 			swp_pager_strategy(bp);
14461c7c3c6aSMatthew Dillon 
14471c7c3c6aSMatthew Dillon 			for (j = 0; j < n; ++j)
14481c7c3c6aSMatthew Dillon 				rtvals[i+j] = VM_PAGER_PEND;
144923955314SAlfred Perlstein 			/* restart outter loop */
14501c7c3c6aSMatthew Dillon 			continue;
145126f9a767SRodney W. Grimes 		}
1452e47ed70bSJohn Dyson 
145326f9a767SRodney W. Grimes 		/*
14541c7c3c6aSMatthew Dillon 		 * synchronous
14551c7c3c6aSMatthew Dillon 		 *
1456c37a77eeSPoul-Henning Kamp 		 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
14571c7c3c6aSMatthew Dillon 		 */
14582c840b1fSAlan Cox 		bp->b_iodone = bdone;
14594b03903aSPoul-Henning Kamp 		swp_pager_strategy(bp);
14601c7c3c6aSMatthew Dillon 
14611c7c3c6aSMatthew Dillon 		/*
14621c7c3c6aSMatthew Dillon 		 * Wait for the sync I/O to complete, then update rtvals.
14631c7c3c6aSMatthew Dillon 		 * We just set the rtvals[] to VM_PAGER_PEND so we can call
14641c7c3c6aSMatthew Dillon 		 * our async completion routine at the end, thus avoiding a
14651c7c3c6aSMatthew Dillon 		 * double-free.
146626f9a767SRodney W. Grimes 		 */
14672c840b1fSAlan Cox 		bwait(bp, PVM, "swwrt");
14681c7c3c6aSMatthew Dillon 		for (j = 0; j < n; ++j)
14691c7c3c6aSMatthew Dillon 			rtvals[i+j] = VM_PAGER_PEND;
14701c7c3c6aSMatthew Dillon 		/*
14711c7c3c6aSMatthew Dillon 		 * Now that we are through with the bp, we can call the
14721c7c3c6aSMatthew Dillon 		 * normal async completion, which frees everything up.
14731c7c3c6aSMatthew Dillon 		 */
14741c7c3c6aSMatthew Dillon 		swp_pager_async_iodone(bp);
14751c7c3c6aSMatthew Dillon 	}
14762e3b314dSAlan Cox 	VM_OBJECT_LOCK(object);
14771c7c3c6aSMatthew Dillon }
14781c7c3c6aSMatthew Dillon 
14791c7c3c6aSMatthew Dillon /*
14801c7c3c6aSMatthew Dillon  *	swp_pager_async_iodone:
14811c7c3c6aSMatthew Dillon  *
14821c7c3c6aSMatthew Dillon  *	Completion routine for asynchronous reads and writes from/to swap.
14831c7c3c6aSMatthew Dillon  *	Also called manually by synchronous code to finish up a bp.
14841c7c3c6aSMatthew Dillon  *
148555144670SAndriy Gapon  *	For READ operations, the pages are VPO_BUSY'd.  For WRITE operations,
148655144670SAndriy Gapon  *	the pages are vm_page_t->busy'd.  For READ operations, we VPO_BUSY
14871c7c3c6aSMatthew Dillon  *	unbusy all pages except the 'main' request page.  For WRITE
14881c7c3c6aSMatthew Dillon  *	operations, we vm_page_t->busy'd unbusy all pages ( we can do this
14891c7c3c6aSMatthew Dillon  *	because we marked them all VM_PAGER_PEND on return from putpages ).
14901c7c3c6aSMatthew Dillon  *
14911c7c3c6aSMatthew Dillon  *	This routine may not block.
14921c7c3c6aSMatthew Dillon  */
14931c7c3c6aSMatthew Dillon static void
14942f249180SPoul-Henning Kamp swp_pager_async_iodone(struct buf *bp)
14951c7c3c6aSMatthew Dillon {
14961c7c3c6aSMatthew Dillon 	int i;
14971c7c3c6aSMatthew Dillon 	vm_object_t object = NULL;
14981c7c3c6aSMatthew Dillon 
14991c7c3c6aSMatthew Dillon 	/*
15001c7c3c6aSMatthew Dillon 	 * report error
15011c7c3c6aSMatthew Dillon 	 */
1502c244d2deSPoul-Henning Kamp 	if (bp->b_ioflags & BIO_ERROR) {
15031c7c3c6aSMatthew Dillon 		printf(
15041c7c3c6aSMatthew Dillon 		    "swap_pager: I/O error - %s failed; blkno %ld,"
15051c7c3c6aSMatthew Dillon 			"size %ld, error %d\n",
150621144e3bSPoul-Henning Kamp 		    ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
15071c7c3c6aSMatthew Dillon 		    (long)bp->b_blkno,
15081c7c3c6aSMatthew Dillon 		    (long)bp->b_bcount,
15091c7c3c6aSMatthew Dillon 		    bp->b_error
15101c7c3c6aSMatthew Dillon 		);
15111c7c3c6aSMatthew Dillon 	}
15121c7c3c6aSMatthew Dillon 
15131c7c3c6aSMatthew Dillon 	/*
151426f9a767SRodney W. Grimes 	 * remove the mapping for kernel virtual
151526f9a767SRodney W. Grimes 	 */
15161c7c3c6aSMatthew Dillon 	pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
151726f9a767SRodney W. Grimes 
151833a609ecSAlan Cox 	if (bp->b_npages) {
151933a609ecSAlan Cox 		object = bp->b_pages[0]->object;
152033a609ecSAlan Cox 		VM_OBJECT_LOCK(object);
152133a609ecSAlan Cox 	}
15222965a453SKip Macy 
152326f9a767SRodney W. Grimes 	/*
15241c7c3c6aSMatthew Dillon 	 * cleanup pages.  If an error occurs writing to swap, we are in
15251c7c3c6aSMatthew Dillon 	 * very serious trouble.  If it happens to be a disk error, though,
15261c7c3c6aSMatthew Dillon 	 * we may be able to recover by reassigning the swap later on.  So
15271c7c3c6aSMatthew Dillon 	 * in this case we remove the m->swapblk assignment for the page
15281c7c3c6aSMatthew Dillon 	 * but do not free it in the rlist.  The errornous block(s) are thus
15291c7c3c6aSMatthew Dillon 	 * never reallocated as swap.  Redirty the page and continue.
153026f9a767SRodney W. Grimes 	 */
15311c7c3c6aSMatthew Dillon 	for (i = 0; i < bp->b_npages; ++i) {
15321c7c3c6aSMatthew Dillon 		vm_page_t m = bp->b_pages[i];
1533e47ed70bSJohn Dyson 
15345786be7cSAlan Cox 		m->oflags &= ~VPO_SWAPINPROG;
1535e47ed70bSJohn Dyson 
1536c244d2deSPoul-Henning Kamp 		if (bp->b_ioflags & BIO_ERROR) {
1537ffc82b0aSJohn Dyson 			/*
15381c7c3c6aSMatthew Dillon 			 * If an error occurs I'd love to throw the swapblk
15391c7c3c6aSMatthew Dillon 			 * away without freeing it back to swapspace, so it
15401c7c3c6aSMatthew Dillon 			 * can never be used again.  But I can't from an
15411c7c3c6aSMatthew Dillon 			 * interrupt.
1542ffc82b0aSJohn Dyson 			 */
154321144e3bSPoul-Henning Kamp 			if (bp->b_iocmd == BIO_READ) {
15441c7c3c6aSMatthew Dillon 				/*
15451c7c3c6aSMatthew Dillon 				 * When reading, reqpage needs to stay
15461c7c3c6aSMatthew Dillon 				 * locked for the parent, but all other
15471c7c3c6aSMatthew Dillon 				 * pages can be freed.  We still want to
15481c7c3c6aSMatthew Dillon 				 * wakeup the parent waiting on the page,
15491c7c3c6aSMatthew Dillon 				 * though.  ( also: pg_reqpage can be -1 and
15501c7c3c6aSMatthew Dillon 				 * not match anything ).
15511c7c3c6aSMatthew Dillon 				 *
15521c7c3c6aSMatthew Dillon 				 * We have to wake specifically requested pages
15535786be7cSAlan Cox 				 * up too because we cleared VPO_SWAPINPROG and
15541c7c3c6aSMatthew Dillon 				 * someone may be waiting for that.
15551c7c3c6aSMatthew Dillon 				 *
15561c7c3c6aSMatthew Dillon 				 * NOTE: for reads, m->dirty will probably
1557956f3135SPhilippe Charnier 				 * be overridden by the original caller of
15581c7c3c6aSMatthew Dillon 				 * getpages so don't play cute tricks here.
15591c7c3c6aSMatthew Dillon 				 */
15601c7c3c6aSMatthew Dillon 				m->valid = 0;
15611c7c3c6aSMatthew Dillon 				if (i != bp->b_pager.pg_reqpage)
1562e86a87e9SKonstantin Belousov 					swp_pager_free_nrpage(m);
15631c7c3c6aSMatthew Dillon 				else
15641c7c3c6aSMatthew Dillon 					vm_page_flash(m);
15651c7c3c6aSMatthew Dillon 				/*
15661c7c3c6aSMatthew Dillon 				 * If i == bp->b_pager.pg_reqpage, do not wake
15671c7c3c6aSMatthew Dillon 				 * the page up.  The caller needs to.
15681c7c3c6aSMatthew Dillon 				 */
15691c7c3c6aSMatthew Dillon 			} else {
15701c7c3c6aSMatthew Dillon 				/*
15711c7c3c6aSMatthew Dillon 				 * If a write error occurs, reactivate page
15721c7c3c6aSMatthew Dillon 				 * so it doesn't clog the inactive list,
15731c7c3c6aSMatthew Dillon 				 * then finish the I/O.
15741c7c3c6aSMatthew Dillon 				 */
15757dbf82dcSMatthew Dillon 				vm_page_dirty(m);
15763c4a2440SAlan Cox 				vm_page_lock(m);
15771c7c3c6aSMatthew Dillon 				vm_page_activate(m);
15783c4a2440SAlan Cox 				vm_page_unlock(m);
15791c7c3c6aSMatthew Dillon 				vm_page_io_finish(m);
15801c7c3c6aSMatthew Dillon 			}
158121144e3bSPoul-Henning Kamp 		} else if (bp->b_iocmd == BIO_READ) {
15821c7c3c6aSMatthew Dillon 			/*
15831c7c3c6aSMatthew Dillon 			 * NOTE: for reads, m->dirty will probably be
1584956f3135SPhilippe Charnier 			 * overridden by the original caller of getpages so
15851c7c3c6aSMatthew Dillon 			 * we cannot set them in order to free the underlying
15861c7c3c6aSMatthew Dillon 			 * swap in a low-swap situation.  I don't think we'd
15871c7c3c6aSMatthew Dillon 			 * want to do that anyway, but it was an optimization
15881c7c3c6aSMatthew Dillon 			 * that existed in the old swapper for a time before
15891c7c3c6aSMatthew Dillon 			 * it got ripped out due to precisely this problem.
15901c7c3c6aSMatthew Dillon 			 *
15911c7c3c6aSMatthew Dillon 			 * If not the requested page then deactivate it.
15921c7c3c6aSMatthew Dillon 			 *
15931c7c3c6aSMatthew Dillon 			 * Note that the requested page, reqpage, is left
15941c7c3c6aSMatthew Dillon 			 * busied, but we still have to wake it up.  The
15951c7c3c6aSMatthew Dillon 			 * other pages are released (unbusied) by
1596a80982c1SAlan Cox 			 * vm_page_wakeup().
15971c7c3c6aSMatthew Dillon 			 */
1598016a3c93SAlan Cox 			KASSERT(!pmap_page_is_mapped(m),
1599016a3c93SAlan Cox 			    ("swp_pager_async_iodone: page %p is mapped", m));
16001c7c3c6aSMatthew Dillon 			m->valid = VM_PAGE_BITS_ALL;
1601016a3c93SAlan Cox 			KASSERT(m->dirty == 0,
1602016a3c93SAlan Cox 			    ("swp_pager_async_iodone: page %p is dirty", m));
16031c7c3c6aSMatthew Dillon 
16041c7c3c6aSMatthew Dillon 			/*
16051c7c3c6aSMatthew Dillon 			 * We have to wake specifically requested pages
16065786be7cSAlan Cox 			 * up too because we cleared VPO_SWAPINPROG and
16071c7c3c6aSMatthew Dillon 			 * could be waiting for it in getpages.  However,
16081c7c3c6aSMatthew Dillon 			 * be sure to not unbusy getpages specifically
16091c7c3c6aSMatthew Dillon 			 * requested page - getpages expects it to be
16101c7c3c6aSMatthew Dillon 			 * left busy.
16111c7c3c6aSMatthew Dillon 			 */
16121c7c3c6aSMatthew Dillon 			if (i != bp->b_pager.pg_reqpage) {
16133c4a2440SAlan Cox 				vm_page_lock(m);
16141c7c3c6aSMatthew Dillon 				vm_page_deactivate(m);
16153c4a2440SAlan Cox 				vm_page_unlock(m);
16161c7c3c6aSMatthew Dillon 				vm_page_wakeup(m);
16173c4a2440SAlan Cox 			} else
16181c7c3c6aSMatthew Dillon 				vm_page_flash(m);
16191c7c3c6aSMatthew Dillon 		} else {
16201c7c3c6aSMatthew Dillon 			/*
1621016a3c93SAlan Cox 			 * For write success, clear the dirty
16221c7c3c6aSMatthew Dillon 			 * status, then finish the I/O ( which decrements the
16231c7c3c6aSMatthew Dillon 			 * busy count and possibly wakes waiter's up ).
16241c7c3c6aSMatthew Dillon 			 */
1625016a3c93SAlan Cox 			KASSERT((m->flags & PG_WRITEABLE) == 0,
1626016a3c93SAlan Cox 			    ("swp_pager_async_iodone: page %p is not write"
1627016a3c93SAlan Cox 			    " protected", m));
1628c52e7044SAlan Cox 			vm_page_undirty(m);
16291c7c3c6aSMatthew Dillon 			vm_page_io_finish(m);
16303c4a2440SAlan Cox 			if (vm_page_count_severe()) {
16313c4a2440SAlan Cox 				vm_page_lock(m);
16322c840b1fSAlan Cox 				vm_page_try_to_cache(m);
16332965a453SKip Macy 				vm_page_unlock(m);
16342965a453SKip Macy 			}
16353c4a2440SAlan Cox 		}
16363c4a2440SAlan Cox 	}
163726f9a767SRodney W. Grimes 
16381c7c3c6aSMatthew Dillon 	/*
16391c7c3c6aSMatthew Dillon 	 * adjust pip.  NOTE: the original parent may still have its own
16401c7c3c6aSMatthew Dillon 	 * pip refs on the object.
16411c7c3c6aSMatthew Dillon 	 */
16420d420ad3SAlan Cox 	if (object != NULL) {
16431c7c3c6aSMatthew Dillon 		vm_object_pip_wakeupn(object, bp->b_npages);
16440d420ad3SAlan Cox 		VM_OBJECT_UNLOCK(object);
16450d420ad3SAlan Cox 	}
164626f9a767SRodney W. Grimes 
16471c7c3c6aSMatthew Dillon 	/*
1648100650deSOlivier Houchard 	 * swapdev_strategy() manually sets b_vp and b_bufobj before calling
1649100650deSOlivier Houchard 	 * bstrategy(). Set them back to NULL now we're done with it, or we'll
1650100650deSOlivier Houchard 	 * trigger a KASSERT in relpbuf().
1651100650deSOlivier Houchard 	 */
1652100650deSOlivier Houchard 	if (bp->b_vp) {
1653100650deSOlivier Houchard 		    bp->b_vp = NULL;
1654100650deSOlivier Houchard 		    bp->b_bufobj = NULL;
1655100650deSOlivier Houchard 	}
1656100650deSOlivier Houchard 	/*
16571c7c3c6aSMatthew Dillon 	 * release the physical I/O buffer
16581c7c3c6aSMatthew Dillon 	 */
1659327f4e83SMatthew Dillon 	relpbuf(
1660327f4e83SMatthew Dillon 	    bp,
166121144e3bSPoul-Henning Kamp 	    ((bp->b_iocmd == BIO_READ) ? &nsw_rcount :
1662327f4e83SMatthew Dillon 		((bp->b_flags & B_ASYNC) ?
1663327f4e83SMatthew Dillon 		    &nsw_wcount_async :
1664327f4e83SMatthew Dillon 		    &nsw_wcount_sync
1665327f4e83SMatthew Dillon 		)
1666327f4e83SMatthew Dillon 	    )
1667327f4e83SMatthew Dillon 	);
166826f9a767SRodney W. Grimes }
16691c7c3c6aSMatthew Dillon 
167092da00bbSMatthew Dillon /*
167192da00bbSMatthew Dillon  *	swap_pager_isswapped:
167292da00bbSMatthew Dillon  *
167392da00bbSMatthew Dillon  *	Return 1 if at least one page in the given object is paged
167492da00bbSMatthew Dillon  *	out to the given swap device.
167592da00bbSMatthew Dillon  *
167692da00bbSMatthew Dillon  *	This routine may not block.
167792da00bbSMatthew Dillon  */
16788f60c087SPoul-Henning Kamp int
16798f60c087SPoul-Henning Kamp swap_pager_isswapped(vm_object_t object, struct swdevt *sp)
16808f60c087SPoul-Henning Kamp {
168192da00bbSMatthew Dillon 	daddr_t index = 0;
168292da00bbSMatthew Dillon 	int bcount;
168392da00bbSMatthew Dillon 	int i;
168492da00bbSMatthew Dillon 
168517cd3642SAlan Cox 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1686f6bcadc4SDavid Schultz 	if (object->type != OBJT_SWAP)
1687f6bcadc4SDavid Schultz 		return (0);
1688f6bcadc4SDavid Schultz 
1689b3fed13eSDavid Schultz 	mtx_lock(&swhash_mtx);
169092da00bbSMatthew Dillon 	for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) {
169192da00bbSMatthew Dillon 		struct swblock *swap;
169292da00bbSMatthew Dillon 
169392da00bbSMatthew Dillon 		if ((swap = *swp_pager_hash(object, index)) != NULL) {
169492da00bbSMatthew Dillon 			for (i = 0; i < SWAP_META_PAGES; ++i) {
1695b3fed13eSDavid Schultz 				if (swp_pager_isondev(swap->swb_pages[i], sp)) {
16967827d9b0SAlan Cox 					mtx_unlock(&swhash_mtx);
1697b3fed13eSDavid Schultz 					return (1);
169892da00bbSMatthew Dillon 				}
169992da00bbSMatthew Dillon 			}
17007827d9b0SAlan Cox 		}
170192da00bbSMatthew Dillon 		index += SWAP_META_PAGES;
170292da00bbSMatthew Dillon 	}
1703b3fed13eSDavid Schultz 	mtx_unlock(&swhash_mtx);
1704b3fed13eSDavid Schultz 	return (0);
170592da00bbSMatthew Dillon }
170692da00bbSMatthew Dillon 
170792da00bbSMatthew Dillon /*
170892da00bbSMatthew Dillon  * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in
170992da00bbSMatthew Dillon  *
171092da00bbSMatthew Dillon  *	This routine dissociates the page at the given index within a
171192da00bbSMatthew Dillon  *	swap block from its backing store, paging it in if necessary.
171292da00bbSMatthew Dillon  *	If the page is paged in, it is placed in the inactive queue,
171392da00bbSMatthew Dillon  *	since it had its backing store ripped out from under it.
171492da00bbSMatthew Dillon  *	We also attempt to swap in all other pages in the swap block,
171592da00bbSMatthew Dillon  *	we only guarantee that the one at the specified index is
171692da00bbSMatthew Dillon  *	paged in.
171792da00bbSMatthew Dillon  *
171892da00bbSMatthew Dillon  *	XXX - The code to page the whole block in doesn't work, so we
171992da00bbSMatthew Dillon  *	      revert to the one-by-one behavior for now.  Sigh.
172092da00bbSMatthew Dillon  */
172162a59e8fSWarner Losh static inline void
1722b3fed13eSDavid Schultz swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex)
172392da00bbSMatthew Dillon {
172492da00bbSMatthew Dillon 	vm_page_t m;
172592da00bbSMatthew Dillon 
172692da00bbSMatthew Dillon 	vm_object_pip_add(object, 1);
1727b3fed13eSDavid Schultz 	m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL|VM_ALLOC_RETRY);
172892da00bbSMatthew Dillon 	if (m->valid == VM_PAGE_BITS_ALL) {
172992da00bbSMatthew Dillon 		vm_object_pip_subtract(object, 1);
173092da00bbSMatthew Dillon 		vm_page_dirty(m);
1731d061cdd5SAlan Cox 		vm_page_lock(m);
1732d061cdd5SAlan Cox 		vm_page_activate(m);
17332965a453SKip Macy 		vm_page_unlock(m);
173466bdd5d6SAlan Cox 		vm_page_wakeup(m);
173592da00bbSMatthew Dillon 		vm_pager_page_unswapped(m);
173692da00bbSMatthew Dillon 		return;
173792da00bbSMatthew Dillon 	}
173892da00bbSMatthew Dillon 
1739b3fed13eSDavid Schultz 	if (swap_pager_getpages(object, &m, 1, 0) != VM_PAGER_OK)
174092da00bbSMatthew Dillon 		panic("swap_pager_force_pagein: read from swap failed");/*XXX*/
174192da00bbSMatthew Dillon 	vm_object_pip_subtract(object, 1);
174292da00bbSMatthew Dillon 	vm_page_dirty(m);
1743db1f085eSAlan Cox 	vm_page_lock(m);
1744db1f085eSAlan Cox 	vm_page_deactivate(m);
17452965a453SKip Macy 	vm_page_unlock(m);
174666bdd5d6SAlan Cox 	vm_page_wakeup(m);
174792da00bbSMatthew Dillon 	vm_pager_page_unswapped(m);
174892da00bbSMatthew Dillon }
174992da00bbSMatthew Dillon 
175092da00bbSMatthew Dillon /*
175192da00bbSMatthew Dillon  *	swap_pager_swapoff:
175292da00bbSMatthew Dillon  *
175392da00bbSMatthew Dillon  *	Page in all of the pages that have been paged out to the
175492da00bbSMatthew Dillon  *	given device.  The corresponding blocks in the bitmap must be
175592da00bbSMatthew Dillon  *	marked as allocated and the device must be flagged SW_CLOSING.
175692da00bbSMatthew Dillon  *	There may be no processes swapped out to the device.
175792da00bbSMatthew Dillon  *
175892da00bbSMatthew Dillon  *	This routine may block.
175992da00bbSMatthew Dillon  */
1760e9c0cc15SPoul-Henning Kamp static void
1761b3fed13eSDavid Schultz swap_pager_swapoff(struct swdevt *sp)
176292da00bbSMatthew Dillon {
176392da00bbSMatthew Dillon 	struct swblock *swap;
17648bc61209SDavid Schultz 	int i, j, retries;
176592da00bbSMatthew Dillon 
176692da00bbSMatthew Dillon 	GIANT_REQUIRED;
176792da00bbSMatthew Dillon 
17688bc61209SDavid Schultz 	retries = 0;
176992da00bbSMatthew Dillon full_rescan:
1770b3fed13eSDavid Schultz 	mtx_lock(&swhash_mtx);
177192da00bbSMatthew Dillon 	for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */
177292da00bbSMatthew Dillon restart:
1773b3fed13eSDavid Schultz 		for (swap = swhash[i]; swap != NULL; swap = swap->swb_hnext) {
1774b3fed13eSDavid Schultz 			vm_object_t object = swap->swb_object;
1775b3fed13eSDavid Schultz 			vm_pindex_t pindex = swap->swb_index;
177692da00bbSMatthew Dillon                         for (j = 0; j < SWAP_META_PAGES; ++j) {
1777b3fed13eSDavid Schultz                                 if (swp_pager_isondev(swap->swb_pages[j], sp)) {
1778b3fed13eSDavid Schultz 					/* avoid deadlock */
1779b3fed13eSDavid Schultz 					if (!VM_OBJECT_TRYLOCK(object)) {
178092da00bbSMatthew Dillon 						break;
1781b3fed13eSDavid Schultz 					} else {
1782b3fed13eSDavid Schultz 						mtx_unlock(&swhash_mtx);
1783b3fed13eSDavid Schultz 						swp_pager_force_pagein(object,
1784b3fed13eSDavid Schultz 						    pindex + j);
1785b3fed13eSDavid Schultz 						VM_OBJECT_UNLOCK(object);
1786b3fed13eSDavid Schultz 						mtx_lock(&swhash_mtx);
178792da00bbSMatthew Dillon 						goto restart;
178892da00bbSMatthew Dillon 					}
1789b3fed13eSDavid Schultz 				}
1790b3fed13eSDavid Schultz                         }
1791b3fed13eSDavid Schultz 		}
179292da00bbSMatthew Dillon 	}
1793d536c58fSAlan Cox 	mtx_unlock(&swhash_mtx);
17948bc61209SDavid Schultz 	if (sp->sw_used) {
179592da00bbSMatthew Dillon 		/*
17968bc61209SDavid Schultz 		 * Objects may be locked or paging to the device being
17978bc61209SDavid Schultz 		 * removed, so we will miss their pages and need to
17988bc61209SDavid Schultz 		 * make another pass.  We have marked this device as
17998bc61209SDavid Schultz 		 * SW_CLOSING, so the activity should finish soon.
180092da00bbSMatthew Dillon 		 */
18018bc61209SDavid Schultz 		retries++;
18028bc61209SDavid Schultz 		if (retries > 100) {
18038bc61209SDavid Schultz 			panic("swapoff: failed to locate %d swap blocks",
18048bc61209SDavid Schultz 			    sp->sw_used);
18058bc61209SDavid Schultz 		}
18064d70511aSJohn Baldwin 		pause("swpoff", hz / 20);
180792da00bbSMatthew Dillon 		goto full_rescan;
180892da00bbSMatthew Dillon 	}
180992da00bbSMatthew Dillon }
181092da00bbSMatthew Dillon 
18111c7c3c6aSMatthew Dillon /************************************************************************
18121c7c3c6aSMatthew Dillon  *				SWAP META DATA 				*
18131c7c3c6aSMatthew Dillon  ************************************************************************
18141c7c3c6aSMatthew Dillon  *
18151c7c3c6aSMatthew Dillon  *	These routines manipulate the swap metadata stored in the
18164dcc5c2dSMatthew Dillon  *	OBJT_SWAP object.  All swp_*() routines must be called at
18174dcc5c2dSMatthew Dillon  *	splvm() because swap can be freed up by the low level vm_page
18184dcc5c2dSMatthew Dillon  *	code which might be called from interrupts beyond what splbio() covers.
18191c7c3c6aSMatthew Dillon  *
18204dcc5c2dSMatthew Dillon  *	Swap metadata is implemented with a global hash and not directly
18214dcc5c2dSMatthew Dillon  *	linked into the object.  Instead the object simply contains
18224dcc5c2dSMatthew Dillon  *	appropriate tracking counters.
18231c7c3c6aSMatthew Dillon  */
18241c7c3c6aSMatthew Dillon 
18251c7c3c6aSMatthew Dillon /*
18261c7c3c6aSMatthew Dillon  * SWP_PAGER_META_BUILD() -	add swap block to swap meta data for object
18271c7c3c6aSMatthew Dillon  *
18281c7c3c6aSMatthew Dillon  *	We first convert the object to a swap object if it is a default
18291c7c3c6aSMatthew Dillon  *	object.
18301c7c3c6aSMatthew Dillon  *
18311c7c3c6aSMatthew Dillon  *	The specified swapblk is added to the object's swap metadata.  If
18321c7c3c6aSMatthew Dillon  *	the swapblk is not valid, it is freed instead.  Any previously
18331c7c3c6aSMatthew Dillon  *	assigned swapblk is freed.
18344dcc5c2dSMatthew Dillon  *
18354dcc5c2dSMatthew Dillon  *	This routine must be called at splvm(), except when used to convert
18364dcc5c2dSMatthew Dillon  *	an OBJT_DEFAULT object into an OBJT_SWAP object.
18371c7c3c6aSMatthew Dillon  */
18381c7c3c6aSMatthew Dillon static void
18392f249180SPoul-Henning Kamp swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
18402f249180SPoul-Henning Kamp {
18411c7c3c6aSMatthew Dillon 	struct swblock *swap;
18421c7c3c6aSMatthew Dillon 	struct swblock **pswap;
184323f09d50SIan Dowse 	int idx;
18441c7c3c6aSMatthew Dillon 
1845ee3dc7d7SAlan Cox 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
18461c7c3c6aSMatthew Dillon 	/*
18471c7c3c6aSMatthew Dillon 	 * Convert default object to swap object if necessary
18481c7c3c6aSMatthew Dillon 	 */
18491c7c3c6aSMatthew Dillon 	if (object->type != OBJT_SWAP) {
18501c7c3c6aSMatthew Dillon 		object->type = OBJT_SWAP;
18511c7c3c6aSMatthew Dillon 		object->un_pager.swp.swp_bcount = 0;
18521c7c3c6aSMatthew Dillon 
18531c7c3c6aSMatthew Dillon 		if (object->handle != NULL) {
1854bd228075SAlan Cox 			mtx_lock(&sw_alloc_mtx);
18551c7c3c6aSMatthew Dillon 			TAILQ_INSERT_TAIL(
18561c7c3c6aSMatthew Dillon 			    NOBJLIST(object->handle),
18571c7c3c6aSMatthew Dillon 			    object,
18581c7c3c6aSMatthew Dillon 			    pager_object_list
18591c7c3c6aSMatthew Dillon 			);
1860a9fa2c05SAlfred Perlstein 			mtx_unlock(&sw_alloc_mtx);
18611c7c3c6aSMatthew Dillon 		}
1862bd228075SAlan Cox 	}
18631c7c3c6aSMatthew Dillon 
18641c7c3c6aSMatthew Dillon 	/*
18651c7c3c6aSMatthew Dillon 	 * Locate hash entry.  If not found create, but if we aren't adding
18664dcc5c2dSMatthew Dillon 	 * anything just return.  If we run out of space in the map we wait
18674dcc5c2dSMatthew Dillon 	 * and, since the hash table may have changed, retry.
18681c7c3c6aSMatthew Dillon 	 */
18694dcc5c2dSMatthew Dillon retry:
18707827d9b0SAlan Cox 	mtx_lock(&swhash_mtx);
187123f09d50SIan Dowse 	pswap = swp_pager_hash(object, pindex);
18721c7c3c6aSMatthew Dillon 
18731c7c3c6aSMatthew Dillon 	if ((swap = *pswap) == NULL) {
18741c7c3c6aSMatthew Dillon 		int i;
18751c7c3c6aSMatthew Dillon 
18761c7c3c6aSMatthew Dillon 		if (swapblk == SWAPBLK_NONE)
18777827d9b0SAlan Cox 			goto done;
18781c7c3c6aSMatthew Dillon 
1879670d17b5SJeff Roberson 		swap = *pswap = uma_zalloc(swap_zone, M_NOWAIT);
18804dcc5c2dSMatthew Dillon 		if (swap == NULL) {
18817827d9b0SAlan Cox 			mtx_unlock(&swhash_mtx);
1882ee3dc7d7SAlan Cox 			VM_OBJECT_UNLOCK(object);
18832025d69bSKonstantin Belousov 			if (uma_zone_exhausted(swap_zone)) {
18842025d69bSKonstantin Belousov 				printf("swap zone exhausted, increase kern.maxswzone\n");
18852025d69bSKonstantin Belousov 				vm_pageout_oom(VM_OOM_SWAPZ);
18862025d69bSKonstantin Belousov 				pause("swzonex", 10);
18872025d69bSKonstantin Belousov 			} else
18884dcc5c2dSMatthew Dillon 				VM_WAIT;
1889ee3dc7d7SAlan Cox 			VM_OBJECT_LOCK(object);
18904dcc5c2dSMatthew Dillon 			goto retry;
18914dcc5c2dSMatthew Dillon 		}
1892670d17b5SJeff Roberson 
18931c7c3c6aSMatthew Dillon 		swap->swb_hnext = NULL;
18941c7c3c6aSMatthew Dillon 		swap->swb_object = object;
189523f09d50SIan Dowse 		swap->swb_index = pindex & ~(vm_pindex_t)SWAP_META_MASK;
18961c7c3c6aSMatthew Dillon 		swap->swb_count = 0;
18971c7c3c6aSMatthew Dillon 
18981c7c3c6aSMatthew Dillon 		++object->un_pager.swp.swp_bcount;
18991c7c3c6aSMatthew Dillon 
19001c7c3c6aSMatthew Dillon 		for (i = 0; i < SWAP_META_PAGES; ++i)
19011c7c3c6aSMatthew Dillon 			swap->swb_pages[i] = SWAPBLK_NONE;
19021c7c3c6aSMatthew Dillon 	}
19031c7c3c6aSMatthew Dillon 
19041c7c3c6aSMatthew Dillon 	/*
19051c7c3c6aSMatthew Dillon 	 * Delete prior contents of metadata
19061c7c3c6aSMatthew Dillon 	 */
190723f09d50SIan Dowse 	idx = pindex & SWAP_META_MASK;
19081c7c3c6aSMatthew Dillon 
190923f09d50SIan Dowse 	if (swap->swb_pages[idx] != SWAPBLK_NONE) {
191023f09d50SIan Dowse 		swp_pager_freeswapspace(swap->swb_pages[idx], 1);
19111c7c3c6aSMatthew Dillon 		--swap->swb_count;
19121c7c3c6aSMatthew Dillon 	}
19131c7c3c6aSMatthew Dillon 
19141c7c3c6aSMatthew Dillon 	/*
19151c7c3c6aSMatthew Dillon 	 * Enter block into metadata
19161c7c3c6aSMatthew Dillon 	 */
191723f09d50SIan Dowse 	swap->swb_pages[idx] = swapblk;
19184dcc5c2dSMatthew Dillon 	if (swapblk != SWAPBLK_NONE)
19191c7c3c6aSMatthew Dillon 		++swap->swb_count;
19207827d9b0SAlan Cox done:
19217827d9b0SAlan Cox 	mtx_unlock(&swhash_mtx);
19221c7c3c6aSMatthew Dillon }
19231c7c3c6aSMatthew Dillon 
19241c7c3c6aSMatthew Dillon /*
19251c7c3c6aSMatthew Dillon  * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
19261c7c3c6aSMatthew Dillon  *
19271c7c3c6aSMatthew Dillon  *	The requested range of blocks is freed, with any associated swap
19281c7c3c6aSMatthew Dillon  *	returned to the swap bitmap.
19291c7c3c6aSMatthew Dillon  *
19301c7c3c6aSMatthew Dillon  *	This routine will free swap metadata structures as they are cleaned
19311c7c3c6aSMatthew Dillon  *	out.  This routine does *NOT* operate on swap metadata associated
19321c7c3c6aSMatthew Dillon  *	with resident pages.
19331c7c3c6aSMatthew Dillon  *
19341c7c3c6aSMatthew Dillon  *	This routine must be called at splvm()
19351c7c3c6aSMatthew Dillon  */
19361c7c3c6aSMatthew Dillon static void
19374dcc5c2dSMatthew Dillon swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count)
19381c7c3c6aSMatthew Dillon {
19392928cef7SAlan Cox 
1940ee3dc7d7SAlan Cox 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
19411c7c3c6aSMatthew Dillon 	if (object->type != OBJT_SWAP)
19421c7c3c6aSMatthew Dillon 		return;
19431c7c3c6aSMatthew Dillon 
19441c7c3c6aSMatthew Dillon 	while (count > 0) {
19451c7c3c6aSMatthew Dillon 		struct swblock **pswap;
19461c7c3c6aSMatthew Dillon 		struct swblock *swap;
19471c7c3c6aSMatthew Dillon 
19487827d9b0SAlan Cox 		mtx_lock(&swhash_mtx);
19491c7c3c6aSMatthew Dillon 		pswap = swp_pager_hash(object, index);
19501c7c3c6aSMatthew Dillon 
19511c7c3c6aSMatthew Dillon 		if ((swap = *pswap) != NULL) {
19521c7c3c6aSMatthew Dillon 			daddr_t v = swap->swb_pages[index & SWAP_META_MASK];
19531c7c3c6aSMatthew Dillon 
19541c7c3c6aSMatthew Dillon 			if (v != SWAPBLK_NONE) {
19551c7c3c6aSMatthew Dillon 				swp_pager_freeswapspace(v, 1);
19561c7c3c6aSMatthew Dillon 				swap->swb_pages[index & SWAP_META_MASK] =
19571c7c3c6aSMatthew Dillon 					SWAPBLK_NONE;
19581c7c3c6aSMatthew Dillon 				if (--swap->swb_count == 0) {
19591c7c3c6aSMatthew Dillon 					*pswap = swap->swb_hnext;
1960670d17b5SJeff Roberson 					uma_zfree(swap_zone, swap);
19611c7c3c6aSMatthew Dillon 					--object->un_pager.swp.swp_bcount;
19621c7c3c6aSMatthew Dillon 				}
19631c7c3c6aSMatthew Dillon 			}
19641c7c3c6aSMatthew Dillon 			--count;
19651c7c3c6aSMatthew Dillon 			++index;
19661c7c3c6aSMatthew Dillon 		} else {
19674dcc5c2dSMatthew Dillon 			int n = SWAP_META_PAGES - (index & SWAP_META_MASK);
19681c7c3c6aSMatthew Dillon 			count -= n;
19691c7c3c6aSMatthew Dillon 			index += n;
19701c7c3c6aSMatthew Dillon 		}
19717827d9b0SAlan Cox 		mtx_unlock(&swhash_mtx);
19721c7c3c6aSMatthew Dillon 	}
19731c7c3c6aSMatthew Dillon }
19741c7c3c6aSMatthew Dillon 
19751c7c3c6aSMatthew Dillon /*
19761c7c3c6aSMatthew Dillon  * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
19771c7c3c6aSMatthew Dillon  *
19781c7c3c6aSMatthew Dillon  *	This routine locates and destroys all swap metadata associated with
19791c7c3c6aSMatthew Dillon  *	an object.
19804dcc5c2dSMatthew Dillon  *
19814dcc5c2dSMatthew Dillon  *	This routine must be called at splvm()
19821c7c3c6aSMatthew Dillon  */
19831c7c3c6aSMatthew Dillon static void
19841c7c3c6aSMatthew Dillon swp_pager_meta_free_all(vm_object_t object)
19851c7c3c6aSMatthew Dillon {
19861c7c3c6aSMatthew Dillon 	daddr_t index = 0;
19871c7c3c6aSMatthew Dillon 
1988ee3dc7d7SAlan Cox 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
19891c7c3c6aSMatthew Dillon 	if (object->type != OBJT_SWAP)
19901c7c3c6aSMatthew Dillon 		return;
19911c7c3c6aSMatthew Dillon 
19921c7c3c6aSMatthew Dillon 	while (object->un_pager.swp.swp_bcount) {
19931c7c3c6aSMatthew Dillon 		struct swblock **pswap;
19941c7c3c6aSMatthew Dillon 		struct swblock *swap;
19951c7c3c6aSMatthew Dillon 
19967827d9b0SAlan Cox 		mtx_lock(&swhash_mtx);
19971c7c3c6aSMatthew Dillon 		pswap = swp_pager_hash(object, index);
19981c7c3c6aSMatthew Dillon 		if ((swap = *pswap) != NULL) {
19991c7c3c6aSMatthew Dillon 			int i;
20001c7c3c6aSMatthew Dillon 
20011c7c3c6aSMatthew Dillon 			for (i = 0; i < SWAP_META_PAGES; ++i) {
20021c7c3c6aSMatthew Dillon 				daddr_t v = swap->swb_pages[i];
20031c7c3c6aSMatthew Dillon 				if (v != SWAPBLK_NONE) {
20041c7c3c6aSMatthew Dillon 					--swap->swb_count;
20054dcc5c2dSMatthew Dillon 					swp_pager_freeswapspace(v, 1);
20061c7c3c6aSMatthew Dillon 				}
20071c7c3c6aSMatthew Dillon 			}
20081c7c3c6aSMatthew Dillon 			if (swap->swb_count != 0)
20091c7c3c6aSMatthew Dillon 				panic("swap_pager_meta_free_all: swb_count != 0");
20101c7c3c6aSMatthew Dillon 			*pswap = swap->swb_hnext;
2011670d17b5SJeff Roberson 			uma_zfree(swap_zone, swap);
20121c7c3c6aSMatthew Dillon 			--object->un_pager.swp.swp_bcount;
20131c7c3c6aSMatthew Dillon 		}
20147827d9b0SAlan Cox 		mtx_unlock(&swhash_mtx);
20151c7c3c6aSMatthew Dillon 		index += SWAP_META_PAGES;
20161c7c3c6aSMatthew Dillon 	}
20171c7c3c6aSMatthew Dillon }
20181c7c3c6aSMatthew Dillon 
20191c7c3c6aSMatthew Dillon /*
20201c7c3c6aSMatthew Dillon  * SWP_PAGER_METACTL() -  misc control of swap and vm_page_t meta data.
20211c7c3c6aSMatthew Dillon  *
20221c7c3c6aSMatthew Dillon  *	This routine is capable of looking up, popping, or freeing
20231c7c3c6aSMatthew Dillon  *	swapblk assignments in the swap meta data or in the vm_page_t.
20241c7c3c6aSMatthew Dillon  *	The routine typically returns the swapblk being looked-up, or popped,
20251c7c3c6aSMatthew Dillon  *	or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block
20261c7c3c6aSMatthew Dillon  *	was invalid.  This routine will automatically free any invalid
20271c7c3c6aSMatthew Dillon  *	meta-data swapblks.
20281c7c3c6aSMatthew Dillon  *
20291c7c3c6aSMatthew Dillon  *	It is not possible to store invalid swapblks in the swap meta data
20301c7c3c6aSMatthew Dillon  *	(other then a literal 'SWAPBLK_NONE'), so we don't bother checking.
20311c7c3c6aSMatthew Dillon  *
20321c7c3c6aSMatthew Dillon  *	When acting on a busy resident page and paging is in progress, we
20331c7c3c6aSMatthew Dillon  *	have to wait until paging is complete but otherwise can act on the
20341c7c3c6aSMatthew Dillon  *	busy page.
20351c7c3c6aSMatthew Dillon  *
20364dcc5c2dSMatthew Dillon  *	This routine must be called at splvm().
20371c7c3c6aSMatthew Dillon  *
20384dcc5c2dSMatthew Dillon  *	SWM_FREE	remove and free swap block from metadata
20391c7c3c6aSMatthew Dillon  *	SWM_POP		remove from meta data but do not free.. pop it out
20401c7c3c6aSMatthew Dillon  */
20411c7c3c6aSMatthew Dillon static daddr_t
20422f249180SPoul-Henning Kamp swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
20432f249180SPoul-Henning Kamp {
20444dcc5c2dSMatthew Dillon 	struct swblock **pswap;
20454dcc5c2dSMatthew Dillon 	struct swblock *swap;
20464dcc5c2dSMatthew Dillon 	daddr_t r1;
204723f09d50SIan Dowse 	int idx;
20484dcc5c2dSMatthew Dillon 
2049c7c8dd7eSAlan Cox 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
20501c7c3c6aSMatthew Dillon 	/*
20511c7c3c6aSMatthew Dillon 	 * The meta data only exists of the object is OBJT_SWAP
20521c7c3c6aSMatthew Dillon 	 * and even then might not be allocated yet.
20531c7c3c6aSMatthew Dillon 	 */
20544dcc5c2dSMatthew Dillon 	if (object->type != OBJT_SWAP)
20551c7c3c6aSMatthew Dillon 		return (SWAPBLK_NONE);
20561c7c3c6aSMatthew Dillon 
20574dcc5c2dSMatthew Dillon 	r1 = SWAPBLK_NONE;
20587827d9b0SAlan Cox 	mtx_lock(&swhash_mtx);
205923f09d50SIan Dowse 	pswap = swp_pager_hash(object, pindex);
20601c7c3c6aSMatthew Dillon 
20611c7c3c6aSMatthew Dillon 	if ((swap = *pswap) != NULL) {
206223f09d50SIan Dowse 		idx = pindex & SWAP_META_MASK;
206323f09d50SIan Dowse 		r1 = swap->swb_pages[idx];
20641c7c3c6aSMatthew Dillon 
20651c7c3c6aSMatthew Dillon 		if (r1 != SWAPBLK_NONE) {
20661c7c3c6aSMatthew Dillon 			if (flags & SWM_FREE) {
20674dcc5c2dSMatthew Dillon 				swp_pager_freeswapspace(r1, 1);
20681c7c3c6aSMatthew Dillon 				r1 = SWAPBLK_NONE;
20691c7c3c6aSMatthew Dillon 			}
20701c7c3c6aSMatthew Dillon 			if (flags & (SWM_FREE|SWM_POP)) {
207123f09d50SIan Dowse 				swap->swb_pages[idx] = SWAPBLK_NONE;
20721c7c3c6aSMatthew Dillon 				if (--swap->swb_count == 0) {
20731c7c3c6aSMatthew Dillon 					*pswap = swap->swb_hnext;
2074670d17b5SJeff Roberson 					uma_zfree(swap_zone, swap);
20751c7c3c6aSMatthew Dillon 					--object->un_pager.swp.swp_bcount;
20761c7c3c6aSMatthew Dillon 				}
20771c7c3c6aSMatthew Dillon 			}
20781c7c3c6aSMatthew Dillon 		}
20791c7c3c6aSMatthew Dillon 	}
20807827d9b0SAlan Cox 	mtx_unlock(&swhash_mtx);
20811c7c3c6aSMatthew Dillon 	return (r1);
20821c7c3c6aSMatthew Dillon }
20831c7c3c6aSMatthew Dillon 
2084e9c0cc15SPoul-Henning Kamp /*
2085e9c0cc15SPoul-Henning Kamp  * System call swapon(name) enables swapping on device name,
2086e9c0cc15SPoul-Henning Kamp  * which must be in the swdevsw.  Return EBUSY
2087e9c0cc15SPoul-Henning Kamp  * if already swapping on this device.
2088e9c0cc15SPoul-Henning Kamp  */
2089e9c0cc15SPoul-Henning Kamp #ifndef _SYS_SYSPROTO_H_
2090e9c0cc15SPoul-Henning Kamp struct swapon_args {
2091e9c0cc15SPoul-Henning Kamp 	char *name;
2092e9c0cc15SPoul-Henning Kamp };
2093e9c0cc15SPoul-Henning Kamp #endif
2094e9c0cc15SPoul-Henning Kamp 
2095e9c0cc15SPoul-Henning Kamp /*
2096e9c0cc15SPoul-Henning Kamp  * MPSAFE
2097e9c0cc15SPoul-Henning Kamp  */
2098e9c0cc15SPoul-Henning Kamp /* ARGSUSED */
2099e9c0cc15SPoul-Henning Kamp int
21002f249180SPoul-Henning Kamp swapon(struct thread *td, struct swapon_args *uap)
2101e9c0cc15SPoul-Henning Kamp {
2102e9c0cc15SPoul-Henning Kamp 	struct vattr attr;
2103e9c0cc15SPoul-Henning Kamp 	struct vnode *vp;
2104e9c0cc15SPoul-Henning Kamp 	struct nameidata nd;
2105e9c0cc15SPoul-Henning Kamp 	int error;
2106e9c0cc15SPoul-Henning Kamp 
2107acd3428bSRobert Watson 	error = priv_check(td, PRIV_SWAPON);
2108e9c0cc15SPoul-Henning Kamp 	if (error)
2109acd3428bSRobert Watson 		return (error);
2110e9c0cc15SPoul-Henning Kamp 
2111acd3428bSRobert Watson 	mtx_lock(&Giant);
2112e9c0cc15SPoul-Henning Kamp 	while (swdev_syscall_active)
2113e9c0cc15SPoul-Henning Kamp 	    tsleep(&swdev_syscall_active, PUSER - 1, "swpon", 0);
2114e9c0cc15SPoul-Henning Kamp 	swdev_syscall_active = 1;
2115e9c0cc15SPoul-Henning Kamp 
2116e9c0cc15SPoul-Henning Kamp 	/*
2117e9c0cc15SPoul-Henning Kamp 	 * Swap metadata may not fit in the KVM if we have physical
2118e9c0cc15SPoul-Henning Kamp 	 * memory of >1GB.
2119e9c0cc15SPoul-Henning Kamp 	 */
2120e9c0cc15SPoul-Henning Kamp 	if (swap_zone == NULL) {
2121e9c0cc15SPoul-Henning Kamp 		error = ENOMEM;
2122e9c0cc15SPoul-Henning Kamp 		goto done;
2123e9c0cc15SPoul-Henning Kamp 	}
2124e9c0cc15SPoul-Henning Kamp 
2125d9135e72SRobert Watson 	NDINIT(&nd, LOOKUP, ISOPEN | FOLLOW | AUDITVNODE1, UIO_USERSPACE,
2126d9135e72SRobert Watson 	    uap->name, td);
2127e9c0cc15SPoul-Henning Kamp 	error = namei(&nd);
2128e9c0cc15SPoul-Henning Kamp 	if (error)
2129e9c0cc15SPoul-Henning Kamp 		goto done;
2130e9c0cc15SPoul-Henning Kamp 
2131e9c0cc15SPoul-Henning Kamp 	NDFREE(&nd, NDF_ONLY_PNBUF);
2132e9c0cc15SPoul-Henning Kamp 	vp = nd.ni_vp;
2133e9c0cc15SPoul-Henning Kamp 
213420da9c2eSPoul-Henning Kamp 	if (vn_isdisk(vp, &error)) {
2135dee34ca4SPoul-Henning Kamp 		error = swapongeom(td, vp);
213620da9c2eSPoul-Henning Kamp 	} else if (vp->v_type == VREG &&
2137e9c0cc15SPoul-Henning Kamp 	    (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
21380359a12eSAttilio Rao 	    (error = VOP_GETATTR(vp, &attr, td->td_ucred)) == 0) {
2139e9c0cc15SPoul-Henning Kamp 		/*
2140e9c0cc15SPoul-Henning Kamp 		 * Allow direct swapping to NFS regular files in the same
2141e9c0cc15SPoul-Henning Kamp 		 * way that nfs_mountroot() sets up diskless swapping.
2142e9c0cc15SPoul-Henning Kamp 		 */
214359efee01SPoul-Henning Kamp 		error = swaponvp(td, vp, attr.va_size / DEV_BSIZE);
2144e9c0cc15SPoul-Henning Kamp 	}
2145e9c0cc15SPoul-Henning Kamp 
2146e9c0cc15SPoul-Henning Kamp 	if (error)
2147e9c0cc15SPoul-Henning Kamp 		vrele(vp);
2148e9c0cc15SPoul-Henning Kamp done:
2149e9c0cc15SPoul-Henning Kamp 	swdev_syscall_active = 0;
2150e9c0cc15SPoul-Henning Kamp 	wakeup_one(&swdev_syscall_active);
2151e9c0cc15SPoul-Henning Kamp 	mtx_unlock(&Giant);
2152e9c0cc15SPoul-Henning Kamp 	return (error);
2153e9c0cc15SPoul-Henning Kamp }
2154e9c0cc15SPoul-Henning Kamp 
215559efee01SPoul-Henning Kamp static void
2156f3732fd1SPoul-Henning Kamp swaponsomething(struct vnode *vp, void *id, u_long nblks, sw_strategy_t *strategy, sw_close_t *close, dev_t dev)
2157e9c0cc15SPoul-Henning Kamp {
21582d9974c1SAlan Cox 	struct swdevt *sp, *tsp;
2159e9c0cc15SPoul-Henning Kamp 	swblk_t dvbase;
21608f60c087SPoul-Henning Kamp 	u_long mblocks;
2161e9c0cc15SPoul-Henning Kamp 
2162e9c0cc15SPoul-Henning Kamp 	/*
2163e9c0cc15SPoul-Henning Kamp 	 * If we go beyond this, we get overflows in the radix
2164e9c0cc15SPoul-Henning Kamp 	 * tree bitmap code.
2165e9c0cc15SPoul-Henning Kamp 	 */
21668f60c087SPoul-Henning Kamp 	mblocks = 0x40000000 / BLIST_META_RADIX;
2167d3dd89abSPoul-Henning Kamp 	if (nblks > mblocks) {
216885fdafb9SPoul-Henning Kamp 		printf("WARNING: reducing size to maximum of %lu blocks per swap unit\n",
2169d3dd89abSPoul-Henning Kamp 			mblocks);
2170d3dd89abSPoul-Henning Kamp 		nblks = mblocks;
2171e9c0cc15SPoul-Henning Kamp 	}
2172e9c0cc15SPoul-Henning Kamp 	/*
2173e9c0cc15SPoul-Henning Kamp 	 * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks.
2174e9c0cc15SPoul-Henning Kamp 	 * First chop nblks off to page-align it, then convert.
2175e9c0cc15SPoul-Henning Kamp 	 *
2176e9c0cc15SPoul-Henning Kamp 	 * sw->sw_nblks is in page-sized chunks now too.
2177e9c0cc15SPoul-Henning Kamp 	 */
2178e9c0cc15SPoul-Henning Kamp 	nblks &= ~(ctodb(1) - 1);
2179e9c0cc15SPoul-Henning Kamp 	nblks = dbtoc(nblks);
2180e9c0cc15SPoul-Henning Kamp 
21818f60c087SPoul-Henning Kamp 	sp = malloc(sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO);
2182dee34ca4SPoul-Henning Kamp 	sp->sw_vp = vp;
2183dee34ca4SPoul-Henning Kamp 	sp->sw_id = id;
2184f3732fd1SPoul-Henning Kamp 	sp->sw_dev = dev;
21858d677ef9SPoul-Henning Kamp 	sp->sw_flags = 0;
2186e9c0cc15SPoul-Henning Kamp 	sp->sw_nblks = nblks;
2187e9c0cc15SPoul-Henning Kamp 	sp->sw_used = 0;
218859efee01SPoul-Henning Kamp 	sp->sw_strategy = strategy;
2189dee34ca4SPoul-Henning Kamp 	sp->sw_close = close;
2190e9c0cc15SPoul-Henning Kamp 
2191c8c7ad92SKip Macy 	sp->sw_blist = blist_create(nblks, M_WAITOK);
2192e9c0cc15SPoul-Henning Kamp 	/*
2193ef3c5abdSPoul-Henning Kamp 	 * Do not free the first two block in order to avoid overwriting
21948f60c087SPoul-Henning Kamp 	 * any bsd label at the front of the partition
2195e9c0cc15SPoul-Henning Kamp 	 */
2196ef3c5abdSPoul-Henning Kamp 	blist_free(sp->sw_blist, 2, nblks - 2);
2197e9c0cc15SPoul-Henning Kamp 
21982d9974c1SAlan Cox 	dvbase = 0;
219920da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
22002d9974c1SAlan Cox 	TAILQ_FOREACH(tsp, &swtailq, sw_list) {
22012d9974c1SAlan Cox 		if (tsp->sw_end >= dvbase) {
22022d9974c1SAlan Cox 			/*
22032d9974c1SAlan Cox 			 * We put one uncovered page between the devices
22042d9974c1SAlan Cox 			 * in order to definitively prevent any cross-device
22052d9974c1SAlan Cox 			 * I/O requests
22062d9974c1SAlan Cox 			 */
22072d9974c1SAlan Cox 			dvbase = tsp->sw_end + 1;
22082d9974c1SAlan Cox 		}
22092d9974c1SAlan Cox 	}
22102d9974c1SAlan Cox 	sp->sw_first = dvbase;
22112d9974c1SAlan Cox 	sp->sw_end = dvbase + nblks;
22128f60c087SPoul-Henning Kamp 	TAILQ_INSERT_TAIL(&swtailq, sp, sw_list);
22138f60c087SPoul-Henning Kamp 	nswapdev++;
22148f60c087SPoul-Henning Kamp 	swap_pager_avail += nblks;
22153364c323SKonstantin Belousov 	swap_total += (vm_ooffset_t)nblks * PAGE_SIZE;
2216d05bc129SAlan Cox 	swp_sizecheck();
2217d05bc129SAlan Cox 	mtx_unlock(&sw_dev_mtx);
221859efee01SPoul-Henning Kamp }
2219e9c0cc15SPoul-Henning Kamp 
2220e9c0cc15SPoul-Henning Kamp /*
2221e9c0cc15SPoul-Henning Kamp  * SYSCALL: swapoff(devname)
2222e9c0cc15SPoul-Henning Kamp  *
2223e9c0cc15SPoul-Henning Kamp  * Disable swapping on the given device.
2224dee34ca4SPoul-Henning Kamp  *
2225dee34ca4SPoul-Henning Kamp  * XXX: Badly designed system call: it should use a device index
2226dee34ca4SPoul-Henning Kamp  * rather than filename as specification.  We keep sw_vp around
2227dee34ca4SPoul-Henning Kamp  * only to make this work.
2228e9c0cc15SPoul-Henning Kamp  */
2229e9c0cc15SPoul-Henning Kamp #ifndef _SYS_SYSPROTO_H_
2230e9c0cc15SPoul-Henning Kamp struct swapoff_args {
2231e9c0cc15SPoul-Henning Kamp 	char *name;
2232e9c0cc15SPoul-Henning Kamp };
2233e9c0cc15SPoul-Henning Kamp #endif
2234e9c0cc15SPoul-Henning Kamp 
2235e9c0cc15SPoul-Henning Kamp /*
2236e9c0cc15SPoul-Henning Kamp  * MPSAFE
2237e9c0cc15SPoul-Henning Kamp  */
2238e9c0cc15SPoul-Henning Kamp /* ARGSUSED */
2239e9c0cc15SPoul-Henning Kamp int
22402f249180SPoul-Henning Kamp swapoff(struct thread *td, struct swapoff_args *uap)
2241e9c0cc15SPoul-Henning Kamp {
2242e9c0cc15SPoul-Henning Kamp 	struct vnode *vp;
2243e9c0cc15SPoul-Henning Kamp 	struct nameidata nd;
2244e9c0cc15SPoul-Henning Kamp 	struct swdevt *sp;
22458f60c087SPoul-Henning Kamp 	int error;
2246e9c0cc15SPoul-Henning Kamp 
2247acd3428bSRobert Watson 	error = priv_check(td, PRIV_SWAPOFF);
2248e9c0cc15SPoul-Henning Kamp 	if (error)
22490909f38aSPawel Jakub Dawidek 		return (error);
2250e9c0cc15SPoul-Henning Kamp 
22510909f38aSPawel Jakub Dawidek 	mtx_lock(&Giant);
2252e9c0cc15SPoul-Henning Kamp 	while (swdev_syscall_active)
2253e9c0cc15SPoul-Henning Kamp 	    tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
2254e9c0cc15SPoul-Henning Kamp 	swdev_syscall_active = 1;
2255e9c0cc15SPoul-Henning Kamp 
2256d9135e72SRobert Watson 	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, UIO_USERSPACE, uap->name,
2257d9135e72SRobert Watson 	    td);
2258e9c0cc15SPoul-Henning Kamp 	error = namei(&nd);
2259e9c0cc15SPoul-Henning Kamp 	if (error)
2260e9c0cc15SPoul-Henning Kamp 		goto done;
2261e9c0cc15SPoul-Henning Kamp 	NDFREE(&nd, NDF_ONLY_PNBUF);
2262e9c0cc15SPoul-Henning Kamp 	vp = nd.ni_vp;
2263e9c0cc15SPoul-Henning Kamp 
226420da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
22658f60c087SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2266dee34ca4SPoul-Henning Kamp 		if (sp->sw_vp == vp)
22670909f38aSPawel Jakub Dawidek 			break;
2268e9c0cc15SPoul-Henning Kamp 	}
226920da9c2eSPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
22700909f38aSPawel Jakub Dawidek 	if (sp == NULL) {
2271e9c0cc15SPoul-Henning Kamp 		error = EINVAL;
2272e9c0cc15SPoul-Henning Kamp 		goto done;
22730909f38aSPawel Jakub Dawidek 	}
227435918c55SChristian S.J. Peron 	error = swapoff_one(sp, td->td_ucred);
22750909f38aSPawel Jakub Dawidek done:
22760909f38aSPawel Jakub Dawidek 	swdev_syscall_active = 0;
22770909f38aSPawel Jakub Dawidek 	wakeup_one(&swdev_syscall_active);
22780909f38aSPawel Jakub Dawidek 	mtx_unlock(&Giant);
22790909f38aSPawel Jakub Dawidek 	return (error);
22800909f38aSPawel Jakub Dawidek }
22810909f38aSPawel Jakub Dawidek 
22820909f38aSPawel Jakub Dawidek static int
228335918c55SChristian S.J. Peron swapoff_one(struct swdevt *sp, struct ucred *cred)
22840909f38aSPawel Jakub Dawidek {
22850909f38aSPawel Jakub Dawidek 	u_long nblks, dvbase;
2286e9c0cc15SPoul-Henning Kamp #ifdef MAC
22870909f38aSPawel Jakub Dawidek 	int error;
2288e9c0cc15SPoul-Henning Kamp #endif
2289e9c0cc15SPoul-Henning Kamp 
22900909f38aSPawel Jakub Dawidek 	mtx_assert(&Giant, MA_OWNED);
22910909f38aSPawel Jakub Dawidek #ifdef MAC
2292cb05b60aSAttilio Rao 	(void) vn_lock(sp->sw_vp, LK_EXCLUSIVE | LK_RETRY);
229335918c55SChristian S.J. Peron 	error = mac_system_check_swapoff(cred, sp->sw_vp);
229422db15c0SAttilio Rao 	(void) VOP_UNLOCK(sp->sw_vp, 0);
22950909f38aSPawel Jakub Dawidek 	if (error != 0)
22960909f38aSPawel Jakub Dawidek 		return (error);
22970909f38aSPawel Jakub Dawidek #endif
2298e9c0cc15SPoul-Henning Kamp 	nblks = sp->sw_nblks;
2299e9c0cc15SPoul-Henning Kamp 
2300e9c0cc15SPoul-Henning Kamp 	/*
2301e9c0cc15SPoul-Henning Kamp 	 * We can turn off this swap device safely only if the
2302e9c0cc15SPoul-Henning Kamp 	 * available virtual memory in the system will fit the amount
2303e9c0cc15SPoul-Henning Kamp 	 * of data we will have to page back in, plus an epsilon so
2304e9c0cc15SPoul-Henning Kamp 	 * the system doesn't become critically low on swap space.
2305e9c0cc15SPoul-Henning Kamp 	 */
23062feb50bfSAttilio Rao 	if (cnt.v_free_count + cnt.v_cache_count + swap_pager_avail <
23072feb50bfSAttilio Rao 	    nblks + nswap_lowat) {
23080909f38aSPawel Jakub Dawidek 		return (ENOMEM);
2309e9c0cc15SPoul-Henning Kamp 	}
2310e9c0cc15SPoul-Henning Kamp 
2311e9c0cc15SPoul-Henning Kamp 	/*
2312e9c0cc15SPoul-Henning Kamp 	 * Prevent further allocations on this device.
2313e9c0cc15SPoul-Henning Kamp 	 */
23142928cef7SAlan Cox 	mtx_lock(&sw_dev_mtx);
2315e9c0cc15SPoul-Henning Kamp 	sp->sw_flags |= SW_CLOSING;
23168f60c087SPoul-Henning Kamp 	for (dvbase = 0; dvbase < sp->sw_end; dvbase += dmmax) {
23178f60c087SPoul-Henning Kamp 		swap_pager_avail -= blist_fill(sp->sw_blist,
23188f60c087SPoul-Henning Kamp 		     dvbase, dmmax);
2319e9c0cc15SPoul-Henning Kamp 	}
23203364c323SKonstantin Belousov 	swap_total -= (vm_ooffset_t)nblks * PAGE_SIZE;
23212928cef7SAlan Cox 	mtx_unlock(&sw_dev_mtx);
2322e9c0cc15SPoul-Henning Kamp 
2323e9c0cc15SPoul-Henning Kamp 	/*
2324e9c0cc15SPoul-Henning Kamp 	 * Page in the contents of the device and close it.
2325e9c0cc15SPoul-Henning Kamp 	 */
2326b3fed13eSDavid Schultz 	swap_pager_swapoff(sp);
2327e9c0cc15SPoul-Henning Kamp 
232835918c55SChristian S.J. Peron 	sp->sw_close(curthread, sp);
232959efee01SPoul-Henning Kamp 	sp->sw_id = NULL;
233020da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
23318f60c087SPoul-Henning Kamp 	TAILQ_REMOVE(&swtailq, sp, sw_list);
23320676a140SAlan Cox 	nswapdev--;
23337dea2c2eSAlan Cox 	if (nswapdev == 0) {
23347dea2c2eSAlan Cox 		swap_pager_full = 2;
23357dea2c2eSAlan Cox 		swap_pager_almost_full = 1;
23367dea2c2eSAlan Cox 	}
23378f60c087SPoul-Henning Kamp 	if (swdevhd == sp)
23388f60c087SPoul-Henning Kamp 		swdevhd = NULL;
2339d05bc129SAlan Cox 	mtx_unlock(&sw_dev_mtx);
23408f60c087SPoul-Henning Kamp 	blist_destroy(sp->sw_blist);
23418f60c087SPoul-Henning Kamp 	free(sp, M_VMPGDATA);
23420909f38aSPawel Jakub Dawidek 	return (0);
23430909f38aSPawel Jakub Dawidek }
2344e9c0cc15SPoul-Henning Kamp 
23450909f38aSPawel Jakub Dawidek void
23460909f38aSPawel Jakub Dawidek swapoff_all(void)
23470909f38aSPawel Jakub Dawidek {
23480909f38aSPawel Jakub Dawidek 	struct swdevt *sp, *spt;
23490909f38aSPawel Jakub Dawidek 	const char *devname;
23500909f38aSPawel Jakub Dawidek 	int error;
23510909f38aSPawel Jakub Dawidek 
23520909f38aSPawel Jakub Dawidek 	mtx_lock(&Giant);
23530909f38aSPawel Jakub Dawidek 	while (swdev_syscall_active)
23540909f38aSPawel Jakub Dawidek 		tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
23550909f38aSPawel Jakub Dawidek 	swdev_syscall_active = 1;
23560909f38aSPawel Jakub Dawidek 
23570909f38aSPawel Jakub Dawidek 	mtx_lock(&sw_dev_mtx);
23580909f38aSPawel Jakub Dawidek 	TAILQ_FOREACH_SAFE(sp, &swtailq, sw_list, spt) {
23590909f38aSPawel Jakub Dawidek 		mtx_unlock(&sw_dev_mtx);
23600909f38aSPawel Jakub Dawidek 		if (vn_isdisk(sp->sw_vp, NULL))
23610909f38aSPawel Jakub Dawidek 			devname = sp->sw_vp->v_rdev->si_name;
23620909f38aSPawel Jakub Dawidek 		else
23630909f38aSPawel Jakub Dawidek 			devname = "[file]";
236435918c55SChristian S.J. Peron 		error = swapoff_one(sp, thread0.td_ucred);
23650909f38aSPawel Jakub Dawidek 		if (error != 0) {
23660909f38aSPawel Jakub Dawidek 			printf("Cannot remove swap device %s (error=%d), "
23670909f38aSPawel Jakub Dawidek 			    "skipping.\n", devname, error);
23680909f38aSPawel Jakub Dawidek 		} else if (bootverbose) {
23690909f38aSPawel Jakub Dawidek 			printf("Swap device %s removed.\n", devname);
23700909f38aSPawel Jakub Dawidek 		}
23710909f38aSPawel Jakub Dawidek 		mtx_lock(&sw_dev_mtx);
23720909f38aSPawel Jakub Dawidek 	}
23730909f38aSPawel Jakub Dawidek 	mtx_unlock(&sw_dev_mtx);
23740909f38aSPawel Jakub Dawidek 
2375e9c0cc15SPoul-Henning Kamp 	swdev_syscall_active = 0;
2376e9c0cc15SPoul-Henning Kamp 	wakeup_one(&swdev_syscall_active);
2377e9c0cc15SPoul-Henning Kamp 	mtx_unlock(&Giant);
2378e9c0cc15SPoul-Henning Kamp }
2379e9c0cc15SPoul-Henning Kamp 
2380567104a1SPoul-Henning Kamp void
2381567104a1SPoul-Henning Kamp swap_pager_status(int *total, int *used)
2382567104a1SPoul-Henning Kamp {
2383567104a1SPoul-Henning Kamp 	struct swdevt *sp;
2384567104a1SPoul-Henning Kamp 
2385567104a1SPoul-Henning Kamp 	*total = 0;
2386567104a1SPoul-Henning Kamp 	*used = 0;
238720da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
23888f60c087SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2389567104a1SPoul-Henning Kamp 		*total += sp->sw_nblks;
2390567104a1SPoul-Henning Kamp 		*used += sp->sw_used;
2391567104a1SPoul-Henning Kamp 	}
239220da9c2eSPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
2393567104a1SPoul-Henning Kamp }
2394567104a1SPoul-Henning Kamp 
2395e9c0cc15SPoul-Henning Kamp static int
2396e9c0cc15SPoul-Henning Kamp sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS)
2397e9c0cc15SPoul-Henning Kamp {
2398e9c0cc15SPoul-Henning Kamp 	int	*name = (int *)arg1;
23998f60c087SPoul-Henning Kamp 	int	error, n;
2400e9c0cc15SPoul-Henning Kamp 	struct xswdev xs;
2401e9c0cc15SPoul-Henning Kamp 	struct swdevt *sp;
2402e9c0cc15SPoul-Henning Kamp 
2403e9c0cc15SPoul-Henning Kamp 	if (arg2 != 1) /* name length */
2404e9c0cc15SPoul-Henning Kamp 		return (EINVAL);
2405e9c0cc15SPoul-Henning Kamp 
24068f60c087SPoul-Henning Kamp 	n = 0;
240720da9c2eSPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
24088f60c087SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2409e9c0cc15SPoul-Henning Kamp 		if (n == *name) {
241020da9c2eSPoul-Henning Kamp 			mtx_unlock(&sw_dev_mtx);
2411e9c0cc15SPoul-Henning Kamp 			xs.xsw_version = XSWDEV_VERSION;
2412f3732fd1SPoul-Henning Kamp 			xs.xsw_dev = sp->sw_dev;
2413e9c0cc15SPoul-Henning Kamp 			xs.xsw_flags = sp->sw_flags;
2414e9c0cc15SPoul-Henning Kamp 			xs.xsw_nblks = sp->sw_nblks;
2415e9c0cc15SPoul-Henning Kamp 			xs.xsw_used = sp->sw_used;
2416e9c0cc15SPoul-Henning Kamp 
2417e9c0cc15SPoul-Henning Kamp 			error = SYSCTL_OUT(req, &xs, sizeof(xs));
2418e9c0cc15SPoul-Henning Kamp 			return (error);
2419e9c0cc15SPoul-Henning Kamp 		}
2420e9c0cc15SPoul-Henning Kamp 		n++;
2421e9c0cc15SPoul-Henning Kamp 	}
242220da9c2eSPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
2423e9c0cc15SPoul-Henning Kamp 	return (ENOENT);
2424e9c0cc15SPoul-Henning Kamp }
2425e9c0cc15SPoul-Henning Kamp 
24268f60c087SPoul-Henning Kamp SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, &nswapdev, 0,
2427e9c0cc15SPoul-Henning Kamp     "Number of swap devices");
2428e9c0cc15SPoul-Henning Kamp SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD, sysctl_vm_swap_info,
2429e9c0cc15SPoul-Henning Kamp     "Swap statistics by device");
2430ec38b344SPoul-Henning Kamp 
2431ec38b344SPoul-Henning Kamp /*
24328eb5a1cdSKonstantin Belousov  * vmspace_swap_count() - count the approximate swap usage in pages for a
2433ec38b344SPoul-Henning Kamp  *			  vmspace.
2434ec38b344SPoul-Henning Kamp  *
2435ec38b344SPoul-Henning Kamp  *	The map must be locked.
2436ec38b344SPoul-Henning Kamp  *
24378eb5a1cdSKonstantin Belousov  *	Swap usage is determined by taking the proportional swap used by
2438ec38b344SPoul-Henning Kamp  *	VM objects backing the VM map.  To make up for fractional losses,
2439ec38b344SPoul-Henning Kamp  *	if the VM object has any swap use at all the associated map entries
2440ec38b344SPoul-Henning Kamp  *	count for at least 1 swap page.
2441ec38b344SPoul-Henning Kamp  */
24422860553aSRebecca Cran long
2443ec38b344SPoul-Henning Kamp vmspace_swap_count(struct vmspace *vmspace)
2444ec38b344SPoul-Henning Kamp {
244565d8409cSRebecca Cran 	vm_map_t map;
2446ec38b344SPoul-Henning Kamp 	vm_map_entry_t cur;
244765d8409cSRebecca Cran 	vm_object_t object;
24482860553aSRebecca Cran 	long count, n;
244965d8409cSRebecca Cran 
245065d8409cSRebecca Cran 	map = &vmspace->vm_map;
245165d8409cSRebecca Cran 	count = 0;
2452ec38b344SPoul-Henning Kamp 
2453ec38b344SPoul-Henning Kamp 	for (cur = map->header.next; cur != &map->header; cur = cur->next) {
2454ec38b344SPoul-Henning Kamp 		if ((cur->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
2455ec38b344SPoul-Henning Kamp 		    (object = cur->object.vm_object) != NULL) {
2456ec38b344SPoul-Henning Kamp 			VM_OBJECT_LOCK(object);
2457ec38b344SPoul-Henning Kamp 			if (object->type == OBJT_SWAP &&
2458ec38b344SPoul-Henning Kamp 			    object->un_pager.swp.swp_bcount != 0) {
245965d8409cSRebecca Cran 				n = (cur->end - cur->start) / PAGE_SIZE;
2460ec38b344SPoul-Henning Kamp 				count += object->un_pager.swp.swp_bcount *
2461ec38b344SPoul-Henning Kamp 				    SWAP_META_PAGES * n / object->size + 1;
2462ec38b344SPoul-Henning Kamp 			}
2463ec38b344SPoul-Henning Kamp 			VM_OBJECT_UNLOCK(object);
2464ec38b344SPoul-Henning Kamp 		}
2465ec38b344SPoul-Henning Kamp 	}
2466ec38b344SPoul-Henning Kamp 	return (count);
2467ec38b344SPoul-Henning Kamp }
2468dee34ca4SPoul-Henning Kamp 
2469dee34ca4SPoul-Henning Kamp /*
2470dee34ca4SPoul-Henning Kamp  * GEOM backend
2471dee34ca4SPoul-Henning Kamp  *
2472dee34ca4SPoul-Henning Kamp  * Swapping onto disk devices.
2473dee34ca4SPoul-Henning Kamp  *
2474dee34ca4SPoul-Henning Kamp  */
2475dee34ca4SPoul-Henning Kamp 
24765721c9c7SPoul-Henning Kamp static g_orphan_t swapgeom_orphan;
24775721c9c7SPoul-Henning Kamp 
2478dee34ca4SPoul-Henning Kamp static struct g_class g_swap_class = {
2479dee34ca4SPoul-Henning Kamp 	.name = "SWAP",
24805721c9c7SPoul-Henning Kamp 	.version = G_VERSION,
24815721c9c7SPoul-Henning Kamp 	.orphan = swapgeom_orphan,
2482dee34ca4SPoul-Henning Kamp };
2483dee34ca4SPoul-Henning Kamp 
2484dee34ca4SPoul-Henning Kamp DECLARE_GEOM_CLASS(g_swap_class, g_class);
2485dee34ca4SPoul-Henning Kamp 
2486dee34ca4SPoul-Henning Kamp 
2487dee34ca4SPoul-Henning Kamp static void
2488dee34ca4SPoul-Henning Kamp swapgeom_done(struct bio *bp2)
2489dee34ca4SPoul-Henning Kamp {
2490dee34ca4SPoul-Henning Kamp 	struct buf *bp;
2491dee34ca4SPoul-Henning Kamp 
2492dee34ca4SPoul-Henning Kamp 	bp = bp2->bio_caller2;
2493c5d3d25eSPoul-Henning Kamp 	bp->b_ioflags = bp2->bio_flags;
2494dee34ca4SPoul-Henning Kamp 	if (bp2->bio_error)
2495dee34ca4SPoul-Henning Kamp 		bp->b_ioflags |= BIO_ERROR;
2496c5d3d25eSPoul-Henning Kamp 	bp->b_resid = bp->b_bcount - bp2->bio_completed;
2497c5d3d25eSPoul-Henning Kamp 	bp->b_error = bp2->bio_error;
2498dee34ca4SPoul-Henning Kamp 	bufdone(bp);
2499dee34ca4SPoul-Henning Kamp 	g_destroy_bio(bp2);
2500dee34ca4SPoul-Henning Kamp }
2501dee34ca4SPoul-Henning Kamp 
2502dee34ca4SPoul-Henning Kamp static void
2503dee34ca4SPoul-Henning Kamp swapgeom_strategy(struct buf *bp, struct swdevt *sp)
2504dee34ca4SPoul-Henning Kamp {
2505dee34ca4SPoul-Henning Kamp 	struct bio *bio;
2506dee34ca4SPoul-Henning Kamp 	struct g_consumer *cp;
2507dee34ca4SPoul-Henning Kamp 
2508dee34ca4SPoul-Henning Kamp 	cp = sp->sw_id;
2509dee34ca4SPoul-Henning Kamp 	if (cp == NULL) {
2510dee34ca4SPoul-Henning Kamp 		bp->b_error = ENXIO;
2511dee34ca4SPoul-Henning Kamp 		bp->b_ioflags |= BIO_ERROR;
2512dee34ca4SPoul-Henning Kamp 		bufdone(bp);
2513dee34ca4SPoul-Henning Kamp 		return;
2514dee34ca4SPoul-Henning Kamp 	}
251511041003SKonstantin Belousov 	if (bp->b_iocmd == BIO_WRITE)
251611041003SKonstantin Belousov 		bio = g_new_bio();
251711041003SKonstantin Belousov 	else
25184f8205e5SPoul-Henning Kamp 		bio = g_alloc_bio();
25194f8205e5SPoul-Henning Kamp 	if (bio == NULL) {
25203e5b6861SPoul-Henning Kamp 		bp->b_error = ENOMEM;
25213e5b6861SPoul-Henning Kamp 		bp->b_ioflags |= BIO_ERROR;
25223e5b6861SPoul-Henning Kamp 		bufdone(bp);
25233e5b6861SPoul-Henning Kamp 		return;
25243e5b6861SPoul-Henning Kamp 	}
252511041003SKonstantin Belousov 
2526dee34ca4SPoul-Henning Kamp 	bio->bio_caller2 = bp;
2527c5d3d25eSPoul-Henning Kamp 	bio->bio_cmd = bp->b_iocmd;
2528c5d3d25eSPoul-Henning Kamp 	bio->bio_data = bp->b_data;
2529dee34ca4SPoul-Henning Kamp 	bio->bio_offset = (bp->b_blkno - sp->sw_first) * PAGE_SIZE;
2530dee34ca4SPoul-Henning Kamp 	bio->bio_length = bp->b_bcount;
2531dee34ca4SPoul-Henning Kamp 	bio->bio_done = swapgeom_done;
2532dee34ca4SPoul-Henning Kamp 	g_io_request(bio, cp);
2533dee34ca4SPoul-Henning Kamp 	return;
2534dee34ca4SPoul-Henning Kamp }
2535dee34ca4SPoul-Henning Kamp 
2536dee34ca4SPoul-Henning Kamp static void
2537dee34ca4SPoul-Henning Kamp swapgeom_orphan(struct g_consumer *cp)
2538dee34ca4SPoul-Henning Kamp {
2539dee34ca4SPoul-Henning Kamp 	struct swdevt *sp;
2540dee34ca4SPoul-Henning Kamp 
2541dee34ca4SPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
2542dee34ca4SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list)
2543dee34ca4SPoul-Henning Kamp 		if (sp->sw_id == cp)
2544dee34ca4SPoul-Henning Kamp 			sp->sw_id = NULL;
2545dee34ca4SPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
2546dee34ca4SPoul-Henning Kamp }
2547dee34ca4SPoul-Henning Kamp 
2548dee34ca4SPoul-Henning Kamp static void
2549dee34ca4SPoul-Henning Kamp swapgeom_close_ev(void *arg, int flags)
2550dee34ca4SPoul-Henning Kamp {
2551dee34ca4SPoul-Henning Kamp 	struct g_consumer *cp;
2552dee34ca4SPoul-Henning Kamp 
2553dee34ca4SPoul-Henning Kamp 	cp = arg;
2554d2bae332SPoul-Henning Kamp 	g_access(cp, -1, -1, 0);
2555dee34ca4SPoul-Henning Kamp 	g_detach(cp);
2556dee34ca4SPoul-Henning Kamp 	g_destroy_consumer(cp);
2557dee34ca4SPoul-Henning Kamp }
2558dee34ca4SPoul-Henning Kamp 
2559dee34ca4SPoul-Henning Kamp static void
2560dee34ca4SPoul-Henning Kamp swapgeom_close(struct thread *td, struct swdevt *sw)
2561dee34ca4SPoul-Henning Kamp {
2562dee34ca4SPoul-Henning Kamp 
2563dee34ca4SPoul-Henning Kamp 	/* XXX: direct call when Giant untangled */
2564dee34ca4SPoul-Henning Kamp 	g_waitfor_event(swapgeom_close_ev, sw->sw_id, M_WAITOK, NULL);
2565dee34ca4SPoul-Henning Kamp }
2566dee34ca4SPoul-Henning Kamp 
2567dee34ca4SPoul-Henning Kamp 
2568dee34ca4SPoul-Henning Kamp struct swh0h0 {
256989c9c53dSPoul-Henning Kamp 	struct cdev *dev;
2570dee34ca4SPoul-Henning Kamp 	struct vnode *vp;
2571dee34ca4SPoul-Henning Kamp 	int	error;
2572dee34ca4SPoul-Henning Kamp };
2573dee34ca4SPoul-Henning Kamp 
2574dee34ca4SPoul-Henning Kamp static void
2575dee34ca4SPoul-Henning Kamp swapongeom_ev(void *arg, int flags)
2576dee34ca4SPoul-Henning Kamp {
2577dee34ca4SPoul-Henning Kamp 	struct swh0h0 *swh;
2578dee34ca4SPoul-Henning Kamp 	struct g_provider *pp;
2579dee34ca4SPoul-Henning Kamp 	struct g_consumer *cp;
2580dee34ca4SPoul-Henning Kamp 	static struct g_geom *gp;
2581dee34ca4SPoul-Henning Kamp 	struct swdevt *sp;
2582dee34ca4SPoul-Henning Kamp 	u_long nblks;
2583dee34ca4SPoul-Henning Kamp 	int error;
2584dee34ca4SPoul-Henning Kamp 
2585dee34ca4SPoul-Henning Kamp 	swh = arg;
2586dee34ca4SPoul-Henning Kamp 	swh->error = 0;
2587dee34ca4SPoul-Henning Kamp 	pp = g_dev_getprovider(swh->dev);
2588dee34ca4SPoul-Henning Kamp 	if (pp == NULL) {
2589dee34ca4SPoul-Henning Kamp 		swh->error = ENODEV;
2590dee34ca4SPoul-Henning Kamp 		return;
2591dee34ca4SPoul-Henning Kamp 	}
2592dee34ca4SPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
2593dee34ca4SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2594dee34ca4SPoul-Henning Kamp 		cp = sp->sw_id;
2595dee34ca4SPoul-Henning Kamp 		if (cp != NULL && cp->provider == pp) {
2596dee34ca4SPoul-Henning Kamp 			mtx_unlock(&sw_dev_mtx);
2597dee34ca4SPoul-Henning Kamp 			swh->error = EBUSY;
2598dee34ca4SPoul-Henning Kamp 			return;
2599dee34ca4SPoul-Henning Kamp 		}
2600dee34ca4SPoul-Henning Kamp 	}
2601dee34ca4SPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
26025721c9c7SPoul-Henning Kamp 	if (gp == NULL)
2603dee34ca4SPoul-Henning Kamp 		gp = g_new_geomf(&g_swap_class, "swap", NULL);
2604dee34ca4SPoul-Henning Kamp 	cp = g_new_consumer(gp);
2605dee34ca4SPoul-Henning Kamp 	g_attach(cp, pp);
2606afeb65e6SPoul-Henning Kamp 	/*
2607afeb65e6SPoul-Henning Kamp 	 * XXX: Everytime you think you can improve the margin for
2608afeb65e6SPoul-Henning Kamp 	 * footshooting, somebody depends on the ability to do so:
2609afeb65e6SPoul-Henning Kamp 	 * savecore(8) wants to write to our swapdev so we cannot
2610afeb65e6SPoul-Henning Kamp 	 * set an exclusive count :-(
2611afeb65e6SPoul-Henning Kamp 	 */
2612d2bae332SPoul-Henning Kamp 	error = g_access(cp, 1, 1, 0);
2613dee34ca4SPoul-Henning Kamp 	if (error) {
2614dee34ca4SPoul-Henning Kamp 		g_detach(cp);
2615dee34ca4SPoul-Henning Kamp 		g_destroy_consumer(cp);
2616dee34ca4SPoul-Henning Kamp 		swh->error = error;
2617dee34ca4SPoul-Henning Kamp 		return;
2618dee34ca4SPoul-Henning Kamp 	}
2619dee34ca4SPoul-Henning Kamp 	nblks = pp->mediasize / DEV_BSIZE;
2620dee34ca4SPoul-Henning Kamp 	swaponsomething(swh->vp, cp, nblks, swapgeom_strategy,
2621dee34ca4SPoul-Henning Kamp 	    swapgeom_close, dev2udev(swh->dev));
2622dee34ca4SPoul-Henning Kamp 	swh->error = 0;
2623dee34ca4SPoul-Henning Kamp 	return;
2624dee34ca4SPoul-Henning Kamp }
2625dee34ca4SPoul-Henning Kamp 
2626dee34ca4SPoul-Henning Kamp static int
2627dee34ca4SPoul-Henning Kamp swapongeom(struct thread *td, struct vnode *vp)
2628dee34ca4SPoul-Henning Kamp {
2629dee34ca4SPoul-Henning Kamp 	int error;
2630dee34ca4SPoul-Henning Kamp 	struct swh0h0 swh;
2631dee34ca4SPoul-Henning Kamp 
2632cb05b60aSAttilio Rao 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2633dee34ca4SPoul-Henning Kamp 
2634dee34ca4SPoul-Henning Kamp 	swh.dev = vp->v_rdev;
2635dee34ca4SPoul-Henning Kamp 	swh.vp = vp;
2636dee34ca4SPoul-Henning Kamp 	swh.error = 0;
2637dee34ca4SPoul-Henning Kamp 	/* XXX: direct call when Giant untangled */
2638dee34ca4SPoul-Henning Kamp 	error = g_waitfor_event(swapongeom_ev, &swh, M_WAITOK, NULL);
2639dee34ca4SPoul-Henning Kamp 	if (!error)
2640dee34ca4SPoul-Henning Kamp 		error = swh.error;
264122db15c0SAttilio Rao 	VOP_UNLOCK(vp, 0);
2642dee34ca4SPoul-Henning Kamp 	return (error);
2643dee34ca4SPoul-Henning Kamp }
2644dee34ca4SPoul-Henning Kamp 
2645dee34ca4SPoul-Henning Kamp /*
2646dee34ca4SPoul-Henning Kamp  * VNODE backend
2647dee34ca4SPoul-Henning Kamp  *
2648dee34ca4SPoul-Henning Kamp  * This is used mainly for network filesystem (read: probably only tested
2649dee34ca4SPoul-Henning Kamp  * with NFS) swapfiles.
2650dee34ca4SPoul-Henning Kamp  *
2651dee34ca4SPoul-Henning Kamp  */
2652dee34ca4SPoul-Henning Kamp 
2653dee34ca4SPoul-Henning Kamp static void
2654dee34ca4SPoul-Henning Kamp swapdev_strategy(struct buf *bp, struct swdevt *sp)
2655dee34ca4SPoul-Henning Kamp {
2656494eb176SPoul-Henning Kamp 	struct vnode *vp2;
2657dee34ca4SPoul-Henning Kamp 
2658dee34ca4SPoul-Henning Kamp 	bp->b_blkno = ctodb(bp->b_blkno - sp->sw_first);
2659dee34ca4SPoul-Henning Kamp 
2660dee34ca4SPoul-Henning Kamp 	vp2 = sp->sw_id;
2661dee34ca4SPoul-Henning Kamp 	vhold(vp2);
2662dee34ca4SPoul-Henning Kamp 	if (bp->b_iocmd == BIO_WRITE) {
26633cfc7651SOlivier Houchard 		if (bp->b_bufobj)
2664494eb176SPoul-Henning Kamp 			bufobj_wdrop(bp->b_bufobj);
2665a76d8f4eSPoul-Henning Kamp 		bufobj_wref(&vp2->v_bufobj);
2666dee34ca4SPoul-Henning Kamp 	}
26673cfc7651SOlivier Houchard 	if (bp->b_bufobj != &vp2->v_bufobj)
26683cfc7651SOlivier Houchard 		bp->b_bufobj = &vp2->v_bufobj;
2669dee34ca4SPoul-Henning Kamp 	bp->b_vp = vp2;
26702c18019fSPoul-Henning Kamp 	bp->b_iooffset = dbtob(bp->b_blkno);
2671b792bebeSPoul-Henning Kamp 	bstrategy(bp);
2672dee34ca4SPoul-Henning Kamp 	return;
2673dee34ca4SPoul-Henning Kamp }
2674dee34ca4SPoul-Henning Kamp 
2675dee34ca4SPoul-Henning Kamp static void
2676dee34ca4SPoul-Henning Kamp swapdev_close(struct thread *td, struct swdevt *sp)
2677dee34ca4SPoul-Henning Kamp {
2678dee34ca4SPoul-Henning Kamp 
2679dee34ca4SPoul-Henning Kamp 	VOP_CLOSE(sp->sw_vp, FREAD | FWRITE, td->td_ucred, td);
2680dee34ca4SPoul-Henning Kamp 	vrele(sp->sw_vp);
2681dee34ca4SPoul-Henning Kamp }
2682dee34ca4SPoul-Henning Kamp 
2683dee34ca4SPoul-Henning Kamp 
2684dee34ca4SPoul-Henning Kamp static int
2685dee34ca4SPoul-Henning Kamp swaponvp(struct thread *td, struct vnode *vp, u_long nblks)
2686dee34ca4SPoul-Henning Kamp {
2687dee34ca4SPoul-Henning Kamp 	struct swdevt *sp;
2688dee34ca4SPoul-Henning Kamp 	int error;
2689dee34ca4SPoul-Henning Kamp 
2690dee34ca4SPoul-Henning Kamp 	if (nblks == 0)
2691dee34ca4SPoul-Henning Kamp 		return (ENXIO);
2692dee34ca4SPoul-Henning Kamp 	mtx_lock(&sw_dev_mtx);
2693dee34ca4SPoul-Henning Kamp 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2694dee34ca4SPoul-Henning Kamp 		if (sp->sw_id == vp) {
2695dee34ca4SPoul-Henning Kamp 			mtx_unlock(&sw_dev_mtx);
2696dee34ca4SPoul-Henning Kamp 			return (EBUSY);
2697dee34ca4SPoul-Henning Kamp 		}
2698dee34ca4SPoul-Henning Kamp 	}
2699dee34ca4SPoul-Henning Kamp 	mtx_unlock(&sw_dev_mtx);
2700dee34ca4SPoul-Henning Kamp 
2701cb05b60aSAttilio Rao 	(void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2702dee34ca4SPoul-Henning Kamp #ifdef MAC
270330d239bcSRobert Watson 	error = mac_system_check_swapon(td->td_ucred, vp);
2704dee34ca4SPoul-Henning Kamp 	if (error == 0)
2705dee34ca4SPoul-Henning Kamp #endif
27069e223287SKonstantin Belousov 		error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, NULL);
270722db15c0SAttilio Rao 	(void) VOP_UNLOCK(vp, 0);
2708dee34ca4SPoul-Henning Kamp 	if (error)
2709dee34ca4SPoul-Henning Kamp 		return (error);
2710dee34ca4SPoul-Henning Kamp 
2711dee34ca4SPoul-Henning Kamp 	swaponsomething(vp, vp, nblks, swapdev_strategy, swapdev_close,
2712f3732fd1SPoul-Henning Kamp 	    NODEV);
2713dee34ca4SPoul-Henning Kamp 	return (0);
2714dee34ca4SPoul-Henning Kamp }
2715