xref: /freebsd/sys/vm/swap_pager.c (revision 9fe48b8076ae9c6dc486d713c468ff03ec056eda)
1 /*-
2  * Copyright (c) 1998 Matthew Dillon,
3  * Copyright (c) 1994 John S. Dyson
4  * Copyright (c) 1990 University of Utah.
5  * Copyright (c) 1982, 1986, 1989, 1993
6  *	The Regents of the University of California.  All rights reserved.
7  *
8  * This code is derived from software contributed to Berkeley by
9  * the Systems Programming Group of the University of Utah Computer
10  * Science Department.
11  *
12  * Redistribution and use in source and binary forms, with or without
13  * modification, are permitted provided that the following conditions
14  * are met:
15  * 1. Redistributions of source code must retain the above copyright
16  *    notice, this list of conditions and the following disclaimer.
17  * 2. Redistributions in binary form must reproduce the above copyright
18  *    notice, this list of conditions and the following disclaimer in the
19  *    documentation and/or other materials provided with the distribution.
20  * 3. All advertising materials mentioning features or use of this software
21  *    must display the following acknowledgement:
22  *	This product includes software developed by the University of
23  *	California, Berkeley and its contributors.
24  * 4. Neither the name of the University nor the names of its contributors
25  *    may be used to endorse or promote products derived from this software
26  *    without specific prior written permission.
27  *
28  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
29  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
32  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38  * SUCH DAMAGE.
39  *
40  *				New Swap System
41  *				Matthew Dillon
42  *
43  * Radix Bitmap 'blists'.
44  *
45  *	- The new swapper uses the new radix bitmap code.  This should scale
46  *	  to arbitrarily small or arbitrarily large swap spaces and an almost
47  *	  arbitrary degree of fragmentation.
48  *
49  * Features:
50  *
51  *	- on the fly reallocation of swap during putpages.  The new system
52  *	  does not try to keep previously allocated swap blocks for dirty
53  *	  pages.
54  *
55  *	- on the fly deallocation of swap
56  *
57  *	- No more garbage collection required.  Unnecessarily allocated swap
58  *	  blocks only exist for dirty vm_page_t's now and these are already
59  *	  cycled (in a high-load system) by the pager.  We also do on-the-fly
60  *	  removal of invalidated swap blocks when a page is destroyed
61  *	  or renamed.
62  *
63  * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
64  *
65  *	@(#)swap_pager.c	8.9 (Berkeley) 3/21/94
66  *	@(#)vm_swap.c	8.5 (Berkeley) 2/17/94
67  */
68 
69 #include <sys/cdefs.h>
70 __FBSDID("$FreeBSD$");
71 
72 #include "opt_swap.h"
73 #include "opt_vm.h"
74 
75 #include <sys/param.h>
76 #include <sys/systm.h>
77 #include <sys/conf.h>
78 #include <sys/kernel.h>
79 #include <sys/priv.h>
80 #include <sys/proc.h>
81 #include <sys/bio.h>
82 #include <sys/buf.h>
83 #include <sys/disk.h>
84 #include <sys/fcntl.h>
85 #include <sys/mount.h>
86 #include <sys/namei.h>
87 #include <sys/vnode.h>
88 #include <sys/malloc.h>
89 #include <sys/racct.h>
90 #include <sys/resource.h>
91 #include <sys/resourcevar.h>
92 #include <sys/rwlock.h>
93 #include <sys/sysctl.h>
94 #include <sys/sysproto.h>
95 #include <sys/blist.h>
96 #include <sys/lock.h>
97 #include <sys/sx.h>
98 #include <sys/vmmeter.h>
99 
100 #include <security/mac/mac_framework.h>
101 
102 #include <vm/vm.h>
103 #include <vm/pmap.h>
104 #include <vm/vm_map.h>
105 #include <vm/vm_kern.h>
106 #include <vm/vm_object.h>
107 #include <vm/vm_page.h>
108 #include <vm/vm_pager.h>
109 #include <vm/vm_pageout.h>
110 #include <vm/vm_param.h>
111 #include <vm/swap_pager.h>
112 #include <vm/vm_extern.h>
113 #include <vm/uma.h>
114 
115 #include <geom/geom.h>
116 
117 /*
118  * SWB_NPAGES must be a power of 2.  It may be set to 1, 2, 4, 8, 16
119  * or 32 pages per allocation.
120  * The 32-page limit is due to the radix code (kern/subr_blist.c).
121  */
122 #ifndef MAX_PAGEOUT_CLUSTER
123 #define MAX_PAGEOUT_CLUSTER 16
124 #endif
125 
126 #if !defined(SWB_NPAGES)
127 #define SWB_NPAGES	MAX_PAGEOUT_CLUSTER
128 #endif
129 
130 /*
131  * The swblock structure maps an object and a small, fixed-size range
132  * of page indices to disk addresses within a swap area.
133  * The collection of these mappings is implemented as a hash table.
134  * Unused disk addresses within a swap area are allocated and managed
135  * using a blist.
136  */
137 #define SWCORRECT(n) (sizeof(void *) * (n) / sizeof(daddr_t))
138 #define SWAP_META_PAGES		(SWB_NPAGES * 2)
139 #define SWAP_META_MASK		(SWAP_META_PAGES - 1)
140 
141 struct swblock {
142 	struct swblock	*swb_hnext;
143 	vm_object_t	swb_object;
144 	vm_pindex_t	swb_index;
145 	int		swb_count;
146 	daddr_t		swb_pages[SWAP_META_PAGES];
147 };
148 
149 static MALLOC_DEFINE(M_VMPGDATA, "vm_pgdata", "swap pager private data");
150 static struct mtx sw_dev_mtx;
151 static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq);
152 static struct swdevt *swdevhd;	/* Allocate from here next */
153 static int nswapdev;		/* Number of swap devices */
154 int swap_pager_avail;
155 static int swdev_syscall_active = 0; /* serialize swap(on|off) */
156 
157 static vm_ooffset_t swap_total;
158 SYSCTL_QUAD(_vm, OID_AUTO, swap_total, CTLFLAG_RD, &swap_total, 0,
159     "Total amount of available swap storage.");
160 static vm_ooffset_t swap_reserved;
161 SYSCTL_QUAD(_vm, OID_AUTO, swap_reserved, CTLFLAG_RD, &swap_reserved, 0,
162     "Amount of swap storage needed to back all allocated anonymous memory.");
163 static int overcommit = 0;
164 SYSCTL_INT(_vm, OID_AUTO, overcommit, CTLFLAG_RW, &overcommit, 0,
165     "Configure virtual memory overcommit behavior. See tuning(7) "
166     "for details.");
167 static unsigned long swzone;
168 SYSCTL_ULONG(_vm, OID_AUTO, swzone, CTLFLAG_RD, &swzone, 0,
169     "Actual size of swap metadata zone");
170 static unsigned long swap_maxpages;
171 SYSCTL_ULONG(_vm, OID_AUTO, swap_maxpages, CTLFLAG_RD, &swap_maxpages, 0,
172     "Maximum amount of swap supported");
173 
174 /* bits from overcommit */
175 #define	SWAP_RESERVE_FORCE_ON		(1 << 0)
176 #define	SWAP_RESERVE_RLIMIT_ON		(1 << 1)
177 #define	SWAP_RESERVE_ALLOW_NONWIRED	(1 << 2)
178 
179 int
180 swap_reserve(vm_ooffset_t incr)
181 {
182 
183 	return (swap_reserve_by_cred(incr, curthread->td_ucred));
184 }
185 
186 int
187 swap_reserve_by_cred(vm_ooffset_t incr, struct ucred *cred)
188 {
189 	vm_ooffset_t r, s;
190 	int res, error;
191 	static int curfail;
192 	static struct timeval lastfail;
193 	struct uidinfo *uip;
194 
195 	uip = cred->cr_ruidinfo;
196 
197 	if (incr & PAGE_MASK)
198 		panic("swap_reserve: & PAGE_MASK");
199 
200 #ifdef RACCT
201 	if (racct_enable) {
202 		PROC_LOCK(curproc);
203 		error = racct_add(curproc, RACCT_SWAP, incr);
204 		PROC_UNLOCK(curproc);
205 		if (error != 0)
206 			return (0);
207 	}
208 #endif
209 
210 	res = 0;
211 	mtx_lock(&sw_dev_mtx);
212 	r = swap_reserved + incr;
213 	if (overcommit & SWAP_RESERVE_ALLOW_NONWIRED) {
214 		s = vm_cnt.v_page_count - vm_cnt.v_free_reserved - vm_cnt.v_wire_count;
215 		s *= PAGE_SIZE;
216 	} else
217 		s = 0;
218 	s += swap_total;
219 	if ((overcommit & SWAP_RESERVE_FORCE_ON) == 0 || r <= s ||
220 	    (error = priv_check(curthread, PRIV_VM_SWAP_NOQUOTA)) == 0) {
221 		res = 1;
222 		swap_reserved = r;
223 	}
224 	mtx_unlock(&sw_dev_mtx);
225 
226 	if (res) {
227 		PROC_LOCK(curproc);
228 		UIDINFO_VMSIZE_LOCK(uip);
229 		if ((overcommit & SWAP_RESERVE_RLIMIT_ON) != 0 &&
230 		    uip->ui_vmsize + incr > lim_cur(curproc, RLIMIT_SWAP) &&
231 		    priv_check(curthread, PRIV_VM_SWAP_NORLIMIT))
232 			res = 0;
233 		else
234 			uip->ui_vmsize += incr;
235 		UIDINFO_VMSIZE_UNLOCK(uip);
236 		PROC_UNLOCK(curproc);
237 		if (!res) {
238 			mtx_lock(&sw_dev_mtx);
239 			swap_reserved -= incr;
240 			mtx_unlock(&sw_dev_mtx);
241 		}
242 	}
243 	if (!res && ppsratecheck(&lastfail, &curfail, 1)) {
244 		printf("uid %d, pid %d: swap reservation for %jd bytes failed\n",
245 		    uip->ui_uid, curproc->p_pid, incr);
246 	}
247 
248 #ifdef RACCT
249 	if (!res) {
250 		PROC_LOCK(curproc);
251 		racct_sub(curproc, RACCT_SWAP, incr);
252 		PROC_UNLOCK(curproc);
253 	}
254 #endif
255 
256 	return (res);
257 }
258 
259 void
260 swap_reserve_force(vm_ooffset_t incr)
261 {
262 	struct uidinfo *uip;
263 
264 	mtx_lock(&sw_dev_mtx);
265 	swap_reserved += incr;
266 	mtx_unlock(&sw_dev_mtx);
267 
268 #ifdef RACCT
269 	PROC_LOCK(curproc);
270 	racct_add_force(curproc, RACCT_SWAP, incr);
271 	PROC_UNLOCK(curproc);
272 #endif
273 
274 	uip = curthread->td_ucred->cr_ruidinfo;
275 	PROC_LOCK(curproc);
276 	UIDINFO_VMSIZE_LOCK(uip);
277 	uip->ui_vmsize += incr;
278 	UIDINFO_VMSIZE_UNLOCK(uip);
279 	PROC_UNLOCK(curproc);
280 }
281 
282 void
283 swap_release(vm_ooffset_t decr)
284 {
285 	struct ucred *cred;
286 
287 	PROC_LOCK(curproc);
288 	cred = curthread->td_ucred;
289 	swap_release_by_cred(decr, cred);
290 	PROC_UNLOCK(curproc);
291 }
292 
293 void
294 swap_release_by_cred(vm_ooffset_t decr, struct ucred *cred)
295 {
296  	struct uidinfo *uip;
297 
298 	uip = cred->cr_ruidinfo;
299 
300 	if (decr & PAGE_MASK)
301 		panic("swap_release: & PAGE_MASK");
302 
303 	mtx_lock(&sw_dev_mtx);
304 	if (swap_reserved < decr)
305 		panic("swap_reserved < decr");
306 	swap_reserved -= decr;
307 	mtx_unlock(&sw_dev_mtx);
308 
309 	UIDINFO_VMSIZE_LOCK(uip);
310 	if (uip->ui_vmsize < decr)
311 		printf("negative vmsize for uid = %d\n", uip->ui_uid);
312 	uip->ui_vmsize -= decr;
313 	UIDINFO_VMSIZE_UNLOCK(uip);
314 
315 	racct_sub_cred(cred, RACCT_SWAP, decr);
316 }
317 
318 static void swapdev_strategy(struct buf *, struct swdevt *sw);
319 
320 #define SWM_FREE	0x02	/* free, period			*/
321 #define SWM_POP		0x04	/* pop out			*/
322 
323 int swap_pager_full = 2;	/* swap space exhaustion (task killing) */
324 static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/
325 static int nsw_rcount;		/* free read buffers			*/
326 static int nsw_wcount_sync;	/* limit write buffers / synchronous	*/
327 static int nsw_wcount_async;	/* limit write buffers / asynchronous	*/
328 static int nsw_wcount_async_max;/* assigned maximum			*/
329 static int nsw_cluster_max;	/* maximum VOP I/O allowed		*/
330 
331 static struct swblock **swhash;
332 static int swhash_mask;
333 static struct mtx swhash_mtx;
334 
335 static int swap_async_max = 4;	/* maximum in-progress async I/O's	*/
336 static struct sx sw_alloc_sx;
337 
338 
339 SYSCTL_INT(_vm, OID_AUTO, swap_async_max,
340 	CTLFLAG_RW, &swap_async_max, 0, "Maximum running async swap ops");
341 
342 /*
343  * "named" and "unnamed" anon region objects.  Try to reduce the overhead
344  * of searching a named list by hashing it just a little.
345  */
346 
347 #define NOBJLISTS		8
348 
349 #define NOBJLIST(handle)	\
350 	(&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)])
351 
352 static struct mtx sw_alloc_mtx;	/* protect list manipulation */
353 static struct pagerlst	swap_pager_object_list[NOBJLISTS];
354 static uma_zone_t	swap_zone;
355 
356 /*
357  * pagerops for OBJT_SWAP - "swap pager".  Some ops are also global procedure
358  * calls hooked from other parts of the VM system and do not appear here.
359  * (see vm/swap_pager.h).
360  */
361 static vm_object_t
362 		swap_pager_alloc(void *handle, vm_ooffset_t size,
363 		    vm_prot_t prot, vm_ooffset_t offset, struct ucred *);
364 static void	swap_pager_dealloc(vm_object_t object);
365 static int	swap_pager_getpages(vm_object_t, vm_page_t *, int, int);
366 static int	swap_pager_getpages_async(vm_object_t, vm_page_t *, int, int,
367     pgo_getpages_iodone_t, void *);
368 static void	swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *);
369 static boolean_t
370 		swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
371 static void	swap_pager_init(void);
372 static void	swap_pager_unswapped(vm_page_t);
373 static void	swap_pager_swapoff(struct swdevt *sp);
374 
375 struct pagerops swappagerops = {
376 	.pgo_init =	swap_pager_init,	/* early system initialization of pager	*/
377 	.pgo_alloc =	swap_pager_alloc,	/* allocate an OBJT_SWAP object		*/
378 	.pgo_dealloc =	swap_pager_dealloc,	/* deallocate an OBJT_SWAP object	*/
379 	.pgo_getpages =	swap_pager_getpages,	/* pagein				*/
380 	.pgo_getpages_async = swap_pager_getpages_async, /* pagein (async)		*/
381 	.pgo_putpages =	swap_pager_putpages,	/* pageout				*/
382 	.pgo_haspage =	swap_pager_haspage,	/* get backing store status for page	*/
383 	.pgo_pageunswapped = swap_pager_unswapped,	/* remove swap related to page		*/
384 };
385 
386 /*
387  * dmmax is in page-sized chunks with the new swap system.  It was
388  * dev-bsized chunks in the old.  dmmax is always a power of 2.
389  *
390  * swap_*() routines are externally accessible.  swp_*() routines are
391  * internal.
392  */
393 static int dmmax;
394 static int nswap_lowat = 128;	/* in pages, swap_pager_almost_full warn */
395 static int nswap_hiwat = 512;	/* in pages, swap_pager_almost_full warn */
396 
397 SYSCTL_INT(_vm, OID_AUTO, dmmax,
398 	CTLFLAG_RD, &dmmax, 0, "Maximum size of a swap block");
399 
400 static void	swp_sizecheck(void);
401 static void	swp_pager_async_iodone(struct buf *bp);
402 static int	swapongeom(struct thread *, struct vnode *);
403 static int	swaponvp(struct thread *, struct vnode *, u_long);
404 static int	swapoff_one(struct swdevt *sp, struct ucred *cred);
405 
406 /*
407  * Swap bitmap functions
408  */
409 static void	swp_pager_freeswapspace(daddr_t blk, int npages);
410 static daddr_t	swp_pager_getswapspace(int npages);
411 
412 /*
413  * Metadata functions
414  */
415 static struct swblock **swp_pager_hash(vm_object_t object, vm_pindex_t index);
416 static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
417 static void swp_pager_meta_free(vm_object_t, vm_pindex_t, daddr_t);
418 static void swp_pager_meta_free_all(vm_object_t);
419 static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);
420 
421 static void
422 swp_pager_free_nrpage(vm_page_t m)
423 {
424 
425 	vm_page_lock(m);
426 	if (m->wire_count == 0)
427 		vm_page_free(m);
428 	vm_page_unlock(m);
429 }
430 
431 /*
432  * SWP_SIZECHECK() -	update swap_pager_full indication
433  *
434  *	update the swap_pager_almost_full indication and warn when we are
435  *	about to run out of swap space, using lowat/hiwat hysteresis.
436  *
437  *	Clear swap_pager_full ( task killing ) indication when lowat is met.
438  *
439  *	No restrictions on call
440  *	This routine may not block.
441  */
442 static void
443 swp_sizecheck(void)
444 {
445 
446 	if (swap_pager_avail < nswap_lowat) {
447 		if (swap_pager_almost_full == 0) {
448 			printf("swap_pager: out of swap space\n");
449 			swap_pager_almost_full = 1;
450 		}
451 	} else {
452 		swap_pager_full = 0;
453 		if (swap_pager_avail > nswap_hiwat)
454 			swap_pager_almost_full = 0;
455 	}
456 }
457 
458 /*
459  * SWP_PAGER_HASH() -	hash swap meta data
460  *
461  *	This is an helper function which hashes the swapblk given
462  *	the object and page index.  It returns a pointer to a pointer
463  *	to the object, or a pointer to a NULL pointer if it could not
464  *	find a swapblk.
465  */
466 static struct swblock **
467 swp_pager_hash(vm_object_t object, vm_pindex_t index)
468 {
469 	struct swblock **pswap;
470 	struct swblock *swap;
471 
472 	index &= ~(vm_pindex_t)SWAP_META_MASK;
473 	pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask];
474 	while ((swap = *pswap) != NULL) {
475 		if (swap->swb_object == object &&
476 		    swap->swb_index == index
477 		) {
478 			break;
479 		}
480 		pswap = &swap->swb_hnext;
481 	}
482 	return (pswap);
483 }
484 
485 /*
486  * SWAP_PAGER_INIT() -	initialize the swap pager!
487  *
488  *	Expected to be started from system init.  NOTE:  This code is run
489  *	before much else so be careful what you depend on.  Most of the VM
490  *	system has yet to be initialized at this point.
491  */
492 static void
493 swap_pager_init(void)
494 {
495 	/*
496 	 * Initialize object lists
497 	 */
498 	int i;
499 
500 	for (i = 0; i < NOBJLISTS; ++i)
501 		TAILQ_INIT(&swap_pager_object_list[i]);
502 	mtx_init(&sw_alloc_mtx, "swap_pager list", NULL, MTX_DEF);
503 	mtx_init(&sw_dev_mtx, "swapdev", NULL, MTX_DEF);
504 
505 	/*
506 	 * Device Stripe, in PAGE_SIZE'd blocks
507 	 */
508 	dmmax = SWB_NPAGES * 2;
509 }
510 
511 /*
512  * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process
513  *
514  *	Expected to be started from pageout process once, prior to entering
515  *	its main loop.
516  */
517 void
518 swap_pager_swap_init(void)
519 {
520 	unsigned long n, n2;
521 
522 	/*
523 	 * Number of in-transit swap bp operations.  Don't
524 	 * exhaust the pbufs completely.  Make sure we
525 	 * initialize workable values (0 will work for hysteresis
526 	 * but it isn't very efficient).
527 	 *
528 	 * The nsw_cluster_max is constrained by the bp->b_pages[]
529 	 * array (MAXPHYS/PAGE_SIZE) and our locally defined
530 	 * MAX_PAGEOUT_CLUSTER.   Also be aware that swap ops are
531 	 * constrained by the swap device interleave stripe size.
532 	 *
533 	 * Currently we hardwire nsw_wcount_async to 4.  This limit is
534 	 * designed to prevent other I/O from having high latencies due to
535 	 * our pageout I/O.  The value 4 works well for one or two active swap
536 	 * devices but is probably a little low if you have more.  Even so,
537 	 * a higher value would probably generate only a limited improvement
538 	 * with three or four active swap devices since the system does not
539 	 * typically have to pageout at extreme bandwidths.   We will want
540 	 * at least 2 per swap devices, and 4 is a pretty good value if you
541 	 * have one NFS swap device due to the command/ack latency over NFS.
542 	 * So it all works out pretty well.
543 	 */
544 	nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER);
545 
546 	mtx_lock(&pbuf_mtx);
547 	nsw_rcount = (nswbuf + 1) / 2;
548 	nsw_wcount_sync = (nswbuf + 3) / 4;
549 	nsw_wcount_async = 4;
550 	nsw_wcount_async_max = nsw_wcount_async;
551 	mtx_unlock(&pbuf_mtx);
552 
553 	/*
554 	 * Initialize our zone.  Right now I'm just guessing on the number
555 	 * we need based on the number of pages in the system.  Each swblock
556 	 * can hold 32 pages, so this is probably overkill.  This reservation
557 	 * is typically limited to around 32MB by default.
558 	 */
559 	n = vm_cnt.v_page_count / 2;
560 	if (maxswzone && n > maxswzone / sizeof(struct swblock))
561 		n = maxswzone / sizeof(struct swblock);
562 	n2 = n;
563 	swap_zone = uma_zcreate("SWAPMETA", sizeof(struct swblock), NULL, NULL,
564 	    NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM);
565 	if (swap_zone == NULL)
566 		panic("failed to create swap_zone.");
567 	do {
568 		if (uma_zone_reserve_kva(swap_zone, n))
569 			break;
570 		/*
571 		 * if the allocation failed, try a zone two thirds the
572 		 * size of the previous attempt.
573 		 */
574 		n -= ((n + 2) / 3);
575 	} while (n > 0);
576 	if (n2 != n)
577 		printf("Swap zone entries reduced from %lu to %lu.\n", n2, n);
578 	swap_maxpages = n * SWAP_META_PAGES;
579 	swzone = n * sizeof(struct swblock);
580 	n2 = n;
581 
582 	/*
583 	 * Initialize our meta-data hash table.  The swapper does not need to
584 	 * be quite as efficient as the VM system, so we do not use an
585 	 * oversized hash table.
586 	 *
587 	 * 	n: 		size of hash table, must be power of 2
588 	 *	swhash_mask:	hash table index mask
589 	 */
590 	for (n = 1; n < n2 / 8; n *= 2)
591 		;
592 	swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO);
593 	swhash_mask = n - 1;
594 	mtx_init(&swhash_mtx, "swap_pager swhash", NULL, MTX_DEF);
595 }
596 
597 /*
598  * SWAP_PAGER_ALLOC() -	allocate a new OBJT_SWAP VM object and instantiate
599  *			its metadata structures.
600  *
601  *	This routine is called from the mmap and fork code to create a new
602  *	OBJT_SWAP object.  We do this by creating an OBJT_DEFAULT object
603  *	and then converting it with swp_pager_meta_build().
604  *
605  *	This routine may block in vm_object_allocate() and create a named
606  *	object lookup race, so we must interlock.
607  *
608  * MPSAFE
609  */
610 static vm_object_t
611 swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
612     vm_ooffset_t offset, struct ucred *cred)
613 {
614 	vm_object_t object;
615 	vm_pindex_t pindex;
616 
617 	pindex = OFF_TO_IDX(offset + PAGE_MASK + size);
618 	if (handle) {
619 		mtx_lock(&Giant);
620 		/*
621 		 * Reference existing named region or allocate new one.  There
622 		 * should not be a race here against swp_pager_meta_build()
623 		 * as called from vm_page_remove() in regards to the lookup
624 		 * of the handle.
625 		 */
626 		sx_xlock(&sw_alloc_sx);
627 		object = vm_pager_object_lookup(NOBJLIST(handle), handle);
628 		if (object == NULL) {
629 			if (cred != NULL) {
630 				if (!swap_reserve_by_cred(size, cred)) {
631 					sx_xunlock(&sw_alloc_sx);
632 					mtx_unlock(&Giant);
633 					return (NULL);
634 				}
635 				crhold(cred);
636 			}
637 			object = vm_object_allocate(OBJT_DEFAULT, pindex);
638 			VM_OBJECT_WLOCK(object);
639 			object->handle = handle;
640 			if (cred != NULL) {
641 				object->cred = cred;
642 				object->charge = size;
643 			}
644 			swp_pager_meta_build(object, 0, SWAPBLK_NONE);
645 			VM_OBJECT_WUNLOCK(object);
646 		}
647 		sx_xunlock(&sw_alloc_sx);
648 		mtx_unlock(&Giant);
649 	} else {
650 		if (cred != NULL) {
651 			if (!swap_reserve_by_cred(size, cred))
652 				return (NULL);
653 			crhold(cred);
654 		}
655 		object = vm_object_allocate(OBJT_DEFAULT, pindex);
656 		VM_OBJECT_WLOCK(object);
657 		if (cred != NULL) {
658 			object->cred = cred;
659 			object->charge = size;
660 		}
661 		swp_pager_meta_build(object, 0, SWAPBLK_NONE);
662 		VM_OBJECT_WUNLOCK(object);
663 	}
664 	return (object);
665 }
666 
667 /*
668  * SWAP_PAGER_DEALLOC() -	remove swap metadata from object
669  *
670  *	The swap backing for the object is destroyed.  The code is
671  *	designed such that we can reinstantiate it later, but this
672  *	routine is typically called only when the entire object is
673  *	about to be destroyed.
674  *
675  *	The object must be locked.
676  */
677 static void
678 swap_pager_dealloc(vm_object_t object)
679 {
680 
681 	/*
682 	 * Remove from list right away so lookups will fail if we block for
683 	 * pageout completion.
684 	 */
685 	if (object->handle != NULL) {
686 		mtx_lock(&sw_alloc_mtx);
687 		TAILQ_REMOVE(NOBJLIST(object->handle), object, pager_object_list);
688 		mtx_unlock(&sw_alloc_mtx);
689 	}
690 
691 	VM_OBJECT_ASSERT_WLOCKED(object);
692 	vm_object_pip_wait(object, "swpdea");
693 
694 	/*
695 	 * Free all remaining metadata.  We only bother to free it from
696 	 * the swap meta data.  We do not attempt to free swapblk's still
697 	 * associated with vm_page_t's for this object.  We do not care
698 	 * if paging is still in progress on some objects.
699 	 */
700 	swp_pager_meta_free_all(object);
701 }
702 
703 /************************************************************************
704  *			SWAP PAGER BITMAP ROUTINES			*
705  ************************************************************************/
706 
707 /*
708  * SWP_PAGER_GETSWAPSPACE() -	allocate raw swap space
709  *
710  *	Allocate swap for the requested number of pages.  The starting
711  *	swap block number (a page index) is returned or SWAPBLK_NONE
712  *	if the allocation failed.
713  *
714  *	Also has the side effect of advising that somebody made a mistake
715  *	when they configured swap and didn't configure enough.
716  *
717  *	This routine may not sleep.
718  *
719  *	We allocate in round-robin fashion from the configured devices.
720  */
721 static daddr_t
722 swp_pager_getswapspace(int npages)
723 {
724 	daddr_t blk;
725 	struct swdevt *sp;
726 	int i;
727 
728 	blk = SWAPBLK_NONE;
729 	mtx_lock(&sw_dev_mtx);
730 	sp = swdevhd;
731 	for (i = 0; i < nswapdev; i++) {
732 		if (sp == NULL)
733 			sp = TAILQ_FIRST(&swtailq);
734 		if (!(sp->sw_flags & SW_CLOSING)) {
735 			blk = blist_alloc(sp->sw_blist, npages);
736 			if (blk != SWAPBLK_NONE) {
737 				blk += sp->sw_first;
738 				sp->sw_used += npages;
739 				swap_pager_avail -= npages;
740 				swp_sizecheck();
741 				swdevhd = TAILQ_NEXT(sp, sw_list);
742 				goto done;
743 			}
744 		}
745 		sp = TAILQ_NEXT(sp, sw_list);
746 	}
747 	if (swap_pager_full != 2) {
748 		printf("swap_pager_getswapspace(%d): failed\n", npages);
749 		swap_pager_full = 2;
750 		swap_pager_almost_full = 1;
751 	}
752 	swdevhd = NULL;
753 done:
754 	mtx_unlock(&sw_dev_mtx);
755 	return (blk);
756 }
757 
758 static int
759 swp_pager_isondev(daddr_t blk, struct swdevt *sp)
760 {
761 
762 	return (blk >= sp->sw_first && blk < sp->sw_end);
763 }
764 
765 static void
766 swp_pager_strategy(struct buf *bp)
767 {
768 	struct swdevt *sp;
769 
770 	mtx_lock(&sw_dev_mtx);
771 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
772 		if (bp->b_blkno >= sp->sw_first && bp->b_blkno < sp->sw_end) {
773 			mtx_unlock(&sw_dev_mtx);
774 			if ((sp->sw_flags & SW_UNMAPPED) != 0 &&
775 			    unmapped_buf_allowed) {
776 				bp->b_kvaalloc = bp->b_data;
777 				bp->b_data = unmapped_buf;
778 				bp->b_kvabase = unmapped_buf;
779 				bp->b_offset = 0;
780 				bp->b_flags |= B_UNMAPPED;
781 			} else {
782 				pmap_qenter((vm_offset_t)bp->b_data,
783 				    &bp->b_pages[0], bp->b_bcount / PAGE_SIZE);
784 			}
785 			sp->sw_strategy(bp, sp);
786 			return;
787 		}
788 	}
789 	panic("Swapdev not found");
790 }
791 
792 
793 /*
794  * SWP_PAGER_FREESWAPSPACE() -	free raw swap space
795  *
796  *	This routine returns the specified swap blocks back to the bitmap.
797  *
798  *	This routine may not sleep.
799  */
800 static void
801 swp_pager_freeswapspace(daddr_t blk, int npages)
802 {
803 	struct swdevt *sp;
804 
805 	mtx_lock(&sw_dev_mtx);
806 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
807 		if (blk >= sp->sw_first && blk < sp->sw_end) {
808 			sp->sw_used -= npages;
809 			/*
810 			 * If we are attempting to stop swapping on
811 			 * this device, we don't want to mark any
812 			 * blocks free lest they be reused.
813 			 */
814 			if ((sp->sw_flags & SW_CLOSING) == 0) {
815 				blist_free(sp->sw_blist, blk - sp->sw_first,
816 				    npages);
817 				swap_pager_avail += npages;
818 				swp_sizecheck();
819 			}
820 			mtx_unlock(&sw_dev_mtx);
821 			return;
822 		}
823 	}
824 	panic("Swapdev not found");
825 }
826 
827 /*
828  * SWAP_PAGER_FREESPACE() -	frees swap blocks associated with a page
829  *				range within an object.
830  *
831  *	This is a globally accessible routine.
832  *
833  *	This routine removes swapblk assignments from swap metadata.
834  *
835  *	The external callers of this routine typically have already destroyed
836  *	or renamed vm_page_t's associated with this range in the object so
837  *	we should be ok.
838  *
839  *	The object must be locked.
840  */
841 void
842 swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size)
843 {
844 
845 	swp_pager_meta_free(object, start, size);
846 }
847 
848 /*
849  * SWAP_PAGER_RESERVE() - reserve swap blocks in object
850  *
851  *	Assigns swap blocks to the specified range within the object.  The
852  *	swap blocks are not zeroed.  Any previous swap assignment is destroyed.
853  *
854  *	Returns 0 on success, -1 on failure.
855  */
856 int
857 swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
858 {
859 	int n = 0;
860 	daddr_t blk = SWAPBLK_NONE;
861 	vm_pindex_t beg = start;	/* save start index */
862 
863 	VM_OBJECT_WLOCK(object);
864 	while (size) {
865 		if (n == 0) {
866 			n = BLIST_MAX_ALLOC;
867 			while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
868 				n >>= 1;
869 				if (n == 0) {
870 					swp_pager_meta_free(object, beg, start - beg);
871 					VM_OBJECT_WUNLOCK(object);
872 					return (-1);
873 				}
874 			}
875 		}
876 		swp_pager_meta_build(object, start, blk);
877 		--size;
878 		++start;
879 		++blk;
880 		--n;
881 	}
882 	swp_pager_meta_free(object, start, n);
883 	VM_OBJECT_WUNLOCK(object);
884 	return (0);
885 }
886 
887 /*
888  * SWAP_PAGER_COPY() -  copy blocks from source pager to destination pager
889  *			and destroy the source.
890  *
891  *	Copy any valid swapblks from the source to the destination.  In
892  *	cases where both the source and destination have a valid swapblk,
893  *	we keep the destination's.
894  *
895  *	This routine is allowed to sleep.  It may sleep allocating metadata
896  *	indirectly through swp_pager_meta_build() or if paging is still in
897  *	progress on the source.
898  *
899  *	The source object contains no vm_page_t's (which is just as well)
900  *
901  *	The source object is of type OBJT_SWAP.
902  *
903  *	The source and destination objects must be locked.
904  *	Both object locks may temporarily be released.
905  */
906 void
907 swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
908     vm_pindex_t offset, int destroysource)
909 {
910 	vm_pindex_t i;
911 
912 	VM_OBJECT_ASSERT_WLOCKED(srcobject);
913 	VM_OBJECT_ASSERT_WLOCKED(dstobject);
914 
915 	/*
916 	 * If destroysource is set, we remove the source object from the
917 	 * swap_pager internal queue now.
918 	 */
919 	if (destroysource) {
920 		if (srcobject->handle != NULL) {
921 			mtx_lock(&sw_alloc_mtx);
922 			TAILQ_REMOVE(
923 			    NOBJLIST(srcobject->handle),
924 			    srcobject,
925 			    pager_object_list
926 			);
927 			mtx_unlock(&sw_alloc_mtx);
928 		}
929 	}
930 
931 	/*
932 	 * transfer source to destination.
933 	 */
934 	for (i = 0; i < dstobject->size; ++i) {
935 		daddr_t dstaddr;
936 
937 		/*
938 		 * Locate (without changing) the swapblk on the destination,
939 		 * unless it is invalid in which case free it silently, or
940 		 * if the destination is a resident page, in which case the
941 		 * source is thrown away.
942 		 */
943 		dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
944 
945 		if (dstaddr == SWAPBLK_NONE) {
946 			/*
947 			 * Destination has no swapblk and is not resident,
948 			 * copy source.
949 			 */
950 			daddr_t srcaddr;
951 
952 			srcaddr = swp_pager_meta_ctl(
953 			    srcobject,
954 			    i + offset,
955 			    SWM_POP
956 			);
957 
958 			if (srcaddr != SWAPBLK_NONE) {
959 				/*
960 				 * swp_pager_meta_build() can sleep.
961 				 */
962 				vm_object_pip_add(srcobject, 1);
963 				VM_OBJECT_WUNLOCK(srcobject);
964 				vm_object_pip_add(dstobject, 1);
965 				swp_pager_meta_build(dstobject, i, srcaddr);
966 				vm_object_pip_wakeup(dstobject);
967 				VM_OBJECT_WLOCK(srcobject);
968 				vm_object_pip_wakeup(srcobject);
969 			}
970 		} else {
971 			/*
972 			 * Destination has valid swapblk or it is represented
973 			 * by a resident page.  We destroy the sourceblock.
974 			 */
975 
976 			swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE);
977 		}
978 	}
979 
980 	/*
981 	 * Free left over swap blocks in source.
982 	 *
983 	 * We have to revert the type to OBJT_DEFAULT so we do not accidently
984 	 * double-remove the object from the swap queues.
985 	 */
986 	if (destroysource) {
987 		swp_pager_meta_free_all(srcobject);
988 		/*
989 		 * Reverting the type is not necessary, the caller is going
990 		 * to destroy srcobject directly, but I'm doing it here
991 		 * for consistency since we've removed the object from its
992 		 * queues.
993 		 */
994 		srcobject->type = OBJT_DEFAULT;
995 	}
996 }
997 
998 /*
999  * SWAP_PAGER_HASPAGE() -	determine if we have good backing store for
1000  *				the requested page.
1001  *
1002  *	We determine whether good backing store exists for the requested
1003  *	page and return TRUE if it does, FALSE if it doesn't.
1004  *
1005  *	If TRUE, we also try to determine how much valid, contiguous backing
1006  *	store exists before and after the requested page within a reasonable
1007  *	distance.  We do not try to restrict it to the swap device stripe
1008  *	(that is handled in getpages/putpages).  It probably isn't worth
1009  *	doing here.
1010  */
1011 static boolean_t
1012 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after)
1013 {
1014 	daddr_t blk0;
1015 
1016 	VM_OBJECT_ASSERT_LOCKED(object);
1017 	/*
1018 	 * do we have good backing store at the requested index ?
1019 	 */
1020 	blk0 = swp_pager_meta_ctl(object, pindex, 0);
1021 
1022 	if (blk0 == SWAPBLK_NONE) {
1023 		if (before)
1024 			*before = 0;
1025 		if (after)
1026 			*after = 0;
1027 		return (FALSE);
1028 	}
1029 
1030 	/*
1031 	 * find backwards-looking contiguous good backing store
1032 	 */
1033 	if (before != NULL) {
1034 		int i;
1035 
1036 		for (i = 1; i < (SWB_NPAGES/2); ++i) {
1037 			daddr_t blk;
1038 
1039 			if (i > pindex)
1040 				break;
1041 			blk = swp_pager_meta_ctl(object, pindex - i, 0);
1042 			if (blk != blk0 - i)
1043 				break;
1044 		}
1045 		*before = (i - 1);
1046 	}
1047 
1048 	/*
1049 	 * find forward-looking contiguous good backing store
1050 	 */
1051 	if (after != NULL) {
1052 		int i;
1053 
1054 		for (i = 1; i < (SWB_NPAGES/2); ++i) {
1055 			daddr_t blk;
1056 
1057 			blk = swp_pager_meta_ctl(object, pindex + i, 0);
1058 			if (blk != blk0 + i)
1059 				break;
1060 		}
1061 		*after = (i - 1);
1062 	}
1063 	return (TRUE);
1064 }
1065 
1066 /*
1067  * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page
1068  *
1069  *	This removes any associated swap backing store, whether valid or
1070  *	not, from the page.
1071  *
1072  *	This routine is typically called when a page is made dirty, at
1073  *	which point any associated swap can be freed.  MADV_FREE also
1074  *	calls us in a special-case situation
1075  *
1076  *	NOTE!!!  If the page is clean and the swap was valid, the caller
1077  *	should make the page dirty before calling this routine.  This routine
1078  *	does NOT change the m->dirty status of the page.  Also: MADV_FREE
1079  *	depends on it.
1080  *
1081  *	This routine may not sleep.
1082  *
1083  *	The object containing the page must be locked.
1084  */
1085 static void
1086 swap_pager_unswapped(vm_page_t m)
1087 {
1088 
1089 	swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE);
1090 }
1091 
1092 /*
1093  * SWAP_PAGER_GETPAGES() - bring pages in from swap
1094  *
1095  *	Attempt to retrieve (m, count) pages from backing store, but make
1096  *	sure we retrieve at least m[reqpage].  We try to load in as large
1097  *	a chunk surrounding m[reqpage] as is contiguous in swap and which
1098  *	belongs to the same object.
1099  *
1100  *	The code is designed for asynchronous operation and
1101  *	immediate-notification of 'reqpage' but tends not to be
1102  *	used that way.  Please do not optimize-out this algorithmic
1103  *	feature, I intend to improve on it in the future.
1104  *
1105  *	The parent has a single vm_object_pip_add() reference prior to
1106  *	calling us and we should return with the same.
1107  *
1108  *	The parent has BUSY'd the pages.  We should return with 'm'
1109  *	left busy, but the others adjusted.
1110  */
1111 static int
1112 swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage)
1113 {
1114 	struct buf *bp;
1115 	vm_page_t mreq;
1116 	int i;
1117 	int j;
1118 	daddr_t blk;
1119 
1120 	mreq = m[reqpage];
1121 
1122 	KASSERT(mreq->object == object,
1123 	    ("swap_pager_getpages: object mismatch %p/%p",
1124 	    object, mreq->object));
1125 
1126 	/*
1127 	 * Calculate range to retrieve.  The pages have already been assigned
1128 	 * their swapblks.  We require a *contiguous* range but we know it to
1129 	 * not span devices.   If we do not supply it, bad things
1130 	 * happen.  Note that blk, iblk & jblk can be SWAPBLK_NONE, but the
1131 	 * loops are set up such that the case(s) are handled implicitly.
1132 	 *
1133 	 * The swp_*() calls must be made with the object locked.
1134 	 */
1135 	blk = swp_pager_meta_ctl(mreq->object, mreq->pindex, 0);
1136 
1137 	for (i = reqpage - 1; i >= 0; --i) {
1138 		daddr_t iblk;
1139 
1140 		iblk = swp_pager_meta_ctl(m[i]->object, m[i]->pindex, 0);
1141 		if (blk != iblk + (reqpage - i))
1142 			break;
1143 	}
1144 	++i;
1145 
1146 	for (j = reqpage + 1; j < count; ++j) {
1147 		daddr_t jblk;
1148 
1149 		jblk = swp_pager_meta_ctl(m[j]->object, m[j]->pindex, 0);
1150 		if (blk != jblk - (j - reqpage))
1151 			break;
1152 	}
1153 
1154 	/*
1155 	 * free pages outside our collection range.   Note: we never free
1156 	 * mreq, it must remain busy throughout.
1157 	 */
1158 	if (0 < i || j < count) {
1159 		int k;
1160 
1161 		for (k = 0; k < i; ++k)
1162 			swp_pager_free_nrpage(m[k]);
1163 		for (k = j; k < count; ++k)
1164 			swp_pager_free_nrpage(m[k]);
1165 	}
1166 
1167 	/*
1168 	 * Return VM_PAGER_FAIL if we have nothing to do.  Return mreq
1169 	 * still busy, but the others unbusied.
1170 	 */
1171 	if (blk == SWAPBLK_NONE)
1172 		return (VM_PAGER_FAIL);
1173 
1174 	/*
1175 	 * Getpbuf() can sleep.
1176 	 */
1177 	VM_OBJECT_WUNLOCK(object);
1178 	/*
1179 	 * Get a swap buffer header to perform the IO
1180 	 */
1181 	bp = getpbuf(&nsw_rcount);
1182 	bp->b_flags |= B_PAGING;
1183 
1184 	bp->b_iocmd = BIO_READ;
1185 	bp->b_iodone = swp_pager_async_iodone;
1186 	bp->b_rcred = crhold(thread0.td_ucred);
1187 	bp->b_wcred = crhold(thread0.td_ucred);
1188 	bp->b_blkno = blk - (reqpage - i);
1189 	bp->b_bcount = PAGE_SIZE * (j - i);
1190 	bp->b_bufsize = PAGE_SIZE * (j - i);
1191 	bp->b_pager.pg_reqpage = reqpage - i;
1192 
1193 	VM_OBJECT_WLOCK(object);
1194 	{
1195 		int k;
1196 
1197 		for (k = i; k < j; ++k) {
1198 			bp->b_pages[k - i] = m[k];
1199 			m[k]->oflags |= VPO_SWAPINPROG;
1200 		}
1201 	}
1202 	bp->b_npages = j - i;
1203 
1204 	PCPU_INC(cnt.v_swapin);
1205 	PCPU_ADD(cnt.v_swappgsin, bp->b_npages);
1206 
1207 	/*
1208 	 * We still hold the lock on mreq, and our automatic completion routine
1209 	 * does not remove it.
1210 	 */
1211 	vm_object_pip_add(object, bp->b_npages);
1212 	VM_OBJECT_WUNLOCK(object);
1213 
1214 	/*
1215 	 * perform the I/O.  NOTE!!!  bp cannot be considered valid after
1216 	 * this point because we automatically release it on completion.
1217 	 * Instead, we look at the one page we are interested in which we
1218 	 * still hold a lock on even through the I/O completion.
1219 	 *
1220 	 * The other pages in our m[] array are also released on completion,
1221 	 * so we cannot assume they are valid anymore either.
1222 	 *
1223 	 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1224 	 */
1225 	BUF_KERNPROC(bp);
1226 	swp_pager_strategy(bp);
1227 
1228 	/*
1229 	 * wait for the page we want to complete.  VPO_SWAPINPROG is always
1230 	 * cleared on completion.  If an I/O error occurs, SWAPBLK_NONE
1231 	 * is set in the meta-data.
1232 	 */
1233 	VM_OBJECT_WLOCK(object);
1234 	while ((mreq->oflags & VPO_SWAPINPROG) != 0) {
1235 		mreq->oflags |= VPO_SWAPSLEEP;
1236 		PCPU_INC(cnt.v_intrans);
1237 		if (VM_OBJECT_SLEEP(object, &object->paging_in_progress, PSWP,
1238 		    "swread", hz * 20)) {
1239 			printf(
1240 "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n",
1241 			    bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount);
1242 		}
1243 	}
1244 
1245 	/*
1246 	 * mreq is left busied after completion, but all the other pages
1247 	 * are freed.  If we had an unrecoverable read error the page will
1248 	 * not be valid.
1249 	 */
1250 	if (mreq->valid != VM_PAGE_BITS_ALL) {
1251 		return (VM_PAGER_ERROR);
1252 	} else {
1253 		return (VM_PAGER_OK);
1254 	}
1255 
1256 	/*
1257 	 * A final note: in a low swap situation, we cannot deallocate swap
1258 	 * and mark a page dirty here because the caller is likely to mark
1259 	 * the page clean when we return, causing the page to possibly revert
1260 	 * to all-zero's later.
1261 	 */
1262 }
1263 
1264 /*
1265  * 	swap_pager_getpages_async():
1266  *
1267  *	Right now this is emulation of asynchronous operation on top of
1268  *	swap_pager_getpages().
1269  */
1270 static int
1271 swap_pager_getpages_async(vm_object_t object, vm_page_t *m, int count,
1272     int reqpage, pgo_getpages_iodone_t iodone, void *arg)
1273 {
1274 	int r, error;
1275 
1276 	r = swap_pager_getpages(object, m, count, reqpage);
1277 	VM_OBJECT_WUNLOCK(object);
1278 	switch (r) {
1279 	case VM_PAGER_OK:
1280 		error = 0;
1281 		break;
1282 	case VM_PAGER_ERROR:
1283 		error = EIO;
1284 		break;
1285 	case VM_PAGER_FAIL:
1286 		error = EINVAL;
1287 		break;
1288 	default:
1289 		panic("unhandled swap_pager_getpages() error %d", r);
1290 	}
1291 	(iodone)(arg, m, count, error);
1292 	VM_OBJECT_WLOCK(object);
1293 
1294 	return (r);
1295 }
1296 
1297 /*
1298  *	swap_pager_putpages:
1299  *
1300  *	Assign swap (if necessary) and initiate I/O on the specified pages.
1301  *
1302  *	We support both OBJT_DEFAULT and OBJT_SWAP objects.  DEFAULT objects
1303  *	are automatically converted to SWAP objects.
1304  *
1305  *	In a low memory situation we may block in VOP_STRATEGY(), but the new
1306  *	vm_page reservation system coupled with properly written VFS devices
1307  *	should ensure that no low-memory deadlock occurs.  This is an area
1308  *	which needs work.
1309  *
1310  *	The parent has N vm_object_pip_add() references prior to
1311  *	calling us and will remove references for rtvals[] that are
1312  *	not set to VM_PAGER_PEND.  We need to remove the rest on I/O
1313  *	completion.
1314  *
1315  *	The parent has soft-busy'd the pages it passes us and will unbusy
1316  *	those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
1317  *	We need to unbusy the rest on I/O completion.
1318  */
1319 void
1320 swap_pager_putpages(vm_object_t object, vm_page_t *m, int count,
1321     int flags, int *rtvals)
1322 {
1323 	int i, n;
1324 	boolean_t sync;
1325 
1326 	if (count && m[0]->object != object) {
1327 		panic("swap_pager_putpages: object mismatch %p/%p",
1328 		    object,
1329 		    m[0]->object
1330 		);
1331 	}
1332 
1333 	/*
1334 	 * Step 1
1335 	 *
1336 	 * Turn object into OBJT_SWAP
1337 	 * check for bogus sysops
1338 	 * force sync if not pageout process
1339 	 */
1340 	if (object->type != OBJT_SWAP)
1341 		swp_pager_meta_build(object, 0, SWAPBLK_NONE);
1342 	VM_OBJECT_WUNLOCK(object);
1343 
1344 	n = 0;
1345 	if (curproc != pageproc)
1346 		sync = TRUE;
1347 	else
1348 		sync = (flags & VM_PAGER_PUT_SYNC) != 0;
1349 
1350 	/*
1351 	 * Step 2
1352 	 *
1353 	 * Update nsw parameters from swap_async_max sysctl values.
1354 	 * Do not let the sysop crash the machine with bogus numbers.
1355 	 */
1356 	mtx_lock(&pbuf_mtx);
1357 	if (swap_async_max != nsw_wcount_async_max) {
1358 		int n;
1359 
1360 		/*
1361 		 * limit range
1362 		 */
1363 		if ((n = swap_async_max) > nswbuf / 2)
1364 			n = nswbuf / 2;
1365 		if (n < 1)
1366 			n = 1;
1367 		swap_async_max = n;
1368 
1369 		/*
1370 		 * Adjust difference ( if possible ).  If the current async
1371 		 * count is too low, we may not be able to make the adjustment
1372 		 * at this time.
1373 		 */
1374 		n -= nsw_wcount_async_max;
1375 		if (nsw_wcount_async + n >= 0) {
1376 			nsw_wcount_async += n;
1377 			nsw_wcount_async_max += n;
1378 			wakeup(&nsw_wcount_async);
1379 		}
1380 	}
1381 	mtx_unlock(&pbuf_mtx);
1382 
1383 	/*
1384 	 * Step 3
1385 	 *
1386 	 * Assign swap blocks and issue I/O.  We reallocate swap on the fly.
1387 	 * The page is left dirty until the pageout operation completes
1388 	 * successfully.
1389 	 */
1390 	for (i = 0; i < count; i += n) {
1391 		int j;
1392 		struct buf *bp;
1393 		daddr_t blk;
1394 
1395 		/*
1396 		 * Maximum I/O size is limited by a number of factors.
1397 		 */
1398 		n = min(BLIST_MAX_ALLOC, count - i);
1399 		n = min(n, nsw_cluster_max);
1400 
1401 		/*
1402 		 * Get biggest block of swap we can.  If we fail, fall
1403 		 * back and try to allocate a smaller block.  Don't go
1404 		 * overboard trying to allocate space if it would overly
1405 		 * fragment swap.
1406 		 */
1407 		while (
1408 		    (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE &&
1409 		    n > 4
1410 		) {
1411 			n >>= 1;
1412 		}
1413 		if (blk == SWAPBLK_NONE) {
1414 			for (j = 0; j < n; ++j)
1415 				rtvals[i+j] = VM_PAGER_FAIL;
1416 			continue;
1417 		}
1418 
1419 		/*
1420 		 * All I/O parameters have been satisfied, build the I/O
1421 		 * request and assign the swap space.
1422 		 */
1423 		if (sync == TRUE) {
1424 			bp = getpbuf(&nsw_wcount_sync);
1425 		} else {
1426 			bp = getpbuf(&nsw_wcount_async);
1427 			bp->b_flags = B_ASYNC;
1428 		}
1429 		bp->b_flags |= B_PAGING;
1430 		bp->b_iocmd = BIO_WRITE;
1431 
1432 		bp->b_rcred = crhold(thread0.td_ucred);
1433 		bp->b_wcred = crhold(thread0.td_ucred);
1434 		bp->b_bcount = PAGE_SIZE * n;
1435 		bp->b_bufsize = PAGE_SIZE * n;
1436 		bp->b_blkno = blk;
1437 
1438 		VM_OBJECT_WLOCK(object);
1439 		for (j = 0; j < n; ++j) {
1440 			vm_page_t mreq = m[i+j];
1441 
1442 			swp_pager_meta_build(
1443 			    mreq->object,
1444 			    mreq->pindex,
1445 			    blk + j
1446 			);
1447 			vm_page_dirty(mreq);
1448 			rtvals[i+j] = VM_PAGER_OK;
1449 
1450 			mreq->oflags |= VPO_SWAPINPROG;
1451 			bp->b_pages[j] = mreq;
1452 		}
1453 		VM_OBJECT_WUNLOCK(object);
1454 		bp->b_npages = n;
1455 		/*
1456 		 * Must set dirty range for NFS to work.
1457 		 */
1458 		bp->b_dirtyoff = 0;
1459 		bp->b_dirtyend = bp->b_bcount;
1460 
1461 		PCPU_INC(cnt.v_swapout);
1462 		PCPU_ADD(cnt.v_swappgsout, bp->b_npages);
1463 
1464 		/*
1465 		 * asynchronous
1466 		 *
1467 		 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1468 		 */
1469 		if (sync == FALSE) {
1470 			bp->b_iodone = swp_pager_async_iodone;
1471 			BUF_KERNPROC(bp);
1472 			swp_pager_strategy(bp);
1473 
1474 			for (j = 0; j < n; ++j)
1475 				rtvals[i+j] = VM_PAGER_PEND;
1476 			/* restart outter loop */
1477 			continue;
1478 		}
1479 
1480 		/*
1481 		 * synchronous
1482 		 *
1483 		 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1484 		 */
1485 		bp->b_iodone = bdone;
1486 		swp_pager_strategy(bp);
1487 
1488 		/*
1489 		 * Wait for the sync I/O to complete, then update rtvals.
1490 		 * We just set the rtvals[] to VM_PAGER_PEND so we can call
1491 		 * our async completion routine at the end, thus avoiding a
1492 		 * double-free.
1493 		 */
1494 		bwait(bp, PVM, "swwrt");
1495 		for (j = 0; j < n; ++j)
1496 			rtvals[i+j] = VM_PAGER_PEND;
1497 		/*
1498 		 * Now that we are through with the bp, we can call the
1499 		 * normal async completion, which frees everything up.
1500 		 */
1501 		swp_pager_async_iodone(bp);
1502 	}
1503 	VM_OBJECT_WLOCK(object);
1504 }
1505 
1506 /*
1507  *	swp_pager_async_iodone:
1508  *
1509  *	Completion routine for asynchronous reads and writes from/to swap.
1510  *	Also called manually by synchronous code to finish up a bp.
1511  *
1512  *	This routine may not sleep.
1513  */
1514 static void
1515 swp_pager_async_iodone(struct buf *bp)
1516 {
1517 	int i;
1518 	vm_object_t object = NULL;
1519 
1520 	/*
1521 	 * report error
1522 	 */
1523 	if (bp->b_ioflags & BIO_ERROR) {
1524 		printf(
1525 		    "swap_pager: I/O error - %s failed; blkno %ld,"
1526 			"size %ld, error %d\n",
1527 		    ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
1528 		    (long)bp->b_blkno,
1529 		    (long)bp->b_bcount,
1530 		    bp->b_error
1531 		);
1532 	}
1533 
1534 	/*
1535 	 * remove the mapping for kernel virtual
1536 	 */
1537 	if ((bp->b_flags & B_UNMAPPED) != 0) {
1538 		bp->b_data = bp->b_kvaalloc;
1539 		bp->b_kvabase = bp->b_kvaalloc;
1540 		bp->b_flags &= ~B_UNMAPPED;
1541 	} else
1542 		pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
1543 
1544 	if (bp->b_npages) {
1545 		object = bp->b_pages[0]->object;
1546 		VM_OBJECT_WLOCK(object);
1547 	}
1548 
1549 	/*
1550 	 * cleanup pages.  If an error occurs writing to swap, we are in
1551 	 * very serious trouble.  If it happens to be a disk error, though,
1552 	 * we may be able to recover by reassigning the swap later on.  So
1553 	 * in this case we remove the m->swapblk assignment for the page
1554 	 * but do not free it in the rlist.  The errornous block(s) are thus
1555 	 * never reallocated as swap.  Redirty the page and continue.
1556 	 */
1557 	for (i = 0; i < bp->b_npages; ++i) {
1558 		vm_page_t m = bp->b_pages[i];
1559 
1560 		m->oflags &= ~VPO_SWAPINPROG;
1561 		if (m->oflags & VPO_SWAPSLEEP) {
1562 			m->oflags &= ~VPO_SWAPSLEEP;
1563 			wakeup(&object->paging_in_progress);
1564 		}
1565 
1566 		if (bp->b_ioflags & BIO_ERROR) {
1567 			/*
1568 			 * If an error occurs I'd love to throw the swapblk
1569 			 * away without freeing it back to swapspace, so it
1570 			 * can never be used again.  But I can't from an
1571 			 * interrupt.
1572 			 */
1573 			if (bp->b_iocmd == BIO_READ) {
1574 				/*
1575 				 * When reading, reqpage needs to stay
1576 				 * locked for the parent, but all other
1577 				 * pages can be freed.  We still want to
1578 				 * wakeup the parent waiting on the page,
1579 				 * though.  ( also: pg_reqpage can be -1 and
1580 				 * not match anything ).
1581 				 *
1582 				 * We have to wake specifically requested pages
1583 				 * up too because we cleared VPO_SWAPINPROG and
1584 				 * someone may be waiting for that.
1585 				 *
1586 				 * NOTE: for reads, m->dirty will probably
1587 				 * be overridden by the original caller of
1588 				 * getpages so don't play cute tricks here.
1589 				 */
1590 				m->valid = 0;
1591 				if (i != bp->b_pager.pg_reqpage)
1592 					swp_pager_free_nrpage(m);
1593 				else {
1594 					vm_page_lock(m);
1595 					vm_page_flash(m);
1596 					vm_page_unlock(m);
1597 				}
1598 				/*
1599 				 * If i == bp->b_pager.pg_reqpage, do not wake
1600 				 * the page up.  The caller needs to.
1601 				 */
1602 			} else {
1603 				/*
1604 				 * If a write error occurs, reactivate page
1605 				 * so it doesn't clog the inactive list,
1606 				 * then finish the I/O.
1607 				 */
1608 				vm_page_dirty(m);
1609 				vm_page_lock(m);
1610 				vm_page_activate(m);
1611 				vm_page_unlock(m);
1612 				vm_page_sunbusy(m);
1613 			}
1614 		} else if (bp->b_iocmd == BIO_READ) {
1615 			/*
1616 			 * NOTE: for reads, m->dirty will probably be
1617 			 * overridden by the original caller of getpages so
1618 			 * we cannot set them in order to free the underlying
1619 			 * swap in a low-swap situation.  I don't think we'd
1620 			 * want to do that anyway, but it was an optimization
1621 			 * that existed in the old swapper for a time before
1622 			 * it got ripped out due to precisely this problem.
1623 			 *
1624 			 * If not the requested page then deactivate it.
1625 			 *
1626 			 * Note that the requested page, reqpage, is left
1627 			 * busied, but we still have to wake it up.  The
1628 			 * other pages are released (unbusied) by
1629 			 * vm_page_xunbusy().
1630 			 */
1631 			KASSERT(!pmap_page_is_mapped(m),
1632 			    ("swp_pager_async_iodone: page %p is mapped", m));
1633 			m->valid = VM_PAGE_BITS_ALL;
1634 			KASSERT(m->dirty == 0,
1635 			    ("swp_pager_async_iodone: page %p is dirty", m));
1636 
1637 			/*
1638 			 * We have to wake specifically requested pages
1639 			 * up too because we cleared VPO_SWAPINPROG and
1640 			 * could be waiting for it in getpages.  However,
1641 			 * be sure to not unbusy getpages specifically
1642 			 * requested page - getpages expects it to be
1643 			 * left busy.
1644 			 */
1645 			if (i != bp->b_pager.pg_reqpage) {
1646 				vm_page_lock(m);
1647 				vm_page_deactivate(m);
1648 				vm_page_unlock(m);
1649 				vm_page_xunbusy(m);
1650 			} else {
1651 				vm_page_lock(m);
1652 				vm_page_flash(m);
1653 				vm_page_unlock(m);
1654 			}
1655 		} else {
1656 			/*
1657 			 * For write success, clear the dirty
1658 			 * status, then finish the I/O ( which decrements the
1659 			 * busy count and possibly wakes waiter's up ).
1660 			 */
1661 			KASSERT(!pmap_page_is_write_mapped(m),
1662 			    ("swp_pager_async_iodone: page %p is not write"
1663 			    " protected", m));
1664 			vm_page_undirty(m);
1665 			vm_page_sunbusy(m);
1666 			if (vm_page_count_severe()) {
1667 				vm_page_lock(m);
1668 				vm_page_try_to_cache(m);
1669 				vm_page_unlock(m);
1670 			}
1671 		}
1672 	}
1673 
1674 	/*
1675 	 * adjust pip.  NOTE: the original parent may still have its own
1676 	 * pip refs on the object.
1677 	 */
1678 	if (object != NULL) {
1679 		vm_object_pip_wakeupn(object, bp->b_npages);
1680 		VM_OBJECT_WUNLOCK(object);
1681 	}
1682 
1683 	/*
1684 	 * swapdev_strategy() manually sets b_vp and b_bufobj before calling
1685 	 * bstrategy(). Set them back to NULL now we're done with it, or we'll
1686 	 * trigger a KASSERT in relpbuf().
1687 	 */
1688 	if (bp->b_vp) {
1689 		    bp->b_vp = NULL;
1690 		    bp->b_bufobj = NULL;
1691 	}
1692 	/*
1693 	 * release the physical I/O buffer
1694 	 */
1695 	relpbuf(
1696 	    bp,
1697 	    ((bp->b_iocmd == BIO_READ) ? &nsw_rcount :
1698 		((bp->b_flags & B_ASYNC) ?
1699 		    &nsw_wcount_async :
1700 		    &nsw_wcount_sync
1701 		)
1702 	    )
1703 	);
1704 }
1705 
1706 /*
1707  *	swap_pager_isswapped:
1708  *
1709  *	Return 1 if at least one page in the given object is paged
1710  *	out to the given swap device.
1711  *
1712  *	This routine may not sleep.
1713  */
1714 int
1715 swap_pager_isswapped(vm_object_t object, struct swdevt *sp)
1716 {
1717 	daddr_t index = 0;
1718 	int bcount;
1719 	int i;
1720 
1721 	VM_OBJECT_ASSERT_WLOCKED(object);
1722 	if (object->type != OBJT_SWAP)
1723 		return (0);
1724 
1725 	mtx_lock(&swhash_mtx);
1726 	for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) {
1727 		struct swblock *swap;
1728 
1729 		if ((swap = *swp_pager_hash(object, index)) != NULL) {
1730 			for (i = 0; i < SWAP_META_PAGES; ++i) {
1731 				if (swp_pager_isondev(swap->swb_pages[i], sp)) {
1732 					mtx_unlock(&swhash_mtx);
1733 					return (1);
1734 				}
1735 			}
1736 		}
1737 		index += SWAP_META_PAGES;
1738 	}
1739 	mtx_unlock(&swhash_mtx);
1740 	return (0);
1741 }
1742 
1743 /*
1744  * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in
1745  *
1746  *	This routine dissociates the page at the given index within a
1747  *	swap block from its backing store, paging it in if necessary.
1748  *	If the page is paged in, it is placed in the inactive queue,
1749  *	since it had its backing store ripped out from under it.
1750  *	We also attempt to swap in all other pages in the swap block,
1751  *	we only guarantee that the one at the specified index is
1752  *	paged in.
1753  *
1754  *	XXX - The code to page the whole block in doesn't work, so we
1755  *	      revert to the one-by-one behavior for now.  Sigh.
1756  */
1757 static inline void
1758 swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex)
1759 {
1760 	vm_page_t m;
1761 
1762 	vm_object_pip_add(object, 1);
1763 	m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL);
1764 	if (m->valid == VM_PAGE_BITS_ALL) {
1765 		vm_object_pip_wakeup(object);
1766 		vm_page_dirty(m);
1767 		vm_page_lock(m);
1768 		vm_page_activate(m);
1769 		vm_page_unlock(m);
1770 		vm_page_xunbusy(m);
1771 		vm_pager_page_unswapped(m);
1772 		return;
1773 	}
1774 
1775 	if (swap_pager_getpages(object, &m, 1, 0) != VM_PAGER_OK)
1776 		panic("swap_pager_force_pagein: read from swap failed");/*XXX*/
1777 	vm_object_pip_wakeup(object);
1778 	vm_page_dirty(m);
1779 	vm_page_lock(m);
1780 	vm_page_deactivate(m);
1781 	vm_page_unlock(m);
1782 	vm_page_xunbusy(m);
1783 	vm_pager_page_unswapped(m);
1784 }
1785 
1786 /*
1787  *	swap_pager_swapoff:
1788  *
1789  *	Page in all of the pages that have been paged out to the
1790  *	given device.  The corresponding blocks in the bitmap must be
1791  *	marked as allocated and the device must be flagged SW_CLOSING.
1792  *	There may be no processes swapped out to the device.
1793  *
1794  *	This routine may block.
1795  */
1796 static void
1797 swap_pager_swapoff(struct swdevt *sp)
1798 {
1799 	struct swblock *swap;
1800 	int i, j, retries;
1801 
1802 	GIANT_REQUIRED;
1803 
1804 	retries = 0;
1805 full_rescan:
1806 	mtx_lock(&swhash_mtx);
1807 	for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */
1808 restart:
1809 		for (swap = swhash[i]; swap != NULL; swap = swap->swb_hnext) {
1810 			vm_object_t object = swap->swb_object;
1811 			vm_pindex_t pindex = swap->swb_index;
1812 			for (j = 0; j < SWAP_META_PAGES; ++j) {
1813 				if (swp_pager_isondev(swap->swb_pages[j], sp)) {
1814 					/* avoid deadlock */
1815 					if (!VM_OBJECT_TRYWLOCK(object)) {
1816 						break;
1817 					} else {
1818 						mtx_unlock(&swhash_mtx);
1819 						swp_pager_force_pagein(object,
1820 						    pindex + j);
1821 						VM_OBJECT_WUNLOCK(object);
1822 						mtx_lock(&swhash_mtx);
1823 						goto restart;
1824 					}
1825 				}
1826 			}
1827 		}
1828 	}
1829 	mtx_unlock(&swhash_mtx);
1830 	if (sp->sw_used) {
1831 		/*
1832 		 * Objects may be locked or paging to the device being
1833 		 * removed, so we will miss their pages and need to
1834 		 * make another pass.  We have marked this device as
1835 		 * SW_CLOSING, so the activity should finish soon.
1836 		 */
1837 		retries++;
1838 		if (retries > 100) {
1839 			panic("swapoff: failed to locate %d swap blocks",
1840 			    sp->sw_used);
1841 		}
1842 		pause("swpoff", hz / 20);
1843 		goto full_rescan;
1844 	}
1845 }
1846 
1847 /************************************************************************
1848  *				SWAP META DATA 				*
1849  ************************************************************************
1850  *
1851  *	These routines manipulate the swap metadata stored in the
1852  *	OBJT_SWAP object.
1853  *
1854  *	Swap metadata is implemented with a global hash and not directly
1855  *	linked into the object.  Instead the object simply contains
1856  *	appropriate tracking counters.
1857  */
1858 
1859 /*
1860  * SWP_PAGER_META_BUILD() -	add swap block to swap meta data for object
1861  *
1862  *	We first convert the object to a swap object if it is a default
1863  *	object.
1864  *
1865  *	The specified swapblk is added to the object's swap metadata.  If
1866  *	the swapblk is not valid, it is freed instead.  Any previously
1867  *	assigned swapblk is freed.
1868  */
1869 static void
1870 swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
1871 {
1872 	static volatile int exhausted;
1873 	struct swblock *swap;
1874 	struct swblock **pswap;
1875 	int idx;
1876 
1877 	VM_OBJECT_ASSERT_WLOCKED(object);
1878 	/*
1879 	 * Convert default object to swap object if necessary
1880 	 */
1881 	if (object->type != OBJT_SWAP) {
1882 		object->type = OBJT_SWAP;
1883 		object->un_pager.swp.swp_bcount = 0;
1884 
1885 		if (object->handle != NULL) {
1886 			mtx_lock(&sw_alloc_mtx);
1887 			TAILQ_INSERT_TAIL(
1888 			    NOBJLIST(object->handle),
1889 			    object,
1890 			    pager_object_list
1891 			);
1892 			mtx_unlock(&sw_alloc_mtx);
1893 		}
1894 	}
1895 
1896 	/*
1897 	 * Locate hash entry.  If not found create, but if we aren't adding
1898 	 * anything just return.  If we run out of space in the map we wait
1899 	 * and, since the hash table may have changed, retry.
1900 	 */
1901 retry:
1902 	mtx_lock(&swhash_mtx);
1903 	pswap = swp_pager_hash(object, pindex);
1904 
1905 	if ((swap = *pswap) == NULL) {
1906 		int i;
1907 
1908 		if (swapblk == SWAPBLK_NONE)
1909 			goto done;
1910 
1911 		swap = *pswap = uma_zalloc(swap_zone, M_NOWAIT |
1912 		    (curproc == pageproc ? M_USE_RESERVE : 0));
1913 		if (swap == NULL) {
1914 			mtx_unlock(&swhash_mtx);
1915 			VM_OBJECT_WUNLOCK(object);
1916 			if (uma_zone_exhausted(swap_zone)) {
1917 				if (atomic_cmpset_int(&exhausted, 0, 1))
1918 					printf("swap zone exhausted, "
1919 					    "increase kern.maxswzone\n");
1920 				vm_pageout_oom(VM_OOM_SWAPZ);
1921 				pause("swzonex", 10);
1922 			} else
1923 				VM_WAIT;
1924 			VM_OBJECT_WLOCK(object);
1925 			goto retry;
1926 		}
1927 
1928 		if (atomic_cmpset_int(&exhausted, 1, 0))
1929 			printf("swap zone ok\n");
1930 
1931 		swap->swb_hnext = NULL;
1932 		swap->swb_object = object;
1933 		swap->swb_index = pindex & ~(vm_pindex_t)SWAP_META_MASK;
1934 		swap->swb_count = 0;
1935 
1936 		++object->un_pager.swp.swp_bcount;
1937 
1938 		for (i = 0; i < SWAP_META_PAGES; ++i)
1939 			swap->swb_pages[i] = SWAPBLK_NONE;
1940 	}
1941 
1942 	/*
1943 	 * Delete prior contents of metadata
1944 	 */
1945 	idx = pindex & SWAP_META_MASK;
1946 
1947 	if (swap->swb_pages[idx] != SWAPBLK_NONE) {
1948 		swp_pager_freeswapspace(swap->swb_pages[idx], 1);
1949 		--swap->swb_count;
1950 	}
1951 
1952 	/*
1953 	 * Enter block into metadata
1954 	 */
1955 	swap->swb_pages[idx] = swapblk;
1956 	if (swapblk != SWAPBLK_NONE)
1957 		++swap->swb_count;
1958 done:
1959 	mtx_unlock(&swhash_mtx);
1960 }
1961 
1962 /*
1963  * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
1964  *
1965  *	The requested range of blocks is freed, with any associated swap
1966  *	returned to the swap bitmap.
1967  *
1968  *	This routine will free swap metadata structures as they are cleaned
1969  *	out.  This routine does *NOT* operate on swap metadata associated
1970  *	with resident pages.
1971  */
1972 static void
1973 swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count)
1974 {
1975 
1976 	VM_OBJECT_ASSERT_LOCKED(object);
1977 	if (object->type != OBJT_SWAP)
1978 		return;
1979 
1980 	while (count > 0) {
1981 		struct swblock **pswap;
1982 		struct swblock *swap;
1983 
1984 		mtx_lock(&swhash_mtx);
1985 		pswap = swp_pager_hash(object, index);
1986 
1987 		if ((swap = *pswap) != NULL) {
1988 			daddr_t v = swap->swb_pages[index & SWAP_META_MASK];
1989 
1990 			if (v != SWAPBLK_NONE) {
1991 				swp_pager_freeswapspace(v, 1);
1992 				swap->swb_pages[index & SWAP_META_MASK] =
1993 					SWAPBLK_NONE;
1994 				if (--swap->swb_count == 0) {
1995 					*pswap = swap->swb_hnext;
1996 					uma_zfree(swap_zone, swap);
1997 					--object->un_pager.swp.swp_bcount;
1998 				}
1999 			}
2000 			--count;
2001 			++index;
2002 		} else {
2003 			int n = SWAP_META_PAGES - (index & SWAP_META_MASK);
2004 			count -= n;
2005 			index += n;
2006 		}
2007 		mtx_unlock(&swhash_mtx);
2008 	}
2009 }
2010 
2011 /*
2012  * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
2013  *
2014  *	This routine locates and destroys all swap metadata associated with
2015  *	an object.
2016  */
2017 static void
2018 swp_pager_meta_free_all(vm_object_t object)
2019 {
2020 	daddr_t index = 0;
2021 
2022 	VM_OBJECT_ASSERT_WLOCKED(object);
2023 	if (object->type != OBJT_SWAP)
2024 		return;
2025 
2026 	while (object->un_pager.swp.swp_bcount) {
2027 		struct swblock **pswap;
2028 		struct swblock *swap;
2029 
2030 		mtx_lock(&swhash_mtx);
2031 		pswap = swp_pager_hash(object, index);
2032 		if ((swap = *pswap) != NULL) {
2033 			int i;
2034 
2035 			for (i = 0; i < SWAP_META_PAGES; ++i) {
2036 				daddr_t v = swap->swb_pages[i];
2037 				if (v != SWAPBLK_NONE) {
2038 					--swap->swb_count;
2039 					swp_pager_freeswapspace(v, 1);
2040 				}
2041 			}
2042 			if (swap->swb_count != 0)
2043 				panic("swap_pager_meta_free_all: swb_count != 0");
2044 			*pswap = swap->swb_hnext;
2045 			uma_zfree(swap_zone, swap);
2046 			--object->un_pager.swp.swp_bcount;
2047 		}
2048 		mtx_unlock(&swhash_mtx);
2049 		index += SWAP_META_PAGES;
2050 	}
2051 }
2052 
2053 /*
2054  * SWP_PAGER_METACTL() -  misc control of swap and vm_page_t meta data.
2055  *
2056  *	This routine is capable of looking up, popping, or freeing
2057  *	swapblk assignments in the swap meta data or in the vm_page_t.
2058  *	The routine typically returns the swapblk being looked-up, or popped,
2059  *	or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block
2060  *	was invalid.  This routine will automatically free any invalid
2061  *	meta-data swapblks.
2062  *
2063  *	It is not possible to store invalid swapblks in the swap meta data
2064  *	(other then a literal 'SWAPBLK_NONE'), so we don't bother checking.
2065  *
2066  *	When acting on a busy resident page and paging is in progress, we
2067  *	have to wait until paging is complete but otherwise can act on the
2068  *	busy page.
2069  *
2070  *	SWM_FREE	remove and free swap block from metadata
2071  *	SWM_POP		remove from meta data but do not free.. pop it out
2072  */
2073 static daddr_t
2074 swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
2075 {
2076 	struct swblock **pswap;
2077 	struct swblock *swap;
2078 	daddr_t r1;
2079 	int idx;
2080 
2081 	VM_OBJECT_ASSERT_LOCKED(object);
2082 	/*
2083 	 * The meta data only exists of the object is OBJT_SWAP
2084 	 * and even then might not be allocated yet.
2085 	 */
2086 	if (object->type != OBJT_SWAP)
2087 		return (SWAPBLK_NONE);
2088 
2089 	r1 = SWAPBLK_NONE;
2090 	mtx_lock(&swhash_mtx);
2091 	pswap = swp_pager_hash(object, pindex);
2092 
2093 	if ((swap = *pswap) != NULL) {
2094 		idx = pindex & SWAP_META_MASK;
2095 		r1 = swap->swb_pages[idx];
2096 
2097 		if (r1 != SWAPBLK_NONE) {
2098 			if (flags & SWM_FREE) {
2099 				swp_pager_freeswapspace(r1, 1);
2100 				r1 = SWAPBLK_NONE;
2101 			}
2102 			if (flags & (SWM_FREE|SWM_POP)) {
2103 				swap->swb_pages[idx] = SWAPBLK_NONE;
2104 				if (--swap->swb_count == 0) {
2105 					*pswap = swap->swb_hnext;
2106 					uma_zfree(swap_zone, swap);
2107 					--object->un_pager.swp.swp_bcount;
2108 				}
2109 			}
2110 		}
2111 	}
2112 	mtx_unlock(&swhash_mtx);
2113 	return (r1);
2114 }
2115 
2116 /*
2117  * System call swapon(name) enables swapping on device name,
2118  * which must be in the swdevsw.  Return EBUSY
2119  * if already swapping on this device.
2120  */
2121 #ifndef _SYS_SYSPROTO_H_
2122 struct swapon_args {
2123 	char *name;
2124 };
2125 #endif
2126 
2127 /*
2128  * MPSAFE
2129  */
2130 /* ARGSUSED */
2131 int
2132 sys_swapon(struct thread *td, struct swapon_args *uap)
2133 {
2134 	struct vattr attr;
2135 	struct vnode *vp;
2136 	struct nameidata nd;
2137 	int error;
2138 
2139 	error = priv_check(td, PRIV_SWAPON);
2140 	if (error)
2141 		return (error);
2142 
2143 	mtx_lock(&Giant);
2144 	while (swdev_syscall_active)
2145 	    tsleep(&swdev_syscall_active, PUSER - 1, "swpon", 0);
2146 	swdev_syscall_active = 1;
2147 
2148 	/*
2149 	 * Swap metadata may not fit in the KVM if we have physical
2150 	 * memory of >1GB.
2151 	 */
2152 	if (swap_zone == NULL) {
2153 		error = ENOMEM;
2154 		goto done;
2155 	}
2156 
2157 	NDINIT(&nd, LOOKUP, ISOPEN | FOLLOW | AUDITVNODE1, UIO_USERSPACE,
2158 	    uap->name, td);
2159 	error = namei(&nd);
2160 	if (error)
2161 		goto done;
2162 
2163 	NDFREE(&nd, NDF_ONLY_PNBUF);
2164 	vp = nd.ni_vp;
2165 
2166 	if (vn_isdisk(vp, &error)) {
2167 		error = swapongeom(td, vp);
2168 	} else if (vp->v_type == VREG &&
2169 	    (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
2170 	    (error = VOP_GETATTR(vp, &attr, td->td_ucred)) == 0) {
2171 		/*
2172 		 * Allow direct swapping to NFS regular files in the same
2173 		 * way that nfs_mountroot() sets up diskless swapping.
2174 		 */
2175 		error = swaponvp(td, vp, attr.va_size / DEV_BSIZE);
2176 	}
2177 
2178 	if (error)
2179 		vrele(vp);
2180 done:
2181 	swdev_syscall_active = 0;
2182 	wakeup_one(&swdev_syscall_active);
2183 	mtx_unlock(&Giant);
2184 	return (error);
2185 }
2186 
2187 /*
2188  * Check that the total amount of swap currently configured does not
2189  * exceed half the theoretical maximum.  If it does, print a warning
2190  * message and return -1; otherwise, return 0.
2191  */
2192 static int
2193 swapon_check_swzone(unsigned long npages)
2194 {
2195 	unsigned long maxpages;
2196 
2197 	/* absolute maximum we can handle assuming 100% efficiency */
2198 	maxpages = uma_zone_get_max(swap_zone) * SWAP_META_PAGES;
2199 
2200 	/* recommend using no more than half that amount */
2201 	if (npages > maxpages / 2) {
2202 		printf("warning: total configured swap (%lu pages) "
2203 		    "exceeds maximum recommended amount (%lu pages).\n",
2204 		    npages, maxpages / 2);
2205 		printf("warning: increase kern.maxswzone "
2206 		    "or reduce amount of swap.\n");
2207 		return (-1);
2208 	}
2209 	return (0);
2210 }
2211 
2212 static void
2213 swaponsomething(struct vnode *vp, void *id, u_long nblks,
2214     sw_strategy_t *strategy, sw_close_t *close, dev_t dev, int flags)
2215 {
2216 	struct swdevt *sp, *tsp;
2217 	swblk_t dvbase;
2218 	u_long mblocks;
2219 
2220 	/*
2221 	 * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks.
2222 	 * First chop nblks off to page-align it, then convert.
2223 	 *
2224 	 * sw->sw_nblks is in page-sized chunks now too.
2225 	 */
2226 	nblks &= ~(ctodb(1) - 1);
2227 	nblks = dbtoc(nblks);
2228 
2229 	/*
2230 	 * If we go beyond this, we get overflows in the radix
2231 	 * tree bitmap code.
2232 	 */
2233 	mblocks = 0x40000000 / BLIST_META_RADIX;
2234 	if (nblks > mblocks) {
2235 		printf(
2236     "WARNING: reducing swap size to maximum of %luMB per unit\n",
2237 		    mblocks / 1024 / 1024 * PAGE_SIZE);
2238 		nblks = mblocks;
2239 	}
2240 
2241 	sp = malloc(sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO);
2242 	sp->sw_vp = vp;
2243 	sp->sw_id = id;
2244 	sp->sw_dev = dev;
2245 	sp->sw_flags = 0;
2246 	sp->sw_nblks = nblks;
2247 	sp->sw_used = 0;
2248 	sp->sw_strategy = strategy;
2249 	sp->sw_close = close;
2250 	sp->sw_flags = flags;
2251 
2252 	sp->sw_blist = blist_create(nblks, M_WAITOK);
2253 	/*
2254 	 * Do not free the first two block in order to avoid overwriting
2255 	 * any bsd label at the front of the partition
2256 	 */
2257 	blist_free(sp->sw_blist, 2, nblks - 2);
2258 
2259 	dvbase = 0;
2260 	mtx_lock(&sw_dev_mtx);
2261 	TAILQ_FOREACH(tsp, &swtailq, sw_list) {
2262 		if (tsp->sw_end >= dvbase) {
2263 			/*
2264 			 * We put one uncovered page between the devices
2265 			 * in order to definitively prevent any cross-device
2266 			 * I/O requests
2267 			 */
2268 			dvbase = tsp->sw_end + 1;
2269 		}
2270 	}
2271 	sp->sw_first = dvbase;
2272 	sp->sw_end = dvbase + nblks;
2273 	TAILQ_INSERT_TAIL(&swtailq, sp, sw_list);
2274 	nswapdev++;
2275 	swap_pager_avail += nblks;
2276 	swap_total += (vm_ooffset_t)nblks * PAGE_SIZE;
2277 	swapon_check_swzone(swap_total / PAGE_SIZE);
2278 	swp_sizecheck();
2279 	mtx_unlock(&sw_dev_mtx);
2280 }
2281 
2282 /*
2283  * SYSCALL: swapoff(devname)
2284  *
2285  * Disable swapping on the given device.
2286  *
2287  * XXX: Badly designed system call: it should use a device index
2288  * rather than filename as specification.  We keep sw_vp around
2289  * only to make this work.
2290  */
2291 #ifndef _SYS_SYSPROTO_H_
2292 struct swapoff_args {
2293 	char *name;
2294 };
2295 #endif
2296 
2297 /*
2298  * MPSAFE
2299  */
2300 /* ARGSUSED */
2301 int
2302 sys_swapoff(struct thread *td, struct swapoff_args *uap)
2303 {
2304 	struct vnode *vp;
2305 	struct nameidata nd;
2306 	struct swdevt *sp;
2307 	int error;
2308 
2309 	error = priv_check(td, PRIV_SWAPOFF);
2310 	if (error)
2311 		return (error);
2312 
2313 	mtx_lock(&Giant);
2314 	while (swdev_syscall_active)
2315 	    tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
2316 	swdev_syscall_active = 1;
2317 
2318 	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, UIO_USERSPACE, uap->name,
2319 	    td);
2320 	error = namei(&nd);
2321 	if (error)
2322 		goto done;
2323 	NDFREE(&nd, NDF_ONLY_PNBUF);
2324 	vp = nd.ni_vp;
2325 
2326 	mtx_lock(&sw_dev_mtx);
2327 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2328 		if (sp->sw_vp == vp)
2329 			break;
2330 	}
2331 	mtx_unlock(&sw_dev_mtx);
2332 	if (sp == NULL) {
2333 		error = EINVAL;
2334 		goto done;
2335 	}
2336 	error = swapoff_one(sp, td->td_ucred);
2337 done:
2338 	swdev_syscall_active = 0;
2339 	wakeup_one(&swdev_syscall_active);
2340 	mtx_unlock(&Giant);
2341 	return (error);
2342 }
2343 
2344 static int
2345 swapoff_one(struct swdevt *sp, struct ucred *cred)
2346 {
2347 	u_long nblks, dvbase;
2348 #ifdef MAC
2349 	int error;
2350 #endif
2351 
2352 	mtx_assert(&Giant, MA_OWNED);
2353 #ifdef MAC
2354 	(void) vn_lock(sp->sw_vp, LK_EXCLUSIVE | LK_RETRY);
2355 	error = mac_system_check_swapoff(cred, sp->sw_vp);
2356 	(void) VOP_UNLOCK(sp->sw_vp, 0);
2357 	if (error != 0)
2358 		return (error);
2359 #endif
2360 	nblks = sp->sw_nblks;
2361 
2362 	/*
2363 	 * We can turn off this swap device safely only if the
2364 	 * available virtual memory in the system will fit the amount
2365 	 * of data we will have to page back in, plus an epsilon so
2366 	 * the system doesn't become critically low on swap space.
2367 	 */
2368 	if (vm_cnt.v_free_count + vm_cnt.v_cache_count + swap_pager_avail <
2369 	    nblks + nswap_lowat) {
2370 		return (ENOMEM);
2371 	}
2372 
2373 	/*
2374 	 * Prevent further allocations on this device.
2375 	 */
2376 	mtx_lock(&sw_dev_mtx);
2377 	sp->sw_flags |= SW_CLOSING;
2378 	for (dvbase = 0; dvbase < sp->sw_end; dvbase += dmmax) {
2379 		swap_pager_avail -= blist_fill(sp->sw_blist,
2380 		     dvbase, dmmax);
2381 	}
2382 	swap_total -= (vm_ooffset_t)nblks * PAGE_SIZE;
2383 	mtx_unlock(&sw_dev_mtx);
2384 
2385 	/*
2386 	 * Page in the contents of the device and close it.
2387 	 */
2388 	swap_pager_swapoff(sp);
2389 
2390 	sp->sw_close(curthread, sp);
2391 	sp->sw_id = NULL;
2392 	mtx_lock(&sw_dev_mtx);
2393 	TAILQ_REMOVE(&swtailq, sp, sw_list);
2394 	nswapdev--;
2395 	if (nswapdev == 0) {
2396 		swap_pager_full = 2;
2397 		swap_pager_almost_full = 1;
2398 	}
2399 	if (swdevhd == sp)
2400 		swdevhd = NULL;
2401 	mtx_unlock(&sw_dev_mtx);
2402 	blist_destroy(sp->sw_blist);
2403 	free(sp, M_VMPGDATA);
2404 	return (0);
2405 }
2406 
2407 void
2408 swapoff_all(void)
2409 {
2410 	struct swdevt *sp, *spt;
2411 	const char *devname;
2412 	int error;
2413 
2414 	mtx_lock(&Giant);
2415 	while (swdev_syscall_active)
2416 		tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
2417 	swdev_syscall_active = 1;
2418 
2419 	mtx_lock(&sw_dev_mtx);
2420 	TAILQ_FOREACH_SAFE(sp, &swtailq, sw_list, spt) {
2421 		mtx_unlock(&sw_dev_mtx);
2422 		if (vn_isdisk(sp->sw_vp, NULL))
2423 			devname = devtoname(sp->sw_vp->v_rdev);
2424 		else
2425 			devname = "[file]";
2426 		error = swapoff_one(sp, thread0.td_ucred);
2427 		if (error != 0) {
2428 			printf("Cannot remove swap device %s (error=%d), "
2429 			    "skipping.\n", devname, error);
2430 		} else if (bootverbose) {
2431 			printf("Swap device %s removed.\n", devname);
2432 		}
2433 		mtx_lock(&sw_dev_mtx);
2434 	}
2435 	mtx_unlock(&sw_dev_mtx);
2436 
2437 	swdev_syscall_active = 0;
2438 	wakeup_one(&swdev_syscall_active);
2439 	mtx_unlock(&Giant);
2440 }
2441 
2442 void
2443 swap_pager_status(int *total, int *used)
2444 {
2445 	struct swdevt *sp;
2446 
2447 	*total = 0;
2448 	*used = 0;
2449 	mtx_lock(&sw_dev_mtx);
2450 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2451 		*total += sp->sw_nblks;
2452 		*used += sp->sw_used;
2453 	}
2454 	mtx_unlock(&sw_dev_mtx);
2455 }
2456 
2457 int
2458 swap_dev_info(int name, struct xswdev *xs, char *devname, size_t len)
2459 {
2460 	struct swdevt *sp;
2461 	const char *tmp_devname;
2462 	int error, n;
2463 
2464 	n = 0;
2465 	error = ENOENT;
2466 	mtx_lock(&sw_dev_mtx);
2467 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2468 		if (n != name) {
2469 			n++;
2470 			continue;
2471 		}
2472 		xs->xsw_version = XSWDEV_VERSION;
2473 		xs->xsw_dev = sp->sw_dev;
2474 		xs->xsw_flags = sp->sw_flags;
2475 		xs->xsw_nblks = sp->sw_nblks;
2476 		xs->xsw_used = sp->sw_used;
2477 		if (devname != NULL) {
2478 			if (vn_isdisk(sp->sw_vp, NULL))
2479 				tmp_devname = devtoname(sp->sw_vp->v_rdev);
2480 			else
2481 				tmp_devname = "[file]";
2482 			strncpy(devname, tmp_devname, len);
2483 		}
2484 		error = 0;
2485 		break;
2486 	}
2487 	mtx_unlock(&sw_dev_mtx);
2488 	return (error);
2489 }
2490 
2491 static int
2492 sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS)
2493 {
2494 	struct xswdev xs;
2495 	int error;
2496 
2497 	if (arg2 != 1)			/* name length */
2498 		return (EINVAL);
2499 	error = swap_dev_info(*(int *)arg1, &xs, NULL, 0);
2500 	if (error != 0)
2501 		return (error);
2502 	error = SYSCTL_OUT(req, &xs, sizeof(xs));
2503 	return (error);
2504 }
2505 
2506 SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, &nswapdev, 0,
2507     "Number of swap devices");
2508 SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD, sysctl_vm_swap_info,
2509     "Swap statistics by device");
2510 
2511 /*
2512  * vmspace_swap_count() - count the approximate swap usage in pages for a
2513  *			  vmspace.
2514  *
2515  *	The map must be locked.
2516  *
2517  *	Swap usage is determined by taking the proportional swap used by
2518  *	VM objects backing the VM map.  To make up for fractional losses,
2519  *	if the VM object has any swap use at all the associated map entries
2520  *	count for at least 1 swap page.
2521  */
2522 long
2523 vmspace_swap_count(struct vmspace *vmspace)
2524 {
2525 	vm_map_t map;
2526 	vm_map_entry_t cur;
2527 	vm_object_t object;
2528 	long count, n;
2529 
2530 	map = &vmspace->vm_map;
2531 	count = 0;
2532 
2533 	for (cur = map->header.next; cur != &map->header; cur = cur->next) {
2534 		if ((cur->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
2535 		    (object = cur->object.vm_object) != NULL) {
2536 			VM_OBJECT_WLOCK(object);
2537 			if (object->type == OBJT_SWAP &&
2538 			    object->un_pager.swp.swp_bcount != 0) {
2539 				n = (cur->end - cur->start) / PAGE_SIZE;
2540 				count += object->un_pager.swp.swp_bcount *
2541 				    SWAP_META_PAGES * n / object->size + 1;
2542 			}
2543 			VM_OBJECT_WUNLOCK(object);
2544 		}
2545 	}
2546 	return (count);
2547 }
2548 
2549 /*
2550  * GEOM backend
2551  *
2552  * Swapping onto disk devices.
2553  *
2554  */
2555 
2556 static g_orphan_t swapgeom_orphan;
2557 
2558 static struct g_class g_swap_class = {
2559 	.name = "SWAP",
2560 	.version = G_VERSION,
2561 	.orphan = swapgeom_orphan,
2562 };
2563 
2564 DECLARE_GEOM_CLASS(g_swap_class, g_class);
2565 
2566 
2567 static void
2568 swapgeom_close_ev(void *arg, int flags)
2569 {
2570 	struct g_consumer *cp;
2571 
2572 	cp = arg;
2573 	g_access(cp, -1, -1, 0);
2574 	g_detach(cp);
2575 	g_destroy_consumer(cp);
2576 }
2577 
2578 static void
2579 swapgeom_done(struct bio *bp2)
2580 {
2581 	struct swdevt *sp;
2582 	struct buf *bp;
2583 	struct g_consumer *cp;
2584 
2585 	bp = bp2->bio_caller2;
2586 	cp = bp2->bio_from;
2587 	bp->b_ioflags = bp2->bio_flags;
2588 	if (bp2->bio_error)
2589 		bp->b_ioflags |= BIO_ERROR;
2590 	bp->b_resid = bp->b_bcount - bp2->bio_completed;
2591 	bp->b_error = bp2->bio_error;
2592 	bufdone(bp);
2593 	mtx_lock(&sw_dev_mtx);
2594 	if ((--cp->index) == 0 && cp->private) {
2595 		if (g_post_event(swapgeom_close_ev, cp, M_NOWAIT, NULL) == 0) {
2596 			sp = bp2->bio_caller1;
2597 			sp->sw_id = NULL;
2598 		}
2599 	}
2600 	mtx_unlock(&sw_dev_mtx);
2601 	g_destroy_bio(bp2);
2602 }
2603 
2604 static void
2605 swapgeom_strategy(struct buf *bp, struct swdevt *sp)
2606 {
2607 	struct bio *bio;
2608 	struct g_consumer *cp;
2609 
2610 	mtx_lock(&sw_dev_mtx);
2611 	cp = sp->sw_id;
2612 	if (cp == NULL) {
2613 		mtx_unlock(&sw_dev_mtx);
2614 		bp->b_error = ENXIO;
2615 		bp->b_ioflags |= BIO_ERROR;
2616 		bufdone(bp);
2617 		return;
2618 	}
2619 	cp->index++;
2620 	mtx_unlock(&sw_dev_mtx);
2621 	if (bp->b_iocmd == BIO_WRITE)
2622 		bio = g_new_bio();
2623 	else
2624 		bio = g_alloc_bio();
2625 	if (bio == NULL) {
2626 		bp->b_error = ENOMEM;
2627 		bp->b_ioflags |= BIO_ERROR;
2628 		bufdone(bp);
2629 		return;
2630 	}
2631 
2632 	bio->bio_caller1 = sp;
2633 	bio->bio_caller2 = bp;
2634 	bio->bio_cmd = bp->b_iocmd;
2635 	bio->bio_offset = (bp->b_blkno - sp->sw_first) * PAGE_SIZE;
2636 	bio->bio_length = bp->b_bcount;
2637 	bio->bio_done = swapgeom_done;
2638 	if ((bp->b_flags & B_UNMAPPED) != 0) {
2639 		bio->bio_ma = bp->b_pages;
2640 		bio->bio_data = unmapped_buf;
2641 		bio->bio_ma_offset = (vm_offset_t)bp->b_offset & PAGE_MASK;
2642 		bio->bio_ma_n = bp->b_npages;
2643 		bio->bio_flags |= BIO_UNMAPPED;
2644 	} else {
2645 		bio->bio_data = bp->b_data;
2646 		bio->bio_ma = NULL;
2647 	}
2648 	g_io_request(bio, cp);
2649 	return;
2650 }
2651 
2652 static void
2653 swapgeom_orphan(struct g_consumer *cp)
2654 {
2655 	struct swdevt *sp;
2656 	int destroy;
2657 
2658 	mtx_lock(&sw_dev_mtx);
2659 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2660 		if (sp->sw_id == cp) {
2661 			sp->sw_flags |= SW_CLOSING;
2662 			break;
2663 		}
2664 	}
2665 	cp->private = (void *)(uintptr_t)1;
2666 	destroy = ((sp != NULL) && (cp->index == 0));
2667 	if (destroy)
2668 		sp->sw_id = NULL;
2669 	mtx_unlock(&sw_dev_mtx);
2670 	if (destroy)
2671 		swapgeom_close_ev(cp, 0);
2672 }
2673 
2674 static void
2675 swapgeom_close(struct thread *td, struct swdevt *sw)
2676 {
2677 	struct g_consumer *cp;
2678 
2679 	mtx_lock(&sw_dev_mtx);
2680 	cp = sw->sw_id;
2681 	sw->sw_id = NULL;
2682 	mtx_unlock(&sw_dev_mtx);
2683 	/* XXX: direct call when Giant untangled */
2684 	if (cp != NULL)
2685 		g_waitfor_event(swapgeom_close_ev, cp, M_WAITOK, NULL);
2686 }
2687 
2688 
2689 struct swh0h0 {
2690 	struct cdev *dev;
2691 	struct vnode *vp;
2692 	int	error;
2693 };
2694 
2695 static void
2696 swapongeom_ev(void *arg, int flags)
2697 {
2698 	struct swh0h0 *swh;
2699 	struct g_provider *pp;
2700 	struct g_consumer *cp;
2701 	static struct g_geom *gp;
2702 	struct swdevt *sp;
2703 	u_long nblks;
2704 	int error;
2705 
2706 	swh = arg;
2707 	swh->error = 0;
2708 	pp = g_dev_getprovider(swh->dev);
2709 	if (pp == NULL) {
2710 		swh->error = ENODEV;
2711 		return;
2712 	}
2713 	mtx_lock(&sw_dev_mtx);
2714 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2715 		cp = sp->sw_id;
2716 		if (cp != NULL && cp->provider == pp) {
2717 			mtx_unlock(&sw_dev_mtx);
2718 			swh->error = EBUSY;
2719 			return;
2720 		}
2721 	}
2722 	mtx_unlock(&sw_dev_mtx);
2723 	if (gp == NULL)
2724 		gp = g_new_geomf(&g_swap_class, "swap");
2725 	cp = g_new_consumer(gp);
2726 	cp->index = 0;		/* Number of active I/Os. */
2727 	cp->private = NULL;	/* Orphanization flag */
2728 	g_attach(cp, pp);
2729 	/*
2730 	 * XXX: Everytime you think you can improve the margin for
2731 	 * footshooting, somebody depends on the ability to do so:
2732 	 * savecore(8) wants to write to our swapdev so we cannot
2733 	 * set an exclusive count :-(
2734 	 */
2735 	error = g_access(cp, 1, 1, 0);
2736 	if (error) {
2737 		g_detach(cp);
2738 		g_destroy_consumer(cp);
2739 		swh->error = error;
2740 		return;
2741 	}
2742 	nblks = pp->mediasize / DEV_BSIZE;
2743 	swaponsomething(swh->vp, cp, nblks, swapgeom_strategy,
2744 	    swapgeom_close, dev2udev(swh->dev),
2745 	    (pp->flags & G_PF_ACCEPT_UNMAPPED) != 0 ? SW_UNMAPPED : 0);
2746 	swh->error = 0;
2747 }
2748 
2749 static int
2750 swapongeom(struct thread *td, struct vnode *vp)
2751 {
2752 	int error;
2753 	struct swh0h0 swh;
2754 
2755 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2756 
2757 	swh.dev = vp->v_rdev;
2758 	swh.vp = vp;
2759 	swh.error = 0;
2760 	/* XXX: direct call when Giant untangled */
2761 	error = g_waitfor_event(swapongeom_ev, &swh, M_WAITOK, NULL);
2762 	if (!error)
2763 		error = swh.error;
2764 	VOP_UNLOCK(vp, 0);
2765 	return (error);
2766 }
2767 
2768 /*
2769  * VNODE backend
2770  *
2771  * This is used mainly for network filesystem (read: probably only tested
2772  * with NFS) swapfiles.
2773  *
2774  */
2775 
2776 static void
2777 swapdev_strategy(struct buf *bp, struct swdevt *sp)
2778 {
2779 	struct vnode *vp2;
2780 
2781 	bp->b_blkno = ctodb(bp->b_blkno - sp->sw_first);
2782 
2783 	vp2 = sp->sw_id;
2784 	vhold(vp2);
2785 	if (bp->b_iocmd == BIO_WRITE) {
2786 		if (bp->b_bufobj)
2787 			bufobj_wdrop(bp->b_bufobj);
2788 		bufobj_wref(&vp2->v_bufobj);
2789 	}
2790 	if (bp->b_bufobj != &vp2->v_bufobj)
2791 		bp->b_bufobj = &vp2->v_bufobj;
2792 	bp->b_vp = vp2;
2793 	bp->b_iooffset = dbtob(bp->b_blkno);
2794 	bstrategy(bp);
2795 	return;
2796 }
2797 
2798 static void
2799 swapdev_close(struct thread *td, struct swdevt *sp)
2800 {
2801 
2802 	VOP_CLOSE(sp->sw_vp, FREAD | FWRITE, td->td_ucred, td);
2803 	vrele(sp->sw_vp);
2804 }
2805 
2806 
2807 static int
2808 swaponvp(struct thread *td, struct vnode *vp, u_long nblks)
2809 {
2810 	struct swdevt *sp;
2811 	int error;
2812 
2813 	if (nblks == 0)
2814 		return (ENXIO);
2815 	mtx_lock(&sw_dev_mtx);
2816 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2817 		if (sp->sw_id == vp) {
2818 			mtx_unlock(&sw_dev_mtx);
2819 			return (EBUSY);
2820 		}
2821 	}
2822 	mtx_unlock(&sw_dev_mtx);
2823 
2824 	(void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2825 #ifdef MAC
2826 	error = mac_system_check_swapon(td->td_ucred, vp);
2827 	if (error == 0)
2828 #endif
2829 		error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, NULL);
2830 	(void) VOP_UNLOCK(vp, 0);
2831 	if (error)
2832 		return (error);
2833 
2834 	swaponsomething(vp, vp, nblks, swapdev_strategy, swapdev_close,
2835 	    NODEV, 0);
2836 	return (0);
2837 }
2838