xref: /freebsd/sys/vm/swap_pager.c (revision c9dbb1cc52b063bbd9ab078a7afc89a8696da659)
1 /*-
2  * Copyright (c) 1998 Matthew Dillon,
3  * Copyright (c) 1994 John S. Dyson
4  * Copyright (c) 1990 University of Utah.
5  * Copyright (c) 1982, 1986, 1989, 1993
6  *	The Regents of the University of California.  All rights reserved.
7  *
8  * This code is derived from software contributed to Berkeley by
9  * the Systems Programming Group of the University of Utah Computer
10  * Science Department.
11  *
12  * Redistribution and use in source and binary forms, with or without
13  * modification, are permitted provided that the following conditions
14  * are met:
15  * 1. Redistributions of source code must retain the above copyright
16  *    notice, this list of conditions and the following disclaimer.
17  * 2. Redistributions in binary form must reproduce the above copyright
18  *    notice, this list of conditions and the following disclaimer in the
19  *    documentation and/or other materials provided with the distribution.
20  * 3. All advertising materials mentioning features or use of this software
21  *    must display the following acknowledgement:
22  *	This product includes software developed by the University of
23  *	California, Berkeley and its contributors.
24  * 4. Neither the name of the University nor the names of its contributors
25  *    may be used to endorse or promote products derived from this software
26  *    without specific prior written permission.
27  *
28  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
29  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
32  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38  * SUCH DAMAGE.
39  *
40  *				New Swap System
41  *				Matthew Dillon
42  *
43  * Radix Bitmap 'blists'.
44  *
45  *	- The new swapper uses the new radix bitmap code.  This should scale
46  *	  to arbitrarily small or arbitrarily large swap spaces and an almost
47  *	  arbitrary degree of fragmentation.
48  *
49  * Features:
50  *
51  *	- on the fly reallocation of swap during putpages.  The new system
52  *	  does not try to keep previously allocated swap blocks for dirty
53  *	  pages.
54  *
55  *	- on the fly deallocation of swap
56  *
57  *	- No more garbage collection required.  Unnecessarily allocated swap
58  *	  blocks only exist for dirty vm_page_t's now and these are already
59  *	  cycled (in a high-load system) by the pager.  We also do on-the-fly
60  *	  removal of invalidated swap blocks when a page is destroyed
61  *	  or renamed.
62  *
63  * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
64  *
65  *	@(#)swap_pager.c	8.9 (Berkeley) 3/21/94
66  *	@(#)vm_swap.c	8.5 (Berkeley) 2/17/94
67  */
68 
69 #include <sys/cdefs.h>
70 __FBSDID("$FreeBSD$");
71 
72 #include "opt_swap.h"
73 #include "opt_vm.h"
74 
75 #include <sys/param.h>
76 #include <sys/systm.h>
77 #include <sys/conf.h>
78 #include <sys/kernel.h>
79 #include <sys/priv.h>
80 #include <sys/proc.h>
81 #include <sys/bio.h>
82 #include <sys/buf.h>
83 #include <sys/disk.h>
84 #include <sys/fcntl.h>
85 #include <sys/mount.h>
86 #include <sys/namei.h>
87 #include <sys/vnode.h>
88 #include <sys/malloc.h>
89 #include <sys/racct.h>
90 #include <sys/resource.h>
91 #include <sys/resourcevar.h>
92 #include <sys/rwlock.h>
93 #include <sys/sysctl.h>
94 #include <sys/sysproto.h>
95 #include <sys/blist.h>
96 #include <sys/lock.h>
97 #include <sys/sx.h>
98 #include <sys/vmmeter.h>
99 
100 #include <security/mac/mac_framework.h>
101 
102 #include <vm/vm.h>
103 #include <vm/pmap.h>
104 #include <vm/vm_map.h>
105 #include <vm/vm_kern.h>
106 #include <vm/vm_object.h>
107 #include <vm/vm_page.h>
108 #include <vm/vm_pager.h>
109 #include <vm/vm_pageout.h>
110 #include <vm/vm_param.h>
111 #include <vm/swap_pager.h>
112 #include <vm/vm_extern.h>
113 #include <vm/uma.h>
114 
115 #include <geom/geom.h>
116 
117 /*
118  * SWB_NPAGES must be a power of 2.  It may be set to 1, 2, 4, 8, 16
119  * or 32 pages per allocation.
120  * The 32-page limit is due to the radix code (kern/subr_blist.c).
121  */
122 #ifndef MAX_PAGEOUT_CLUSTER
123 #define MAX_PAGEOUT_CLUSTER 16
124 #endif
125 
126 #if !defined(SWB_NPAGES)
127 #define SWB_NPAGES	MAX_PAGEOUT_CLUSTER
128 #endif
129 
130 /*
131  * The swblock structure maps an object and a small, fixed-size range
132  * of page indices to disk addresses within a swap area.
133  * The collection of these mappings is implemented as a hash table.
134  * Unused disk addresses within a swap area are allocated and managed
135  * using a blist.
136  */
137 #define SWCORRECT(n) (sizeof(void *) * (n) / sizeof(daddr_t))
138 #define SWAP_META_PAGES		(SWB_NPAGES * 2)
139 #define SWAP_META_MASK		(SWAP_META_PAGES - 1)
140 
141 struct swblock {
142 	struct swblock	*swb_hnext;
143 	vm_object_t	swb_object;
144 	vm_pindex_t	swb_index;
145 	int		swb_count;
146 	daddr_t		swb_pages[SWAP_META_PAGES];
147 };
148 
149 static MALLOC_DEFINE(M_VMPGDATA, "vm_pgdata", "swap pager private data");
150 static struct mtx sw_dev_mtx;
151 static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq);
152 static struct swdevt *swdevhd;	/* Allocate from here next */
153 static int nswapdev;		/* Number of swap devices */
154 int swap_pager_avail;
155 static int swdev_syscall_active = 0; /* serialize swap(on|off) */
156 
157 static vm_ooffset_t swap_total;
158 SYSCTL_QUAD(_vm, OID_AUTO, swap_total, CTLFLAG_RD, &swap_total, 0,
159     "Total amount of available swap storage.");
160 static vm_ooffset_t swap_reserved;
161 SYSCTL_QUAD(_vm, OID_AUTO, swap_reserved, CTLFLAG_RD, &swap_reserved, 0,
162     "Amount of swap storage needed to back all allocated anonymous memory.");
163 static int overcommit = 0;
164 SYSCTL_INT(_vm, OID_AUTO, overcommit, CTLFLAG_RW, &overcommit, 0,
165     "Configure virtual memory overcommit behavior. See tuning(7) "
166     "for details.");
167 static unsigned long swzone;
168 SYSCTL_ULONG(_vm, OID_AUTO, swzone, CTLFLAG_RD, &swzone, 0,
169     "Actual size of swap metadata zone");
170 static unsigned long swap_maxpages;
171 SYSCTL_ULONG(_vm, OID_AUTO, swap_maxpages, CTLFLAG_RD, &swap_maxpages, 0,
172     "Maximum amount of swap supported");
173 
174 /* bits from overcommit */
175 #define	SWAP_RESERVE_FORCE_ON		(1 << 0)
176 #define	SWAP_RESERVE_RLIMIT_ON		(1 << 1)
177 #define	SWAP_RESERVE_ALLOW_NONWIRED	(1 << 2)
178 
179 int
180 swap_reserve(vm_ooffset_t incr)
181 {
182 
183 	return (swap_reserve_by_cred(incr, curthread->td_ucred));
184 }
185 
186 int
187 swap_reserve_by_cred(vm_ooffset_t incr, struct ucred *cred)
188 {
189 	vm_ooffset_t r, s;
190 	int res, error;
191 	static int curfail;
192 	static struct timeval lastfail;
193 	struct uidinfo *uip;
194 
195 	uip = cred->cr_ruidinfo;
196 
197 	if (incr & PAGE_MASK)
198 		panic("swap_reserve: & PAGE_MASK");
199 
200 #ifdef RACCT
201 	if (racct_enable) {
202 		PROC_LOCK(curproc);
203 		error = racct_add(curproc, RACCT_SWAP, incr);
204 		PROC_UNLOCK(curproc);
205 		if (error != 0)
206 			return (0);
207 	}
208 #endif
209 
210 	res = 0;
211 	mtx_lock(&sw_dev_mtx);
212 	r = swap_reserved + incr;
213 	if (overcommit & SWAP_RESERVE_ALLOW_NONWIRED) {
214 		s = vm_cnt.v_page_count - vm_cnt.v_free_reserved - vm_cnt.v_wire_count;
215 		s *= PAGE_SIZE;
216 	} else
217 		s = 0;
218 	s += swap_total;
219 	if ((overcommit & SWAP_RESERVE_FORCE_ON) == 0 || r <= s ||
220 	    (error = priv_check(curthread, PRIV_VM_SWAP_NOQUOTA)) == 0) {
221 		res = 1;
222 		swap_reserved = r;
223 	}
224 	mtx_unlock(&sw_dev_mtx);
225 
226 	if (res) {
227 		UIDINFO_VMSIZE_LOCK(uip);
228 		if ((overcommit & SWAP_RESERVE_RLIMIT_ON) != 0 &&
229 		    uip->ui_vmsize + incr > lim_cur(curthread, RLIMIT_SWAP) &&
230 		    priv_check(curthread, PRIV_VM_SWAP_NORLIMIT))
231 			res = 0;
232 		else
233 			uip->ui_vmsize += incr;
234 		UIDINFO_VMSIZE_UNLOCK(uip);
235 		if (!res) {
236 			mtx_lock(&sw_dev_mtx);
237 			swap_reserved -= incr;
238 			mtx_unlock(&sw_dev_mtx);
239 		}
240 	}
241 	if (!res && ppsratecheck(&lastfail, &curfail, 1)) {
242 		printf("uid %d, pid %d: swap reservation for %jd bytes failed\n",
243 		    uip->ui_uid, curproc->p_pid, incr);
244 	}
245 
246 #ifdef RACCT
247 	if (!res) {
248 		PROC_LOCK(curproc);
249 		racct_sub(curproc, RACCT_SWAP, incr);
250 		PROC_UNLOCK(curproc);
251 	}
252 #endif
253 
254 	return (res);
255 }
256 
257 void
258 swap_reserve_force(vm_ooffset_t incr)
259 {
260 	struct uidinfo *uip;
261 
262 	mtx_lock(&sw_dev_mtx);
263 	swap_reserved += incr;
264 	mtx_unlock(&sw_dev_mtx);
265 
266 #ifdef RACCT
267 	PROC_LOCK(curproc);
268 	racct_add_force(curproc, RACCT_SWAP, incr);
269 	PROC_UNLOCK(curproc);
270 #endif
271 
272 	uip = curthread->td_ucred->cr_ruidinfo;
273 	PROC_LOCK(curproc);
274 	UIDINFO_VMSIZE_LOCK(uip);
275 	uip->ui_vmsize += incr;
276 	UIDINFO_VMSIZE_UNLOCK(uip);
277 	PROC_UNLOCK(curproc);
278 }
279 
280 void
281 swap_release(vm_ooffset_t decr)
282 {
283 	struct ucred *cred;
284 
285 	PROC_LOCK(curproc);
286 	cred = curthread->td_ucred;
287 	swap_release_by_cred(decr, cred);
288 	PROC_UNLOCK(curproc);
289 }
290 
291 void
292 swap_release_by_cred(vm_ooffset_t decr, struct ucred *cred)
293 {
294  	struct uidinfo *uip;
295 
296 	uip = cred->cr_ruidinfo;
297 
298 	if (decr & PAGE_MASK)
299 		panic("swap_release: & PAGE_MASK");
300 
301 	mtx_lock(&sw_dev_mtx);
302 	if (swap_reserved < decr)
303 		panic("swap_reserved < decr");
304 	swap_reserved -= decr;
305 	mtx_unlock(&sw_dev_mtx);
306 
307 	UIDINFO_VMSIZE_LOCK(uip);
308 	if (uip->ui_vmsize < decr)
309 		printf("negative vmsize for uid = %d\n", uip->ui_uid);
310 	uip->ui_vmsize -= decr;
311 	UIDINFO_VMSIZE_UNLOCK(uip);
312 
313 	racct_sub_cred(cred, RACCT_SWAP, decr);
314 }
315 
316 static void swapdev_strategy(struct buf *, struct swdevt *sw);
317 
318 #define SWM_FREE	0x02	/* free, period			*/
319 #define SWM_POP		0x04	/* pop out			*/
320 
321 int swap_pager_full = 2;	/* swap space exhaustion (task killing) */
322 static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/
323 static int nsw_rcount;		/* free read buffers			*/
324 static int nsw_wcount_sync;	/* limit write buffers / synchronous	*/
325 static int nsw_wcount_async;	/* limit write buffers / asynchronous	*/
326 static int nsw_wcount_async_max;/* assigned maximum			*/
327 static int nsw_cluster_max;	/* maximum VOP I/O allowed		*/
328 
329 static int sysctl_swap_async_max(SYSCTL_HANDLER_ARGS);
330 SYSCTL_PROC(_vm, OID_AUTO, swap_async_max, CTLTYPE_INT | CTLFLAG_RW,
331     NULL, 0, sysctl_swap_async_max, "I", "Maximum running async swap ops");
332 
333 static struct swblock **swhash;
334 static int swhash_mask;
335 static struct mtx swhash_mtx;
336 
337 static struct sx sw_alloc_sx;
338 
339 /*
340  * "named" and "unnamed" anon region objects.  Try to reduce the overhead
341  * of searching a named list by hashing it just a little.
342  */
343 
344 #define NOBJLISTS		8
345 
346 #define NOBJLIST(handle)	\
347 	(&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)])
348 
349 static struct mtx sw_alloc_mtx;	/* protect list manipulation */
350 static struct pagerlst	swap_pager_object_list[NOBJLISTS];
351 static uma_zone_t	swap_zone;
352 
353 /*
354  * pagerops for OBJT_SWAP - "swap pager".  Some ops are also global procedure
355  * calls hooked from other parts of the VM system and do not appear here.
356  * (see vm/swap_pager.h).
357  */
358 static vm_object_t
359 		swap_pager_alloc(void *handle, vm_ooffset_t size,
360 		    vm_prot_t prot, vm_ooffset_t offset, struct ucred *);
361 static void	swap_pager_dealloc(vm_object_t object);
362 static int	swap_pager_getpages(vm_object_t, vm_page_t *, int, int);
363 static int	swap_pager_getpages_async(vm_object_t, vm_page_t *, int, int,
364     pgo_getpages_iodone_t, void *);
365 static void	swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *);
366 static boolean_t
367 		swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
368 static void	swap_pager_init(void);
369 static void	swap_pager_unswapped(vm_page_t);
370 static void	swap_pager_swapoff(struct swdevt *sp);
371 
372 struct pagerops swappagerops = {
373 	.pgo_init =	swap_pager_init,	/* early system initialization of pager	*/
374 	.pgo_alloc =	swap_pager_alloc,	/* allocate an OBJT_SWAP object		*/
375 	.pgo_dealloc =	swap_pager_dealloc,	/* deallocate an OBJT_SWAP object	*/
376 	.pgo_getpages =	swap_pager_getpages,	/* pagein				*/
377 	.pgo_getpages_async = swap_pager_getpages_async, /* pagein (async)		*/
378 	.pgo_putpages =	swap_pager_putpages,	/* pageout				*/
379 	.pgo_haspage =	swap_pager_haspage,	/* get backing store status for page	*/
380 	.pgo_pageunswapped = swap_pager_unswapped,	/* remove swap related to page		*/
381 };
382 
383 /*
384  * dmmax is in page-sized chunks with the new swap system.  It was
385  * dev-bsized chunks in the old.  dmmax is always a power of 2.
386  *
387  * swap_*() routines are externally accessible.  swp_*() routines are
388  * internal.
389  */
390 static int dmmax;
391 static int nswap_lowat = 128;	/* in pages, swap_pager_almost_full warn */
392 static int nswap_hiwat = 512;	/* in pages, swap_pager_almost_full warn */
393 
394 SYSCTL_INT(_vm, OID_AUTO, dmmax,
395 	CTLFLAG_RD, &dmmax, 0, "Maximum size of a swap block");
396 
397 static void	swp_sizecheck(void);
398 static void	swp_pager_async_iodone(struct buf *bp);
399 static int	swapongeom(struct thread *, struct vnode *);
400 static int	swaponvp(struct thread *, struct vnode *, u_long);
401 static int	swapoff_one(struct swdevt *sp, struct ucred *cred);
402 
403 /*
404  * Swap bitmap functions
405  */
406 static void	swp_pager_freeswapspace(daddr_t blk, int npages);
407 static daddr_t	swp_pager_getswapspace(int npages);
408 
409 /*
410  * Metadata functions
411  */
412 static struct swblock **swp_pager_hash(vm_object_t object, vm_pindex_t index);
413 static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
414 static void swp_pager_meta_free(vm_object_t, vm_pindex_t, daddr_t);
415 static void swp_pager_meta_free_all(vm_object_t);
416 static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);
417 
418 static void
419 swp_pager_free_nrpage(vm_page_t m)
420 {
421 
422 	vm_page_lock(m);
423 	if (m->wire_count == 0)
424 		vm_page_free(m);
425 	vm_page_unlock(m);
426 }
427 
428 /*
429  * SWP_SIZECHECK() -	update swap_pager_full indication
430  *
431  *	update the swap_pager_almost_full indication and warn when we are
432  *	about to run out of swap space, using lowat/hiwat hysteresis.
433  *
434  *	Clear swap_pager_full ( task killing ) indication when lowat is met.
435  *
436  *	No restrictions on call
437  *	This routine may not block.
438  */
439 static void
440 swp_sizecheck(void)
441 {
442 
443 	if (swap_pager_avail < nswap_lowat) {
444 		if (swap_pager_almost_full == 0) {
445 			printf("swap_pager: out of swap space\n");
446 			swap_pager_almost_full = 1;
447 		}
448 	} else {
449 		swap_pager_full = 0;
450 		if (swap_pager_avail > nswap_hiwat)
451 			swap_pager_almost_full = 0;
452 	}
453 }
454 
455 /*
456  * SWP_PAGER_HASH() -	hash swap meta data
457  *
458  *	This is an helper function which hashes the swapblk given
459  *	the object and page index.  It returns a pointer to a pointer
460  *	to the object, or a pointer to a NULL pointer if it could not
461  *	find a swapblk.
462  */
463 static struct swblock **
464 swp_pager_hash(vm_object_t object, vm_pindex_t index)
465 {
466 	struct swblock **pswap;
467 	struct swblock *swap;
468 
469 	index &= ~(vm_pindex_t)SWAP_META_MASK;
470 	pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask];
471 	while ((swap = *pswap) != NULL) {
472 		if (swap->swb_object == object &&
473 		    swap->swb_index == index
474 		) {
475 			break;
476 		}
477 		pswap = &swap->swb_hnext;
478 	}
479 	return (pswap);
480 }
481 
482 /*
483  * SWAP_PAGER_INIT() -	initialize the swap pager!
484  *
485  *	Expected to be started from system init.  NOTE:  This code is run
486  *	before much else so be careful what you depend on.  Most of the VM
487  *	system has yet to be initialized at this point.
488  */
489 static void
490 swap_pager_init(void)
491 {
492 	/*
493 	 * Initialize object lists
494 	 */
495 	int i;
496 
497 	for (i = 0; i < NOBJLISTS; ++i)
498 		TAILQ_INIT(&swap_pager_object_list[i]);
499 	mtx_init(&sw_alloc_mtx, "swap_pager list", NULL, MTX_DEF);
500 	mtx_init(&sw_dev_mtx, "swapdev", NULL, MTX_DEF);
501 
502 	/*
503 	 * Device Stripe, in PAGE_SIZE'd blocks
504 	 */
505 	dmmax = SWB_NPAGES * 2;
506 }
507 
508 /*
509  * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process
510  *
511  *	Expected to be started from pageout process once, prior to entering
512  *	its main loop.
513  */
514 void
515 swap_pager_swap_init(void)
516 {
517 	unsigned long n, n2;
518 
519 	/*
520 	 * Number of in-transit swap bp operations.  Don't
521 	 * exhaust the pbufs completely.  Make sure we
522 	 * initialize workable values (0 will work for hysteresis
523 	 * but it isn't very efficient).
524 	 *
525 	 * The nsw_cluster_max is constrained by the bp->b_pages[]
526 	 * array (MAXPHYS/PAGE_SIZE) and our locally defined
527 	 * MAX_PAGEOUT_CLUSTER.   Also be aware that swap ops are
528 	 * constrained by the swap device interleave stripe size.
529 	 *
530 	 * Currently we hardwire nsw_wcount_async to 4.  This limit is
531 	 * designed to prevent other I/O from having high latencies due to
532 	 * our pageout I/O.  The value 4 works well for one or two active swap
533 	 * devices but is probably a little low if you have more.  Even so,
534 	 * a higher value would probably generate only a limited improvement
535 	 * with three or four active swap devices since the system does not
536 	 * typically have to pageout at extreme bandwidths.   We will want
537 	 * at least 2 per swap devices, and 4 is a pretty good value if you
538 	 * have one NFS swap device due to the command/ack latency over NFS.
539 	 * So it all works out pretty well.
540 	 */
541 	nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER);
542 
543 	mtx_lock(&pbuf_mtx);
544 	nsw_rcount = (nswbuf + 1) / 2;
545 	nsw_wcount_sync = (nswbuf + 3) / 4;
546 	nsw_wcount_async = 4;
547 	nsw_wcount_async_max = nsw_wcount_async;
548 	mtx_unlock(&pbuf_mtx);
549 
550 	/*
551 	 * Initialize our zone.  Right now I'm just guessing on the number
552 	 * we need based on the number of pages in the system.  Each swblock
553 	 * can hold 32 pages, so this is probably overkill.  This reservation
554 	 * is typically limited to around 32MB by default.
555 	 */
556 	n = vm_cnt.v_page_count / 2;
557 	if (maxswzone && n > maxswzone / sizeof(struct swblock))
558 		n = maxswzone / sizeof(struct swblock);
559 	n2 = n;
560 	swap_zone = uma_zcreate("SWAPMETA", sizeof(struct swblock), NULL, NULL,
561 	    NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM);
562 	if (swap_zone == NULL)
563 		panic("failed to create swap_zone.");
564 	do {
565 		if (uma_zone_reserve_kva(swap_zone, n))
566 			break;
567 		/*
568 		 * if the allocation failed, try a zone two thirds the
569 		 * size of the previous attempt.
570 		 */
571 		n -= ((n + 2) / 3);
572 	} while (n > 0);
573 	if (n2 != n)
574 		printf("Swap zone entries reduced from %lu to %lu.\n", n2, n);
575 	swap_maxpages = n * SWAP_META_PAGES;
576 	swzone = n * sizeof(struct swblock);
577 	n2 = n;
578 
579 	/*
580 	 * Initialize our meta-data hash table.  The swapper does not need to
581 	 * be quite as efficient as the VM system, so we do not use an
582 	 * oversized hash table.
583 	 *
584 	 * 	n: 		size of hash table, must be power of 2
585 	 *	swhash_mask:	hash table index mask
586 	 */
587 	for (n = 1; n < n2 / 8; n *= 2)
588 		;
589 	swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO);
590 	swhash_mask = n - 1;
591 	mtx_init(&swhash_mtx, "swap_pager swhash", NULL, MTX_DEF);
592 }
593 
594 /*
595  * SWAP_PAGER_ALLOC() -	allocate a new OBJT_SWAP VM object and instantiate
596  *			its metadata structures.
597  *
598  *	This routine is called from the mmap and fork code to create a new
599  *	OBJT_SWAP object.  We do this by creating an OBJT_DEFAULT object
600  *	and then converting it with swp_pager_meta_build().
601  *
602  *	This routine may block in vm_object_allocate() and create a named
603  *	object lookup race, so we must interlock.
604  *
605  * MPSAFE
606  */
607 static vm_object_t
608 swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
609     vm_ooffset_t offset, struct ucred *cred)
610 {
611 	vm_object_t object;
612 	vm_pindex_t pindex;
613 
614 	pindex = OFF_TO_IDX(offset + PAGE_MASK + size);
615 	if (handle) {
616 		mtx_lock(&Giant);
617 		/*
618 		 * Reference existing named region or allocate new one.  There
619 		 * should not be a race here against swp_pager_meta_build()
620 		 * as called from vm_page_remove() in regards to the lookup
621 		 * of the handle.
622 		 */
623 		sx_xlock(&sw_alloc_sx);
624 		object = vm_pager_object_lookup(NOBJLIST(handle), handle);
625 		if (object == NULL) {
626 			if (cred != NULL) {
627 				if (!swap_reserve_by_cred(size, cred)) {
628 					sx_xunlock(&sw_alloc_sx);
629 					mtx_unlock(&Giant);
630 					return (NULL);
631 				}
632 				crhold(cred);
633 			}
634 			object = vm_object_allocate(OBJT_DEFAULT, pindex);
635 			VM_OBJECT_WLOCK(object);
636 			object->handle = handle;
637 			if (cred != NULL) {
638 				object->cred = cred;
639 				object->charge = size;
640 			}
641 			swp_pager_meta_build(object, 0, SWAPBLK_NONE);
642 			VM_OBJECT_WUNLOCK(object);
643 		}
644 		sx_xunlock(&sw_alloc_sx);
645 		mtx_unlock(&Giant);
646 	} else {
647 		if (cred != NULL) {
648 			if (!swap_reserve_by_cred(size, cred))
649 				return (NULL);
650 			crhold(cred);
651 		}
652 		object = vm_object_allocate(OBJT_DEFAULT, pindex);
653 		VM_OBJECT_WLOCK(object);
654 		if (cred != NULL) {
655 			object->cred = cred;
656 			object->charge = size;
657 		}
658 		swp_pager_meta_build(object, 0, SWAPBLK_NONE);
659 		VM_OBJECT_WUNLOCK(object);
660 	}
661 	return (object);
662 }
663 
664 /*
665  * SWAP_PAGER_DEALLOC() -	remove swap metadata from object
666  *
667  *	The swap backing for the object is destroyed.  The code is
668  *	designed such that we can reinstantiate it later, but this
669  *	routine is typically called only when the entire object is
670  *	about to be destroyed.
671  *
672  *	The object must be locked.
673  */
674 static void
675 swap_pager_dealloc(vm_object_t object)
676 {
677 
678 	/*
679 	 * Remove from list right away so lookups will fail if we block for
680 	 * pageout completion.
681 	 */
682 	if (object->handle != NULL) {
683 		mtx_lock(&sw_alloc_mtx);
684 		TAILQ_REMOVE(NOBJLIST(object->handle), object, pager_object_list);
685 		mtx_unlock(&sw_alloc_mtx);
686 	}
687 
688 	VM_OBJECT_ASSERT_WLOCKED(object);
689 	vm_object_pip_wait(object, "swpdea");
690 
691 	/*
692 	 * Free all remaining metadata.  We only bother to free it from
693 	 * the swap meta data.  We do not attempt to free swapblk's still
694 	 * associated with vm_page_t's for this object.  We do not care
695 	 * if paging is still in progress on some objects.
696 	 */
697 	swp_pager_meta_free_all(object);
698 	object->handle = NULL;
699 	object->type = OBJT_DEAD;
700 }
701 
702 /************************************************************************
703  *			SWAP PAGER BITMAP ROUTINES			*
704  ************************************************************************/
705 
706 /*
707  * SWP_PAGER_GETSWAPSPACE() -	allocate raw swap space
708  *
709  *	Allocate swap for the requested number of pages.  The starting
710  *	swap block number (a page index) is returned or SWAPBLK_NONE
711  *	if the allocation failed.
712  *
713  *	Also has the side effect of advising that somebody made a mistake
714  *	when they configured swap and didn't configure enough.
715  *
716  *	This routine may not sleep.
717  *
718  *	We allocate in round-robin fashion from the configured devices.
719  */
720 static daddr_t
721 swp_pager_getswapspace(int npages)
722 {
723 	daddr_t blk;
724 	struct swdevt *sp;
725 	int i;
726 
727 	blk = SWAPBLK_NONE;
728 	mtx_lock(&sw_dev_mtx);
729 	sp = swdevhd;
730 	for (i = 0; i < nswapdev; i++) {
731 		if (sp == NULL)
732 			sp = TAILQ_FIRST(&swtailq);
733 		if (!(sp->sw_flags & SW_CLOSING)) {
734 			blk = blist_alloc(sp->sw_blist, npages);
735 			if (blk != SWAPBLK_NONE) {
736 				blk += sp->sw_first;
737 				sp->sw_used += npages;
738 				swap_pager_avail -= npages;
739 				swp_sizecheck();
740 				swdevhd = TAILQ_NEXT(sp, sw_list);
741 				goto done;
742 			}
743 		}
744 		sp = TAILQ_NEXT(sp, sw_list);
745 	}
746 	if (swap_pager_full != 2) {
747 		printf("swap_pager_getswapspace(%d): failed\n", npages);
748 		swap_pager_full = 2;
749 		swap_pager_almost_full = 1;
750 	}
751 	swdevhd = NULL;
752 done:
753 	mtx_unlock(&sw_dev_mtx);
754 	return (blk);
755 }
756 
757 static int
758 swp_pager_isondev(daddr_t blk, struct swdevt *sp)
759 {
760 
761 	return (blk >= sp->sw_first && blk < sp->sw_end);
762 }
763 
764 static void
765 swp_pager_strategy(struct buf *bp)
766 {
767 	struct swdevt *sp;
768 
769 	mtx_lock(&sw_dev_mtx);
770 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
771 		if (bp->b_blkno >= sp->sw_first && bp->b_blkno < sp->sw_end) {
772 			mtx_unlock(&sw_dev_mtx);
773 			if ((sp->sw_flags & SW_UNMAPPED) != 0 &&
774 			    unmapped_buf_allowed) {
775 				bp->b_data = unmapped_buf;
776 				bp->b_offset = 0;
777 			} else {
778 				pmap_qenter((vm_offset_t)bp->b_data,
779 				    &bp->b_pages[0], bp->b_bcount / PAGE_SIZE);
780 			}
781 			sp->sw_strategy(bp, sp);
782 			return;
783 		}
784 	}
785 	panic("Swapdev not found");
786 }
787 
788 
789 /*
790  * SWP_PAGER_FREESWAPSPACE() -	free raw swap space
791  *
792  *	This routine returns the specified swap blocks back to the bitmap.
793  *
794  *	This routine may not sleep.
795  */
796 static void
797 swp_pager_freeswapspace(daddr_t blk, int npages)
798 {
799 	struct swdevt *sp;
800 
801 	mtx_lock(&sw_dev_mtx);
802 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
803 		if (blk >= sp->sw_first && blk < sp->sw_end) {
804 			sp->sw_used -= npages;
805 			/*
806 			 * If we are attempting to stop swapping on
807 			 * this device, we don't want to mark any
808 			 * blocks free lest they be reused.
809 			 */
810 			if ((sp->sw_flags & SW_CLOSING) == 0) {
811 				blist_free(sp->sw_blist, blk - sp->sw_first,
812 				    npages);
813 				swap_pager_avail += npages;
814 				swp_sizecheck();
815 			}
816 			mtx_unlock(&sw_dev_mtx);
817 			return;
818 		}
819 	}
820 	panic("Swapdev not found");
821 }
822 
823 /*
824  * SWAP_PAGER_FREESPACE() -	frees swap blocks associated with a page
825  *				range within an object.
826  *
827  *	This is a globally accessible routine.
828  *
829  *	This routine removes swapblk assignments from swap metadata.
830  *
831  *	The external callers of this routine typically have already destroyed
832  *	or renamed vm_page_t's associated with this range in the object so
833  *	we should be ok.
834  *
835  *	The object must be locked.
836  */
837 void
838 swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size)
839 {
840 
841 	swp_pager_meta_free(object, start, size);
842 }
843 
844 /*
845  * SWAP_PAGER_RESERVE() - reserve swap blocks in object
846  *
847  *	Assigns swap blocks to the specified range within the object.  The
848  *	swap blocks are not zeroed.  Any previous swap assignment is destroyed.
849  *
850  *	Returns 0 on success, -1 on failure.
851  */
852 int
853 swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
854 {
855 	int n = 0;
856 	daddr_t blk = SWAPBLK_NONE;
857 	vm_pindex_t beg = start;	/* save start index */
858 
859 	VM_OBJECT_WLOCK(object);
860 	while (size) {
861 		if (n == 0) {
862 			n = BLIST_MAX_ALLOC;
863 			while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
864 				n >>= 1;
865 				if (n == 0) {
866 					swp_pager_meta_free(object, beg, start - beg);
867 					VM_OBJECT_WUNLOCK(object);
868 					return (-1);
869 				}
870 			}
871 		}
872 		swp_pager_meta_build(object, start, blk);
873 		--size;
874 		++start;
875 		++blk;
876 		--n;
877 	}
878 	swp_pager_meta_free(object, start, n);
879 	VM_OBJECT_WUNLOCK(object);
880 	return (0);
881 }
882 
883 /*
884  * SWAP_PAGER_COPY() -  copy blocks from source pager to destination pager
885  *			and destroy the source.
886  *
887  *	Copy any valid swapblks from the source to the destination.  In
888  *	cases where both the source and destination have a valid swapblk,
889  *	we keep the destination's.
890  *
891  *	This routine is allowed to sleep.  It may sleep allocating metadata
892  *	indirectly through swp_pager_meta_build() or if paging is still in
893  *	progress on the source.
894  *
895  *	The source object contains no vm_page_t's (which is just as well)
896  *
897  *	The source object is of type OBJT_SWAP.
898  *
899  *	The source and destination objects must be locked.
900  *	Both object locks may temporarily be released.
901  */
902 void
903 swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
904     vm_pindex_t offset, int destroysource)
905 {
906 	vm_pindex_t i;
907 
908 	VM_OBJECT_ASSERT_WLOCKED(srcobject);
909 	VM_OBJECT_ASSERT_WLOCKED(dstobject);
910 
911 	/*
912 	 * If destroysource is set, we remove the source object from the
913 	 * swap_pager internal queue now.
914 	 */
915 	if (destroysource) {
916 		if (srcobject->handle != NULL) {
917 			mtx_lock(&sw_alloc_mtx);
918 			TAILQ_REMOVE(
919 			    NOBJLIST(srcobject->handle),
920 			    srcobject,
921 			    pager_object_list
922 			);
923 			mtx_unlock(&sw_alloc_mtx);
924 		}
925 	}
926 
927 	/*
928 	 * transfer source to destination.
929 	 */
930 	for (i = 0; i < dstobject->size; ++i) {
931 		daddr_t dstaddr;
932 
933 		/*
934 		 * Locate (without changing) the swapblk on the destination,
935 		 * unless it is invalid in which case free it silently, or
936 		 * if the destination is a resident page, in which case the
937 		 * source is thrown away.
938 		 */
939 		dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
940 
941 		if (dstaddr == SWAPBLK_NONE) {
942 			/*
943 			 * Destination has no swapblk and is not resident,
944 			 * copy source.
945 			 */
946 			daddr_t srcaddr;
947 
948 			srcaddr = swp_pager_meta_ctl(
949 			    srcobject,
950 			    i + offset,
951 			    SWM_POP
952 			);
953 
954 			if (srcaddr != SWAPBLK_NONE) {
955 				/*
956 				 * swp_pager_meta_build() can sleep.
957 				 */
958 				vm_object_pip_add(srcobject, 1);
959 				VM_OBJECT_WUNLOCK(srcobject);
960 				vm_object_pip_add(dstobject, 1);
961 				swp_pager_meta_build(dstobject, i, srcaddr);
962 				vm_object_pip_wakeup(dstobject);
963 				VM_OBJECT_WLOCK(srcobject);
964 				vm_object_pip_wakeup(srcobject);
965 			}
966 		} else {
967 			/*
968 			 * Destination has valid swapblk or it is represented
969 			 * by a resident page.  We destroy the sourceblock.
970 			 */
971 
972 			swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE);
973 		}
974 	}
975 
976 	/*
977 	 * Free left over swap blocks in source.
978 	 *
979 	 * We have to revert the type to OBJT_DEFAULT so we do not accidently
980 	 * double-remove the object from the swap queues.
981 	 */
982 	if (destroysource) {
983 		swp_pager_meta_free_all(srcobject);
984 		/*
985 		 * Reverting the type is not necessary, the caller is going
986 		 * to destroy srcobject directly, but I'm doing it here
987 		 * for consistency since we've removed the object from its
988 		 * queues.
989 		 */
990 		srcobject->type = OBJT_DEFAULT;
991 	}
992 }
993 
994 /*
995  * SWAP_PAGER_HASPAGE() -	determine if we have good backing store for
996  *				the requested page.
997  *
998  *	We determine whether good backing store exists for the requested
999  *	page and return TRUE if it does, FALSE if it doesn't.
1000  *
1001  *	If TRUE, we also try to determine how much valid, contiguous backing
1002  *	store exists before and after the requested page within a reasonable
1003  *	distance.  We do not try to restrict it to the swap device stripe
1004  *	(that is handled in getpages/putpages).  It probably isn't worth
1005  *	doing here.
1006  */
1007 static boolean_t
1008 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after)
1009 {
1010 	daddr_t blk0;
1011 
1012 	VM_OBJECT_ASSERT_LOCKED(object);
1013 	/*
1014 	 * do we have good backing store at the requested index ?
1015 	 */
1016 	blk0 = swp_pager_meta_ctl(object, pindex, 0);
1017 
1018 	if (blk0 == SWAPBLK_NONE) {
1019 		if (before)
1020 			*before = 0;
1021 		if (after)
1022 			*after = 0;
1023 		return (FALSE);
1024 	}
1025 
1026 	/*
1027 	 * find backwards-looking contiguous good backing store
1028 	 */
1029 	if (before != NULL) {
1030 		int i;
1031 
1032 		for (i = 1; i < (SWB_NPAGES/2); ++i) {
1033 			daddr_t blk;
1034 
1035 			if (i > pindex)
1036 				break;
1037 			blk = swp_pager_meta_ctl(object, pindex - i, 0);
1038 			if (blk != blk0 - i)
1039 				break;
1040 		}
1041 		*before = (i - 1);
1042 	}
1043 
1044 	/*
1045 	 * find forward-looking contiguous good backing store
1046 	 */
1047 	if (after != NULL) {
1048 		int i;
1049 
1050 		for (i = 1; i < (SWB_NPAGES/2); ++i) {
1051 			daddr_t blk;
1052 
1053 			blk = swp_pager_meta_ctl(object, pindex + i, 0);
1054 			if (blk != blk0 + i)
1055 				break;
1056 		}
1057 		*after = (i - 1);
1058 	}
1059 	return (TRUE);
1060 }
1061 
1062 /*
1063  * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page
1064  *
1065  *	This removes any associated swap backing store, whether valid or
1066  *	not, from the page.
1067  *
1068  *	This routine is typically called when a page is made dirty, at
1069  *	which point any associated swap can be freed.  MADV_FREE also
1070  *	calls us in a special-case situation
1071  *
1072  *	NOTE!!!  If the page is clean and the swap was valid, the caller
1073  *	should make the page dirty before calling this routine.  This routine
1074  *	does NOT change the m->dirty status of the page.  Also: MADV_FREE
1075  *	depends on it.
1076  *
1077  *	This routine may not sleep.
1078  *
1079  *	The object containing the page must be locked.
1080  */
1081 static void
1082 swap_pager_unswapped(vm_page_t m)
1083 {
1084 
1085 	swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE);
1086 }
1087 
1088 /*
1089  * SWAP_PAGER_GETPAGES() - bring pages in from swap
1090  *
1091  *	Attempt to retrieve (m, count) pages from backing store, but make
1092  *	sure we retrieve at least m[reqpage].  We try to load in as large
1093  *	a chunk surrounding m[reqpage] as is contiguous in swap and which
1094  *	belongs to the same object.
1095  *
1096  *	The code is designed for asynchronous operation and
1097  *	immediate-notification of 'reqpage' but tends not to be
1098  *	used that way.  Please do not optimize-out this algorithmic
1099  *	feature, I intend to improve on it in the future.
1100  *
1101  *	The parent has a single vm_object_pip_add() reference prior to
1102  *	calling us and we should return with the same.
1103  *
1104  *	The parent has BUSY'd the pages.  We should return with 'm'
1105  *	left busy, but the others adjusted.
1106  */
1107 static int
1108 swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage)
1109 {
1110 	struct buf *bp;
1111 	vm_page_t mreq;
1112 	int i;
1113 	int j;
1114 	daddr_t blk;
1115 
1116 	mreq = m[reqpage];
1117 
1118 	/*
1119 	 * Calculate range to retrieve.  The pages have already been assigned
1120 	 * their swapblks.  We require a *contiguous* range but we know it to
1121 	 * not span devices.   If we do not supply it, bad things
1122 	 * happen.  Note that blk, iblk & jblk can be SWAPBLK_NONE, but the
1123 	 * loops are set up such that the case(s) are handled implicitly.
1124 	 *
1125 	 * The swp_*() calls must be made with the object locked.
1126 	 */
1127 	blk = swp_pager_meta_ctl(mreq->object, mreq->pindex, 0);
1128 
1129 	for (i = reqpage - 1; i >= 0; --i) {
1130 		daddr_t iblk;
1131 
1132 		iblk = swp_pager_meta_ctl(m[i]->object, m[i]->pindex, 0);
1133 		if (blk != iblk + (reqpage - i))
1134 			break;
1135 	}
1136 	++i;
1137 
1138 	for (j = reqpage + 1; j < count; ++j) {
1139 		daddr_t jblk;
1140 
1141 		jblk = swp_pager_meta_ctl(m[j]->object, m[j]->pindex, 0);
1142 		if (blk != jblk - (j - reqpage))
1143 			break;
1144 	}
1145 
1146 	/*
1147 	 * free pages outside our collection range.   Note: we never free
1148 	 * mreq, it must remain busy throughout.
1149 	 */
1150 	if (0 < i || j < count) {
1151 		int k;
1152 
1153 		for (k = 0; k < i; ++k)
1154 			swp_pager_free_nrpage(m[k]);
1155 		for (k = j; k < count; ++k)
1156 			swp_pager_free_nrpage(m[k]);
1157 	}
1158 
1159 	/*
1160 	 * Return VM_PAGER_FAIL if we have nothing to do.  Return mreq
1161 	 * still busy, but the others unbusied.
1162 	 */
1163 	if (blk == SWAPBLK_NONE)
1164 		return (VM_PAGER_FAIL);
1165 
1166 	/*
1167 	 * Getpbuf() can sleep.
1168 	 */
1169 	VM_OBJECT_WUNLOCK(object);
1170 	/*
1171 	 * Get a swap buffer header to perform the IO
1172 	 */
1173 	bp = getpbuf(&nsw_rcount);
1174 	bp->b_flags |= B_PAGING;
1175 
1176 	bp->b_iocmd = BIO_READ;
1177 	bp->b_iodone = swp_pager_async_iodone;
1178 	bp->b_rcred = crhold(thread0.td_ucred);
1179 	bp->b_wcred = crhold(thread0.td_ucred);
1180 	bp->b_blkno = blk - (reqpage - i);
1181 	bp->b_bcount = PAGE_SIZE * (j - i);
1182 	bp->b_bufsize = PAGE_SIZE * (j - i);
1183 	bp->b_pager.pg_reqpage = reqpage - i;
1184 
1185 	VM_OBJECT_WLOCK(object);
1186 	{
1187 		int k;
1188 
1189 		for (k = i; k < j; ++k) {
1190 			bp->b_pages[k - i] = m[k];
1191 			m[k]->oflags |= VPO_SWAPINPROG;
1192 		}
1193 	}
1194 	bp->b_npages = j - i;
1195 
1196 	PCPU_INC(cnt.v_swapin);
1197 	PCPU_ADD(cnt.v_swappgsin, bp->b_npages);
1198 
1199 	/*
1200 	 * We still hold the lock on mreq, and our automatic completion routine
1201 	 * does not remove it.
1202 	 */
1203 	vm_object_pip_add(object, bp->b_npages);
1204 	VM_OBJECT_WUNLOCK(object);
1205 
1206 	/*
1207 	 * perform the I/O.  NOTE!!!  bp cannot be considered valid after
1208 	 * this point because we automatically release it on completion.
1209 	 * Instead, we look at the one page we are interested in which we
1210 	 * still hold a lock on even through the I/O completion.
1211 	 *
1212 	 * The other pages in our m[] array are also released on completion,
1213 	 * so we cannot assume they are valid anymore either.
1214 	 *
1215 	 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1216 	 */
1217 	BUF_KERNPROC(bp);
1218 	swp_pager_strategy(bp);
1219 
1220 	/*
1221 	 * wait for the page we want to complete.  VPO_SWAPINPROG is always
1222 	 * cleared on completion.  If an I/O error occurs, SWAPBLK_NONE
1223 	 * is set in the meta-data.
1224 	 */
1225 	VM_OBJECT_WLOCK(object);
1226 	while ((mreq->oflags & VPO_SWAPINPROG) != 0) {
1227 		mreq->oflags |= VPO_SWAPSLEEP;
1228 		PCPU_INC(cnt.v_intrans);
1229 		if (VM_OBJECT_SLEEP(object, &object->paging_in_progress, PSWP,
1230 		    "swread", hz * 20)) {
1231 			printf(
1232 "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n",
1233 			    bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount);
1234 		}
1235 	}
1236 
1237 	/*
1238 	 * mreq is left busied after completion, but all the other pages
1239 	 * are freed.  If we had an unrecoverable read error the page will
1240 	 * not be valid.
1241 	 */
1242 	if (mreq->valid != VM_PAGE_BITS_ALL) {
1243 		return (VM_PAGER_ERROR);
1244 	} else {
1245 		return (VM_PAGER_OK);
1246 	}
1247 
1248 	/*
1249 	 * A final note: in a low swap situation, we cannot deallocate swap
1250 	 * and mark a page dirty here because the caller is likely to mark
1251 	 * the page clean when we return, causing the page to possibly revert
1252 	 * to all-zero's later.
1253 	 */
1254 }
1255 
1256 /*
1257  * 	swap_pager_getpages_async():
1258  *
1259  *	Right now this is emulation of asynchronous operation on top of
1260  *	swap_pager_getpages().
1261  */
1262 static int
1263 swap_pager_getpages_async(vm_object_t object, vm_page_t *m, int count,
1264     int reqpage, pgo_getpages_iodone_t iodone, void *arg)
1265 {
1266 	int r, error;
1267 
1268 	r = swap_pager_getpages(object, m, count, reqpage);
1269 	VM_OBJECT_WUNLOCK(object);
1270 	switch (r) {
1271 	case VM_PAGER_OK:
1272 		error = 0;
1273 		break;
1274 	case VM_PAGER_ERROR:
1275 		error = EIO;
1276 		break;
1277 	case VM_PAGER_FAIL:
1278 		error = EINVAL;
1279 		break;
1280 	default:
1281 		panic("unhandled swap_pager_getpages() error %d", r);
1282 	}
1283 	(iodone)(arg, m, count, error);
1284 	VM_OBJECT_WLOCK(object);
1285 
1286 	return (r);
1287 }
1288 
1289 /*
1290  *	swap_pager_putpages:
1291  *
1292  *	Assign swap (if necessary) and initiate I/O on the specified pages.
1293  *
1294  *	We support both OBJT_DEFAULT and OBJT_SWAP objects.  DEFAULT objects
1295  *	are automatically converted to SWAP objects.
1296  *
1297  *	In a low memory situation we may block in VOP_STRATEGY(), but the new
1298  *	vm_page reservation system coupled with properly written VFS devices
1299  *	should ensure that no low-memory deadlock occurs.  This is an area
1300  *	which needs work.
1301  *
1302  *	The parent has N vm_object_pip_add() references prior to
1303  *	calling us and will remove references for rtvals[] that are
1304  *	not set to VM_PAGER_PEND.  We need to remove the rest on I/O
1305  *	completion.
1306  *
1307  *	The parent has soft-busy'd the pages it passes us and will unbusy
1308  *	those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
1309  *	We need to unbusy the rest on I/O completion.
1310  */
1311 void
1312 swap_pager_putpages(vm_object_t object, vm_page_t *m, int count,
1313     int flags, int *rtvals)
1314 {
1315 	int i, n;
1316 	boolean_t sync;
1317 
1318 	if (count && m[0]->object != object) {
1319 		panic("swap_pager_putpages: object mismatch %p/%p",
1320 		    object,
1321 		    m[0]->object
1322 		);
1323 	}
1324 
1325 	/*
1326 	 * Step 1
1327 	 *
1328 	 * Turn object into OBJT_SWAP
1329 	 * check for bogus sysops
1330 	 * force sync if not pageout process
1331 	 */
1332 	if (object->type != OBJT_SWAP)
1333 		swp_pager_meta_build(object, 0, SWAPBLK_NONE);
1334 	VM_OBJECT_WUNLOCK(object);
1335 
1336 	n = 0;
1337 	if (curproc != pageproc)
1338 		sync = TRUE;
1339 	else
1340 		sync = (flags & VM_PAGER_PUT_SYNC) != 0;
1341 
1342 	/*
1343 	 * Step 2
1344 	 *
1345 	 * Assign swap blocks and issue I/O.  We reallocate swap on the fly.
1346 	 * The page is left dirty until the pageout operation completes
1347 	 * successfully.
1348 	 */
1349 	for (i = 0; i < count; i += n) {
1350 		int j;
1351 		struct buf *bp;
1352 		daddr_t blk;
1353 
1354 		/*
1355 		 * Maximum I/O size is limited by a number of factors.
1356 		 */
1357 		n = min(BLIST_MAX_ALLOC, count - i);
1358 		n = min(n, nsw_cluster_max);
1359 
1360 		/*
1361 		 * Get biggest block of swap we can.  If we fail, fall
1362 		 * back and try to allocate a smaller block.  Don't go
1363 		 * overboard trying to allocate space if it would overly
1364 		 * fragment swap.
1365 		 */
1366 		while (
1367 		    (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE &&
1368 		    n > 4
1369 		) {
1370 			n >>= 1;
1371 		}
1372 		if (blk == SWAPBLK_NONE) {
1373 			for (j = 0; j < n; ++j)
1374 				rtvals[i+j] = VM_PAGER_FAIL;
1375 			continue;
1376 		}
1377 
1378 		/*
1379 		 * All I/O parameters have been satisfied, build the I/O
1380 		 * request and assign the swap space.
1381 		 */
1382 		if (sync == TRUE) {
1383 			bp = getpbuf(&nsw_wcount_sync);
1384 		} else {
1385 			bp = getpbuf(&nsw_wcount_async);
1386 			bp->b_flags = B_ASYNC;
1387 		}
1388 		bp->b_flags |= B_PAGING;
1389 		bp->b_iocmd = BIO_WRITE;
1390 
1391 		bp->b_rcred = crhold(thread0.td_ucred);
1392 		bp->b_wcred = crhold(thread0.td_ucred);
1393 		bp->b_bcount = PAGE_SIZE * n;
1394 		bp->b_bufsize = PAGE_SIZE * n;
1395 		bp->b_blkno = blk;
1396 
1397 		VM_OBJECT_WLOCK(object);
1398 		for (j = 0; j < n; ++j) {
1399 			vm_page_t mreq = m[i+j];
1400 
1401 			swp_pager_meta_build(
1402 			    mreq->object,
1403 			    mreq->pindex,
1404 			    blk + j
1405 			);
1406 			vm_page_dirty(mreq);
1407 			mreq->oflags |= VPO_SWAPINPROG;
1408 			bp->b_pages[j] = mreq;
1409 		}
1410 		VM_OBJECT_WUNLOCK(object);
1411 		bp->b_npages = n;
1412 		/*
1413 		 * Must set dirty range for NFS to work.
1414 		 */
1415 		bp->b_dirtyoff = 0;
1416 		bp->b_dirtyend = bp->b_bcount;
1417 
1418 		PCPU_INC(cnt.v_swapout);
1419 		PCPU_ADD(cnt.v_swappgsout, bp->b_npages);
1420 
1421 		/*
1422 		 * We unconditionally set rtvals[] to VM_PAGER_PEND so that we
1423 		 * can call the async completion routine at the end of a
1424 		 * synchronous I/O operation.  Otherwise, our caller would
1425 		 * perform duplicate unbusy and wakeup operations on the page
1426 		 * and object, respectively.
1427 		 */
1428 		for (j = 0; j < n; j++)
1429 			rtvals[i + j] = VM_PAGER_PEND;
1430 
1431 		/*
1432 		 * asynchronous
1433 		 *
1434 		 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1435 		 */
1436 		if (sync == FALSE) {
1437 			bp->b_iodone = swp_pager_async_iodone;
1438 			BUF_KERNPROC(bp);
1439 			swp_pager_strategy(bp);
1440 			continue;
1441 		}
1442 
1443 		/*
1444 		 * synchronous
1445 		 *
1446 		 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1447 		 */
1448 		bp->b_iodone = bdone;
1449 		swp_pager_strategy(bp);
1450 
1451 		/*
1452 		 * Wait for the sync I/O to complete.
1453 		 */
1454 		bwait(bp, PVM, "swwrt");
1455 
1456 		/*
1457 		 * Now that we are through with the bp, we can call the
1458 		 * normal async completion, which frees everything up.
1459 		 */
1460 		swp_pager_async_iodone(bp);
1461 	}
1462 	VM_OBJECT_WLOCK(object);
1463 }
1464 
1465 /*
1466  *	swp_pager_async_iodone:
1467  *
1468  *	Completion routine for asynchronous reads and writes from/to swap.
1469  *	Also called manually by synchronous code to finish up a bp.
1470  *
1471  *	This routine may not sleep.
1472  */
1473 static void
1474 swp_pager_async_iodone(struct buf *bp)
1475 {
1476 	int i;
1477 	vm_object_t object = NULL;
1478 
1479 	/*
1480 	 * report error
1481 	 */
1482 	if (bp->b_ioflags & BIO_ERROR) {
1483 		printf(
1484 		    "swap_pager: I/O error - %s failed; blkno %ld,"
1485 			"size %ld, error %d\n",
1486 		    ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
1487 		    (long)bp->b_blkno,
1488 		    (long)bp->b_bcount,
1489 		    bp->b_error
1490 		);
1491 	}
1492 
1493 	/*
1494 	 * remove the mapping for kernel virtual
1495 	 */
1496 	if (buf_mapped(bp))
1497 		pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
1498 	else
1499 		bp->b_data = bp->b_kvabase;
1500 
1501 	if (bp->b_npages) {
1502 		object = bp->b_pages[0]->object;
1503 		VM_OBJECT_WLOCK(object);
1504 	}
1505 
1506 	/*
1507 	 * cleanup pages.  If an error occurs writing to swap, we are in
1508 	 * very serious trouble.  If it happens to be a disk error, though,
1509 	 * we may be able to recover by reassigning the swap later on.  So
1510 	 * in this case we remove the m->swapblk assignment for the page
1511 	 * but do not free it in the rlist.  The errornous block(s) are thus
1512 	 * never reallocated as swap.  Redirty the page and continue.
1513 	 */
1514 	for (i = 0; i < bp->b_npages; ++i) {
1515 		vm_page_t m = bp->b_pages[i];
1516 
1517 		m->oflags &= ~VPO_SWAPINPROG;
1518 		if (m->oflags & VPO_SWAPSLEEP) {
1519 			m->oflags &= ~VPO_SWAPSLEEP;
1520 			wakeup(&object->paging_in_progress);
1521 		}
1522 
1523 		if (bp->b_ioflags & BIO_ERROR) {
1524 			/*
1525 			 * If an error occurs I'd love to throw the swapblk
1526 			 * away without freeing it back to swapspace, so it
1527 			 * can never be used again.  But I can't from an
1528 			 * interrupt.
1529 			 */
1530 			if (bp->b_iocmd == BIO_READ) {
1531 				/*
1532 				 * When reading, reqpage needs to stay
1533 				 * locked for the parent, but all other
1534 				 * pages can be freed.  We still want to
1535 				 * wakeup the parent waiting on the page,
1536 				 * though.  ( also: pg_reqpage can be -1 and
1537 				 * not match anything ).
1538 				 *
1539 				 * We have to wake specifically requested pages
1540 				 * up too because we cleared VPO_SWAPINPROG and
1541 				 * someone may be waiting for that.
1542 				 *
1543 				 * NOTE: for reads, m->dirty will probably
1544 				 * be overridden by the original caller of
1545 				 * getpages so don't play cute tricks here.
1546 				 */
1547 				m->valid = 0;
1548 				if (i != bp->b_pager.pg_reqpage)
1549 					swp_pager_free_nrpage(m);
1550 				else {
1551 					vm_page_lock(m);
1552 					vm_page_flash(m);
1553 					vm_page_unlock(m);
1554 				}
1555 				/*
1556 				 * If i == bp->b_pager.pg_reqpage, do not wake
1557 				 * the page up.  The caller needs to.
1558 				 */
1559 			} else {
1560 				/*
1561 				 * If a write error occurs, reactivate page
1562 				 * so it doesn't clog the inactive list,
1563 				 * then finish the I/O.
1564 				 */
1565 				vm_page_dirty(m);
1566 				vm_page_lock(m);
1567 				vm_page_activate(m);
1568 				vm_page_unlock(m);
1569 				vm_page_sunbusy(m);
1570 			}
1571 		} else if (bp->b_iocmd == BIO_READ) {
1572 			/*
1573 			 * NOTE: for reads, m->dirty will probably be
1574 			 * overridden by the original caller of getpages so
1575 			 * we cannot set them in order to free the underlying
1576 			 * swap in a low-swap situation.  I don't think we'd
1577 			 * want to do that anyway, but it was an optimization
1578 			 * that existed in the old swapper for a time before
1579 			 * it got ripped out due to precisely this problem.
1580 			 *
1581 			 * If not the requested page then deactivate it.
1582 			 *
1583 			 * Note that the requested page, reqpage, is left
1584 			 * busied, but we still have to wake it up.  The
1585 			 * other pages are released (unbusied) by
1586 			 * vm_page_xunbusy().
1587 			 */
1588 			KASSERT(!pmap_page_is_mapped(m),
1589 			    ("swp_pager_async_iodone: page %p is mapped", m));
1590 			m->valid = VM_PAGE_BITS_ALL;
1591 			KASSERT(m->dirty == 0,
1592 			    ("swp_pager_async_iodone: page %p is dirty", m));
1593 
1594 			/*
1595 			 * We have to wake specifically requested pages
1596 			 * up too because we cleared VPO_SWAPINPROG and
1597 			 * could be waiting for it in getpages.  However,
1598 			 * be sure to not unbusy getpages specifically
1599 			 * requested page - getpages expects it to be
1600 			 * left busy.
1601 			 */
1602 			if (i != bp->b_pager.pg_reqpage) {
1603 				vm_page_lock(m);
1604 				vm_page_deactivate(m);
1605 				vm_page_unlock(m);
1606 				vm_page_xunbusy(m);
1607 			} else {
1608 				vm_page_lock(m);
1609 				vm_page_flash(m);
1610 				vm_page_unlock(m);
1611 			}
1612 		} else {
1613 			/*
1614 			 * For write success, clear the dirty
1615 			 * status, then finish the I/O ( which decrements the
1616 			 * busy count and possibly wakes waiter's up ).
1617 			 */
1618 			KASSERT(!pmap_page_is_write_mapped(m),
1619 			    ("swp_pager_async_iodone: page %p is not write"
1620 			    " protected", m));
1621 			vm_page_undirty(m);
1622 			vm_page_sunbusy(m);
1623 			if (vm_page_count_severe()) {
1624 				vm_page_lock(m);
1625 				vm_page_try_to_cache(m);
1626 				vm_page_unlock(m);
1627 			}
1628 		}
1629 	}
1630 
1631 	/*
1632 	 * adjust pip.  NOTE: the original parent may still have its own
1633 	 * pip refs on the object.
1634 	 */
1635 	if (object != NULL) {
1636 		vm_object_pip_wakeupn(object, bp->b_npages);
1637 		VM_OBJECT_WUNLOCK(object);
1638 	}
1639 
1640 	/*
1641 	 * swapdev_strategy() manually sets b_vp and b_bufobj before calling
1642 	 * bstrategy(). Set them back to NULL now we're done with it, or we'll
1643 	 * trigger a KASSERT in relpbuf().
1644 	 */
1645 	if (bp->b_vp) {
1646 		    bp->b_vp = NULL;
1647 		    bp->b_bufobj = NULL;
1648 	}
1649 	/*
1650 	 * release the physical I/O buffer
1651 	 */
1652 	relpbuf(
1653 	    bp,
1654 	    ((bp->b_iocmd == BIO_READ) ? &nsw_rcount :
1655 		((bp->b_flags & B_ASYNC) ?
1656 		    &nsw_wcount_async :
1657 		    &nsw_wcount_sync
1658 		)
1659 	    )
1660 	);
1661 }
1662 
1663 /*
1664  *	swap_pager_isswapped:
1665  *
1666  *	Return 1 if at least one page in the given object is paged
1667  *	out to the given swap device.
1668  *
1669  *	This routine may not sleep.
1670  */
1671 int
1672 swap_pager_isswapped(vm_object_t object, struct swdevt *sp)
1673 {
1674 	daddr_t index = 0;
1675 	int bcount;
1676 	int i;
1677 
1678 	VM_OBJECT_ASSERT_WLOCKED(object);
1679 	if (object->type != OBJT_SWAP)
1680 		return (0);
1681 
1682 	mtx_lock(&swhash_mtx);
1683 	for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) {
1684 		struct swblock *swap;
1685 
1686 		if ((swap = *swp_pager_hash(object, index)) != NULL) {
1687 			for (i = 0; i < SWAP_META_PAGES; ++i) {
1688 				if (swp_pager_isondev(swap->swb_pages[i], sp)) {
1689 					mtx_unlock(&swhash_mtx);
1690 					return (1);
1691 				}
1692 			}
1693 		}
1694 		index += SWAP_META_PAGES;
1695 	}
1696 	mtx_unlock(&swhash_mtx);
1697 	return (0);
1698 }
1699 
1700 /*
1701  * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in
1702  *
1703  *	This routine dissociates the page at the given index within a
1704  *	swap block from its backing store, paging it in if necessary.
1705  *	If the page is paged in, it is placed in the inactive queue,
1706  *	since it had its backing store ripped out from under it.
1707  *	We also attempt to swap in all other pages in the swap block,
1708  *	we only guarantee that the one at the specified index is
1709  *	paged in.
1710  *
1711  *	XXX - The code to page the whole block in doesn't work, so we
1712  *	      revert to the one-by-one behavior for now.  Sigh.
1713  */
1714 static inline void
1715 swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex)
1716 {
1717 	vm_page_t m;
1718 
1719 	vm_object_pip_add(object, 1);
1720 	m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL);
1721 	if (m->valid == VM_PAGE_BITS_ALL) {
1722 		vm_object_pip_wakeup(object);
1723 		vm_page_dirty(m);
1724 		vm_page_lock(m);
1725 		vm_page_activate(m);
1726 		vm_page_unlock(m);
1727 		vm_page_xunbusy(m);
1728 		vm_pager_page_unswapped(m);
1729 		return;
1730 	}
1731 
1732 	if (swap_pager_getpages(object, &m, 1, 0) != VM_PAGER_OK)
1733 		panic("swap_pager_force_pagein: read from swap failed");/*XXX*/
1734 	vm_object_pip_wakeup(object);
1735 	vm_page_dirty(m);
1736 	vm_page_lock(m);
1737 	vm_page_deactivate(m);
1738 	vm_page_unlock(m);
1739 	vm_page_xunbusy(m);
1740 	vm_pager_page_unswapped(m);
1741 }
1742 
1743 /*
1744  *	swap_pager_swapoff:
1745  *
1746  *	Page in all of the pages that have been paged out to the
1747  *	given device.  The corresponding blocks in the bitmap must be
1748  *	marked as allocated and the device must be flagged SW_CLOSING.
1749  *	There may be no processes swapped out to the device.
1750  *
1751  *	This routine may block.
1752  */
1753 static void
1754 swap_pager_swapoff(struct swdevt *sp)
1755 {
1756 	struct swblock *swap;
1757 	int i, j, retries;
1758 
1759 	GIANT_REQUIRED;
1760 
1761 	retries = 0;
1762 full_rescan:
1763 	mtx_lock(&swhash_mtx);
1764 	for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */
1765 restart:
1766 		for (swap = swhash[i]; swap != NULL; swap = swap->swb_hnext) {
1767 			vm_object_t object = swap->swb_object;
1768 			vm_pindex_t pindex = swap->swb_index;
1769 			for (j = 0; j < SWAP_META_PAGES; ++j) {
1770 				if (swp_pager_isondev(swap->swb_pages[j], sp)) {
1771 					/* avoid deadlock */
1772 					if (!VM_OBJECT_TRYWLOCK(object)) {
1773 						break;
1774 					} else {
1775 						mtx_unlock(&swhash_mtx);
1776 						swp_pager_force_pagein(object,
1777 						    pindex + j);
1778 						VM_OBJECT_WUNLOCK(object);
1779 						mtx_lock(&swhash_mtx);
1780 						goto restart;
1781 					}
1782 				}
1783 			}
1784 		}
1785 	}
1786 	mtx_unlock(&swhash_mtx);
1787 	if (sp->sw_used) {
1788 		/*
1789 		 * Objects may be locked or paging to the device being
1790 		 * removed, so we will miss their pages and need to
1791 		 * make another pass.  We have marked this device as
1792 		 * SW_CLOSING, so the activity should finish soon.
1793 		 */
1794 		retries++;
1795 		if (retries > 100) {
1796 			panic("swapoff: failed to locate %d swap blocks",
1797 			    sp->sw_used);
1798 		}
1799 		pause("swpoff", hz / 20);
1800 		goto full_rescan;
1801 	}
1802 }
1803 
1804 /************************************************************************
1805  *				SWAP META DATA 				*
1806  ************************************************************************
1807  *
1808  *	These routines manipulate the swap metadata stored in the
1809  *	OBJT_SWAP object.
1810  *
1811  *	Swap metadata is implemented with a global hash and not directly
1812  *	linked into the object.  Instead the object simply contains
1813  *	appropriate tracking counters.
1814  */
1815 
1816 /*
1817  * SWP_PAGER_META_BUILD() -	add swap block to swap meta data for object
1818  *
1819  *	We first convert the object to a swap object if it is a default
1820  *	object.
1821  *
1822  *	The specified swapblk is added to the object's swap metadata.  If
1823  *	the swapblk is not valid, it is freed instead.  Any previously
1824  *	assigned swapblk is freed.
1825  */
1826 static void
1827 swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
1828 {
1829 	static volatile int exhausted;
1830 	struct swblock *swap;
1831 	struct swblock **pswap;
1832 	int idx;
1833 
1834 	VM_OBJECT_ASSERT_WLOCKED(object);
1835 	/*
1836 	 * Convert default object to swap object if necessary
1837 	 */
1838 	if (object->type != OBJT_SWAP) {
1839 		object->type = OBJT_SWAP;
1840 		object->un_pager.swp.swp_bcount = 0;
1841 
1842 		if (object->handle != NULL) {
1843 			mtx_lock(&sw_alloc_mtx);
1844 			TAILQ_INSERT_TAIL(
1845 			    NOBJLIST(object->handle),
1846 			    object,
1847 			    pager_object_list
1848 			);
1849 			mtx_unlock(&sw_alloc_mtx);
1850 		}
1851 	}
1852 
1853 	/*
1854 	 * Locate hash entry.  If not found create, but if we aren't adding
1855 	 * anything just return.  If we run out of space in the map we wait
1856 	 * and, since the hash table may have changed, retry.
1857 	 */
1858 retry:
1859 	mtx_lock(&swhash_mtx);
1860 	pswap = swp_pager_hash(object, pindex);
1861 
1862 	if ((swap = *pswap) == NULL) {
1863 		int i;
1864 
1865 		if (swapblk == SWAPBLK_NONE)
1866 			goto done;
1867 
1868 		swap = *pswap = uma_zalloc(swap_zone, M_NOWAIT |
1869 		    (curproc == pageproc ? M_USE_RESERVE : 0));
1870 		if (swap == NULL) {
1871 			mtx_unlock(&swhash_mtx);
1872 			VM_OBJECT_WUNLOCK(object);
1873 			if (uma_zone_exhausted(swap_zone)) {
1874 				if (atomic_cmpset_int(&exhausted, 0, 1))
1875 					printf("swap zone exhausted, "
1876 					    "increase kern.maxswzone\n");
1877 				vm_pageout_oom(VM_OOM_SWAPZ);
1878 				pause("swzonex", 10);
1879 			} else
1880 				VM_WAIT;
1881 			VM_OBJECT_WLOCK(object);
1882 			goto retry;
1883 		}
1884 
1885 		if (atomic_cmpset_int(&exhausted, 1, 0))
1886 			printf("swap zone ok\n");
1887 
1888 		swap->swb_hnext = NULL;
1889 		swap->swb_object = object;
1890 		swap->swb_index = pindex & ~(vm_pindex_t)SWAP_META_MASK;
1891 		swap->swb_count = 0;
1892 
1893 		++object->un_pager.swp.swp_bcount;
1894 
1895 		for (i = 0; i < SWAP_META_PAGES; ++i)
1896 			swap->swb_pages[i] = SWAPBLK_NONE;
1897 	}
1898 
1899 	/*
1900 	 * Delete prior contents of metadata
1901 	 */
1902 	idx = pindex & SWAP_META_MASK;
1903 
1904 	if (swap->swb_pages[idx] != SWAPBLK_NONE) {
1905 		swp_pager_freeswapspace(swap->swb_pages[idx], 1);
1906 		--swap->swb_count;
1907 	}
1908 
1909 	/*
1910 	 * Enter block into metadata
1911 	 */
1912 	swap->swb_pages[idx] = swapblk;
1913 	if (swapblk != SWAPBLK_NONE)
1914 		++swap->swb_count;
1915 done:
1916 	mtx_unlock(&swhash_mtx);
1917 }
1918 
1919 /*
1920  * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
1921  *
1922  *	The requested range of blocks is freed, with any associated swap
1923  *	returned to the swap bitmap.
1924  *
1925  *	This routine will free swap metadata structures as they are cleaned
1926  *	out.  This routine does *NOT* operate on swap metadata associated
1927  *	with resident pages.
1928  */
1929 static void
1930 swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count)
1931 {
1932 
1933 	VM_OBJECT_ASSERT_LOCKED(object);
1934 	if (object->type != OBJT_SWAP)
1935 		return;
1936 
1937 	while (count > 0) {
1938 		struct swblock **pswap;
1939 		struct swblock *swap;
1940 
1941 		mtx_lock(&swhash_mtx);
1942 		pswap = swp_pager_hash(object, index);
1943 
1944 		if ((swap = *pswap) != NULL) {
1945 			daddr_t v = swap->swb_pages[index & SWAP_META_MASK];
1946 
1947 			if (v != SWAPBLK_NONE) {
1948 				swp_pager_freeswapspace(v, 1);
1949 				swap->swb_pages[index & SWAP_META_MASK] =
1950 					SWAPBLK_NONE;
1951 				if (--swap->swb_count == 0) {
1952 					*pswap = swap->swb_hnext;
1953 					uma_zfree(swap_zone, swap);
1954 					--object->un_pager.swp.swp_bcount;
1955 				}
1956 			}
1957 			--count;
1958 			++index;
1959 		} else {
1960 			int n = SWAP_META_PAGES - (index & SWAP_META_MASK);
1961 			count -= n;
1962 			index += n;
1963 		}
1964 		mtx_unlock(&swhash_mtx);
1965 	}
1966 }
1967 
1968 /*
1969  * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
1970  *
1971  *	This routine locates and destroys all swap metadata associated with
1972  *	an object.
1973  */
1974 static void
1975 swp_pager_meta_free_all(vm_object_t object)
1976 {
1977 	daddr_t index = 0;
1978 
1979 	VM_OBJECT_ASSERT_WLOCKED(object);
1980 	if (object->type != OBJT_SWAP)
1981 		return;
1982 
1983 	while (object->un_pager.swp.swp_bcount) {
1984 		struct swblock **pswap;
1985 		struct swblock *swap;
1986 
1987 		mtx_lock(&swhash_mtx);
1988 		pswap = swp_pager_hash(object, index);
1989 		if ((swap = *pswap) != NULL) {
1990 			int i;
1991 
1992 			for (i = 0; i < SWAP_META_PAGES; ++i) {
1993 				daddr_t v = swap->swb_pages[i];
1994 				if (v != SWAPBLK_NONE) {
1995 					--swap->swb_count;
1996 					swp_pager_freeswapspace(v, 1);
1997 				}
1998 			}
1999 			if (swap->swb_count != 0)
2000 				panic("swap_pager_meta_free_all: swb_count != 0");
2001 			*pswap = swap->swb_hnext;
2002 			uma_zfree(swap_zone, swap);
2003 			--object->un_pager.swp.swp_bcount;
2004 		}
2005 		mtx_unlock(&swhash_mtx);
2006 		index += SWAP_META_PAGES;
2007 	}
2008 }
2009 
2010 /*
2011  * SWP_PAGER_METACTL() -  misc control of swap and vm_page_t meta data.
2012  *
2013  *	This routine is capable of looking up, popping, or freeing
2014  *	swapblk assignments in the swap meta data or in the vm_page_t.
2015  *	The routine typically returns the swapblk being looked-up, or popped,
2016  *	or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block
2017  *	was invalid.  This routine will automatically free any invalid
2018  *	meta-data swapblks.
2019  *
2020  *	It is not possible to store invalid swapblks in the swap meta data
2021  *	(other then a literal 'SWAPBLK_NONE'), so we don't bother checking.
2022  *
2023  *	When acting on a busy resident page and paging is in progress, we
2024  *	have to wait until paging is complete but otherwise can act on the
2025  *	busy page.
2026  *
2027  *	SWM_FREE	remove and free swap block from metadata
2028  *	SWM_POP		remove from meta data but do not free.. pop it out
2029  */
2030 static daddr_t
2031 swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
2032 {
2033 	struct swblock **pswap;
2034 	struct swblock *swap;
2035 	daddr_t r1;
2036 	int idx;
2037 
2038 	VM_OBJECT_ASSERT_LOCKED(object);
2039 	/*
2040 	 * The meta data only exists of the object is OBJT_SWAP
2041 	 * and even then might not be allocated yet.
2042 	 */
2043 	if (object->type != OBJT_SWAP)
2044 		return (SWAPBLK_NONE);
2045 
2046 	r1 = SWAPBLK_NONE;
2047 	mtx_lock(&swhash_mtx);
2048 	pswap = swp_pager_hash(object, pindex);
2049 
2050 	if ((swap = *pswap) != NULL) {
2051 		idx = pindex & SWAP_META_MASK;
2052 		r1 = swap->swb_pages[idx];
2053 
2054 		if (r1 != SWAPBLK_NONE) {
2055 			if (flags & SWM_FREE) {
2056 				swp_pager_freeswapspace(r1, 1);
2057 				r1 = SWAPBLK_NONE;
2058 			}
2059 			if (flags & (SWM_FREE|SWM_POP)) {
2060 				swap->swb_pages[idx] = SWAPBLK_NONE;
2061 				if (--swap->swb_count == 0) {
2062 					*pswap = swap->swb_hnext;
2063 					uma_zfree(swap_zone, swap);
2064 					--object->un_pager.swp.swp_bcount;
2065 				}
2066 			}
2067 		}
2068 	}
2069 	mtx_unlock(&swhash_mtx);
2070 	return (r1);
2071 }
2072 
2073 /*
2074  * System call swapon(name) enables swapping on device name,
2075  * which must be in the swdevsw.  Return EBUSY
2076  * if already swapping on this device.
2077  */
2078 #ifndef _SYS_SYSPROTO_H_
2079 struct swapon_args {
2080 	char *name;
2081 };
2082 #endif
2083 
2084 /*
2085  * MPSAFE
2086  */
2087 /* ARGSUSED */
2088 int
2089 sys_swapon(struct thread *td, struct swapon_args *uap)
2090 {
2091 	struct vattr attr;
2092 	struct vnode *vp;
2093 	struct nameidata nd;
2094 	int error;
2095 
2096 	error = priv_check(td, PRIV_SWAPON);
2097 	if (error)
2098 		return (error);
2099 
2100 	mtx_lock(&Giant);
2101 	while (swdev_syscall_active)
2102 	    tsleep(&swdev_syscall_active, PUSER - 1, "swpon", 0);
2103 	swdev_syscall_active = 1;
2104 
2105 	/*
2106 	 * Swap metadata may not fit in the KVM if we have physical
2107 	 * memory of >1GB.
2108 	 */
2109 	if (swap_zone == NULL) {
2110 		error = ENOMEM;
2111 		goto done;
2112 	}
2113 
2114 	NDINIT(&nd, LOOKUP, ISOPEN | FOLLOW | AUDITVNODE1, UIO_USERSPACE,
2115 	    uap->name, td);
2116 	error = namei(&nd);
2117 	if (error)
2118 		goto done;
2119 
2120 	NDFREE(&nd, NDF_ONLY_PNBUF);
2121 	vp = nd.ni_vp;
2122 
2123 	if (vn_isdisk(vp, &error)) {
2124 		error = swapongeom(td, vp);
2125 	} else if (vp->v_type == VREG &&
2126 	    (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
2127 	    (error = VOP_GETATTR(vp, &attr, td->td_ucred)) == 0) {
2128 		/*
2129 		 * Allow direct swapping to NFS regular files in the same
2130 		 * way that nfs_mountroot() sets up diskless swapping.
2131 		 */
2132 		error = swaponvp(td, vp, attr.va_size / DEV_BSIZE);
2133 	}
2134 
2135 	if (error)
2136 		vrele(vp);
2137 done:
2138 	swdev_syscall_active = 0;
2139 	wakeup_one(&swdev_syscall_active);
2140 	mtx_unlock(&Giant);
2141 	return (error);
2142 }
2143 
2144 /*
2145  * Check that the total amount of swap currently configured does not
2146  * exceed half the theoretical maximum.  If it does, print a warning
2147  * message and return -1; otherwise, return 0.
2148  */
2149 static int
2150 swapon_check_swzone(unsigned long npages)
2151 {
2152 	unsigned long maxpages;
2153 
2154 	/* absolute maximum we can handle assuming 100% efficiency */
2155 	maxpages = uma_zone_get_max(swap_zone) * SWAP_META_PAGES;
2156 
2157 	/* recommend using no more than half that amount */
2158 	if (npages > maxpages / 2) {
2159 		printf("warning: total configured swap (%lu pages) "
2160 		    "exceeds maximum recommended amount (%lu pages).\n",
2161 		    npages, maxpages / 2);
2162 		printf("warning: increase kern.maxswzone "
2163 		    "or reduce amount of swap.\n");
2164 		return (-1);
2165 	}
2166 	return (0);
2167 }
2168 
2169 static void
2170 swaponsomething(struct vnode *vp, void *id, u_long nblks,
2171     sw_strategy_t *strategy, sw_close_t *close, dev_t dev, int flags)
2172 {
2173 	struct swdevt *sp, *tsp;
2174 	swblk_t dvbase;
2175 	u_long mblocks;
2176 
2177 	/*
2178 	 * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks.
2179 	 * First chop nblks off to page-align it, then convert.
2180 	 *
2181 	 * sw->sw_nblks is in page-sized chunks now too.
2182 	 */
2183 	nblks &= ~(ctodb(1) - 1);
2184 	nblks = dbtoc(nblks);
2185 
2186 	/*
2187 	 * If we go beyond this, we get overflows in the radix
2188 	 * tree bitmap code.
2189 	 */
2190 	mblocks = 0x40000000 / BLIST_META_RADIX;
2191 	if (nblks > mblocks) {
2192 		printf(
2193     "WARNING: reducing swap size to maximum of %luMB per unit\n",
2194 		    mblocks / 1024 / 1024 * PAGE_SIZE);
2195 		nblks = mblocks;
2196 	}
2197 
2198 	sp = malloc(sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO);
2199 	sp->sw_vp = vp;
2200 	sp->sw_id = id;
2201 	sp->sw_dev = dev;
2202 	sp->sw_flags = 0;
2203 	sp->sw_nblks = nblks;
2204 	sp->sw_used = 0;
2205 	sp->sw_strategy = strategy;
2206 	sp->sw_close = close;
2207 	sp->sw_flags = flags;
2208 
2209 	sp->sw_blist = blist_create(nblks, M_WAITOK);
2210 	/*
2211 	 * Do not free the first two block in order to avoid overwriting
2212 	 * any bsd label at the front of the partition
2213 	 */
2214 	blist_free(sp->sw_blist, 2, nblks - 2);
2215 
2216 	dvbase = 0;
2217 	mtx_lock(&sw_dev_mtx);
2218 	TAILQ_FOREACH(tsp, &swtailq, sw_list) {
2219 		if (tsp->sw_end >= dvbase) {
2220 			/*
2221 			 * We put one uncovered page between the devices
2222 			 * in order to definitively prevent any cross-device
2223 			 * I/O requests
2224 			 */
2225 			dvbase = tsp->sw_end + 1;
2226 		}
2227 	}
2228 	sp->sw_first = dvbase;
2229 	sp->sw_end = dvbase + nblks;
2230 	TAILQ_INSERT_TAIL(&swtailq, sp, sw_list);
2231 	nswapdev++;
2232 	swap_pager_avail += nblks;
2233 	swap_total += (vm_ooffset_t)nblks * PAGE_SIZE;
2234 	swapon_check_swzone(swap_total / PAGE_SIZE);
2235 	swp_sizecheck();
2236 	mtx_unlock(&sw_dev_mtx);
2237 }
2238 
2239 /*
2240  * SYSCALL: swapoff(devname)
2241  *
2242  * Disable swapping on the given device.
2243  *
2244  * XXX: Badly designed system call: it should use a device index
2245  * rather than filename as specification.  We keep sw_vp around
2246  * only to make this work.
2247  */
2248 #ifndef _SYS_SYSPROTO_H_
2249 struct swapoff_args {
2250 	char *name;
2251 };
2252 #endif
2253 
2254 /*
2255  * MPSAFE
2256  */
2257 /* ARGSUSED */
2258 int
2259 sys_swapoff(struct thread *td, struct swapoff_args *uap)
2260 {
2261 	struct vnode *vp;
2262 	struct nameidata nd;
2263 	struct swdevt *sp;
2264 	int error;
2265 
2266 	error = priv_check(td, PRIV_SWAPOFF);
2267 	if (error)
2268 		return (error);
2269 
2270 	mtx_lock(&Giant);
2271 	while (swdev_syscall_active)
2272 	    tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
2273 	swdev_syscall_active = 1;
2274 
2275 	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, UIO_USERSPACE, uap->name,
2276 	    td);
2277 	error = namei(&nd);
2278 	if (error)
2279 		goto done;
2280 	NDFREE(&nd, NDF_ONLY_PNBUF);
2281 	vp = nd.ni_vp;
2282 
2283 	mtx_lock(&sw_dev_mtx);
2284 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2285 		if (sp->sw_vp == vp)
2286 			break;
2287 	}
2288 	mtx_unlock(&sw_dev_mtx);
2289 	if (sp == NULL) {
2290 		error = EINVAL;
2291 		goto done;
2292 	}
2293 	error = swapoff_one(sp, td->td_ucred);
2294 done:
2295 	swdev_syscall_active = 0;
2296 	wakeup_one(&swdev_syscall_active);
2297 	mtx_unlock(&Giant);
2298 	return (error);
2299 }
2300 
2301 static int
2302 swapoff_one(struct swdevt *sp, struct ucred *cred)
2303 {
2304 	u_long nblks, dvbase;
2305 #ifdef MAC
2306 	int error;
2307 #endif
2308 
2309 	mtx_assert(&Giant, MA_OWNED);
2310 #ifdef MAC
2311 	(void) vn_lock(sp->sw_vp, LK_EXCLUSIVE | LK_RETRY);
2312 	error = mac_system_check_swapoff(cred, sp->sw_vp);
2313 	(void) VOP_UNLOCK(sp->sw_vp, 0);
2314 	if (error != 0)
2315 		return (error);
2316 #endif
2317 	nblks = sp->sw_nblks;
2318 
2319 	/*
2320 	 * We can turn off this swap device safely only if the
2321 	 * available virtual memory in the system will fit the amount
2322 	 * of data we will have to page back in, plus an epsilon so
2323 	 * the system doesn't become critically low on swap space.
2324 	 */
2325 	if (vm_cnt.v_free_count + vm_cnt.v_cache_count + swap_pager_avail <
2326 	    nblks + nswap_lowat) {
2327 		return (ENOMEM);
2328 	}
2329 
2330 	/*
2331 	 * Prevent further allocations on this device.
2332 	 */
2333 	mtx_lock(&sw_dev_mtx);
2334 	sp->sw_flags |= SW_CLOSING;
2335 	for (dvbase = 0; dvbase < sp->sw_end; dvbase += dmmax) {
2336 		swap_pager_avail -= blist_fill(sp->sw_blist,
2337 		     dvbase, dmmax);
2338 	}
2339 	swap_total -= (vm_ooffset_t)nblks * PAGE_SIZE;
2340 	mtx_unlock(&sw_dev_mtx);
2341 
2342 	/*
2343 	 * Page in the contents of the device and close it.
2344 	 */
2345 	swap_pager_swapoff(sp);
2346 
2347 	sp->sw_close(curthread, sp);
2348 	mtx_lock(&sw_dev_mtx);
2349 	sp->sw_id = NULL;
2350 	TAILQ_REMOVE(&swtailq, sp, sw_list);
2351 	nswapdev--;
2352 	if (nswapdev == 0) {
2353 		swap_pager_full = 2;
2354 		swap_pager_almost_full = 1;
2355 	}
2356 	if (swdevhd == sp)
2357 		swdevhd = NULL;
2358 	mtx_unlock(&sw_dev_mtx);
2359 	blist_destroy(sp->sw_blist);
2360 	free(sp, M_VMPGDATA);
2361 	return (0);
2362 }
2363 
2364 void
2365 swapoff_all(void)
2366 {
2367 	struct swdevt *sp, *spt;
2368 	const char *devname;
2369 	int error;
2370 
2371 	mtx_lock(&Giant);
2372 	while (swdev_syscall_active)
2373 		tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
2374 	swdev_syscall_active = 1;
2375 
2376 	mtx_lock(&sw_dev_mtx);
2377 	TAILQ_FOREACH_SAFE(sp, &swtailq, sw_list, spt) {
2378 		mtx_unlock(&sw_dev_mtx);
2379 		if (vn_isdisk(sp->sw_vp, NULL))
2380 			devname = devtoname(sp->sw_vp->v_rdev);
2381 		else
2382 			devname = "[file]";
2383 		error = swapoff_one(sp, thread0.td_ucred);
2384 		if (error != 0) {
2385 			printf("Cannot remove swap device %s (error=%d), "
2386 			    "skipping.\n", devname, error);
2387 		} else if (bootverbose) {
2388 			printf("Swap device %s removed.\n", devname);
2389 		}
2390 		mtx_lock(&sw_dev_mtx);
2391 	}
2392 	mtx_unlock(&sw_dev_mtx);
2393 
2394 	swdev_syscall_active = 0;
2395 	wakeup_one(&swdev_syscall_active);
2396 	mtx_unlock(&Giant);
2397 }
2398 
2399 void
2400 swap_pager_status(int *total, int *used)
2401 {
2402 	struct swdevt *sp;
2403 
2404 	*total = 0;
2405 	*used = 0;
2406 	mtx_lock(&sw_dev_mtx);
2407 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2408 		*total += sp->sw_nblks;
2409 		*used += sp->sw_used;
2410 	}
2411 	mtx_unlock(&sw_dev_mtx);
2412 }
2413 
2414 int
2415 swap_dev_info(int name, struct xswdev *xs, char *devname, size_t len)
2416 {
2417 	struct swdevt *sp;
2418 	const char *tmp_devname;
2419 	int error, n;
2420 
2421 	n = 0;
2422 	error = ENOENT;
2423 	mtx_lock(&sw_dev_mtx);
2424 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2425 		if (n != name) {
2426 			n++;
2427 			continue;
2428 		}
2429 		xs->xsw_version = XSWDEV_VERSION;
2430 		xs->xsw_dev = sp->sw_dev;
2431 		xs->xsw_flags = sp->sw_flags;
2432 		xs->xsw_nblks = sp->sw_nblks;
2433 		xs->xsw_used = sp->sw_used;
2434 		if (devname != NULL) {
2435 			if (vn_isdisk(sp->sw_vp, NULL))
2436 				tmp_devname = devtoname(sp->sw_vp->v_rdev);
2437 			else
2438 				tmp_devname = "[file]";
2439 			strncpy(devname, tmp_devname, len);
2440 		}
2441 		error = 0;
2442 		break;
2443 	}
2444 	mtx_unlock(&sw_dev_mtx);
2445 	return (error);
2446 }
2447 
2448 static int
2449 sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS)
2450 {
2451 	struct xswdev xs;
2452 	int error;
2453 
2454 	if (arg2 != 1)			/* name length */
2455 		return (EINVAL);
2456 	error = swap_dev_info(*(int *)arg1, &xs, NULL, 0);
2457 	if (error != 0)
2458 		return (error);
2459 	error = SYSCTL_OUT(req, &xs, sizeof(xs));
2460 	return (error);
2461 }
2462 
2463 SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, &nswapdev, 0,
2464     "Number of swap devices");
2465 SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD, sysctl_vm_swap_info,
2466     "Swap statistics by device");
2467 
2468 /*
2469  * vmspace_swap_count() - count the approximate swap usage in pages for a
2470  *			  vmspace.
2471  *
2472  *	The map must be locked.
2473  *
2474  *	Swap usage is determined by taking the proportional swap used by
2475  *	VM objects backing the VM map.  To make up for fractional losses,
2476  *	if the VM object has any swap use at all the associated map entries
2477  *	count for at least 1 swap page.
2478  */
2479 long
2480 vmspace_swap_count(struct vmspace *vmspace)
2481 {
2482 	vm_map_t map;
2483 	vm_map_entry_t cur;
2484 	vm_object_t object;
2485 	long count, n;
2486 
2487 	map = &vmspace->vm_map;
2488 	count = 0;
2489 
2490 	for (cur = map->header.next; cur != &map->header; cur = cur->next) {
2491 		if ((cur->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
2492 		    (object = cur->object.vm_object) != NULL) {
2493 			VM_OBJECT_WLOCK(object);
2494 			if (object->type == OBJT_SWAP &&
2495 			    object->un_pager.swp.swp_bcount != 0) {
2496 				n = (cur->end - cur->start) / PAGE_SIZE;
2497 				count += object->un_pager.swp.swp_bcount *
2498 				    SWAP_META_PAGES * n / object->size + 1;
2499 			}
2500 			VM_OBJECT_WUNLOCK(object);
2501 		}
2502 	}
2503 	return (count);
2504 }
2505 
2506 /*
2507  * GEOM backend
2508  *
2509  * Swapping onto disk devices.
2510  *
2511  */
2512 
2513 static g_orphan_t swapgeom_orphan;
2514 
2515 static struct g_class g_swap_class = {
2516 	.name = "SWAP",
2517 	.version = G_VERSION,
2518 	.orphan = swapgeom_orphan,
2519 };
2520 
2521 DECLARE_GEOM_CLASS(g_swap_class, g_class);
2522 
2523 
2524 static void
2525 swapgeom_close_ev(void *arg, int flags)
2526 {
2527 	struct g_consumer *cp;
2528 
2529 	cp = arg;
2530 	g_access(cp, -1, -1, 0);
2531 	g_detach(cp);
2532 	g_destroy_consumer(cp);
2533 }
2534 
2535 /*
2536  * Add a reference to the g_consumer for an inflight transaction.
2537  */
2538 static void
2539 swapgeom_acquire(struct g_consumer *cp)
2540 {
2541 
2542 	mtx_assert(&sw_dev_mtx, MA_OWNED);
2543 	cp->index++;
2544 }
2545 
2546 /*
2547  * Remove a reference from the g_consumer. Post a close event if
2548  * all referneces go away.
2549  */
2550 static void
2551 swapgeom_release(struct g_consumer *cp, struct swdevt *sp)
2552 {
2553 
2554 	mtx_assert(&sw_dev_mtx, MA_OWNED);
2555 	cp->index--;
2556 	if (cp->index == 0) {
2557 		if (g_post_event(swapgeom_close_ev, cp, M_NOWAIT, NULL) == 0)
2558 			sp->sw_id = NULL;
2559 	}
2560 }
2561 
2562 static void
2563 swapgeom_done(struct bio *bp2)
2564 {
2565 	struct swdevt *sp;
2566 	struct buf *bp;
2567 	struct g_consumer *cp;
2568 
2569 	bp = bp2->bio_caller2;
2570 	cp = bp2->bio_from;
2571 	bp->b_ioflags = bp2->bio_flags;
2572 	if (bp2->bio_error)
2573 		bp->b_ioflags |= BIO_ERROR;
2574 	bp->b_resid = bp->b_bcount - bp2->bio_completed;
2575 	bp->b_error = bp2->bio_error;
2576 	bufdone(bp);
2577 	sp = bp2->bio_caller1;
2578 	mtx_lock(&sw_dev_mtx);
2579 	swapgeom_release(cp, sp);
2580 	mtx_unlock(&sw_dev_mtx);
2581 	g_destroy_bio(bp2);
2582 }
2583 
2584 static void
2585 swapgeom_strategy(struct buf *bp, struct swdevt *sp)
2586 {
2587 	struct bio *bio;
2588 	struct g_consumer *cp;
2589 
2590 	mtx_lock(&sw_dev_mtx);
2591 	cp = sp->sw_id;
2592 	if (cp == NULL) {
2593 		mtx_unlock(&sw_dev_mtx);
2594 		bp->b_error = ENXIO;
2595 		bp->b_ioflags |= BIO_ERROR;
2596 		bufdone(bp);
2597 		return;
2598 	}
2599 	swapgeom_acquire(cp);
2600 	mtx_unlock(&sw_dev_mtx);
2601 	if (bp->b_iocmd == BIO_WRITE)
2602 		bio = g_new_bio();
2603 	else
2604 		bio = g_alloc_bio();
2605 	if (bio == NULL) {
2606 		mtx_lock(&sw_dev_mtx);
2607 		swapgeom_release(cp, sp);
2608 		mtx_unlock(&sw_dev_mtx);
2609 		bp->b_error = ENOMEM;
2610 		bp->b_ioflags |= BIO_ERROR;
2611 		bufdone(bp);
2612 		return;
2613 	}
2614 
2615 	bio->bio_caller1 = sp;
2616 	bio->bio_caller2 = bp;
2617 	bio->bio_cmd = bp->b_iocmd;
2618 	bio->bio_offset = (bp->b_blkno - sp->sw_first) * PAGE_SIZE;
2619 	bio->bio_length = bp->b_bcount;
2620 	bio->bio_done = swapgeom_done;
2621 	if (!buf_mapped(bp)) {
2622 		bio->bio_ma = bp->b_pages;
2623 		bio->bio_data = unmapped_buf;
2624 		bio->bio_ma_offset = (vm_offset_t)bp->b_offset & PAGE_MASK;
2625 		bio->bio_ma_n = bp->b_npages;
2626 		bio->bio_flags |= BIO_UNMAPPED;
2627 	} else {
2628 		bio->bio_data = bp->b_data;
2629 		bio->bio_ma = NULL;
2630 	}
2631 	g_io_request(bio, cp);
2632 	return;
2633 }
2634 
2635 static void
2636 swapgeom_orphan(struct g_consumer *cp)
2637 {
2638 	struct swdevt *sp;
2639 	int destroy;
2640 
2641 	mtx_lock(&sw_dev_mtx);
2642 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2643 		if (sp->sw_id == cp) {
2644 			sp->sw_flags |= SW_CLOSING;
2645 			break;
2646 		}
2647 	}
2648 	/*
2649 	 * Drop reference we were created with. Do directly since we're in a
2650 	 * special context where we don't have to queue the call to
2651 	 * swapgeom_close_ev().
2652 	 */
2653 	cp->index--;
2654 	destroy = ((sp != NULL) && (cp->index == 0));
2655 	if (destroy)
2656 		sp->sw_id = NULL;
2657 	mtx_unlock(&sw_dev_mtx);
2658 	if (destroy)
2659 		swapgeom_close_ev(cp, 0);
2660 }
2661 
2662 static void
2663 swapgeom_close(struct thread *td, struct swdevt *sw)
2664 {
2665 	struct g_consumer *cp;
2666 
2667 	mtx_lock(&sw_dev_mtx);
2668 	cp = sw->sw_id;
2669 	sw->sw_id = NULL;
2670 	mtx_unlock(&sw_dev_mtx);
2671 	/* XXX: direct call when Giant untangled */
2672 	if (cp != NULL)
2673 		g_waitfor_event(swapgeom_close_ev, cp, M_WAITOK, NULL);
2674 }
2675 
2676 
2677 struct swh0h0 {
2678 	struct cdev *dev;
2679 	struct vnode *vp;
2680 	int	error;
2681 };
2682 
2683 static void
2684 swapongeom_ev(void *arg, int flags)
2685 {
2686 	struct swh0h0 *swh;
2687 	struct g_provider *pp;
2688 	struct g_consumer *cp;
2689 	static struct g_geom *gp;
2690 	struct swdevt *sp;
2691 	u_long nblks;
2692 	int error;
2693 
2694 	swh = arg;
2695 	swh->error = 0;
2696 	pp = g_dev_getprovider(swh->dev);
2697 	if (pp == NULL) {
2698 		swh->error = ENODEV;
2699 		return;
2700 	}
2701 	mtx_lock(&sw_dev_mtx);
2702 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2703 		cp = sp->sw_id;
2704 		if (cp != NULL && cp->provider == pp) {
2705 			mtx_unlock(&sw_dev_mtx);
2706 			swh->error = EBUSY;
2707 			return;
2708 		}
2709 	}
2710 	mtx_unlock(&sw_dev_mtx);
2711 	if (gp == NULL)
2712 		gp = g_new_geomf(&g_swap_class, "swap");
2713 	cp = g_new_consumer(gp);
2714 	cp->index = 1;		/* Number of active I/Os, plus one for being active. */
2715 	cp->flags |=  G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
2716 	g_attach(cp, pp);
2717 	/*
2718 	 * XXX: Everytime you think you can improve the margin for
2719 	 * footshooting, somebody depends on the ability to do so:
2720 	 * savecore(8) wants to write to our swapdev so we cannot
2721 	 * set an exclusive count :-(
2722 	 */
2723 	error = g_access(cp, 1, 1, 0);
2724 	if (error) {
2725 		g_detach(cp);
2726 		g_destroy_consumer(cp);
2727 		swh->error = error;
2728 		return;
2729 	}
2730 	nblks = pp->mediasize / DEV_BSIZE;
2731 	swaponsomething(swh->vp, cp, nblks, swapgeom_strategy,
2732 	    swapgeom_close, dev2udev(swh->dev),
2733 	    (pp->flags & G_PF_ACCEPT_UNMAPPED) != 0 ? SW_UNMAPPED : 0);
2734 	swh->error = 0;
2735 }
2736 
2737 static int
2738 swapongeom(struct thread *td, struct vnode *vp)
2739 {
2740 	int error;
2741 	struct swh0h0 swh;
2742 
2743 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2744 
2745 	swh.dev = vp->v_rdev;
2746 	swh.vp = vp;
2747 	swh.error = 0;
2748 	/* XXX: direct call when Giant untangled */
2749 	error = g_waitfor_event(swapongeom_ev, &swh, M_WAITOK, NULL);
2750 	if (!error)
2751 		error = swh.error;
2752 	VOP_UNLOCK(vp, 0);
2753 	return (error);
2754 }
2755 
2756 /*
2757  * VNODE backend
2758  *
2759  * This is used mainly for network filesystem (read: probably only tested
2760  * with NFS) swapfiles.
2761  *
2762  */
2763 
2764 static void
2765 swapdev_strategy(struct buf *bp, struct swdevt *sp)
2766 {
2767 	struct vnode *vp2;
2768 
2769 	bp->b_blkno = ctodb(bp->b_blkno - sp->sw_first);
2770 
2771 	vp2 = sp->sw_id;
2772 	vhold(vp2);
2773 	if (bp->b_iocmd == BIO_WRITE) {
2774 		if (bp->b_bufobj)
2775 			bufobj_wdrop(bp->b_bufobj);
2776 		bufobj_wref(&vp2->v_bufobj);
2777 	}
2778 	if (bp->b_bufobj != &vp2->v_bufobj)
2779 		bp->b_bufobj = &vp2->v_bufobj;
2780 	bp->b_vp = vp2;
2781 	bp->b_iooffset = dbtob(bp->b_blkno);
2782 	bstrategy(bp);
2783 	return;
2784 }
2785 
2786 static void
2787 swapdev_close(struct thread *td, struct swdevt *sp)
2788 {
2789 
2790 	VOP_CLOSE(sp->sw_vp, FREAD | FWRITE, td->td_ucred, td);
2791 	vrele(sp->sw_vp);
2792 }
2793 
2794 
2795 static int
2796 swaponvp(struct thread *td, struct vnode *vp, u_long nblks)
2797 {
2798 	struct swdevt *sp;
2799 	int error;
2800 
2801 	if (nblks == 0)
2802 		return (ENXIO);
2803 	mtx_lock(&sw_dev_mtx);
2804 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2805 		if (sp->sw_id == vp) {
2806 			mtx_unlock(&sw_dev_mtx);
2807 			return (EBUSY);
2808 		}
2809 	}
2810 	mtx_unlock(&sw_dev_mtx);
2811 
2812 	(void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2813 #ifdef MAC
2814 	error = mac_system_check_swapon(td->td_ucred, vp);
2815 	if (error == 0)
2816 #endif
2817 		error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, NULL);
2818 	(void) VOP_UNLOCK(vp, 0);
2819 	if (error)
2820 		return (error);
2821 
2822 	swaponsomething(vp, vp, nblks, swapdev_strategy, swapdev_close,
2823 	    NODEV, 0);
2824 	return (0);
2825 }
2826 
2827 static int
2828 sysctl_swap_async_max(SYSCTL_HANDLER_ARGS)
2829 {
2830 	int error, new, n;
2831 
2832 	new = nsw_wcount_async_max;
2833 	error = sysctl_handle_int(oidp, &new, 0, req);
2834 	if (error != 0 || req->newptr == NULL)
2835 		return (error);
2836 
2837 	if (new > nswbuf / 2 || new < 1)
2838 		return (EINVAL);
2839 
2840 	mtx_lock(&pbuf_mtx);
2841 	while (nsw_wcount_async_max != new) {
2842 		/*
2843 		 * Adjust difference.  If the current async count is too low,
2844 		 * we will need to sqeeze our update slowly in.  Sleep with a
2845 		 * higher priority than getpbuf() to finish faster.
2846 		 */
2847 		n = new - nsw_wcount_async_max;
2848 		if (nsw_wcount_async + n >= 0) {
2849 			nsw_wcount_async += n;
2850 			nsw_wcount_async_max += n;
2851 			wakeup(&nsw_wcount_async);
2852 		} else {
2853 			nsw_wcount_async_max -= nsw_wcount_async;
2854 			nsw_wcount_async = 0;
2855 			msleep(&nsw_wcount_async, &pbuf_mtx, PSWP,
2856 			    "swpsysctl", 0);
2857 		}
2858 	}
2859 	mtx_unlock(&pbuf_mtx);
2860 
2861 	return (0);
2862 }
2863