xref: /freebsd/sys/vm/swap_pager.c (revision f5f7c05209ca2c3748fd8b27c5e80ffad49120eb)
1 /*-
2  * Copyright (c) 1998 Matthew Dillon,
3  * Copyright (c) 1994 John S. Dyson
4  * Copyright (c) 1990 University of Utah.
5  * Copyright (c) 1982, 1986, 1989, 1993
6  *	The Regents of the University of California.  All rights reserved.
7  *
8  * This code is derived from software contributed to Berkeley by
9  * the Systems Programming Group of the University of Utah Computer
10  * Science Department.
11  *
12  * Redistribution and use in source and binary forms, with or without
13  * modification, are permitted provided that the following conditions
14  * are met:
15  * 1. Redistributions of source code must retain the above copyright
16  *    notice, this list of conditions and the following disclaimer.
17  * 2. Redistributions in binary form must reproduce the above copyright
18  *    notice, this list of conditions and the following disclaimer in the
19  *    documentation and/or other materials provided with the distribution.
20  * 3. All advertising materials mentioning features or use of this software
21  *    must display the following acknowledgement:
22  *	This product includes software developed by the University of
23  *	California, Berkeley and its contributors.
24  * 4. Neither the name of the University nor the names of its contributors
25  *    may be used to endorse or promote products derived from this software
26  *    without specific prior written permission.
27  *
28  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
29  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
32  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38  * SUCH DAMAGE.
39  *
40  *				New Swap System
41  *				Matthew Dillon
42  *
43  * Radix Bitmap 'blists'.
44  *
45  *	- The new swapper uses the new radix bitmap code.  This should scale
46  *	  to arbitrarily small or arbitrarily large swap spaces and an almost
47  *	  arbitrary degree of fragmentation.
48  *
49  * Features:
50  *
51  *	- on the fly reallocation of swap during putpages.  The new system
52  *	  does not try to keep previously allocated swap blocks for dirty
53  *	  pages.
54  *
55  *	- on the fly deallocation of swap
56  *
57  *	- No more garbage collection required.  Unnecessarily allocated swap
58  *	  blocks only exist for dirty vm_page_t's now and these are already
59  *	  cycled (in a high-load system) by the pager.  We also do on-the-fly
60  *	  removal of invalidated swap blocks when a page is destroyed
61  *	  or renamed.
62  *
63  * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
64  *
65  *	@(#)swap_pager.c	8.9 (Berkeley) 3/21/94
66  *	@(#)vm_swap.c	8.5 (Berkeley) 2/17/94
67  */
68 
69 #include <sys/cdefs.h>
70 __FBSDID("$FreeBSD$");
71 
72 #include "opt_swap.h"
73 #include "opt_vm.h"
74 
75 #include <sys/param.h>
76 #include <sys/systm.h>
77 #include <sys/conf.h>
78 #include <sys/kernel.h>
79 #include <sys/priv.h>
80 #include <sys/proc.h>
81 #include <sys/bio.h>
82 #include <sys/buf.h>
83 #include <sys/disk.h>
84 #include <sys/fcntl.h>
85 #include <sys/mount.h>
86 #include <sys/namei.h>
87 #include <sys/vnode.h>
88 #include <sys/malloc.h>
89 #include <sys/racct.h>
90 #include <sys/resource.h>
91 #include <sys/resourcevar.h>
92 #include <sys/sysctl.h>
93 #include <sys/sysproto.h>
94 #include <sys/blist.h>
95 #include <sys/lock.h>
96 #include <sys/sx.h>
97 #include <sys/vmmeter.h>
98 
99 #include <security/mac/mac_framework.h>
100 
101 #include <vm/vm.h>
102 #include <vm/pmap.h>
103 #include <vm/vm_map.h>
104 #include <vm/vm_kern.h>
105 #include <vm/vm_object.h>
106 #include <vm/vm_page.h>
107 #include <vm/vm_pager.h>
108 #include <vm/vm_pageout.h>
109 #include <vm/vm_param.h>
110 #include <vm/swap_pager.h>
111 #include <vm/vm_extern.h>
112 #include <vm/uma.h>
113 
114 #include <geom/geom.h>
115 
116 /*
117  * SWB_NPAGES must be a power of 2.  It may be set to 1, 2, 4, 8, 16
118  * or 32 pages per allocation.
119  * The 32-page limit is due to the radix code (kern/subr_blist.c).
120  */
121 #ifndef MAX_PAGEOUT_CLUSTER
122 #define MAX_PAGEOUT_CLUSTER 16
123 #endif
124 
125 #if !defined(SWB_NPAGES)
126 #define SWB_NPAGES	MAX_PAGEOUT_CLUSTER
127 #endif
128 
129 /*
130  * The swblock structure maps an object and a small, fixed-size range
131  * of page indices to disk addresses within a swap area.
132  * The collection of these mappings is implemented as a hash table.
133  * Unused disk addresses within a swap area are allocated and managed
134  * using a blist.
135  */
136 #define SWCORRECT(n) (sizeof(void *) * (n) / sizeof(daddr_t))
137 #define SWAP_META_PAGES		(SWB_NPAGES * 2)
138 #define SWAP_META_MASK		(SWAP_META_PAGES - 1)
139 
140 struct swblock {
141 	struct swblock	*swb_hnext;
142 	vm_object_t	swb_object;
143 	vm_pindex_t	swb_index;
144 	int		swb_count;
145 	daddr_t		swb_pages[SWAP_META_PAGES];
146 };
147 
148 static MALLOC_DEFINE(M_VMPGDATA, "vm_pgdata", "swap pager private data");
149 static struct mtx sw_dev_mtx;
150 static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq);
151 static struct swdevt *swdevhd;	/* Allocate from here next */
152 static int nswapdev;		/* Number of swap devices */
153 int swap_pager_avail;
154 static int swdev_syscall_active = 0; /* serialize swap(on|off) */
155 
156 static vm_ooffset_t swap_total;
157 SYSCTL_QUAD(_vm, OID_AUTO, swap_total, CTLFLAG_RD, &swap_total, 0,
158     "Total amount of available swap storage.");
159 static vm_ooffset_t swap_reserved;
160 SYSCTL_QUAD(_vm, OID_AUTO, swap_reserved, CTLFLAG_RD, &swap_reserved, 0,
161     "Amount of swap storage needed to back all allocated anonymous memory.");
162 static int overcommit = 0;
163 SYSCTL_INT(_vm, OID_AUTO, overcommit, CTLFLAG_RW, &overcommit, 0,
164     "Configure virtual memory overcommit behavior. See tuning(7) "
165     "for details.");
166 
167 /* bits from overcommit */
168 #define	SWAP_RESERVE_FORCE_ON		(1 << 0)
169 #define	SWAP_RESERVE_RLIMIT_ON		(1 << 1)
170 #define	SWAP_RESERVE_ALLOW_NONWIRED	(1 << 2)
171 
172 int
173 swap_reserve(vm_ooffset_t incr)
174 {
175 
176 	return (swap_reserve_by_cred(incr, curthread->td_ucred));
177 }
178 
179 int
180 swap_reserve_by_cred(vm_ooffset_t incr, struct ucred *cred)
181 {
182 	vm_ooffset_t r, s;
183 	int res, error;
184 	static int curfail;
185 	static struct timeval lastfail;
186 	struct uidinfo *uip;
187 
188 	uip = cred->cr_ruidinfo;
189 
190 	if (incr & PAGE_MASK)
191 		panic("swap_reserve: & PAGE_MASK");
192 
193 #ifdef RACCT
194 	PROC_LOCK(curproc);
195 	error = racct_add(curproc, RACCT_SWAP, incr);
196 	PROC_UNLOCK(curproc);
197 	if (error != 0)
198 		return (0);
199 #endif
200 
201 	res = 0;
202 	mtx_lock(&sw_dev_mtx);
203 	r = swap_reserved + incr;
204 	if (overcommit & SWAP_RESERVE_ALLOW_NONWIRED) {
205 		s = cnt.v_page_count - cnt.v_free_reserved - cnt.v_wire_count;
206 		s *= PAGE_SIZE;
207 	} else
208 		s = 0;
209 	s += swap_total;
210 	if ((overcommit & SWAP_RESERVE_FORCE_ON) == 0 || r <= s ||
211 	    (error = priv_check(curthread, PRIV_VM_SWAP_NOQUOTA)) == 0) {
212 		res = 1;
213 		swap_reserved = r;
214 	}
215 	mtx_unlock(&sw_dev_mtx);
216 
217 	if (res) {
218 		PROC_LOCK(curproc);
219 		UIDINFO_VMSIZE_LOCK(uip);
220 		if ((overcommit & SWAP_RESERVE_RLIMIT_ON) != 0 &&
221 		    uip->ui_vmsize + incr > lim_cur(curproc, RLIMIT_SWAP) &&
222 		    priv_check(curthread, PRIV_VM_SWAP_NORLIMIT))
223 			res = 0;
224 		else
225 			uip->ui_vmsize += incr;
226 		UIDINFO_VMSIZE_UNLOCK(uip);
227 		PROC_UNLOCK(curproc);
228 		if (!res) {
229 			mtx_lock(&sw_dev_mtx);
230 			swap_reserved -= incr;
231 			mtx_unlock(&sw_dev_mtx);
232 		}
233 	}
234 	if (!res && ppsratecheck(&lastfail, &curfail, 1)) {
235 		printf("uid %d, pid %d: swap reservation for %jd bytes failed\n",
236 		    uip->ui_uid, curproc->p_pid, incr);
237 	}
238 
239 #ifdef RACCT
240 	if (!res) {
241 		PROC_LOCK(curproc);
242 		racct_sub(curproc, RACCT_SWAP, incr);
243 		PROC_UNLOCK(curproc);
244 	}
245 #endif
246 
247 	return (res);
248 }
249 
250 void
251 swap_reserve_force(vm_ooffset_t incr)
252 {
253 	struct uidinfo *uip;
254 
255 	mtx_lock(&sw_dev_mtx);
256 	swap_reserved += incr;
257 	mtx_unlock(&sw_dev_mtx);
258 
259 #ifdef RACCT
260 	PROC_LOCK(curproc);
261 	racct_add_force(curproc, RACCT_SWAP, incr);
262 	PROC_UNLOCK(curproc);
263 #endif
264 
265 	uip = curthread->td_ucred->cr_ruidinfo;
266 	PROC_LOCK(curproc);
267 	UIDINFO_VMSIZE_LOCK(uip);
268 	uip->ui_vmsize += incr;
269 	UIDINFO_VMSIZE_UNLOCK(uip);
270 	PROC_UNLOCK(curproc);
271 }
272 
273 void
274 swap_release(vm_ooffset_t decr)
275 {
276 	struct ucred *cred;
277 
278 	PROC_LOCK(curproc);
279 	cred = curthread->td_ucred;
280 	swap_release_by_cred(decr, cred);
281 	PROC_UNLOCK(curproc);
282 }
283 
284 void
285 swap_release_by_cred(vm_ooffset_t decr, struct ucred *cred)
286 {
287  	struct uidinfo *uip;
288 
289 	uip = cred->cr_ruidinfo;
290 
291 	if (decr & PAGE_MASK)
292 		panic("swap_release: & PAGE_MASK");
293 
294 	mtx_lock(&sw_dev_mtx);
295 	if (swap_reserved < decr)
296 		panic("swap_reserved < decr");
297 	swap_reserved -= decr;
298 	mtx_unlock(&sw_dev_mtx);
299 
300 	UIDINFO_VMSIZE_LOCK(uip);
301 	if (uip->ui_vmsize < decr)
302 		printf("negative vmsize for uid = %d\n", uip->ui_uid);
303 	uip->ui_vmsize -= decr;
304 	UIDINFO_VMSIZE_UNLOCK(uip);
305 
306 	racct_sub_cred(cred, RACCT_SWAP, decr);
307 }
308 
309 static void swapdev_strategy(struct buf *, struct swdevt *sw);
310 
311 #define SWM_FREE	0x02	/* free, period			*/
312 #define SWM_POP		0x04	/* pop out			*/
313 
314 int swap_pager_full = 2;	/* swap space exhaustion (task killing) */
315 static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/
316 static int nsw_rcount;		/* free read buffers			*/
317 static int nsw_wcount_sync;	/* limit write buffers / synchronous	*/
318 static int nsw_wcount_async;	/* limit write buffers / asynchronous	*/
319 static int nsw_wcount_async_max;/* assigned maximum			*/
320 static int nsw_cluster_max;	/* maximum VOP I/O allowed		*/
321 
322 static struct swblock **swhash;
323 static int swhash_mask;
324 static struct mtx swhash_mtx;
325 
326 static int swap_async_max = 4;	/* maximum in-progress async I/O's	*/
327 static struct sx sw_alloc_sx;
328 
329 
330 SYSCTL_INT(_vm, OID_AUTO, swap_async_max,
331 	CTLFLAG_RW, &swap_async_max, 0, "Maximum running async swap ops");
332 
333 /*
334  * "named" and "unnamed" anon region objects.  Try to reduce the overhead
335  * of searching a named list by hashing it just a little.
336  */
337 
338 #define NOBJLISTS		8
339 
340 #define NOBJLIST(handle)	\
341 	(&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)])
342 
343 static struct mtx sw_alloc_mtx;	/* protect list manipulation */
344 static struct pagerlst	swap_pager_object_list[NOBJLISTS];
345 static uma_zone_t	swap_zone;
346 static struct vm_object	swap_zone_obj;
347 
348 /*
349  * pagerops for OBJT_SWAP - "swap pager".  Some ops are also global procedure
350  * calls hooked from other parts of the VM system and do not appear here.
351  * (see vm/swap_pager.h).
352  */
353 static vm_object_t
354 		swap_pager_alloc(void *handle, vm_ooffset_t size,
355 		    vm_prot_t prot, vm_ooffset_t offset, struct ucred *);
356 static void	swap_pager_dealloc(vm_object_t object);
357 static int	swap_pager_getpages(vm_object_t, vm_page_t *, int, int);
358 static void	swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *);
359 static boolean_t
360 		swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
361 static void	swap_pager_init(void);
362 static void	swap_pager_unswapped(vm_page_t);
363 static void	swap_pager_swapoff(struct swdevt *sp);
364 
365 struct pagerops swappagerops = {
366 	.pgo_init =	swap_pager_init,	/* early system initialization of pager	*/
367 	.pgo_alloc =	swap_pager_alloc,	/* allocate an OBJT_SWAP object		*/
368 	.pgo_dealloc =	swap_pager_dealloc,	/* deallocate an OBJT_SWAP object	*/
369 	.pgo_getpages =	swap_pager_getpages,	/* pagein				*/
370 	.pgo_putpages =	swap_pager_putpages,	/* pageout				*/
371 	.pgo_haspage =	swap_pager_haspage,	/* get backing store status for page	*/
372 	.pgo_pageunswapped = swap_pager_unswapped,	/* remove swap related to page		*/
373 };
374 
375 /*
376  * dmmax is in page-sized chunks with the new swap system.  It was
377  * dev-bsized chunks in the old.  dmmax is always a power of 2.
378  *
379  * swap_*() routines are externally accessible.  swp_*() routines are
380  * internal.
381  */
382 static int dmmax;
383 static int nswap_lowat = 128;	/* in pages, swap_pager_almost_full warn */
384 static int nswap_hiwat = 512;	/* in pages, swap_pager_almost_full warn */
385 
386 SYSCTL_INT(_vm, OID_AUTO, dmmax,
387 	CTLFLAG_RD, &dmmax, 0, "Maximum size of a swap block");
388 
389 static void	swp_sizecheck(void);
390 static void	swp_pager_async_iodone(struct buf *bp);
391 static int	swapongeom(struct thread *, struct vnode *);
392 static int	swaponvp(struct thread *, struct vnode *, u_long);
393 static int	swapoff_one(struct swdevt *sp, struct ucred *cred);
394 
395 /*
396  * Swap bitmap functions
397  */
398 static void	swp_pager_freeswapspace(daddr_t blk, int npages);
399 static daddr_t	swp_pager_getswapspace(int npages);
400 
401 /*
402  * Metadata functions
403  */
404 static struct swblock **swp_pager_hash(vm_object_t object, vm_pindex_t index);
405 static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
406 static void swp_pager_meta_free(vm_object_t, vm_pindex_t, daddr_t);
407 static void swp_pager_meta_free_all(vm_object_t);
408 static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);
409 
410 static void
411 swp_pager_free_nrpage(vm_page_t m)
412 {
413 
414 	vm_page_lock(m);
415 	if (m->wire_count == 0)
416 		vm_page_free(m);
417 	vm_page_unlock(m);
418 }
419 
420 /*
421  * SWP_SIZECHECK() -	update swap_pager_full indication
422  *
423  *	update the swap_pager_almost_full indication and warn when we are
424  *	about to run out of swap space, using lowat/hiwat hysteresis.
425  *
426  *	Clear swap_pager_full ( task killing ) indication when lowat is met.
427  *
428  *	No restrictions on call
429  *	This routine may not block.
430  */
431 static void
432 swp_sizecheck(void)
433 {
434 
435 	if (swap_pager_avail < nswap_lowat) {
436 		if (swap_pager_almost_full == 0) {
437 			printf("swap_pager: out of swap space\n");
438 			swap_pager_almost_full = 1;
439 		}
440 	} else {
441 		swap_pager_full = 0;
442 		if (swap_pager_avail > nswap_hiwat)
443 			swap_pager_almost_full = 0;
444 	}
445 }
446 
447 /*
448  * SWP_PAGER_HASH() -	hash swap meta data
449  *
450  *	This is an helper function which hashes the swapblk given
451  *	the object and page index.  It returns a pointer to a pointer
452  *	to the object, or a pointer to a NULL pointer if it could not
453  *	find a swapblk.
454  */
455 static struct swblock **
456 swp_pager_hash(vm_object_t object, vm_pindex_t index)
457 {
458 	struct swblock **pswap;
459 	struct swblock *swap;
460 
461 	index &= ~(vm_pindex_t)SWAP_META_MASK;
462 	pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask];
463 	while ((swap = *pswap) != NULL) {
464 		if (swap->swb_object == object &&
465 		    swap->swb_index == index
466 		) {
467 			break;
468 		}
469 		pswap = &swap->swb_hnext;
470 	}
471 	return (pswap);
472 }
473 
474 /*
475  * SWAP_PAGER_INIT() -	initialize the swap pager!
476  *
477  *	Expected to be started from system init.  NOTE:  This code is run
478  *	before much else so be careful what you depend on.  Most of the VM
479  *	system has yet to be initialized at this point.
480  */
481 static void
482 swap_pager_init(void)
483 {
484 	/*
485 	 * Initialize object lists
486 	 */
487 	int i;
488 
489 	for (i = 0; i < NOBJLISTS; ++i)
490 		TAILQ_INIT(&swap_pager_object_list[i]);
491 	mtx_init(&sw_alloc_mtx, "swap_pager list", NULL, MTX_DEF);
492 	mtx_init(&sw_dev_mtx, "swapdev", NULL, MTX_DEF);
493 
494 	/*
495 	 * Device Stripe, in PAGE_SIZE'd blocks
496 	 */
497 	dmmax = SWB_NPAGES * 2;
498 }
499 
500 /*
501  * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process
502  *
503  *	Expected to be started from pageout process once, prior to entering
504  *	its main loop.
505  */
506 void
507 swap_pager_swap_init(void)
508 {
509 	int n, n2;
510 
511 	/*
512 	 * Number of in-transit swap bp operations.  Don't
513 	 * exhaust the pbufs completely.  Make sure we
514 	 * initialize workable values (0 will work for hysteresis
515 	 * but it isn't very efficient).
516 	 *
517 	 * The nsw_cluster_max is constrained by the bp->b_pages[]
518 	 * array (MAXPHYS/PAGE_SIZE) and our locally defined
519 	 * MAX_PAGEOUT_CLUSTER.   Also be aware that swap ops are
520 	 * constrained by the swap device interleave stripe size.
521 	 *
522 	 * Currently we hardwire nsw_wcount_async to 4.  This limit is
523 	 * designed to prevent other I/O from having high latencies due to
524 	 * our pageout I/O.  The value 4 works well for one or two active swap
525 	 * devices but is probably a little low if you have more.  Even so,
526 	 * a higher value would probably generate only a limited improvement
527 	 * with three or four active swap devices since the system does not
528 	 * typically have to pageout at extreme bandwidths.   We will want
529 	 * at least 2 per swap devices, and 4 is a pretty good value if you
530 	 * have one NFS swap device due to the command/ack latency over NFS.
531 	 * So it all works out pretty well.
532 	 */
533 	nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER);
534 
535 	mtx_lock(&pbuf_mtx);
536 	nsw_rcount = (nswbuf + 1) / 2;
537 	nsw_wcount_sync = (nswbuf + 3) / 4;
538 	nsw_wcount_async = 4;
539 	nsw_wcount_async_max = nsw_wcount_async;
540 	mtx_unlock(&pbuf_mtx);
541 
542 	/*
543 	 * Initialize our zone.  Right now I'm just guessing on the number
544 	 * we need based on the number of pages in the system.  Each swblock
545 	 * can hold 16 pages, so this is probably overkill.  This reservation
546 	 * is typically limited to around 32MB by default.
547 	 */
548 	n = cnt.v_page_count / 2;
549 	if (maxswzone && n > maxswzone / sizeof(struct swblock))
550 		n = maxswzone / sizeof(struct swblock);
551 	n2 = n;
552 	swap_zone = uma_zcreate("SWAPMETA", sizeof(struct swblock), NULL, NULL,
553 	    NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM);
554 	if (swap_zone == NULL)
555 		panic("failed to create swap_zone.");
556 	do {
557 		if (uma_zone_set_obj(swap_zone, &swap_zone_obj, n))
558 			break;
559 		/*
560 		 * if the allocation failed, try a zone two thirds the
561 		 * size of the previous attempt.
562 		 */
563 		n -= ((n + 2) / 3);
564 	} while (n > 0);
565 	if (n2 != n)
566 		printf("Swap zone entries reduced from %d to %d.\n", n2, n);
567 	n2 = n;
568 
569 	/*
570 	 * Initialize our meta-data hash table.  The swapper does not need to
571 	 * be quite as efficient as the VM system, so we do not use an
572 	 * oversized hash table.
573 	 *
574 	 * 	n: 		size of hash table, must be power of 2
575 	 *	swhash_mask:	hash table index mask
576 	 */
577 	for (n = 1; n < n2 / 8; n *= 2)
578 		;
579 	swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO);
580 	swhash_mask = n - 1;
581 	mtx_init(&swhash_mtx, "swap_pager swhash", NULL, MTX_DEF);
582 }
583 
584 /*
585  * SWAP_PAGER_ALLOC() -	allocate a new OBJT_SWAP VM object and instantiate
586  *			its metadata structures.
587  *
588  *	This routine is called from the mmap and fork code to create a new
589  *	OBJT_SWAP object.  We do this by creating an OBJT_DEFAULT object
590  *	and then converting it with swp_pager_meta_build().
591  *
592  *	This routine may block in vm_object_allocate() and create a named
593  *	object lookup race, so we must interlock.
594  *
595  * MPSAFE
596  */
597 static vm_object_t
598 swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
599     vm_ooffset_t offset, struct ucred *cred)
600 {
601 	vm_object_t object;
602 	vm_pindex_t pindex;
603 
604 	pindex = OFF_TO_IDX(offset + PAGE_MASK + size);
605 	if (handle) {
606 		mtx_lock(&Giant);
607 		/*
608 		 * Reference existing named region or allocate new one.  There
609 		 * should not be a race here against swp_pager_meta_build()
610 		 * as called from vm_page_remove() in regards to the lookup
611 		 * of the handle.
612 		 */
613 		sx_xlock(&sw_alloc_sx);
614 		object = vm_pager_object_lookup(NOBJLIST(handle), handle);
615 		if (object == NULL) {
616 			if (cred != NULL) {
617 				if (!swap_reserve_by_cred(size, cred)) {
618 					sx_xunlock(&sw_alloc_sx);
619 					mtx_unlock(&Giant);
620 					return (NULL);
621 				}
622 				crhold(cred);
623 			}
624 			object = vm_object_allocate(OBJT_DEFAULT, pindex);
625 			VM_OBJECT_LOCK(object);
626 			object->handle = handle;
627 			if (cred != NULL) {
628 				object->cred = cred;
629 				object->charge = size;
630 			}
631 			swp_pager_meta_build(object, 0, SWAPBLK_NONE);
632 			VM_OBJECT_UNLOCK(object);
633 		}
634 		sx_xunlock(&sw_alloc_sx);
635 		mtx_unlock(&Giant);
636 	} else {
637 		if (cred != NULL) {
638 			if (!swap_reserve_by_cred(size, cred))
639 				return (NULL);
640 			crhold(cred);
641 		}
642 		object = vm_object_allocate(OBJT_DEFAULT, pindex);
643 		VM_OBJECT_LOCK(object);
644 		if (cred != NULL) {
645 			object->cred = cred;
646 			object->charge = size;
647 		}
648 		swp_pager_meta_build(object, 0, SWAPBLK_NONE);
649 		VM_OBJECT_UNLOCK(object);
650 	}
651 	return (object);
652 }
653 
654 /*
655  * SWAP_PAGER_DEALLOC() -	remove swap metadata from object
656  *
657  *	The swap backing for the object is destroyed.  The code is
658  *	designed such that we can reinstantiate it later, but this
659  *	routine is typically called only when the entire object is
660  *	about to be destroyed.
661  *
662  *	The object must be locked.
663  */
664 static void
665 swap_pager_dealloc(vm_object_t object)
666 {
667 
668 	/*
669 	 * Remove from list right away so lookups will fail if we block for
670 	 * pageout completion.
671 	 */
672 	if (object->handle != NULL) {
673 		mtx_lock(&sw_alloc_mtx);
674 		TAILQ_REMOVE(NOBJLIST(object->handle), object, pager_object_list);
675 		mtx_unlock(&sw_alloc_mtx);
676 	}
677 
678 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
679 	vm_object_pip_wait(object, "swpdea");
680 
681 	/*
682 	 * Free all remaining metadata.  We only bother to free it from
683 	 * the swap meta data.  We do not attempt to free swapblk's still
684 	 * associated with vm_page_t's for this object.  We do not care
685 	 * if paging is still in progress on some objects.
686 	 */
687 	swp_pager_meta_free_all(object);
688 }
689 
690 /************************************************************************
691  *			SWAP PAGER BITMAP ROUTINES			*
692  ************************************************************************/
693 
694 /*
695  * SWP_PAGER_GETSWAPSPACE() -	allocate raw swap space
696  *
697  *	Allocate swap for the requested number of pages.  The starting
698  *	swap block number (a page index) is returned or SWAPBLK_NONE
699  *	if the allocation failed.
700  *
701  *	Also has the side effect of advising that somebody made a mistake
702  *	when they configured swap and didn't configure enough.
703  *
704  *	This routine may not sleep.
705  *
706  *	We allocate in round-robin fashion from the configured devices.
707  */
708 static daddr_t
709 swp_pager_getswapspace(int npages)
710 {
711 	daddr_t blk;
712 	struct swdevt *sp;
713 	int i;
714 
715 	blk = SWAPBLK_NONE;
716 	mtx_lock(&sw_dev_mtx);
717 	sp = swdevhd;
718 	for (i = 0; i < nswapdev; i++) {
719 		if (sp == NULL)
720 			sp = TAILQ_FIRST(&swtailq);
721 		if (!(sp->sw_flags & SW_CLOSING)) {
722 			blk = blist_alloc(sp->sw_blist, npages);
723 			if (blk != SWAPBLK_NONE) {
724 				blk += sp->sw_first;
725 				sp->sw_used += npages;
726 				swap_pager_avail -= npages;
727 				swp_sizecheck();
728 				swdevhd = TAILQ_NEXT(sp, sw_list);
729 				goto done;
730 			}
731 		}
732 		sp = TAILQ_NEXT(sp, sw_list);
733 	}
734 	if (swap_pager_full != 2) {
735 		printf("swap_pager_getswapspace(%d): failed\n", npages);
736 		swap_pager_full = 2;
737 		swap_pager_almost_full = 1;
738 	}
739 	swdevhd = NULL;
740 done:
741 	mtx_unlock(&sw_dev_mtx);
742 	return (blk);
743 }
744 
745 static int
746 swp_pager_isondev(daddr_t blk, struct swdevt *sp)
747 {
748 
749 	return (blk >= sp->sw_first && blk < sp->sw_end);
750 }
751 
752 static void
753 swp_pager_strategy(struct buf *bp)
754 {
755 	struct swdevt *sp;
756 
757 	mtx_lock(&sw_dev_mtx);
758 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
759 		if (bp->b_blkno >= sp->sw_first && bp->b_blkno < sp->sw_end) {
760 			mtx_unlock(&sw_dev_mtx);
761 			sp->sw_strategy(bp, sp);
762 			return;
763 		}
764 	}
765 	panic("Swapdev not found");
766 }
767 
768 
769 /*
770  * SWP_PAGER_FREESWAPSPACE() -	free raw swap space
771  *
772  *	This routine returns the specified swap blocks back to the bitmap.
773  *
774  *	This routine may not sleep.
775  */
776 static void
777 swp_pager_freeswapspace(daddr_t blk, int npages)
778 {
779 	struct swdevt *sp;
780 
781 	mtx_lock(&sw_dev_mtx);
782 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
783 		if (blk >= sp->sw_first && blk < sp->sw_end) {
784 			sp->sw_used -= npages;
785 			/*
786 			 * If we are attempting to stop swapping on
787 			 * this device, we don't want to mark any
788 			 * blocks free lest they be reused.
789 			 */
790 			if ((sp->sw_flags & SW_CLOSING) == 0) {
791 				blist_free(sp->sw_blist, blk - sp->sw_first,
792 				    npages);
793 				swap_pager_avail += npages;
794 				swp_sizecheck();
795 			}
796 			mtx_unlock(&sw_dev_mtx);
797 			return;
798 		}
799 	}
800 	panic("Swapdev not found");
801 }
802 
803 /*
804  * SWAP_PAGER_FREESPACE() -	frees swap blocks associated with a page
805  *				range within an object.
806  *
807  *	This is a globally accessible routine.
808  *
809  *	This routine removes swapblk assignments from swap metadata.
810  *
811  *	The external callers of this routine typically have already destroyed
812  *	or renamed vm_page_t's associated with this range in the object so
813  *	we should be ok.
814  */
815 void
816 swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size)
817 {
818 
819 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
820 	swp_pager_meta_free(object, start, size);
821 }
822 
823 /*
824  * SWAP_PAGER_RESERVE() - reserve swap blocks in object
825  *
826  *	Assigns swap blocks to the specified range within the object.  The
827  *	swap blocks are not zerod.  Any previous swap assignment is destroyed.
828  *
829  *	Returns 0 on success, -1 on failure.
830  */
831 int
832 swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
833 {
834 	int n = 0;
835 	daddr_t blk = SWAPBLK_NONE;
836 	vm_pindex_t beg = start;	/* save start index */
837 
838 	VM_OBJECT_LOCK(object);
839 	while (size) {
840 		if (n == 0) {
841 			n = BLIST_MAX_ALLOC;
842 			while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
843 				n >>= 1;
844 				if (n == 0) {
845 					swp_pager_meta_free(object, beg, start - beg);
846 					VM_OBJECT_UNLOCK(object);
847 					return (-1);
848 				}
849 			}
850 		}
851 		swp_pager_meta_build(object, start, blk);
852 		--size;
853 		++start;
854 		++blk;
855 		--n;
856 	}
857 	swp_pager_meta_free(object, start, n);
858 	VM_OBJECT_UNLOCK(object);
859 	return (0);
860 }
861 
862 /*
863  * SWAP_PAGER_COPY() -  copy blocks from source pager to destination pager
864  *			and destroy the source.
865  *
866  *	Copy any valid swapblks from the source to the destination.  In
867  *	cases where both the source and destination have a valid swapblk,
868  *	we keep the destination's.
869  *
870  *	This routine is allowed to sleep.  It may sleep allocating metadata
871  *	indirectly through swp_pager_meta_build() or if paging is still in
872  *	progress on the source.
873  *
874  *	The source object contains no vm_page_t's (which is just as well)
875  *
876  *	The source object is of type OBJT_SWAP.
877  *
878  *	The source and destination objects must be locked.
879  *	Both object locks may temporarily be released.
880  */
881 void
882 swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
883     vm_pindex_t offset, int destroysource)
884 {
885 	vm_pindex_t i;
886 
887 	VM_OBJECT_LOCK_ASSERT(srcobject, MA_OWNED);
888 	VM_OBJECT_LOCK_ASSERT(dstobject, MA_OWNED);
889 
890 	/*
891 	 * If destroysource is set, we remove the source object from the
892 	 * swap_pager internal queue now.
893 	 */
894 	if (destroysource) {
895 		if (srcobject->handle != NULL) {
896 			mtx_lock(&sw_alloc_mtx);
897 			TAILQ_REMOVE(
898 			    NOBJLIST(srcobject->handle),
899 			    srcobject,
900 			    pager_object_list
901 			);
902 			mtx_unlock(&sw_alloc_mtx);
903 		}
904 	}
905 
906 	/*
907 	 * transfer source to destination.
908 	 */
909 	for (i = 0; i < dstobject->size; ++i) {
910 		daddr_t dstaddr;
911 
912 		/*
913 		 * Locate (without changing) the swapblk on the destination,
914 		 * unless it is invalid in which case free it silently, or
915 		 * if the destination is a resident page, in which case the
916 		 * source is thrown away.
917 		 */
918 		dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
919 
920 		if (dstaddr == SWAPBLK_NONE) {
921 			/*
922 			 * Destination has no swapblk and is not resident,
923 			 * copy source.
924 			 */
925 			daddr_t srcaddr;
926 
927 			srcaddr = swp_pager_meta_ctl(
928 			    srcobject,
929 			    i + offset,
930 			    SWM_POP
931 			);
932 
933 			if (srcaddr != SWAPBLK_NONE) {
934 				/*
935 				 * swp_pager_meta_build() can sleep.
936 				 */
937 				vm_object_pip_add(srcobject, 1);
938 				VM_OBJECT_UNLOCK(srcobject);
939 				vm_object_pip_add(dstobject, 1);
940 				swp_pager_meta_build(dstobject, i, srcaddr);
941 				vm_object_pip_wakeup(dstobject);
942 				VM_OBJECT_LOCK(srcobject);
943 				vm_object_pip_wakeup(srcobject);
944 			}
945 		} else {
946 			/*
947 			 * Destination has valid swapblk or it is represented
948 			 * by a resident page.  We destroy the sourceblock.
949 			 */
950 
951 			swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE);
952 		}
953 	}
954 
955 	/*
956 	 * Free left over swap blocks in source.
957 	 *
958 	 * We have to revert the type to OBJT_DEFAULT so we do not accidently
959 	 * double-remove the object from the swap queues.
960 	 */
961 	if (destroysource) {
962 		swp_pager_meta_free_all(srcobject);
963 		/*
964 		 * Reverting the type is not necessary, the caller is going
965 		 * to destroy srcobject directly, but I'm doing it here
966 		 * for consistency since we've removed the object from its
967 		 * queues.
968 		 */
969 		srcobject->type = OBJT_DEFAULT;
970 	}
971 }
972 
973 /*
974  * SWAP_PAGER_HASPAGE() -	determine if we have good backing store for
975  *				the requested page.
976  *
977  *	We determine whether good backing store exists for the requested
978  *	page and return TRUE if it does, FALSE if it doesn't.
979  *
980  *	If TRUE, we also try to determine how much valid, contiguous backing
981  *	store exists before and after the requested page within a reasonable
982  *	distance.  We do not try to restrict it to the swap device stripe
983  *	(that is handled in getpages/putpages).  It probably isn't worth
984  *	doing here.
985  */
986 static boolean_t
987 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after)
988 {
989 	daddr_t blk0;
990 
991 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
992 	/*
993 	 * do we have good backing store at the requested index ?
994 	 */
995 	blk0 = swp_pager_meta_ctl(object, pindex, 0);
996 
997 	if (blk0 == SWAPBLK_NONE) {
998 		if (before)
999 			*before = 0;
1000 		if (after)
1001 			*after = 0;
1002 		return (FALSE);
1003 	}
1004 
1005 	/*
1006 	 * find backwards-looking contiguous good backing store
1007 	 */
1008 	if (before != NULL) {
1009 		int i;
1010 
1011 		for (i = 1; i < (SWB_NPAGES/2); ++i) {
1012 			daddr_t blk;
1013 
1014 			if (i > pindex)
1015 				break;
1016 			blk = swp_pager_meta_ctl(object, pindex - i, 0);
1017 			if (blk != blk0 - i)
1018 				break;
1019 		}
1020 		*before = (i - 1);
1021 	}
1022 
1023 	/*
1024 	 * find forward-looking contiguous good backing store
1025 	 */
1026 	if (after != NULL) {
1027 		int i;
1028 
1029 		for (i = 1; i < (SWB_NPAGES/2); ++i) {
1030 			daddr_t blk;
1031 
1032 			blk = swp_pager_meta_ctl(object, pindex + i, 0);
1033 			if (blk != blk0 + i)
1034 				break;
1035 		}
1036 		*after = (i - 1);
1037 	}
1038 	return (TRUE);
1039 }
1040 
1041 /*
1042  * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page
1043  *
1044  *	This removes any associated swap backing store, whether valid or
1045  *	not, from the page.
1046  *
1047  *	This routine is typically called when a page is made dirty, at
1048  *	which point any associated swap can be freed.  MADV_FREE also
1049  *	calls us in a special-case situation
1050  *
1051  *	NOTE!!!  If the page is clean and the swap was valid, the caller
1052  *	should make the page dirty before calling this routine.  This routine
1053  *	does NOT change the m->dirty status of the page.  Also: MADV_FREE
1054  *	depends on it.
1055  *
1056  *	This routine may not sleep.
1057  */
1058 static void
1059 swap_pager_unswapped(vm_page_t m)
1060 {
1061 
1062 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1063 	swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE);
1064 }
1065 
1066 /*
1067  * SWAP_PAGER_GETPAGES() - bring pages in from swap
1068  *
1069  *	Attempt to retrieve (m, count) pages from backing store, but make
1070  *	sure we retrieve at least m[reqpage].  We try to load in as large
1071  *	a chunk surrounding m[reqpage] as is contiguous in swap and which
1072  *	belongs to the same object.
1073  *
1074  *	The code is designed for asynchronous operation and
1075  *	immediate-notification of 'reqpage' but tends not to be
1076  *	used that way.  Please do not optimize-out this algorithmic
1077  *	feature, I intend to improve on it in the future.
1078  *
1079  *	The parent has a single vm_object_pip_add() reference prior to
1080  *	calling us and we should return with the same.
1081  *
1082  *	The parent has BUSY'd the pages.  We should return with 'm'
1083  *	left busy, but the others adjusted.
1084  */
1085 static int
1086 swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage)
1087 {
1088 	struct buf *bp;
1089 	vm_page_t mreq;
1090 	int i;
1091 	int j;
1092 	daddr_t blk;
1093 
1094 	mreq = m[reqpage];
1095 
1096 	KASSERT(mreq->object == object,
1097 	    ("swap_pager_getpages: object mismatch %p/%p",
1098 	    object, mreq->object));
1099 
1100 	/*
1101 	 * Calculate range to retrieve.  The pages have already been assigned
1102 	 * their swapblks.  We require a *contiguous* range but we know it to
1103 	 * not span devices.   If we do not supply it, bad things
1104 	 * happen.  Note that blk, iblk & jblk can be SWAPBLK_NONE, but the
1105 	 * loops are set up such that the case(s) are handled implicitly.
1106 	 *
1107 	 * The swp_*() calls must be made with the object locked.
1108 	 */
1109 	blk = swp_pager_meta_ctl(mreq->object, mreq->pindex, 0);
1110 
1111 	for (i = reqpage - 1; i >= 0; --i) {
1112 		daddr_t iblk;
1113 
1114 		iblk = swp_pager_meta_ctl(m[i]->object, m[i]->pindex, 0);
1115 		if (blk != iblk + (reqpage - i))
1116 			break;
1117 	}
1118 	++i;
1119 
1120 	for (j = reqpage + 1; j < count; ++j) {
1121 		daddr_t jblk;
1122 
1123 		jblk = swp_pager_meta_ctl(m[j]->object, m[j]->pindex, 0);
1124 		if (blk != jblk - (j - reqpage))
1125 			break;
1126 	}
1127 
1128 	/*
1129 	 * free pages outside our collection range.   Note: we never free
1130 	 * mreq, it must remain busy throughout.
1131 	 */
1132 	if (0 < i || j < count) {
1133 		int k;
1134 
1135 		for (k = 0; k < i; ++k)
1136 			swp_pager_free_nrpage(m[k]);
1137 		for (k = j; k < count; ++k)
1138 			swp_pager_free_nrpage(m[k]);
1139 	}
1140 
1141 	/*
1142 	 * Return VM_PAGER_FAIL if we have nothing to do.  Return mreq
1143 	 * still busy, but the others unbusied.
1144 	 */
1145 	if (blk == SWAPBLK_NONE)
1146 		return (VM_PAGER_FAIL);
1147 
1148 	/*
1149 	 * Getpbuf() can sleep.
1150 	 */
1151 	VM_OBJECT_UNLOCK(object);
1152 	/*
1153 	 * Get a swap buffer header to perform the IO
1154 	 */
1155 	bp = getpbuf(&nsw_rcount);
1156 	bp->b_flags |= B_PAGING;
1157 
1158 	/*
1159 	 * map our page(s) into kva for input
1160 	 */
1161 	pmap_qenter((vm_offset_t)bp->b_data, m + i, j - i);
1162 
1163 	bp->b_iocmd = BIO_READ;
1164 	bp->b_iodone = swp_pager_async_iodone;
1165 	bp->b_rcred = crhold(thread0.td_ucred);
1166 	bp->b_wcred = crhold(thread0.td_ucred);
1167 	bp->b_blkno = blk - (reqpage - i);
1168 	bp->b_bcount = PAGE_SIZE * (j - i);
1169 	bp->b_bufsize = PAGE_SIZE * (j - i);
1170 	bp->b_pager.pg_reqpage = reqpage - i;
1171 
1172 	VM_OBJECT_LOCK(object);
1173 	{
1174 		int k;
1175 
1176 		for (k = i; k < j; ++k) {
1177 			bp->b_pages[k - i] = m[k];
1178 			m[k]->oflags |= VPO_SWAPINPROG;
1179 		}
1180 	}
1181 	bp->b_npages = j - i;
1182 
1183 	PCPU_INC(cnt.v_swapin);
1184 	PCPU_ADD(cnt.v_swappgsin, bp->b_npages);
1185 
1186 	/*
1187 	 * We still hold the lock on mreq, and our automatic completion routine
1188 	 * does not remove it.
1189 	 */
1190 	vm_object_pip_add(object, bp->b_npages);
1191 	VM_OBJECT_UNLOCK(object);
1192 
1193 	/*
1194 	 * perform the I/O.  NOTE!!!  bp cannot be considered valid after
1195 	 * this point because we automatically release it on completion.
1196 	 * Instead, we look at the one page we are interested in which we
1197 	 * still hold a lock on even through the I/O completion.
1198 	 *
1199 	 * The other pages in our m[] array are also released on completion,
1200 	 * so we cannot assume they are valid anymore either.
1201 	 *
1202 	 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1203 	 */
1204 	BUF_KERNPROC(bp);
1205 	swp_pager_strategy(bp);
1206 
1207 	/*
1208 	 * wait for the page we want to complete.  VPO_SWAPINPROG is always
1209 	 * cleared on completion.  If an I/O error occurs, SWAPBLK_NONE
1210 	 * is set in the meta-data.
1211 	 */
1212 	VM_OBJECT_LOCK(object);
1213 	while ((mreq->oflags & VPO_SWAPINPROG) != 0) {
1214 		mreq->oflags |= VPO_WANTED;
1215 		PCPU_INC(cnt.v_intrans);
1216 		if (msleep(mreq, VM_OBJECT_MTX(object), PSWP, "swread", hz*20)) {
1217 			printf(
1218 "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n",
1219 			    bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount);
1220 		}
1221 	}
1222 
1223 	/*
1224 	 * mreq is left busied after completion, but all the other pages
1225 	 * are freed.  If we had an unrecoverable read error the page will
1226 	 * not be valid.
1227 	 */
1228 	if (mreq->valid != VM_PAGE_BITS_ALL) {
1229 		return (VM_PAGER_ERROR);
1230 	} else {
1231 		return (VM_PAGER_OK);
1232 	}
1233 
1234 	/*
1235 	 * A final note: in a low swap situation, we cannot deallocate swap
1236 	 * and mark a page dirty here because the caller is likely to mark
1237 	 * the page clean when we return, causing the page to possibly revert
1238 	 * to all-zero's later.
1239 	 */
1240 }
1241 
1242 /*
1243  *	swap_pager_putpages:
1244  *
1245  *	Assign swap (if necessary) and initiate I/O on the specified pages.
1246  *
1247  *	We support both OBJT_DEFAULT and OBJT_SWAP objects.  DEFAULT objects
1248  *	are automatically converted to SWAP objects.
1249  *
1250  *	In a low memory situation we may block in VOP_STRATEGY(), but the new
1251  *	vm_page reservation system coupled with properly written VFS devices
1252  *	should ensure that no low-memory deadlock occurs.  This is an area
1253  *	which needs work.
1254  *
1255  *	The parent has N vm_object_pip_add() references prior to
1256  *	calling us and will remove references for rtvals[] that are
1257  *	not set to VM_PAGER_PEND.  We need to remove the rest on I/O
1258  *	completion.
1259  *
1260  *	The parent has soft-busy'd the pages it passes us and will unbusy
1261  *	those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
1262  *	We need to unbusy the rest on I/O completion.
1263  */
1264 void
1265 swap_pager_putpages(vm_object_t object, vm_page_t *m, int count,
1266     boolean_t sync, int *rtvals)
1267 {
1268 	int i;
1269 	int n = 0;
1270 
1271 	if (count && m[0]->object != object) {
1272 		panic("swap_pager_putpages: object mismatch %p/%p",
1273 		    object,
1274 		    m[0]->object
1275 		);
1276 	}
1277 
1278 	/*
1279 	 * Step 1
1280 	 *
1281 	 * Turn object into OBJT_SWAP
1282 	 * check for bogus sysops
1283 	 * force sync if not pageout process
1284 	 */
1285 	if (object->type != OBJT_SWAP)
1286 		swp_pager_meta_build(object, 0, SWAPBLK_NONE);
1287 	VM_OBJECT_UNLOCK(object);
1288 
1289 	if (curproc != pageproc)
1290 		sync = TRUE;
1291 
1292 	/*
1293 	 * Step 2
1294 	 *
1295 	 * Update nsw parameters from swap_async_max sysctl values.
1296 	 * Do not let the sysop crash the machine with bogus numbers.
1297 	 */
1298 	mtx_lock(&pbuf_mtx);
1299 	if (swap_async_max != nsw_wcount_async_max) {
1300 		int n;
1301 
1302 		/*
1303 		 * limit range
1304 		 */
1305 		if ((n = swap_async_max) > nswbuf / 2)
1306 			n = nswbuf / 2;
1307 		if (n < 1)
1308 			n = 1;
1309 		swap_async_max = n;
1310 
1311 		/*
1312 		 * Adjust difference ( if possible ).  If the current async
1313 		 * count is too low, we may not be able to make the adjustment
1314 		 * at this time.
1315 		 */
1316 		n -= nsw_wcount_async_max;
1317 		if (nsw_wcount_async + n >= 0) {
1318 			nsw_wcount_async += n;
1319 			nsw_wcount_async_max += n;
1320 			wakeup(&nsw_wcount_async);
1321 		}
1322 	}
1323 	mtx_unlock(&pbuf_mtx);
1324 
1325 	/*
1326 	 * Step 3
1327 	 *
1328 	 * Assign swap blocks and issue I/O.  We reallocate swap on the fly.
1329 	 * The page is left dirty until the pageout operation completes
1330 	 * successfully.
1331 	 */
1332 	for (i = 0; i < count; i += n) {
1333 		int j;
1334 		struct buf *bp;
1335 		daddr_t blk;
1336 
1337 		/*
1338 		 * Maximum I/O size is limited by a number of factors.
1339 		 */
1340 		n = min(BLIST_MAX_ALLOC, count - i);
1341 		n = min(n, nsw_cluster_max);
1342 
1343 		/*
1344 		 * Get biggest block of swap we can.  If we fail, fall
1345 		 * back and try to allocate a smaller block.  Don't go
1346 		 * overboard trying to allocate space if it would overly
1347 		 * fragment swap.
1348 		 */
1349 		while (
1350 		    (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE &&
1351 		    n > 4
1352 		) {
1353 			n >>= 1;
1354 		}
1355 		if (blk == SWAPBLK_NONE) {
1356 			for (j = 0; j < n; ++j)
1357 				rtvals[i+j] = VM_PAGER_FAIL;
1358 			continue;
1359 		}
1360 
1361 		/*
1362 		 * All I/O parameters have been satisfied, build the I/O
1363 		 * request and assign the swap space.
1364 		 */
1365 		if (sync == TRUE) {
1366 			bp = getpbuf(&nsw_wcount_sync);
1367 		} else {
1368 			bp = getpbuf(&nsw_wcount_async);
1369 			bp->b_flags = B_ASYNC;
1370 		}
1371 		bp->b_flags |= B_PAGING;
1372 		bp->b_iocmd = BIO_WRITE;
1373 
1374 		pmap_qenter((vm_offset_t)bp->b_data, &m[i], n);
1375 
1376 		bp->b_rcred = crhold(thread0.td_ucred);
1377 		bp->b_wcred = crhold(thread0.td_ucred);
1378 		bp->b_bcount = PAGE_SIZE * n;
1379 		bp->b_bufsize = PAGE_SIZE * n;
1380 		bp->b_blkno = blk;
1381 
1382 		VM_OBJECT_LOCK(object);
1383 		for (j = 0; j < n; ++j) {
1384 			vm_page_t mreq = m[i+j];
1385 
1386 			swp_pager_meta_build(
1387 			    mreq->object,
1388 			    mreq->pindex,
1389 			    blk + j
1390 			);
1391 			vm_page_dirty(mreq);
1392 			rtvals[i+j] = VM_PAGER_OK;
1393 
1394 			mreq->oflags |= VPO_SWAPINPROG;
1395 			bp->b_pages[j] = mreq;
1396 		}
1397 		VM_OBJECT_UNLOCK(object);
1398 		bp->b_npages = n;
1399 		/*
1400 		 * Must set dirty range for NFS to work.
1401 		 */
1402 		bp->b_dirtyoff = 0;
1403 		bp->b_dirtyend = bp->b_bcount;
1404 
1405 		PCPU_INC(cnt.v_swapout);
1406 		PCPU_ADD(cnt.v_swappgsout, bp->b_npages);
1407 
1408 		/*
1409 		 * asynchronous
1410 		 *
1411 		 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1412 		 */
1413 		if (sync == FALSE) {
1414 			bp->b_iodone = swp_pager_async_iodone;
1415 			BUF_KERNPROC(bp);
1416 			swp_pager_strategy(bp);
1417 
1418 			for (j = 0; j < n; ++j)
1419 				rtvals[i+j] = VM_PAGER_PEND;
1420 			/* restart outter loop */
1421 			continue;
1422 		}
1423 
1424 		/*
1425 		 * synchronous
1426 		 *
1427 		 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1428 		 */
1429 		bp->b_iodone = bdone;
1430 		swp_pager_strategy(bp);
1431 
1432 		/*
1433 		 * Wait for the sync I/O to complete, then update rtvals.
1434 		 * We just set the rtvals[] to VM_PAGER_PEND so we can call
1435 		 * our async completion routine at the end, thus avoiding a
1436 		 * double-free.
1437 		 */
1438 		bwait(bp, PVM, "swwrt");
1439 		for (j = 0; j < n; ++j)
1440 			rtvals[i+j] = VM_PAGER_PEND;
1441 		/*
1442 		 * Now that we are through with the bp, we can call the
1443 		 * normal async completion, which frees everything up.
1444 		 */
1445 		swp_pager_async_iodone(bp);
1446 	}
1447 	VM_OBJECT_LOCK(object);
1448 }
1449 
1450 /*
1451  *	swp_pager_async_iodone:
1452  *
1453  *	Completion routine for asynchronous reads and writes from/to swap.
1454  *	Also called manually by synchronous code to finish up a bp.
1455  *
1456  *	For READ operations, the pages are VPO_BUSY'd.  For WRITE operations,
1457  *	the pages are vm_page_t->busy'd.  For READ operations, we VPO_BUSY
1458  *	unbusy all pages except the 'main' request page.  For WRITE
1459  *	operations, we vm_page_t->busy'd unbusy all pages ( we can do this
1460  *	because we marked them all VM_PAGER_PEND on return from putpages ).
1461  *
1462  *	This routine may not sleep.
1463  */
1464 static void
1465 swp_pager_async_iodone(struct buf *bp)
1466 {
1467 	int i;
1468 	vm_object_t object = NULL;
1469 
1470 	/*
1471 	 * report error
1472 	 */
1473 	if (bp->b_ioflags & BIO_ERROR) {
1474 		printf(
1475 		    "swap_pager: I/O error - %s failed; blkno %ld,"
1476 			"size %ld, error %d\n",
1477 		    ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
1478 		    (long)bp->b_blkno,
1479 		    (long)bp->b_bcount,
1480 		    bp->b_error
1481 		);
1482 	}
1483 
1484 	/*
1485 	 * remove the mapping for kernel virtual
1486 	 */
1487 	pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
1488 
1489 	if (bp->b_npages) {
1490 		object = bp->b_pages[0]->object;
1491 		VM_OBJECT_LOCK(object);
1492 	}
1493 
1494 	/*
1495 	 * cleanup pages.  If an error occurs writing to swap, we are in
1496 	 * very serious trouble.  If it happens to be a disk error, though,
1497 	 * we may be able to recover by reassigning the swap later on.  So
1498 	 * in this case we remove the m->swapblk assignment for the page
1499 	 * but do not free it in the rlist.  The errornous block(s) are thus
1500 	 * never reallocated as swap.  Redirty the page and continue.
1501 	 */
1502 	for (i = 0; i < bp->b_npages; ++i) {
1503 		vm_page_t m = bp->b_pages[i];
1504 
1505 		m->oflags &= ~VPO_SWAPINPROG;
1506 
1507 		if (bp->b_ioflags & BIO_ERROR) {
1508 			/*
1509 			 * If an error occurs I'd love to throw the swapblk
1510 			 * away without freeing it back to swapspace, so it
1511 			 * can never be used again.  But I can't from an
1512 			 * interrupt.
1513 			 */
1514 			if (bp->b_iocmd == BIO_READ) {
1515 				/*
1516 				 * When reading, reqpage needs to stay
1517 				 * locked for the parent, but all other
1518 				 * pages can be freed.  We still want to
1519 				 * wakeup the parent waiting on the page,
1520 				 * though.  ( also: pg_reqpage can be -1 and
1521 				 * not match anything ).
1522 				 *
1523 				 * We have to wake specifically requested pages
1524 				 * up too because we cleared VPO_SWAPINPROG and
1525 				 * someone may be waiting for that.
1526 				 *
1527 				 * NOTE: for reads, m->dirty will probably
1528 				 * be overridden by the original caller of
1529 				 * getpages so don't play cute tricks here.
1530 				 */
1531 				m->valid = 0;
1532 				if (i != bp->b_pager.pg_reqpage)
1533 					swp_pager_free_nrpage(m);
1534 				else
1535 					vm_page_flash(m);
1536 				/*
1537 				 * If i == bp->b_pager.pg_reqpage, do not wake
1538 				 * the page up.  The caller needs to.
1539 				 */
1540 			} else {
1541 				/*
1542 				 * If a write error occurs, reactivate page
1543 				 * so it doesn't clog the inactive list,
1544 				 * then finish the I/O.
1545 				 */
1546 				vm_page_dirty(m);
1547 				vm_page_lock(m);
1548 				vm_page_activate(m);
1549 				vm_page_unlock(m);
1550 				vm_page_io_finish(m);
1551 			}
1552 		} else if (bp->b_iocmd == BIO_READ) {
1553 			/*
1554 			 * NOTE: for reads, m->dirty will probably be
1555 			 * overridden by the original caller of getpages so
1556 			 * we cannot set them in order to free the underlying
1557 			 * swap in a low-swap situation.  I don't think we'd
1558 			 * want to do that anyway, but it was an optimization
1559 			 * that existed in the old swapper for a time before
1560 			 * it got ripped out due to precisely this problem.
1561 			 *
1562 			 * If not the requested page then deactivate it.
1563 			 *
1564 			 * Note that the requested page, reqpage, is left
1565 			 * busied, but we still have to wake it up.  The
1566 			 * other pages are released (unbusied) by
1567 			 * vm_page_wakeup().
1568 			 */
1569 			KASSERT(!pmap_page_is_mapped(m),
1570 			    ("swp_pager_async_iodone: page %p is mapped", m));
1571 			m->valid = VM_PAGE_BITS_ALL;
1572 			KASSERT(m->dirty == 0,
1573 			    ("swp_pager_async_iodone: page %p is dirty", m));
1574 
1575 			/*
1576 			 * We have to wake specifically requested pages
1577 			 * up too because we cleared VPO_SWAPINPROG and
1578 			 * could be waiting for it in getpages.  However,
1579 			 * be sure to not unbusy getpages specifically
1580 			 * requested page - getpages expects it to be
1581 			 * left busy.
1582 			 */
1583 			if (i != bp->b_pager.pg_reqpage) {
1584 				vm_page_lock(m);
1585 				vm_page_deactivate(m);
1586 				vm_page_unlock(m);
1587 				vm_page_wakeup(m);
1588 			} else
1589 				vm_page_flash(m);
1590 		} else {
1591 			/*
1592 			 * For write success, clear the dirty
1593 			 * status, then finish the I/O ( which decrements the
1594 			 * busy count and possibly wakes waiter's up ).
1595 			 */
1596 			KASSERT(!pmap_page_is_write_mapped(m),
1597 			    ("swp_pager_async_iodone: page %p is not write"
1598 			    " protected", m));
1599 			vm_page_undirty(m);
1600 			vm_page_io_finish(m);
1601 			if (vm_page_count_severe()) {
1602 				vm_page_lock(m);
1603 				vm_page_try_to_cache(m);
1604 				vm_page_unlock(m);
1605 			}
1606 		}
1607 	}
1608 
1609 	/*
1610 	 * adjust pip.  NOTE: the original parent may still have its own
1611 	 * pip refs on the object.
1612 	 */
1613 	if (object != NULL) {
1614 		vm_object_pip_wakeupn(object, bp->b_npages);
1615 		VM_OBJECT_UNLOCK(object);
1616 	}
1617 
1618 	/*
1619 	 * swapdev_strategy() manually sets b_vp and b_bufobj before calling
1620 	 * bstrategy(). Set them back to NULL now we're done with it, or we'll
1621 	 * trigger a KASSERT in relpbuf().
1622 	 */
1623 	if (bp->b_vp) {
1624 		    bp->b_vp = NULL;
1625 		    bp->b_bufobj = NULL;
1626 	}
1627 	/*
1628 	 * release the physical I/O buffer
1629 	 */
1630 	relpbuf(
1631 	    bp,
1632 	    ((bp->b_iocmd == BIO_READ) ? &nsw_rcount :
1633 		((bp->b_flags & B_ASYNC) ?
1634 		    &nsw_wcount_async :
1635 		    &nsw_wcount_sync
1636 		)
1637 	    )
1638 	);
1639 }
1640 
1641 /*
1642  *	swap_pager_isswapped:
1643  *
1644  *	Return 1 if at least one page in the given object is paged
1645  *	out to the given swap device.
1646  *
1647  *	This routine may not sleep.
1648  */
1649 int
1650 swap_pager_isswapped(vm_object_t object, struct swdevt *sp)
1651 {
1652 	daddr_t index = 0;
1653 	int bcount;
1654 	int i;
1655 
1656 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1657 	if (object->type != OBJT_SWAP)
1658 		return (0);
1659 
1660 	mtx_lock(&swhash_mtx);
1661 	for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) {
1662 		struct swblock *swap;
1663 
1664 		if ((swap = *swp_pager_hash(object, index)) != NULL) {
1665 			for (i = 0; i < SWAP_META_PAGES; ++i) {
1666 				if (swp_pager_isondev(swap->swb_pages[i], sp)) {
1667 					mtx_unlock(&swhash_mtx);
1668 					return (1);
1669 				}
1670 			}
1671 		}
1672 		index += SWAP_META_PAGES;
1673 	}
1674 	mtx_unlock(&swhash_mtx);
1675 	return (0);
1676 }
1677 
1678 /*
1679  * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in
1680  *
1681  *	This routine dissociates the page at the given index within a
1682  *	swap block from its backing store, paging it in if necessary.
1683  *	If the page is paged in, it is placed in the inactive queue,
1684  *	since it had its backing store ripped out from under it.
1685  *	We also attempt to swap in all other pages in the swap block,
1686  *	we only guarantee that the one at the specified index is
1687  *	paged in.
1688  *
1689  *	XXX - The code to page the whole block in doesn't work, so we
1690  *	      revert to the one-by-one behavior for now.  Sigh.
1691  */
1692 static inline void
1693 swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex)
1694 {
1695 	vm_page_t m;
1696 
1697 	vm_object_pip_add(object, 1);
1698 	m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL|VM_ALLOC_RETRY);
1699 	if (m->valid == VM_PAGE_BITS_ALL) {
1700 		vm_object_pip_subtract(object, 1);
1701 		vm_page_dirty(m);
1702 		vm_page_lock(m);
1703 		vm_page_activate(m);
1704 		vm_page_unlock(m);
1705 		vm_page_wakeup(m);
1706 		vm_pager_page_unswapped(m);
1707 		return;
1708 	}
1709 
1710 	if (swap_pager_getpages(object, &m, 1, 0) != VM_PAGER_OK)
1711 		panic("swap_pager_force_pagein: read from swap failed");/*XXX*/
1712 	vm_object_pip_subtract(object, 1);
1713 	vm_page_dirty(m);
1714 	vm_page_lock(m);
1715 	vm_page_deactivate(m);
1716 	vm_page_unlock(m);
1717 	vm_page_wakeup(m);
1718 	vm_pager_page_unswapped(m);
1719 }
1720 
1721 /*
1722  *	swap_pager_swapoff:
1723  *
1724  *	Page in all of the pages that have been paged out to the
1725  *	given device.  The corresponding blocks in the bitmap must be
1726  *	marked as allocated and the device must be flagged SW_CLOSING.
1727  *	There may be no processes swapped out to the device.
1728  *
1729  *	This routine may block.
1730  */
1731 static void
1732 swap_pager_swapoff(struct swdevt *sp)
1733 {
1734 	struct swblock *swap;
1735 	int i, j, retries;
1736 
1737 	GIANT_REQUIRED;
1738 
1739 	retries = 0;
1740 full_rescan:
1741 	mtx_lock(&swhash_mtx);
1742 	for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */
1743 restart:
1744 		for (swap = swhash[i]; swap != NULL; swap = swap->swb_hnext) {
1745 			vm_object_t object = swap->swb_object;
1746 			vm_pindex_t pindex = swap->swb_index;
1747 			for (j = 0; j < SWAP_META_PAGES; ++j) {
1748 				if (swp_pager_isondev(swap->swb_pages[j], sp)) {
1749 					/* avoid deadlock */
1750 					if (!VM_OBJECT_TRYLOCK(object)) {
1751 						break;
1752 					} else {
1753 						mtx_unlock(&swhash_mtx);
1754 						swp_pager_force_pagein(object,
1755 						    pindex + j);
1756 						VM_OBJECT_UNLOCK(object);
1757 						mtx_lock(&swhash_mtx);
1758 						goto restart;
1759 					}
1760 				}
1761 			}
1762 		}
1763 	}
1764 	mtx_unlock(&swhash_mtx);
1765 	if (sp->sw_used) {
1766 		/*
1767 		 * Objects may be locked or paging to the device being
1768 		 * removed, so we will miss their pages and need to
1769 		 * make another pass.  We have marked this device as
1770 		 * SW_CLOSING, so the activity should finish soon.
1771 		 */
1772 		retries++;
1773 		if (retries > 100) {
1774 			panic("swapoff: failed to locate %d swap blocks",
1775 			    sp->sw_used);
1776 		}
1777 		pause("swpoff", hz / 20);
1778 		goto full_rescan;
1779 	}
1780 }
1781 
1782 /************************************************************************
1783  *				SWAP META DATA 				*
1784  ************************************************************************
1785  *
1786  *	These routines manipulate the swap metadata stored in the
1787  *	OBJT_SWAP object.
1788  *
1789  *	Swap metadata is implemented with a global hash and not directly
1790  *	linked into the object.  Instead the object simply contains
1791  *	appropriate tracking counters.
1792  */
1793 
1794 /*
1795  * SWP_PAGER_META_BUILD() -	add swap block to swap meta data for object
1796  *
1797  *	We first convert the object to a swap object if it is a default
1798  *	object.
1799  *
1800  *	The specified swapblk is added to the object's swap metadata.  If
1801  *	the swapblk is not valid, it is freed instead.  Any previously
1802  *	assigned swapblk is freed.
1803  */
1804 static void
1805 swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
1806 {
1807 	static volatile int exhausted;
1808 	struct swblock *swap;
1809 	struct swblock **pswap;
1810 	int idx;
1811 
1812 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1813 	/*
1814 	 * Convert default object to swap object if necessary
1815 	 */
1816 	if (object->type != OBJT_SWAP) {
1817 		object->type = OBJT_SWAP;
1818 		object->un_pager.swp.swp_bcount = 0;
1819 
1820 		if (object->handle != NULL) {
1821 			mtx_lock(&sw_alloc_mtx);
1822 			TAILQ_INSERT_TAIL(
1823 			    NOBJLIST(object->handle),
1824 			    object,
1825 			    pager_object_list
1826 			);
1827 			mtx_unlock(&sw_alloc_mtx);
1828 		}
1829 	}
1830 
1831 	/*
1832 	 * Locate hash entry.  If not found create, but if we aren't adding
1833 	 * anything just return.  If we run out of space in the map we wait
1834 	 * and, since the hash table may have changed, retry.
1835 	 */
1836 retry:
1837 	mtx_lock(&swhash_mtx);
1838 	pswap = swp_pager_hash(object, pindex);
1839 
1840 	if ((swap = *pswap) == NULL) {
1841 		int i;
1842 
1843 		if (swapblk == SWAPBLK_NONE)
1844 			goto done;
1845 
1846 		swap = *pswap = uma_zalloc(swap_zone, M_NOWAIT);
1847 		if (swap == NULL) {
1848 			mtx_unlock(&swhash_mtx);
1849 			VM_OBJECT_UNLOCK(object);
1850 			if (uma_zone_exhausted(swap_zone)) {
1851 				if (atomic_cmpset_int(&exhausted, 0, 1))
1852 					printf("swap zone exhausted, "
1853 					    "increase kern.maxswzone\n");
1854 				vm_pageout_oom(VM_OOM_SWAPZ);
1855 				pause("swzonex", 10);
1856 			} else
1857 				VM_WAIT;
1858 			VM_OBJECT_LOCK(object);
1859 			goto retry;
1860 		}
1861 
1862 		if (atomic_cmpset_int(&exhausted, 1, 0))
1863 			printf("swap zone ok\n");
1864 
1865 		swap->swb_hnext = NULL;
1866 		swap->swb_object = object;
1867 		swap->swb_index = pindex & ~(vm_pindex_t)SWAP_META_MASK;
1868 		swap->swb_count = 0;
1869 
1870 		++object->un_pager.swp.swp_bcount;
1871 
1872 		for (i = 0; i < SWAP_META_PAGES; ++i)
1873 			swap->swb_pages[i] = SWAPBLK_NONE;
1874 	}
1875 
1876 	/*
1877 	 * Delete prior contents of metadata
1878 	 */
1879 	idx = pindex & SWAP_META_MASK;
1880 
1881 	if (swap->swb_pages[idx] != SWAPBLK_NONE) {
1882 		swp_pager_freeswapspace(swap->swb_pages[idx], 1);
1883 		--swap->swb_count;
1884 	}
1885 
1886 	/*
1887 	 * Enter block into metadata
1888 	 */
1889 	swap->swb_pages[idx] = swapblk;
1890 	if (swapblk != SWAPBLK_NONE)
1891 		++swap->swb_count;
1892 done:
1893 	mtx_unlock(&swhash_mtx);
1894 }
1895 
1896 /*
1897  * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
1898  *
1899  *	The requested range of blocks is freed, with any associated swap
1900  *	returned to the swap bitmap.
1901  *
1902  *	This routine will free swap metadata structures as they are cleaned
1903  *	out.  This routine does *NOT* operate on swap metadata associated
1904  *	with resident pages.
1905  */
1906 static void
1907 swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count)
1908 {
1909 
1910 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1911 	if (object->type != OBJT_SWAP)
1912 		return;
1913 
1914 	while (count > 0) {
1915 		struct swblock **pswap;
1916 		struct swblock *swap;
1917 
1918 		mtx_lock(&swhash_mtx);
1919 		pswap = swp_pager_hash(object, index);
1920 
1921 		if ((swap = *pswap) != NULL) {
1922 			daddr_t v = swap->swb_pages[index & SWAP_META_MASK];
1923 
1924 			if (v != SWAPBLK_NONE) {
1925 				swp_pager_freeswapspace(v, 1);
1926 				swap->swb_pages[index & SWAP_META_MASK] =
1927 					SWAPBLK_NONE;
1928 				if (--swap->swb_count == 0) {
1929 					*pswap = swap->swb_hnext;
1930 					uma_zfree(swap_zone, swap);
1931 					--object->un_pager.swp.swp_bcount;
1932 				}
1933 			}
1934 			--count;
1935 			++index;
1936 		} else {
1937 			int n = SWAP_META_PAGES - (index & SWAP_META_MASK);
1938 			count -= n;
1939 			index += n;
1940 		}
1941 		mtx_unlock(&swhash_mtx);
1942 	}
1943 }
1944 
1945 /*
1946  * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
1947  *
1948  *	This routine locates and destroys all swap metadata associated with
1949  *	an object.
1950  */
1951 static void
1952 swp_pager_meta_free_all(vm_object_t object)
1953 {
1954 	daddr_t index = 0;
1955 
1956 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1957 	if (object->type != OBJT_SWAP)
1958 		return;
1959 
1960 	while (object->un_pager.swp.swp_bcount) {
1961 		struct swblock **pswap;
1962 		struct swblock *swap;
1963 
1964 		mtx_lock(&swhash_mtx);
1965 		pswap = swp_pager_hash(object, index);
1966 		if ((swap = *pswap) != NULL) {
1967 			int i;
1968 
1969 			for (i = 0; i < SWAP_META_PAGES; ++i) {
1970 				daddr_t v = swap->swb_pages[i];
1971 				if (v != SWAPBLK_NONE) {
1972 					--swap->swb_count;
1973 					swp_pager_freeswapspace(v, 1);
1974 				}
1975 			}
1976 			if (swap->swb_count != 0)
1977 				panic("swap_pager_meta_free_all: swb_count != 0");
1978 			*pswap = swap->swb_hnext;
1979 			uma_zfree(swap_zone, swap);
1980 			--object->un_pager.swp.swp_bcount;
1981 		}
1982 		mtx_unlock(&swhash_mtx);
1983 		index += SWAP_META_PAGES;
1984 	}
1985 }
1986 
1987 /*
1988  * SWP_PAGER_METACTL() -  misc control of swap and vm_page_t meta data.
1989  *
1990  *	This routine is capable of looking up, popping, or freeing
1991  *	swapblk assignments in the swap meta data or in the vm_page_t.
1992  *	The routine typically returns the swapblk being looked-up, or popped,
1993  *	or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block
1994  *	was invalid.  This routine will automatically free any invalid
1995  *	meta-data swapblks.
1996  *
1997  *	It is not possible to store invalid swapblks in the swap meta data
1998  *	(other then a literal 'SWAPBLK_NONE'), so we don't bother checking.
1999  *
2000  *	When acting on a busy resident page and paging is in progress, we
2001  *	have to wait until paging is complete but otherwise can act on the
2002  *	busy page.
2003  *
2004  *	SWM_FREE	remove and free swap block from metadata
2005  *	SWM_POP		remove from meta data but do not free.. pop it out
2006  */
2007 static daddr_t
2008 swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
2009 {
2010 	struct swblock **pswap;
2011 	struct swblock *swap;
2012 	daddr_t r1;
2013 	int idx;
2014 
2015 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
2016 	/*
2017 	 * The meta data only exists of the object is OBJT_SWAP
2018 	 * and even then might not be allocated yet.
2019 	 */
2020 	if (object->type != OBJT_SWAP)
2021 		return (SWAPBLK_NONE);
2022 
2023 	r1 = SWAPBLK_NONE;
2024 	mtx_lock(&swhash_mtx);
2025 	pswap = swp_pager_hash(object, pindex);
2026 
2027 	if ((swap = *pswap) != NULL) {
2028 		idx = pindex & SWAP_META_MASK;
2029 		r1 = swap->swb_pages[idx];
2030 
2031 		if (r1 != SWAPBLK_NONE) {
2032 			if (flags & SWM_FREE) {
2033 				swp_pager_freeswapspace(r1, 1);
2034 				r1 = SWAPBLK_NONE;
2035 			}
2036 			if (flags & (SWM_FREE|SWM_POP)) {
2037 				swap->swb_pages[idx] = SWAPBLK_NONE;
2038 				if (--swap->swb_count == 0) {
2039 					*pswap = swap->swb_hnext;
2040 					uma_zfree(swap_zone, swap);
2041 					--object->un_pager.swp.swp_bcount;
2042 				}
2043 			}
2044 		}
2045 	}
2046 	mtx_unlock(&swhash_mtx);
2047 	return (r1);
2048 }
2049 
2050 /*
2051  * System call swapon(name) enables swapping on device name,
2052  * which must be in the swdevsw.  Return EBUSY
2053  * if already swapping on this device.
2054  */
2055 #ifndef _SYS_SYSPROTO_H_
2056 struct swapon_args {
2057 	char *name;
2058 };
2059 #endif
2060 
2061 /*
2062  * MPSAFE
2063  */
2064 /* ARGSUSED */
2065 int
2066 sys_swapon(struct thread *td, struct swapon_args *uap)
2067 {
2068 	struct vattr attr;
2069 	struct vnode *vp;
2070 	struct nameidata nd;
2071 	int error;
2072 
2073 	error = priv_check(td, PRIV_SWAPON);
2074 	if (error)
2075 		return (error);
2076 
2077 	mtx_lock(&Giant);
2078 	while (swdev_syscall_active)
2079 	    tsleep(&swdev_syscall_active, PUSER - 1, "swpon", 0);
2080 	swdev_syscall_active = 1;
2081 
2082 	/*
2083 	 * Swap metadata may not fit in the KVM if we have physical
2084 	 * memory of >1GB.
2085 	 */
2086 	if (swap_zone == NULL) {
2087 		error = ENOMEM;
2088 		goto done;
2089 	}
2090 
2091 	NDINIT(&nd, LOOKUP, ISOPEN | FOLLOW | AUDITVNODE1, UIO_USERSPACE,
2092 	    uap->name, td);
2093 	error = namei(&nd);
2094 	if (error)
2095 		goto done;
2096 
2097 	NDFREE(&nd, NDF_ONLY_PNBUF);
2098 	vp = nd.ni_vp;
2099 
2100 	if (vn_isdisk(vp, &error)) {
2101 		error = swapongeom(td, vp);
2102 	} else if (vp->v_type == VREG &&
2103 	    (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
2104 	    (error = VOP_GETATTR(vp, &attr, td->td_ucred)) == 0) {
2105 		/*
2106 		 * Allow direct swapping to NFS regular files in the same
2107 		 * way that nfs_mountroot() sets up diskless swapping.
2108 		 */
2109 		error = swaponvp(td, vp, attr.va_size / DEV_BSIZE);
2110 	}
2111 
2112 	if (error)
2113 		vrele(vp);
2114 done:
2115 	swdev_syscall_active = 0;
2116 	wakeup_one(&swdev_syscall_active);
2117 	mtx_unlock(&Giant);
2118 	return (error);
2119 }
2120 
2121 /*
2122  * Check that the total amount of swap currently configured does not
2123  * exceed half the theoretical maximum.  If it does, print a warning
2124  * message and return -1; otherwise, return 0.
2125  */
2126 static int
2127 swapon_check_swzone(unsigned long npages)
2128 {
2129 	unsigned long maxpages;
2130 
2131 	/* absolute maximum we can handle assuming 100% efficiency */
2132 	maxpages = uma_zone_get_max(swap_zone) * SWAP_META_PAGES;
2133 
2134 	/* recommend using no more than half that amount */
2135 	if (npages > maxpages / 2) {
2136 		printf("warning: total configured swap (%lu pages) "
2137 		    "exceeds maximum recommended amount (%lu pages).\n",
2138 		    npages, maxpages / 2);
2139 		printf("warning: increase kern.maxswzone "
2140 		    "or reduce amount of swap.\n");
2141 		return (-1);
2142 	}
2143 	return (0);
2144 }
2145 
2146 static void
2147 swaponsomething(struct vnode *vp, void *id, u_long nblks, sw_strategy_t *strategy, sw_close_t *close, dev_t dev)
2148 {
2149 	struct swdevt *sp, *tsp;
2150 	swblk_t dvbase;
2151 	u_long mblocks;
2152 
2153 	/*
2154 	 * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks.
2155 	 * First chop nblks off to page-align it, then convert.
2156 	 *
2157 	 * sw->sw_nblks is in page-sized chunks now too.
2158 	 */
2159 	nblks &= ~(ctodb(1) - 1);
2160 	nblks = dbtoc(nblks);
2161 
2162 	/*
2163 	 * If we go beyond this, we get overflows in the radix
2164 	 * tree bitmap code.
2165 	 */
2166 	mblocks = 0x40000000 / BLIST_META_RADIX;
2167 	if (nblks > mblocks) {
2168 		printf(
2169     "WARNING: reducing swap size to maximum of %luMB per unit\n",
2170 		    mblocks / 1024 / 1024 * PAGE_SIZE);
2171 		nblks = mblocks;
2172 	}
2173 
2174 	sp = malloc(sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO);
2175 	sp->sw_vp = vp;
2176 	sp->sw_id = id;
2177 	sp->sw_dev = dev;
2178 	sp->sw_flags = 0;
2179 	sp->sw_nblks = nblks;
2180 	sp->sw_used = 0;
2181 	sp->sw_strategy = strategy;
2182 	sp->sw_close = close;
2183 
2184 	sp->sw_blist = blist_create(nblks, M_WAITOK);
2185 	/*
2186 	 * Do not free the first two block in order to avoid overwriting
2187 	 * any bsd label at the front of the partition
2188 	 */
2189 	blist_free(sp->sw_blist, 2, nblks - 2);
2190 
2191 	dvbase = 0;
2192 	mtx_lock(&sw_dev_mtx);
2193 	TAILQ_FOREACH(tsp, &swtailq, sw_list) {
2194 		if (tsp->sw_end >= dvbase) {
2195 			/*
2196 			 * We put one uncovered page between the devices
2197 			 * in order to definitively prevent any cross-device
2198 			 * I/O requests
2199 			 */
2200 			dvbase = tsp->sw_end + 1;
2201 		}
2202 	}
2203 	sp->sw_first = dvbase;
2204 	sp->sw_end = dvbase + nblks;
2205 	TAILQ_INSERT_TAIL(&swtailq, sp, sw_list);
2206 	nswapdev++;
2207 	swap_pager_avail += nblks;
2208 	swap_total += (vm_ooffset_t)nblks * PAGE_SIZE;
2209 	swapon_check_swzone(swap_total / PAGE_SIZE);
2210 	swp_sizecheck();
2211 	mtx_unlock(&sw_dev_mtx);
2212 }
2213 
2214 /*
2215  * SYSCALL: swapoff(devname)
2216  *
2217  * Disable swapping on the given device.
2218  *
2219  * XXX: Badly designed system call: it should use a device index
2220  * rather than filename as specification.  We keep sw_vp around
2221  * only to make this work.
2222  */
2223 #ifndef _SYS_SYSPROTO_H_
2224 struct swapoff_args {
2225 	char *name;
2226 };
2227 #endif
2228 
2229 /*
2230  * MPSAFE
2231  */
2232 /* ARGSUSED */
2233 int
2234 sys_swapoff(struct thread *td, struct swapoff_args *uap)
2235 {
2236 	struct vnode *vp;
2237 	struct nameidata nd;
2238 	struct swdevt *sp;
2239 	int error;
2240 
2241 	error = priv_check(td, PRIV_SWAPOFF);
2242 	if (error)
2243 		return (error);
2244 
2245 	mtx_lock(&Giant);
2246 	while (swdev_syscall_active)
2247 	    tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
2248 	swdev_syscall_active = 1;
2249 
2250 	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, UIO_USERSPACE, uap->name,
2251 	    td);
2252 	error = namei(&nd);
2253 	if (error)
2254 		goto done;
2255 	NDFREE(&nd, NDF_ONLY_PNBUF);
2256 	vp = nd.ni_vp;
2257 
2258 	mtx_lock(&sw_dev_mtx);
2259 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2260 		if (sp->sw_vp == vp)
2261 			break;
2262 	}
2263 	mtx_unlock(&sw_dev_mtx);
2264 	if (sp == NULL) {
2265 		error = EINVAL;
2266 		goto done;
2267 	}
2268 	error = swapoff_one(sp, td->td_ucred);
2269 done:
2270 	swdev_syscall_active = 0;
2271 	wakeup_one(&swdev_syscall_active);
2272 	mtx_unlock(&Giant);
2273 	return (error);
2274 }
2275 
2276 static int
2277 swapoff_one(struct swdevt *sp, struct ucred *cred)
2278 {
2279 	u_long nblks, dvbase;
2280 #ifdef MAC
2281 	int error;
2282 #endif
2283 
2284 	mtx_assert(&Giant, MA_OWNED);
2285 #ifdef MAC
2286 	(void) vn_lock(sp->sw_vp, LK_EXCLUSIVE | LK_RETRY);
2287 	error = mac_system_check_swapoff(cred, sp->sw_vp);
2288 	(void) VOP_UNLOCK(sp->sw_vp, 0);
2289 	if (error != 0)
2290 		return (error);
2291 #endif
2292 	nblks = sp->sw_nblks;
2293 
2294 	/*
2295 	 * We can turn off this swap device safely only if the
2296 	 * available virtual memory in the system will fit the amount
2297 	 * of data we will have to page back in, plus an epsilon so
2298 	 * the system doesn't become critically low on swap space.
2299 	 */
2300 	if (cnt.v_free_count + cnt.v_cache_count + swap_pager_avail <
2301 	    nblks + nswap_lowat) {
2302 		return (ENOMEM);
2303 	}
2304 
2305 	/*
2306 	 * Prevent further allocations on this device.
2307 	 */
2308 	mtx_lock(&sw_dev_mtx);
2309 	sp->sw_flags |= SW_CLOSING;
2310 	for (dvbase = 0; dvbase < sp->sw_end; dvbase += dmmax) {
2311 		swap_pager_avail -= blist_fill(sp->sw_blist,
2312 		     dvbase, dmmax);
2313 	}
2314 	swap_total -= (vm_ooffset_t)nblks * PAGE_SIZE;
2315 	mtx_unlock(&sw_dev_mtx);
2316 
2317 	/*
2318 	 * Page in the contents of the device and close it.
2319 	 */
2320 	swap_pager_swapoff(sp);
2321 
2322 	sp->sw_close(curthread, sp);
2323 	sp->sw_id = NULL;
2324 	mtx_lock(&sw_dev_mtx);
2325 	TAILQ_REMOVE(&swtailq, sp, sw_list);
2326 	nswapdev--;
2327 	if (nswapdev == 0) {
2328 		swap_pager_full = 2;
2329 		swap_pager_almost_full = 1;
2330 	}
2331 	if (swdevhd == sp)
2332 		swdevhd = NULL;
2333 	mtx_unlock(&sw_dev_mtx);
2334 	blist_destroy(sp->sw_blist);
2335 	free(sp, M_VMPGDATA);
2336 	return (0);
2337 }
2338 
2339 void
2340 swapoff_all(void)
2341 {
2342 	struct swdevt *sp, *spt;
2343 	const char *devname;
2344 	int error;
2345 
2346 	mtx_lock(&Giant);
2347 	while (swdev_syscall_active)
2348 		tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
2349 	swdev_syscall_active = 1;
2350 
2351 	mtx_lock(&sw_dev_mtx);
2352 	TAILQ_FOREACH_SAFE(sp, &swtailq, sw_list, spt) {
2353 		mtx_unlock(&sw_dev_mtx);
2354 		if (vn_isdisk(sp->sw_vp, NULL))
2355 			devname = devtoname(sp->sw_vp->v_rdev);
2356 		else
2357 			devname = "[file]";
2358 		error = swapoff_one(sp, thread0.td_ucred);
2359 		if (error != 0) {
2360 			printf("Cannot remove swap device %s (error=%d), "
2361 			    "skipping.\n", devname, error);
2362 		} else if (bootverbose) {
2363 			printf("Swap device %s removed.\n", devname);
2364 		}
2365 		mtx_lock(&sw_dev_mtx);
2366 	}
2367 	mtx_unlock(&sw_dev_mtx);
2368 
2369 	swdev_syscall_active = 0;
2370 	wakeup_one(&swdev_syscall_active);
2371 	mtx_unlock(&Giant);
2372 }
2373 
2374 void
2375 swap_pager_status(int *total, int *used)
2376 {
2377 	struct swdevt *sp;
2378 
2379 	*total = 0;
2380 	*used = 0;
2381 	mtx_lock(&sw_dev_mtx);
2382 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2383 		*total += sp->sw_nblks;
2384 		*used += sp->sw_used;
2385 	}
2386 	mtx_unlock(&sw_dev_mtx);
2387 }
2388 
2389 int
2390 swap_dev_info(int name, struct xswdev *xs, char *devname, size_t len)
2391 {
2392 	struct swdevt *sp;
2393 	const char *tmp_devname;
2394 	int error, n;
2395 
2396 	n = 0;
2397 	error = ENOENT;
2398 	mtx_lock(&sw_dev_mtx);
2399 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2400 		if (n != name) {
2401 			n++;
2402 			continue;
2403 		}
2404 		xs->xsw_version = XSWDEV_VERSION;
2405 		xs->xsw_dev = sp->sw_dev;
2406 		xs->xsw_flags = sp->sw_flags;
2407 		xs->xsw_nblks = sp->sw_nblks;
2408 		xs->xsw_used = sp->sw_used;
2409 		if (devname != NULL) {
2410 			if (vn_isdisk(sp->sw_vp, NULL))
2411 				tmp_devname = devtoname(sp->sw_vp->v_rdev);
2412 			else
2413 				tmp_devname = "[file]";
2414 			strncpy(devname, tmp_devname, len);
2415 		}
2416 		error = 0;
2417 		break;
2418 	}
2419 	mtx_unlock(&sw_dev_mtx);
2420 	return (error);
2421 }
2422 
2423 static int
2424 sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS)
2425 {
2426 	struct xswdev xs;
2427 	int error;
2428 
2429 	if (arg2 != 1)			/* name length */
2430 		return (EINVAL);
2431 	error = swap_dev_info(*(int *)arg1, &xs, NULL, 0);
2432 	if (error != 0)
2433 		return (error);
2434 	error = SYSCTL_OUT(req, &xs, sizeof(xs));
2435 	return (error);
2436 }
2437 
2438 SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, &nswapdev, 0,
2439     "Number of swap devices");
2440 SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD, sysctl_vm_swap_info,
2441     "Swap statistics by device");
2442 
2443 /*
2444  * vmspace_swap_count() - count the approximate swap usage in pages for a
2445  *			  vmspace.
2446  *
2447  *	The map must be locked.
2448  *
2449  *	Swap usage is determined by taking the proportional swap used by
2450  *	VM objects backing the VM map.  To make up for fractional losses,
2451  *	if the VM object has any swap use at all the associated map entries
2452  *	count for at least 1 swap page.
2453  */
2454 long
2455 vmspace_swap_count(struct vmspace *vmspace)
2456 {
2457 	vm_map_t map;
2458 	vm_map_entry_t cur;
2459 	vm_object_t object;
2460 	long count, n;
2461 
2462 	map = &vmspace->vm_map;
2463 	count = 0;
2464 
2465 	for (cur = map->header.next; cur != &map->header; cur = cur->next) {
2466 		if ((cur->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
2467 		    (object = cur->object.vm_object) != NULL) {
2468 			VM_OBJECT_LOCK(object);
2469 			if (object->type == OBJT_SWAP &&
2470 			    object->un_pager.swp.swp_bcount != 0) {
2471 				n = (cur->end - cur->start) / PAGE_SIZE;
2472 				count += object->un_pager.swp.swp_bcount *
2473 				    SWAP_META_PAGES * n / object->size + 1;
2474 			}
2475 			VM_OBJECT_UNLOCK(object);
2476 		}
2477 	}
2478 	return (count);
2479 }
2480 
2481 /*
2482  * GEOM backend
2483  *
2484  * Swapping onto disk devices.
2485  *
2486  */
2487 
2488 static g_orphan_t swapgeom_orphan;
2489 
2490 static struct g_class g_swap_class = {
2491 	.name = "SWAP",
2492 	.version = G_VERSION,
2493 	.orphan = swapgeom_orphan,
2494 };
2495 
2496 DECLARE_GEOM_CLASS(g_swap_class, g_class);
2497 
2498 
2499 static void
2500 swapgeom_done(struct bio *bp2)
2501 {
2502 	struct buf *bp;
2503 
2504 	bp = bp2->bio_caller2;
2505 	bp->b_ioflags = bp2->bio_flags;
2506 	if (bp2->bio_error)
2507 		bp->b_ioflags |= BIO_ERROR;
2508 	bp->b_resid = bp->b_bcount - bp2->bio_completed;
2509 	bp->b_error = bp2->bio_error;
2510 	bufdone(bp);
2511 	g_destroy_bio(bp2);
2512 }
2513 
2514 static void
2515 swapgeom_strategy(struct buf *bp, struct swdevt *sp)
2516 {
2517 	struct bio *bio;
2518 	struct g_consumer *cp;
2519 
2520 	cp = sp->sw_id;
2521 	if (cp == NULL) {
2522 		bp->b_error = ENXIO;
2523 		bp->b_ioflags |= BIO_ERROR;
2524 		bufdone(bp);
2525 		return;
2526 	}
2527 	if (bp->b_iocmd == BIO_WRITE)
2528 		bio = g_new_bio();
2529 	else
2530 		bio = g_alloc_bio();
2531 	if (bio == NULL) {
2532 		bp->b_error = ENOMEM;
2533 		bp->b_ioflags |= BIO_ERROR;
2534 		bufdone(bp);
2535 		return;
2536 	}
2537 
2538 	bio->bio_caller2 = bp;
2539 	bio->bio_cmd = bp->b_iocmd;
2540 	bio->bio_data = bp->b_data;
2541 	bio->bio_offset = (bp->b_blkno - sp->sw_first) * PAGE_SIZE;
2542 	bio->bio_length = bp->b_bcount;
2543 	bio->bio_done = swapgeom_done;
2544 	g_io_request(bio, cp);
2545 	return;
2546 }
2547 
2548 static void
2549 swapgeom_orphan(struct g_consumer *cp)
2550 {
2551 	struct swdevt *sp;
2552 
2553 	mtx_lock(&sw_dev_mtx);
2554 	TAILQ_FOREACH(sp, &swtailq, sw_list)
2555 		if (sp->sw_id == cp)
2556 			sp->sw_flags |= SW_CLOSING;
2557 	mtx_unlock(&sw_dev_mtx);
2558 }
2559 
2560 static void
2561 swapgeom_close_ev(void *arg, int flags)
2562 {
2563 	struct g_consumer *cp;
2564 
2565 	cp = arg;
2566 	g_access(cp, -1, -1, 0);
2567 	g_detach(cp);
2568 	g_destroy_consumer(cp);
2569 }
2570 
2571 static void
2572 swapgeom_close(struct thread *td, struct swdevt *sw)
2573 {
2574 
2575 	/* XXX: direct call when Giant untangled */
2576 	g_waitfor_event(swapgeom_close_ev, sw->sw_id, M_WAITOK, NULL);
2577 }
2578 
2579 
2580 struct swh0h0 {
2581 	struct cdev *dev;
2582 	struct vnode *vp;
2583 	int	error;
2584 };
2585 
2586 static void
2587 swapongeom_ev(void *arg, int flags)
2588 {
2589 	struct swh0h0 *swh;
2590 	struct g_provider *pp;
2591 	struct g_consumer *cp;
2592 	static struct g_geom *gp;
2593 	struct swdevt *sp;
2594 	u_long nblks;
2595 	int error;
2596 
2597 	swh = arg;
2598 	swh->error = 0;
2599 	pp = g_dev_getprovider(swh->dev);
2600 	if (pp == NULL) {
2601 		swh->error = ENODEV;
2602 		return;
2603 	}
2604 	mtx_lock(&sw_dev_mtx);
2605 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2606 		cp = sp->sw_id;
2607 		if (cp != NULL && cp->provider == pp) {
2608 			mtx_unlock(&sw_dev_mtx);
2609 			swh->error = EBUSY;
2610 			return;
2611 		}
2612 	}
2613 	mtx_unlock(&sw_dev_mtx);
2614 	if (gp == NULL)
2615 		gp = g_new_geomf(&g_swap_class, "swap");
2616 	cp = g_new_consumer(gp);
2617 	g_attach(cp, pp);
2618 	/*
2619 	 * XXX: Everytime you think you can improve the margin for
2620 	 * footshooting, somebody depends on the ability to do so:
2621 	 * savecore(8) wants to write to our swapdev so we cannot
2622 	 * set an exclusive count :-(
2623 	 */
2624 	error = g_access(cp, 1, 1, 0);
2625 	if (error) {
2626 		g_detach(cp);
2627 		g_destroy_consumer(cp);
2628 		swh->error = error;
2629 		return;
2630 	}
2631 	nblks = pp->mediasize / DEV_BSIZE;
2632 	swaponsomething(swh->vp, cp, nblks, swapgeom_strategy,
2633 	    swapgeom_close, dev2udev(swh->dev));
2634 	swh->error = 0;
2635 	return;
2636 }
2637 
2638 static int
2639 swapongeom(struct thread *td, struct vnode *vp)
2640 {
2641 	int error;
2642 	struct swh0h0 swh;
2643 
2644 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2645 
2646 	swh.dev = vp->v_rdev;
2647 	swh.vp = vp;
2648 	swh.error = 0;
2649 	/* XXX: direct call when Giant untangled */
2650 	error = g_waitfor_event(swapongeom_ev, &swh, M_WAITOK, NULL);
2651 	if (!error)
2652 		error = swh.error;
2653 	VOP_UNLOCK(vp, 0);
2654 	return (error);
2655 }
2656 
2657 /*
2658  * VNODE backend
2659  *
2660  * This is used mainly for network filesystem (read: probably only tested
2661  * with NFS) swapfiles.
2662  *
2663  */
2664 
2665 static void
2666 swapdev_strategy(struct buf *bp, struct swdevt *sp)
2667 {
2668 	struct vnode *vp2;
2669 
2670 	bp->b_blkno = ctodb(bp->b_blkno - sp->sw_first);
2671 
2672 	vp2 = sp->sw_id;
2673 	vhold(vp2);
2674 	if (bp->b_iocmd == BIO_WRITE) {
2675 		if (bp->b_bufobj)
2676 			bufobj_wdrop(bp->b_bufobj);
2677 		bufobj_wref(&vp2->v_bufobj);
2678 	}
2679 	if (bp->b_bufobj != &vp2->v_bufobj)
2680 		bp->b_bufobj = &vp2->v_bufobj;
2681 	bp->b_vp = vp2;
2682 	bp->b_iooffset = dbtob(bp->b_blkno);
2683 	bstrategy(bp);
2684 	return;
2685 }
2686 
2687 static void
2688 swapdev_close(struct thread *td, struct swdevt *sp)
2689 {
2690 
2691 	VOP_CLOSE(sp->sw_vp, FREAD | FWRITE, td->td_ucred, td);
2692 	vrele(sp->sw_vp);
2693 }
2694 
2695 
2696 static int
2697 swaponvp(struct thread *td, struct vnode *vp, u_long nblks)
2698 {
2699 	struct swdevt *sp;
2700 	int error;
2701 
2702 	if (nblks == 0)
2703 		return (ENXIO);
2704 	mtx_lock(&sw_dev_mtx);
2705 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2706 		if (sp->sw_id == vp) {
2707 			mtx_unlock(&sw_dev_mtx);
2708 			return (EBUSY);
2709 		}
2710 	}
2711 	mtx_unlock(&sw_dev_mtx);
2712 
2713 	(void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2714 #ifdef MAC
2715 	error = mac_system_check_swapon(td->td_ucred, vp);
2716 	if (error == 0)
2717 #endif
2718 		error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, NULL);
2719 	(void) VOP_UNLOCK(vp, 0);
2720 	if (error)
2721 		return (error);
2722 
2723 	swaponsomething(vp, vp, nblks, swapdev_strategy, swapdev_close,
2724 	    NODEV);
2725 	return (0);
2726 }
2727