xref: /freebsd/sys/vm/swap_pager.c (revision 195ebc7e9e4b129de810833791a19dfb4349d6a9)
1 /*-
2  * Copyright (c) 1998 Matthew Dillon,
3  * Copyright (c) 1994 John S. Dyson
4  * Copyright (c) 1990 University of Utah.
5  * Copyright (c) 1982, 1986, 1989, 1993
6  *	The Regents of the University of California.  All rights reserved.
7  *
8  * This code is derived from software contributed to Berkeley by
9  * the Systems Programming Group of the University of Utah Computer
10  * Science Department.
11  *
12  * Redistribution and use in source and binary forms, with or without
13  * modification, are permitted provided that the following conditions
14  * are met:
15  * 1. Redistributions of source code must retain the above copyright
16  *    notice, this list of conditions and the following disclaimer.
17  * 2. Redistributions in binary form must reproduce the above copyright
18  *    notice, this list of conditions and the following disclaimer in the
19  *    documentation and/or other materials provided with the distribution.
20  * 3. All advertising materials mentioning features or use of this software
21  *    must display the following acknowledgement:
22  *	This product includes software developed by the University of
23  *	California, Berkeley and its contributors.
24  * 4. Neither the name of the University nor the names of its contributors
25  *    may be used to endorse or promote products derived from this software
26  *    without specific prior written permission.
27  *
28  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
29  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
32  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38  * SUCH DAMAGE.
39  *
40  *				New Swap System
41  *				Matthew Dillon
42  *
43  * Radix Bitmap 'blists'.
44  *
45  *	- The new swapper uses the new radix bitmap code.  This should scale
46  *	  to arbitrarily small or arbitrarily large swap spaces and an almost
47  *	  arbitrary degree of fragmentation.
48  *
49  * Features:
50  *
51  *	- on the fly reallocation of swap during putpages.  The new system
52  *	  does not try to keep previously allocated swap blocks for dirty
53  *	  pages.
54  *
55  *	- on the fly deallocation of swap
56  *
57  *	- No more garbage collection required.  Unnecessarily allocated swap
58  *	  blocks only exist for dirty vm_page_t's now and these are already
59  *	  cycled (in a high-load system) by the pager.  We also do on-the-fly
60  *	  removal of invalidated swap blocks when a page is destroyed
61  *	  or renamed.
62  *
63  * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
64  *
65  *	@(#)swap_pager.c	8.9 (Berkeley) 3/21/94
66  *	@(#)vm_swap.c	8.5 (Berkeley) 2/17/94
67  */
68 
69 #include <sys/cdefs.h>
70 __FBSDID("$FreeBSD$");
71 
72 #include "opt_mac.h"
73 #include "opt_swap.h"
74 #include "opt_vm.h"
75 
76 #include <sys/param.h>
77 #include <sys/systm.h>
78 #include <sys/conf.h>
79 #include <sys/kernel.h>
80 #include <sys/priv.h>
81 #include <sys/proc.h>
82 #include <sys/bio.h>
83 #include <sys/buf.h>
84 #include <sys/disk.h>
85 #include <sys/fcntl.h>
86 #include <sys/mount.h>
87 #include <sys/namei.h>
88 #include <sys/vnode.h>
89 #include <sys/malloc.h>
90 #include <sys/sysctl.h>
91 #include <sys/sysproto.h>
92 #include <sys/blist.h>
93 #include <sys/lock.h>
94 #include <sys/sx.h>
95 #include <sys/vmmeter.h>
96 
97 #include <security/mac/mac_framework.h>
98 
99 #include <vm/vm.h>
100 #include <vm/pmap.h>
101 #include <vm/vm_map.h>
102 #include <vm/vm_kern.h>
103 #include <vm/vm_object.h>
104 #include <vm/vm_page.h>
105 #include <vm/vm_pager.h>
106 #include <vm/vm_pageout.h>
107 #include <vm/vm_param.h>
108 #include <vm/swap_pager.h>
109 #include <vm/vm_extern.h>
110 #include <vm/uma.h>
111 
112 #include <geom/geom.h>
113 
114 /*
115  * SWB_NPAGES must be a power of 2.  It may be set to 1, 2, 4, 8, or 16
116  * pages per allocation.  We recommend you stick with the default of 8.
117  * The 16-page limit is due to the radix code (kern/subr_blist.c).
118  */
119 #ifndef MAX_PAGEOUT_CLUSTER
120 #define MAX_PAGEOUT_CLUSTER 16
121 #endif
122 
123 #if !defined(SWB_NPAGES)
124 #define SWB_NPAGES	MAX_PAGEOUT_CLUSTER
125 #endif
126 
127 /*
128  * Piecemeal swap metadata structure.  Swap is stored in a radix tree.
129  *
130  * If SWB_NPAGES is 8 and sizeof(char *) == sizeof(daddr_t), our radix
131  * is basically 8.  Assuming PAGE_SIZE == 4096, one tree level represents
132  * 32K worth of data, two levels represent 256K, three levels represent
133  * 2 MBytes.   This is acceptable.
134  *
135  * Overall memory utilization is about the same as the old swap structure.
136  */
137 #define SWCORRECT(n) (sizeof(void *) * (n) / sizeof(daddr_t))
138 #define SWAP_META_PAGES		(SWB_NPAGES * 2)
139 #define SWAP_META_MASK		(SWAP_META_PAGES - 1)
140 
141 struct swblock {
142 	struct swblock	*swb_hnext;
143 	vm_object_t	swb_object;
144 	vm_pindex_t	swb_index;
145 	int		swb_count;
146 	daddr_t		swb_pages[SWAP_META_PAGES];
147 };
148 
149 static struct mtx sw_dev_mtx;
150 static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq);
151 static struct swdevt *swdevhd;	/* Allocate from here next */
152 static int nswapdev;		/* Number of swap devices */
153 int swap_pager_avail;
154 static int swdev_syscall_active = 0; /* serialize swap(on|off) */
155 
156 static void swapdev_strategy(struct buf *, struct swdevt *sw);
157 
158 #define SWM_FREE	0x02	/* free, period			*/
159 #define SWM_POP		0x04	/* pop out			*/
160 
161 int swap_pager_full = 2;	/* swap space exhaustion (task killing) */
162 static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/
163 static int nsw_rcount;		/* free read buffers			*/
164 static int nsw_wcount_sync;	/* limit write buffers / synchronous	*/
165 static int nsw_wcount_async;	/* limit write buffers / asynchronous	*/
166 static int nsw_wcount_async_max;/* assigned maximum			*/
167 static int nsw_cluster_max;	/* maximum VOP I/O allowed		*/
168 
169 static struct swblock **swhash;
170 static int swhash_mask;
171 static struct mtx swhash_mtx;
172 
173 static int swap_async_max = 4;	/* maximum in-progress async I/O's	*/
174 static struct sx sw_alloc_sx;
175 
176 
177 SYSCTL_INT(_vm, OID_AUTO, swap_async_max,
178         CTLFLAG_RW, &swap_async_max, 0, "Maximum running async swap ops");
179 
180 /*
181  * "named" and "unnamed" anon region objects.  Try to reduce the overhead
182  * of searching a named list by hashing it just a little.
183  */
184 
185 #define NOBJLISTS		8
186 
187 #define NOBJLIST(handle)	\
188 	(&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)])
189 
190 static struct mtx sw_alloc_mtx;	/* protect list manipulation */
191 static struct pagerlst	swap_pager_object_list[NOBJLISTS];
192 static uma_zone_t	swap_zone;
193 static struct vm_object	swap_zone_obj;
194 
195 /*
196  * pagerops for OBJT_SWAP - "swap pager".  Some ops are also global procedure
197  * calls hooked from other parts of the VM system and do not appear here.
198  * (see vm/swap_pager.h).
199  */
200 static vm_object_t
201 		swap_pager_alloc(void *handle, vm_ooffset_t size,
202 				      vm_prot_t prot, vm_ooffset_t offset);
203 static void	swap_pager_dealloc(vm_object_t object);
204 static int	swap_pager_getpages(vm_object_t, vm_page_t *, int, int);
205 static void	swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *);
206 static boolean_t
207 		swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
208 static void	swap_pager_init(void);
209 static void	swap_pager_unswapped(vm_page_t);
210 static void	swap_pager_swapoff(struct swdevt *sp);
211 
212 struct pagerops swappagerops = {
213 	.pgo_init =	swap_pager_init,	/* early system initialization of pager	*/
214 	.pgo_alloc =	swap_pager_alloc,	/* allocate an OBJT_SWAP object		*/
215 	.pgo_dealloc =	swap_pager_dealloc,	/* deallocate an OBJT_SWAP object	*/
216 	.pgo_getpages =	swap_pager_getpages,	/* pagein				*/
217 	.pgo_putpages =	swap_pager_putpages,	/* pageout				*/
218 	.pgo_haspage =	swap_pager_haspage,	/* get backing store status for page	*/
219 	.pgo_pageunswapped = swap_pager_unswapped,	/* remove swap related to page		*/
220 };
221 
222 /*
223  * dmmax is in page-sized chunks with the new swap system.  It was
224  * dev-bsized chunks in the old.  dmmax is always a power of 2.
225  *
226  * swap_*() routines are externally accessible.  swp_*() routines are
227  * internal.
228  */
229 static int dmmax;
230 static int nswap_lowat = 128;	/* in pages, swap_pager_almost_full warn */
231 static int nswap_hiwat = 512;	/* in pages, swap_pager_almost_full warn */
232 
233 SYSCTL_INT(_vm, OID_AUTO, dmmax,
234 	CTLFLAG_RD, &dmmax, 0, "Maximum size of a swap block");
235 
236 static void	swp_sizecheck(void);
237 static void	swp_pager_async_iodone(struct buf *bp);
238 static int	swapongeom(struct thread *, struct vnode *);
239 static int	swaponvp(struct thread *, struct vnode *, u_long);
240 static int	swapoff_one(struct swdevt *sp, struct ucred *cred);
241 
242 /*
243  * Swap bitmap functions
244  */
245 static void	swp_pager_freeswapspace(daddr_t blk, int npages);
246 static daddr_t	swp_pager_getswapspace(int npages);
247 
248 /*
249  * Metadata functions
250  */
251 static struct swblock **swp_pager_hash(vm_object_t object, vm_pindex_t index);
252 static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
253 static void swp_pager_meta_free(vm_object_t, vm_pindex_t, daddr_t);
254 static void swp_pager_meta_free_all(vm_object_t);
255 static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);
256 
257 /*
258  * SWP_SIZECHECK() -	update swap_pager_full indication
259  *
260  *	update the swap_pager_almost_full indication and warn when we are
261  *	about to run out of swap space, using lowat/hiwat hysteresis.
262  *
263  *	Clear swap_pager_full ( task killing ) indication when lowat is met.
264  *
265  *	No restrictions on call
266  *	This routine may not block.
267  *	This routine must be called at splvm()
268  */
269 static void
270 swp_sizecheck(void)
271 {
272 
273 	if (swap_pager_avail < nswap_lowat) {
274 		if (swap_pager_almost_full == 0) {
275 			printf("swap_pager: out of swap space\n");
276 			swap_pager_almost_full = 1;
277 		}
278 	} else {
279 		swap_pager_full = 0;
280 		if (swap_pager_avail > nswap_hiwat)
281 			swap_pager_almost_full = 0;
282 	}
283 }
284 
285 /*
286  * SWP_PAGER_HASH() -	hash swap meta data
287  *
288  *	This is an helper function which hashes the swapblk given
289  *	the object and page index.  It returns a pointer to a pointer
290  *	to the object, or a pointer to a NULL pointer if it could not
291  *	find a swapblk.
292  *
293  *	This routine must be called at splvm().
294  */
295 static struct swblock **
296 swp_pager_hash(vm_object_t object, vm_pindex_t index)
297 {
298 	struct swblock **pswap;
299 	struct swblock *swap;
300 
301 	index &= ~(vm_pindex_t)SWAP_META_MASK;
302 	pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask];
303 	while ((swap = *pswap) != NULL) {
304 		if (swap->swb_object == object &&
305 		    swap->swb_index == index
306 		) {
307 			break;
308 		}
309 		pswap = &swap->swb_hnext;
310 	}
311 	return (pswap);
312 }
313 
314 /*
315  * SWAP_PAGER_INIT() -	initialize the swap pager!
316  *
317  *	Expected to be started from system init.  NOTE:  This code is run
318  *	before much else so be careful what you depend on.  Most of the VM
319  *	system has yet to be initialized at this point.
320  */
321 static void
322 swap_pager_init(void)
323 {
324 	/*
325 	 * Initialize object lists
326 	 */
327 	int i;
328 
329 	for (i = 0; i < NOBJLISTS; ++i)
330 		TAILQ_INIT(&swap_pager_object_list[i]);
331 	mtx_init(&sw_alloc_mtx, "swap_pager list", NULL, MTX_DEF);
332 	mtx_init(&sw_dev_mtx, "swapdev", NULL, MTX_DEF);
333 
334 	/*
335 	 * Device Stripe, in PAGE_SIZE'd blocks
336 	 */
337 	dmmax = SWB_NPAGES * 2;
338 }
339 
340 /*
341  * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process
342  *
343  *	Expected to be started from pageout process once, prior to entering
344  *	its main loop.
345  */
346 void
347 swap_pager_swap_init(void)
348 {
349 	int n, n2;
350 
351 	/*
352 	 * Number of in-transit swap bp operations.  Don't
353 	 * exhaust the pbufs completely.  Make sure we
354 	 * initialize workable values (0 will work for hysteresis
355 	 * but it isn't very efficient).
356 	 *
357 	 * The nsw_cluster_max is constrained by the bp->b_pages[]
358 	 * array (MAXPHYS/PAGE_SIZE) and our locally defined
359 	 * MAX_PAGEOUT_CLUSTER.   Also be aware that swap ops are
360 	 * constrained by the swap device interleave stripe size.
361 	 *
362 	 * Currently we hardwire nsw_wcount_async to 4.  This limit is
363 	 * designed to prevent other I/O from having high latencies due to
364 	 * our pageout I/O.  The value 4 works well for one or two active swap
365 	 * devices but is probably a little low if you have more.  Even so,
366 	 * a higher value would probably generate only a limited improvement
367 	 * with three or four active swap devices since the system does not
368 	 * typically have to pageout at extreme bandwidths.   We will want
369 	 * at least 2 per swap devices, and 4 is a pretty good value if you
370 	 * have one NFS swap device due to the command/ack latency over NFS.
371 	 * So it all works out pretty well.
372 	 */
373 	nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER);
374 
375 	mtx_lock(&pbuf_mtx);
376 	nsw_rcount = (nswbuf + 1) / 2;
377 	nsw_wcount_sync = (nswbuf + 3) / 4;
378 	nsw_wcount_async = 4;
379 	nsw_wcount_async_max = nsw_wcount_async;
380 	mtx_unlock(&pbuf_mtx);
381 
382 	/*
383 	 * Initialize our zone.  Right now I'm just guessing on the number
384 	 * we need based on the number of pages in the system.  Each swblock
385 	 * can hold 16 pages, so this is probably overkill.  This reservation
386 	 * is typically limited to around 32MB by default.
387 	 */
388 	n = cnt.v_page_count / 2;
389 	if (maxswzone && n > maxswzone / sizeof(struct swblock))
390 		n = maxswzone / sizeof(struct swblock);
391 	n2 = n;
392 	swap_zone = uma_zcreate("SWAPMETA", sizeof(struct swblock), NULL, NULL,
393 	    NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM);
394 	if (swap_zone == NULL)
395 		panic("failed to create swap_zone.");
396 	do {
397 		if (uma_zone_set_obj(swap_zone, &swap_zone_obj, n))
398 			break;
399 		/*
400 		 * if the allocation failed, try a zone two thirds the
401 		 * size of the previous attempt.
402 		 */
403 		n -= ((n + 2) / 3);
404 	} while (n > 0);
405 	if (n2 != n)
406 		printf("Swap zone entries reduced from %d to %d.\n", n2, n);
407 	n2 = n;
408 
409 	/*
410 	 * Initialize our meta-data hash table.  The swapper does not need to
411 	 * be quite as efficient as the VM system, so we do not use an
412 	 * oversized hash table.
413 	 *
414 	 * 	n: 		size of hash table, must be power of 2
415 	 *	swhash_mask:	hash table index mask
416 	 */
417 	for (n = 1; n < n2 / 8; n *= 2)
418 		;
419 	swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO);
420 	swhash_mask = n - 1;
421 	mtx_init(&swhash_mtx, "swap_pager swhash", NULL, MTX_DEF);
422 }
423 
424 /*
425  * SWAP_PAGER_ALLOC() -	allocate a new OBJT_SWAP VM object and instantiate
426  *			its metadata structures.
427  *
428  *	This routine is called from the mmap and fork code to create a new
429  *	OBJT_SWAP object.  We do this by creating an OBJT_DEFAULT object
430  *	and then converting it with swp_pager_meta_build().
431  *
432  *	This routine may block in vm_object_allocate() and create a named
433  *	object lookup race, so we must interlock.   We must also run at
434  *	splvm() for the object lookup to handle races with interrupts, but
435  *	we do not have to maintain splvm() in between the lookup and the
436  *	add because (I believe) it is not possible to attempt to create
437  *	a new swap object w/handle when a default object with that handle
438  *	already exists.
439  *
440  * MPSAFE
441  */
442 static vm_object_t
443 swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
444 		 vm_ooffset_t offset)
445 {
446 	vm_object_t object;
447 	vm_pindex_t pindex;
448 
449 	pindex = OFF_TO_IDX(offset + PAGE_MASK + size);
450 
451 	if (handle) {
452 		mtx_lock(&Giant);
453 		/*
454 		 * Reference existing named region or allocate new one.  There
455 		 * should not be a race here against swp_pager_meta_build()
456 		 * as called from vm_page_remove() in regards to the lookup
457 		 * of the handle.
458 		 */
459 		sx_xlock(&sw_alloc_sx);
460 		object = vm_pager_object_lookup(NOBJLIST(handle), handle);
461 
462 		if (object == NULL) {
463 			object = vm_object_allocate(OBJT_DEFAULT, pindex);
464 			object->handle = handle;
465 
466 			VM_OBJECT_LOCK(object);
467 			swp_pager_meta_build(object, 0, SWAPBLK_NONE);
468 			VM_OBJECT_UNLOCK(object);
469 		}
470 		sx_xunlock(&sw_alloc_sx);
471 		mtx_unlock(&Giant);
472 	} else {
473 		object = vm_object_allocate(OBJT_DEFAULT, pindex);
474 
475 		VM_OBJECT_LOCK(object);
476 		swp_pager_meta_build(object, 0, SWAPBLK_NONE);
477 		VM_OBJECT_UNLOCK(object);
478 	}
479 	return (object);
480 }
481 
482 /*
483  * SWAP_PAGER_DEALLOC() -	remove swap metadata from object
484  *
485  *	The swap backing for the object is destroyed.  The code is
486  *	designed such that we can reinstantiate it later, but this
487  *	routine is typically called only when the entire object is
488  *	about to be destroyed.
489  *
490  *	This routine may block, but no longer does.
491  *
492  *	The object must be locked or unreferenceable.
493  */
494 static void
495 swap_pager_dealloc(vm_object_t object)
496 {
497 
498 	/*
499 	 * Remove from list right away so lookups will fail if we block for
500 	 * pageout completion.
501 	 */
502 	if (object->handle != NULL) {
503 		mtx_lock(&sw_alloc_mtx);
504 		TAILQ_REMOVE(NOBJLIST(object->handle), object, pager_object_list);
505 		mtx_unlock(&sw_alloc_mtx);
506 	}
507 
508 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
509 	vm_object_pip_wait(object, "swpdea");
510 
511 	/*
512 	 * Free all remaining metadata.  We only bother to free it from
513 	 * the swap meta data.  We do not attempt to free swapblk's still
514 	 * associated with vm_page_t's for this object.  We do not care
515 	 * if paging is still in progress on some objects.
516 	 */
517 	swp_pager_meta_free_all(object);
518 }
519 
520 /************************************************************************
521  *			SWAP PAGER BITMAP ROUTINES			*
522  ************************************************************************/
523 
524 /*
525  * SWP_PAGER_GETSWAPSPACE() -	allocate raw swap space
526  *
527  *	Allocate swap for the requested number of pages.  The starting
528  *	swap block number (a page index) is returned or SWAPBLK_NONE
529  *	if the allocation failed.
530  *
531  *	Also has the side effect of advising that somebody made a mistake
532  *	when they configured swap and didn't configure enough.
533  *
534  *	Must be called at splvm() to avoid races with bitmap frees from
535  *	vm_page_remove() aka swap_pager_page_removed().
536  *
537  *	This routine may not block
538  *	This routine must be called at splvm().
539  *
540  *	We allocate in round-robin fashion from the configured devices.
541  */
542 static daddr_t
543 swp_pager_getswapspace(int npages)
544 {
545 	daddr_t blk;
546 	struct swdevt *sp;
547 	int i;
548 
549 	blk = SWAPBLK_NONE;
550 	mtx_lock(&sw_dev_mtx);
551 	sp = swdevhd;
552 	for (i = 0; i < nswapdev; i++) {
553 		if (sp == NULL)
554 			sp = TAILQ_FIRST(&swtailq);
555 		if (!(sp->sw_flags & SW_CLOSING)) {
556 			blk = blist_alloc(sp->sw_blist, npages);
557 			if (blk != SWAPBLK_NONE) {
558 				blk += sp->sw_first;
559 				sp->sw_used += npages;
560 				swap_pager_avail -= npages;
561 				swp_sizecheck();
562 				swdevhd = TAILQ_NEXT(sp, sw_list);
563 				goto done;
564 			}
565 		}
566 		sp = TAILQ_NEXT(sp, sw_list);
567 	}
568 	if (swap_pager_full != 2) {
569 		printf("swap_pager_getswapspace(%d): failed\n", npages);
570 		swap_pager_full = 2;
571 		swap_pager_almost_full = 1;
572 	}
573 	swdevhd = NULL;
574 done:
575 	mtx_unlock(&sw_dev_mtx);
576 	return (blk);
577 }
578 
579 static int
580 swp_pager_isondev(daddr_t blk, struct swdevt *sp)
581 {
582 
583 	return (blk >= sp->sw_first && blk < sp->sw_end);
584 }
585 
586 static void
587 swp_pager_strategy(struct buf *bp)
588 {
589 	struct swdevt *sp;
590 
591 	mtx_lock(&sw_dev_mtx);
592 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
593 		if (bp->b_blkno >= sp->sw_first && bp->b_blkno < sp->sw_end) {
594 			mtx_unlock(&sw_dev_mtx);
595 			sp->sw_strategy(bp, sp);
596 			return;
597 		}
598 	}
599 	panic("Swapdev not found");
600 }
601 
602 
603 /*
604  * SWP_PAGER_FREESWAPSPACE() -	free raw swap space
605  *
606  *	This routine returns the specified swap blocks back to the bitmap.
607  *
608  *	Note:  This routine may not block (it could in the old swap code),
609  *	and through the use of the new blist routines it does not block.
610  *
611  *	We must be called at splvm() to avoid races with bitmap frees from
612  *	vm_page_remove() aka swap_pager_page_removed().
613  *
614  *	This routine may not block
615  *	This routine must be called at splvm().
616  */
617 static void
618 swp_pager_freeswapspace(daddr_t blk, int npages)
619 {
620 	struct swdevt *sp;
621 
622 	mtx_lock(&sw_dev_mtx);
623 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
624 		if (blk >= sp->sw_first && blk < sp->sw_end) {
625 			sp->sw_used -= npages;
626 			/*
627 			 * If we are attempting to stop swapping on
628 			 * this device, we don't want to mark any
629 			 * blocks free lest they be reused.
630 			 */
631 			if ((sp->sw_flags & SW_CLOSING) == 0) {
632 				blist_free(sp->sw_blist, blk - sp->sw_first,
633 				    npages);
634 				swap_pager_avail += npages;
635 				swp_sizecheck();
636 			}
637 			mtx_unlock(&sw_dev_mtx);
638 			return;
639 		}
640 	}
641 	panic("Swapdev not found");
642 }
643 
644 /*
645  * SWAP_PAGER_FREESPACE() -	frees swap blocks associated with a page
646  *				range within an object.
647  *
648  *	This is a globally accessible routine.
649  *
650  *	This routine removes swapblk assignments from swap metadata.
651  *
652  *	The external callers of this routine typically have already destroyed
653  *	or renamed vm_page_t's associated with this range in the object so
654  *	we should be ok.
655  *
656  *	This routine may be called at any spl.  We up our spl to splvm temporarily
657  *	in order to perform the metadata removal.
658  */
659 void
660 swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size)
661 {
662 
663 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
664 	swp_pager_meta_free(object, start, size);
665 }
666 
667 /*
668  * SWAP_PAGER_RESERVE() - reserve swap blocks in object
669  *
670  *	Assigns swap blocks to the specified range within the object.  The
671  *	swap blocks are not zerod.  Any previous swap assignment is destroyed.
672  *
673  *	Returns 0 on success, -1 on failure.
674  */
675 int
676 swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
677 {
678 	int n = 0;
679 	daddr_t blk = SWAPBLK_NONE;
680 	vm_pindex_t beg = start;	/* save start index */
681 
682 	VM_OBJECT_LOCK(object);
683 	while (size) {
684 		if (n == 0) {
685 			n = BLIST_MAX_ALLOC;
686 			while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
687 				n >>= 1;
688 				if (n == 0) {
689 					swp_pager_meta_free(object, beg, start - beg);
690 					VM_OBJECT_UNLOCK(object);
691 					return (-1);
692 				}
693 			}
694 		}
695 		swp_pager_meta_build(object, start, blk);
696 		--size;
697 		++start;
698 		++blk;
699 		--n;
700 	}
701 	swp_pager_meta_free(object, start, n);
702 	VM_OBJECT_UNLOCK(object);
703 	return (0);
704 }
705 
706 /*
707  * SWAP_PAGER_COPY() -  copy blocks from source pager to destination pager
708  *			and destroy the source.
709  *
710  *	Copy any valid swapblks from the source to the destination.  In
711  *	cases where both the source and destination have a valid swapblk,
712  *	we keep the destination's.
713  *
714  *	This routine is allowed to block.  It may block allocating metadata
715  *	indirectly through swp_pager_meta_build() or if paging is still in
716  *	progress on the source.
717  *
718  *	This routine can be called at any spl
719  *
720  *	XXX vm_page_collapse() kinda expects us not to block because we
721  *	supposedly do not need to allocate memory, but for the moment we
722  *	*may* have to get a little memory from the zone allocator, but
723  *	it is taken from the interrupt memory.  We should be ok.
724  *
725  *	The source object contains no vm_page_t's (which is just as well)
726  *
727  *	The source object is of type OBJT_SWAP.
728  *
729  *	The source and destination objects must be locked or
730  *	inaccessible (XXX are they ?)
731  */
732 void
733 swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
734     vm_pindex_t offset, int destroysource)
735 {
736 	vm_pindex_t i;
737 
738 	VM_OBJECT_LOCK_ASSERT(srcobject, MA_OWNED);
739 	VM_OBJECT_LOCK_ASSERT(dstobject, MA_OWNED);
740 
741 	/*
742 	 * If destroysource is set, we remove the source object from the
743 	 * swap_pager internal queue now.
744 	 */
745 	if (destroysource) {
746 		if (srcobject->handle != NULL) {
747 			mtx_lock(&sw_alloc_mtx);
748 			TAILQ_REMOVE(
749 			    NOBJLIST(srcobject->handle),
750 			    srcobject,
751 			    pager_object_list
752 			);
753 			mtx_unlock(&sw_alloc_mtx);
754 		}
755 	}
756 
757 	/*
758 	 * transfer source to destination.
759 	 */
760 	for (i = 0; i < dstobject->size; ++i) {
761 		daddr_t dstaddr;
762 
763 		/*
764 		 * Locate (without changing) the swapblk on the destination,
765 		 * unless it is invalid in which case free it silently, or
766 		 * if the destination is a resident page, in which case the
767 		 * source is thrown away.
768 		 */
769 		dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
770 
771 		if (dstaddr == SWAPBLK_NONE) {
772 			/*
773 			 * Destination has no swapblk and is not resident,
774 			 * copy source.
775 			 */
776 			daddr_t srcaddr;
777 
778 			srcaddr = swp_pager_meta_ctl(
779 			    srcobject,
780 			    i + offset,
781 			    SWM_POP
782 			);
783 
784 			if (srcaddr != SWAPBLK_NONE) {
785 				/*
786 				 * swp_pager_meta_build() can sleep.
787 				 */
788 				vm_object_pip_add(srcobject, 1);
789 				VM_OBJECT_UNLOCK(srcobject);
790 				vm_object_pip_add(dstobject, 1);
791 				swp_pager_meta_build(dstobject, i, srcaddr);
792 				vm_object_pip_wakeup(dstobject);
793 				VM_OBJECT_LOCK(srcobject);
794 				vm_object_pip_wakeup(srcobject);
795 			}
796 		} else {
797 			/*
798 			 * Destination has valid swapblk or it is represented
799 			 * by a resident page.  We destroy the sourceblock.
800 			 */
801 
802 			swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE);
803 		}
804 	}
805 
806 	/*
807 	 * Free left over swap blocks in source.
808 	 *
809 	 * We have to revert the type to OBJT_DEFAULT so we do not accidently
810 	 * double-remove the object from the swap queues.
811 	 */
812 	if (destroysource) {
813 		swp_pager_meta_free_all(srcobject);
814 		/*
815 		 * Reverting the type is not necessary, the caller is going
816 		 * to destroy srcobject directly, but I'm doing it here
817 		 * for consistency since we've removed the object from its
818 		 * queues.
819 		 */
820 		srcobject->type = OBJT_DEFAULT;
821 	}
822 }
823 
824 /*
825  * SWAP_PAGER_HASPAGE() -	determine if we have good backing store for
826  *				the requested page.
827  *
828  *	We determine whether good backing store exists for the requested
829  *	page and return TRUE if it does, FALSE if it doesn't.
830  *
831  *	If TRUE, we also try to determine how much valid, contiguous backing
832  *	store exists before and after the requested page within a reasonable
833  *	distance.  We do not try to restrict it to the swap device stripe
834  *	(that is handled in getpages/putpages).  It probably isn't worth
835  *	doing here.
836  */
837 static boolean_t
838 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after)
839 {
840 	daddr_t blk0;
841 
842 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
843 	/*
844 	 * do we have good backing store at the requested index ?
845 	 */
846 	blk0 = swp_pager_meta_ctl(object, pindex, 0);
847 
848 	if (blk0 == SWAPBLK_NONE) {
849 		if (before)
850 			*before = 0;
851 		if (after)
852 			*after = 0;
853 		return (FALSE);
854 	}
855 
856 	/*
857 	 * find backwards-looking contiguous good backing store
858 	 */
859 	if (before != NULL) {
860 		int i;
861 
862 		for (i = 1; i < (SWB_NPAGES/2); ++i) {
863 			daddr_t blk;
864 
865 			if (i > pindex)
866 				break;
867 			blk = swp_pager_meta_ctl(object, pindex - i, 0);
868 			if (blk != blk0 - i)
869 				break;
870 		}
871 		*before = (i - 1);
872 	}
873 
874 	/*
875 	 * find forward-looking contiguous good backing store
876 	 */
877 	if (after != NULL) {
878 		int i;
879 
880 		for (i = 1; i < (SWB_NPAGES/2); ++i) {
881 			daddr_t blk;
882 
883 			blk = swp_pager_meta_ctl(object, pindex + i, 0);
884 			if (blk != blk0 + i)
885 				break;
886 		}
887 		*after = (i - 1);
888 	}
889 	return (TRUE);
890 }
891 
892 /*
893  * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page
894  *
895  *	This removes any associated swap backing store, whether valid or
896  *	not, from the page.
897  *
898  *	This routine is typically called when a page is made dirty, at
899  *	which point any associated swap can be freed.  MADV_FREE also
900  *	calls us in a special-case situation
901  *
902  *	NOTE!!!  If the page is clean and the swap was valid, the caller
903  *	should make the page dirty before calling this routine.  This routine
904  *	does NOT change the m->dirty status of the page.  Also: MADV_FREE
905  *	depends on it.
906  *
907  *	This routine may not block
908  *	This routine must be called at splvm()
909  */
910 static void
911 swap_pager_unswapped(vm_page_t m)
912 {
913 
914 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
915 	swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE);
916 }
917 
918 /*
919  * SWAP_PAGER_GETPAGES() - bring pages in from swap
920  *
921  *	Attempt to retrieve (m, count) pages from backing store, but make
922  *	sure we retrieve at least m[reqpage].  We try to load in as large
923  *	a chunk surrounding m[reqpage] as is contiguous in swap and which
924  *	belongs to the same object.
925  *
926  *	The code is designed for asynchronous operation and
927  *	immediate-notification of 'reqpage' but tends not to be
928  *	used that way.  Please do not optimize-out this algorithmic
929  *	feature, I intend to improve on it in the future.
930  *
931  *	The parent has a single vm_object_pip_add() reference prior to
932  *	calling us and we should return with the same.
933  *
934  *	The parent has BUSY'd the pages.  We should return with 'm'
935  *	left busy, but the others adjusted.
936  */
937 static int
938 swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage)
939 {
940 	struct buf *bp;
941 	vm_page_t mreq;
942 	int i;
943 	int j;
944 	daddr_t blk;
945 
946 	mreq = m[reqpage];
947 
948 	KASSERT(mreq->object == object,
949 	    ("swap_pager_getpages: object mismatch %p/%p",
950 	    object, mreq->object));
951 
952 	/*
953 	 * Calculate range to retrieve.  The pages have already been assigned
954 	 * their swapblks.  We require a *contiguous* range but we know it to
955 	 * not span devices.   If we do not supply it, bad things
956 	 * happen.  Note that blk, iblk & jblk can be SWAPBLK_NONE, but the
957 	 * loops are set up such that the case(s) are handled implicitly.
958 	 *
959 	 * The swp_*() calls must be made at splvm().  vm_page_free() does
960 	 * not need to be, but it will go a little faster if it is.
961 	 */
962 	blk = swp_pager_meta_ctl(mreq->object, mreq->pindex, 0);
963 
964 	for (i = reqpage - 1; i >= 0; --i) {
965 		daddr_t iblk;
966 
967 		iblk = swp_pager_meta_ctl(m[i]->object, m[i]->pindex, 0);
968 		if (blk != iblk + (reqpage - i))
969 			break;
970 	}
971 	++i;
972 
973 	for (j = reqpage + 1; j < count; ++j) {
974 		daddr_t jblk;
975 
976 		jblk = swp_pager_meta_ctl(m[j]->object, m[j]->pindex, 0);
977 		if (blk != jblk - (j - reqpage))
978 			break;
979 	}
980 
981 	/*
982 	 * free pages outside our collection range.   Note: we never free
983 	 * mreq, it must remain busy throughout.
984 	 */
985 	if (0 < i || j < count) {
986 		int k;
987 
988 		vm_page_lock_queues();
989 		for (k = 0; k < i; ++k)
990 			vm_page_free(m[k]);
991 		for (k = j; k < count; ++k)
992 			vm_page_free(m[k]);
993 		vm_page_unlock_queues();
994 	}
995 
996 	/*
997 	 * Return VM_PAGER_FAIL if we have nothing to do.  Return mreq
998 	 * still busy, but the others unbusied.
999 	 */
1000 	if (blk == SWAPBLK_NONE)
1001 		return (VM_PAGER_FAIL);
1002 
1003 	/*
1004 	 * Getpbuf() can sleep.
1005 	 */
1006 	VM_OBJECT_UNLOCK(object);
1007 	/*
1008 	 * Get a swap buffer header to perform the IO
1009 	 */
1010 	bp = getpbuf(&nsw_rcount);
1011 	bp->b_flags |= B_PAGING;
1012 
1013 	/*
1014 	 * map our page(s) into kva for input
1015 	 */
1016 	pmap_qenter((vm_offset_t)bp->b_data, m + i, j - i);
1017 
1018 	bp->b_iocmd = BIO_READ;
1019 	bp->b_iodone = swp_pager_async_iodone;
1020 	bp->b_rcred = crhold(thread0.td_ucred);
1021 	bp->b_wcred = crhold(thread0.td_ucred);
1022 	bp->b_blkno = blk - (reqpage - i);
1023 	bp->b_bcount = PAGE_SIZE * (j - i);
1024 	bp->b_bufsize = PAGE_SIZE * (j - i);
1025 	bp->b_pager.pg_reqpage = reqpage - i;
1026 
1027 	VM_OBJECT_LOCK(object);
1028 	{
1029 		int k;
1030 
1031 		for (k = i; k < j; ++k) {
1032 			bp->b_pages[k - i] = m[k];
1033 			m[k]->oflags |= VPO_SWAPINPROG;
1034 		}
1035 	}
1036 	bp->b_npages = j - i;
1037 
1038 	PCPU_INC(cnt.v_swapin);
1039 	PCPU_ADD(cnt.v_swappgsin, bp->b_npages);
1040 
1041 	/*
1042 	 * We still hold the lock on mreq, and our automatic completion routine
1043 	 * does not remove it.
1044 	 */
1045 	vm_object_pip_add(object, bp->b_npages);
1046 	VM_OBJECT_UNLOCK(object);
1047 
1048 	/*
1049 	 * perform the I/O.  NOTE!!!  bp cannot be considered valid after
1050 	 * this point because we automatically release it on completion.
1051 	 * Instead, we look at the one page we are interested in which we
1052 	 * still hold a lock on even through the I/O completion.
1053 	 *
1054 	 * The other pages in our m[] array are also released on completion,
1055 	 * so we cannot assume they are valid anymore either.
1056 	 *
1057 	 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1058 	 */
1059 	BUF_KERNPROC(bp);
1060 	swp_pager_strategy(bp);
1061 
1062 	/*
1063 	 * wait for the page we want to complete.  VPO_SWAPINPROG is always
1064 	 * cleared on completion.  If an I/O error occurs, SWAPBLK_NONE
1065 	 * is set in the meta-data.
1066 	 */
1067 	VM_OBJECT_LOCK(object);
1068 	while ((mreq->oflags & VPO_SWAPINPROG) != 0) {
1069 		mreq->oflags |= VPO_WANTED;
1070 		vm_page_lock_queues();
1071 		vm_page_flag_set(mreq, PG_REFERENCED);
1072 		vm_page_unlock_queues();
1073 		PCPU_INC(cnt.v_intrans);
1074 		if (msleep(mreq, VM_OBJECT_MTX(object), PSWP, "swread", hz*20)) {
1075 			printf(
1076 "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n",
1077 			    bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount);
1078 		}
1079 	}
1080 
1081 	/*
1082 	 * mreq is left busied after completion, but all the other pages
1083 	 * are freed.  If we had an unrecoverable read error the page will
1084 	 * not be valid.
1085 	 */
1086 	if (mreq->valid != VM_PAGE_BITS_ALL) {
1087 		return (VM_PAGER_ERROR);
1088 	} else {
1089 		return (VM_PAGER_OK);
1090 	}
1091 
1092 	/*
1093 	 * A final note: in a low swap situation, we cannot deallocate swap
1094 	 * and mark a page dirty here because the caller is likely to mark
1095 	 * the page clean when we return, causing the page to possibly revert
1096 	 * to all-zero's later.
1097 	 */
1098 }
1099 
1100 /*
1101  *	swap_pager_putpages:
1102  *
1103  *	Assign swap (if necessary) and initiate I/O on the specified pages.
1104  *
1105  *	We support both OBJT_DEFAULT and OBJT_SWAP objects.  DEFAULT objects
1106  *	are automatically converted to SWAP objects.
1107  *
1108  *	In a low memory situation we may block in VOP_STRATEGY(), but the new
1109  *	vm_page reservation system coupled with properly written VFS devices
1110  *	should ensure that no low-memory deadlock occurs.  This is an area
1111  *	which needs work.
1112  *
1113  *	The parent has N vm_object_pip_add() references prior to
1114  *	calling us and will remove references for rtvals[] that are
1115  *	not set to VM_PAGER_PEND.  We need to remove the rest on I/O
1116  *	completion.
1117  *
1118  *	The parent has soft-busy'd the pages it passes us and will unbusy
1119  *	those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
1120  *	We need to unbusy the rest on I/O completion.
1121  */
1122 void
1123 swap_pager_putpages(vm_object_t object, vm_page_t *m, int count,
1124     boolean_t sync, int *rtvals)
1125 {
1126 	int i;
1127 	int n = 0;
1128 
1129 	if (count && m[0]->object != object) {
1130 		panic("swap_pager_putpages: object mismatch %p/%p",
1131 		    object,
1132 		    m[0]->object
1133 		);
1134 	}
1135 
1136 	/*
1137 	 * Step 1
1138 	 *
1139 	 * Turn object into OBJT_SWAP
1140 	 * check for bogus sysops
1141 	 * force sync if not pageout process
1142 	 */
1143 	if (object->type != OBJT_SWAP)
1144 		swp_pager_meta_build(object, 0, SWAPBLK_NONE);
1145 	VM_OBJECT_UNLOCK(object);
1146 
1147 	if (curproc != pageproc)
1148 		sync = TRUE;
1149 
1150 	/*
1151 	 * Step 2
1152 	 *
1153 	 * Update nsw parameters from swap_async_max sysctl values.
1154 	 * Do not let the sysop crash the machine with bogus numbers.
1155 	 */
1156 	mtx_lock(&pbuf_mtx);
1157 	if (swap_async_max != nsw_wcount_async_max) {
1158 		int n;
1159 
1160 		/*
1161 		 * limit range
1162 		 */
1163 		if ((n = swap_async_max) > nswbuf / 2)
1164 			n = nswbuf / 2;
1165 		if (n < 1)
1166 			n = 1;
1167 		swap_async_max = n;
1168 
1169 		/*
1170 		 * Adjust difference ( if possible ).  If the current async
1171 		 * count is too low, we may not be able to make the adjustment
1172 		 * at this time.
1173 		 */
1174 		n -= nsw_wcount_async_max;
1175 		if (nsw_wcount_async + n >= 0) {
1176 			nsw_wcount_async += n;
1177 			nsw_wcount_async_max += n;
1178 			wakeup(&nsw_wcount_async);
1179 		}
1180 	}
1181 	mtx_unlock(&pbuf_mtx);
1182 
1183 	/*
1184 	 * Step 3
1185 	 *
1186 	 * Assign swap blocks and issue I/O.  We reallocate swap on the fly.
1187 	 * The page is left dirty until the pageout operation completes
1188 	 * successfully.
1189 	 */
1190 	for (i = 0; i < count; i += n) {
1191 		int j;
1192 		struct buf *bp;
1193 		daddr_t blk;
1194 
1195 		/*
1196 		 * Maximum I/O size is limited by a number of factors.
1197 		 */
1198 		n = min(BLIST_MAX_ALLOC, count - i);
1199 		n = min(n, nsw_cluster_max);
1200 
1201 		/*
1202 		 * Get biggest block of swap we can.  If we fail, fall
1203 		 * back and try to allocate a smaller block.  Don't go
1204 		 * overboard trying to allocate space if it would overly
1205 		 * fragment swap.
1206 		 */
1207 		while (
1208 		    (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE &&
1209 		    n > 4
1210 		) {
1211 			n >>= 1;
1212 		}
1213 		if (blk == SWAPBLK_NONE) {
1214 			for (j = 0; j < n; ++j)
1215 				rtvals[i+j] = VM_PAGER_FAIL;
1216 			continue;
1217 		}
1218 
1219 		/*
1220 		 * All I/O parameters have been satisfied, build the I/O
1221 		 * request and assign the swap space.
1222 		 */
1223 		if (sync == TRUE) {
1224 			bp = getpbuf(&nsw_wcount_sync);
1225 		} else {
1226 			bp = getpbuf(&nsw_wcount_async);
1227 			bp->b_flags = B_ASYNC;
1228 		}
1229 		bp->b_flags |= B_PAGING;
1230 		bp->b_iocmd = BIO_WRITE;
1231 
1232 		pmap_qenter((vm_offset_t)bp->b_data, &m[i], n);
1233 
1234 		bp->b_rcred = crhold(thread0.td_ucred);
1235 		bp->b_wcred = crhold(thread0.td_ucred);
1236 		bp->b_bcount = PAGE_SIZE * n;
1237 		bp->b_bufsize = PAGE_SIZE * n;
1238 		bp->b_blkno = blk;
1239 
1240 		VM_OBJECT_LOCK(object);
1241 		for (j = 0; j < n; ++j) {
1242 			vm_page_t mreq = m[i+j];
1243 
1244 			swp_pager_meta_build(
1245 			    mreq->object,
1246 			    mreq->pindex,
1247 			    blk + j
1248 			);
1249 			vm_page_dirty(mreq);
1250 			rtvals[i+j] = VM_PAGER_OK;
1251 
1252 			mreq->oflags |= VPO_SWAPINPROG;
1253 			bp->b_pages[j] = mreq;
1254 		}
1255 		VM_OBJECT_UNLOCK(object);
1256 		bp->b_npages = n;
1257 		/*
1258 		 * Must set dirty range for NFS to work.
1259 		 */
1260 		bp->b_dirtyoff = 0;
1261 		bp->b_dirtyend = bp->b_bcount;
1262 
1263 		PCPU_INC(cnt.v_swapout);
1264 		PCPU_ADD(cnt.v_swappgsout, bp->b_npages);
1265 
1266 		/*
1267 		 * asynchronous
1268 		 *
1269 		 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1270 		 */
1271 		if (sync == FALSE) {
1272 			bp->b_iodone = swp_pager_async_iodone;
1273 			BUF_KERNPROC(bp);
1274 			swp_pager_strategy(bp);
1275 
1276 			for (j = 0; j < n; ++j)
1277 				rtvals[i+j] = VM_PAGER_PEND;
1278 			/* restart outter loop */
1279 			continue;
1280 		}
1281 
1282 		/*
1283 		 * synchronous
1284 		 *
1285 		 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1286 		 */
1287 		bp->b_iodone = bdone;
1288 		swp_pager_strategy(bp);
1289 
1290 		/*
1291 		 * Wait for the sync I/O to complete, then update rtvals.
1292 		 * We just set the rtvals[] to VM_PAGER_PEND so we can call
1293 		 * our async completion routine at the end, thus avoiding a
1294 		 * double-free.
1295 		 */
1296 		bwait(bp, PVM, "swwrt");
1297 		for (j = 0; j < n; ++j)
1298 			rtvals[i+j] = VM_PAGER_PEND;
1299 		/*
1300 		 * Now that we are through with the bp, we can call the
1301 		 * normal async completion, which frees everything up.
1302 		 */
1303 		swp_pager_async_iodone(bp);
1304 	}
1305 	VM_OBJECT_LOCK(object);
1306 }
1307 
1308 /*
1309  *	swp_pager_async_iodone:
1310  *
1311  *	Completion routine for asynchronous reads and writes from/to swap.
1312  *	Also called manually by synchronous code to finish up a bp.
1313  *
1314  *	For READ operations, the pages are PG_BUSY'd.  For WRITE operations,
1315  *	the pages are vm_page_t->busy'd.  For READ operations, we PG_BUSY
1316  *	unbusy all pages except the 'main' request page.  For WRITE
1317  *	operations, we vm_page_t->busy'd unbusy all pages ( we can do this
1318  *	because we marked them all VM_PAGER_PEND on return from putpages ).
1319  *
1320  *	This routine may not block.
1321  */
1322 static void
1323 swp_pager_async_iodone(struct buf *bp)
1324 {
1325 	int i;
1326 	vm_object_t object = NULL;
1327 
1328 	/*
1329 	 * report error
1330 	 */
1331 	if (bp->b_ioflags & BIO_ERROR) {
1332 		printf(
1333 		    "swap_pager: I/O error - %s failed; blkno %ld,"
1334 			"size %ld, error %d\n",
1335 		    ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
1336 		    (long)bp->b_blkno,
1337 		    (long)bp->b_bcount,
1338 		    bp->b_error
1339 		);
1340 	}
1341 
1342 	/*
1343 	 * remove the mapping for kernel virtual
1344 	 */
1345 	pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
1346 
1347 	if (bp->b_npages) {
1348 		object = bp->b_pages[0]->object;
1349 		VM_OBJECT_LOCK(object);
1350 	}
1351 	vm_page_lock_queues();
1352 	/*
1353 	 * cleanup pages.  If an error occurs writing to swap, we are in
1354 	 * very serious trouble.  If it happens to be a disk error, though,
1355 	 * we may be able to recover by reassigning the swap later on.  So
1356 	 * in this case we remove the m->swapblk assignment for the page
1357 	 * but do not free it in the rlist.  The errornous block(s) are thus
1358 	 * never reallocated as swap.  Redirty the page and continue.
1359 	 */
1360 	for (i = 0; i < bp->b_npages; ++i) {
1361 		vm_page_t m = bp->b_pages[i];
1362 
1363 		m->oflags &= ~VPO_SWAPINPROG;
1364 
1365 		if (bp->b_ioflags & BIO_ERROR) {
1366 			/*
1367 			 * If an error occurs I'd love to throw the swapblk
1368 			 * away without freeing it back to swapspace, so it
1369 			 * can never be used again.  But I can't from an
1370 			 * interrupt.
1371 			 */
1372 			if (bp->b_iocmd == BIO_READ) {
1373 				/*
1374 				 * When reading, reqpage needs to stay
1375 				 * locked for the parent, but all other
1376 				 * pages can be freed.  We still want to
1377 				 * wakeup the parent waiting on the page,
1378 				 * though.  ( also: pg_reqpage can be -1 and
1379 				 * not match anything ).
1380 				 *
1381 				 * We have to wake specifically requested pages
1382 				 * up too because we cleared VPO_SWAPINPROG and
1383 				 * someone may be waiting for that.
1384 				 *
1385 				 * NOTE: for reads, m->dirty will probably
1386 				 * be overridden by the original caller of
1387 				 * getpages so don't play cute tricks here.
1388 				 */
1389 				m->valid = 0;
1390 				if (i != bp->b_pager.pg_reqpage)
1391 					vm_page_free(m);
1392 				else
1393 					vm_page_flash(m);
1394 				/*
1395 				 * If i == bp->b_pager.pg_reqpage, do not wake
1396 				 * the page up.  The caller needs to.
1397 				 */
1398 			} else {
1399 				/*
1400 				 * If a write error occurs, reactivate page
1401 				 * so it doesn't clog the inactive list,
1402 				 * then finish the I/O.
1403 				 */
1404 				vm_page_dirty(m);
1405 				vm_page_activate(m);
1406 				vm_page_io_finish(m);
1407 			}
1408 		} else if (bp->b_iocmd == BIO_READ) {
1409 			/*
1410 			 * NOTE: for reads, m->dirty will probably be
1411 			 * overridden by the original caller of getpages so
1412 			 * we cannot set them in order to free the underlying
1413 			 * swap in a low-swap situation.  I don't think we'd
1414 			 * want to do that anyway, but it was an optimization
1415 			 * that existed in the old swapper for a time before
1416 			 * it got ripped out due to precisely this problem.
1417 			 *
1418 			 * If not the requested page then deactivate it.
1419 			 *
1420 			 * Note that the requested page, reqpage, is left
1421 			 * busied, but we still have to wake it up.  The
1422 			 * other pages are released (unbusied) by
1423 			 * vm_page_wakeup().
1424 			 */
1425 			KASSERT(!pmap_page_is_mapped(m),
1426 			    ("swp_pager_async_iodone: page %p is mapped", m));
1427 			m->valid = VM_PAGE_BITS_ALL;
1428 			KASSERT(m->dirty == 0,
1429 			    ("swp_pager_async_iodone: page %p is dirty", m));
1430 
1431 			/*
1432 			 * We have to wake specifically requested pages
1433 			 * up too because we cleared VPO_SWAPINPROG and
1434 			 * could be waiting for it in getpages.  However,
1435 			 * be sure to not unbusy getpages specifically
1436 			 * requested page - getpages expects it to be
1437 			 * left busy.
1438 			 */
1439 			if (i != bp->b_pager.pg_reqpage) {
1440 				vm_page_deactivate(m);
1441 				vm_page_wakeup(m);
1442 			} else {
1443 				vm_page_flash(m);
1444 			}
1445 		} else {
1446 			/*
1447 			 * For write success, clear the dirty
1448 			 * status, then finish the I/O ( which decrements the
1449 			 * busy count and possibly wakes waiter's up ).
1450 			 */
1451 			KASSERT((m->flags & PG_WRITEABLE) == 0,
1452 			    ("swp_pager_async_iodone: page %p is not write"
1453 			    " protected", m));
1454 			vm_page_undirty(m);
1455 			vm_page_io_finish(m);
1456 			if (vm_page_count_severe())
1457 				vm_page_try_to_cache(m);
1458 		}
1459 	}
1460 	vm_page_unlock_queues();
1461 
1462 	/*
1463 	 * adjust pip.  NOTE: the original parent may still have its own
1464 	 * pip refs on the object.
1465 	 */
1466 	if (object != NULL) {
1467 		vm_object_pip_wakeupn(object, bp->b_npages);
1468 		VM_OBJECT_UNLOCK(object);
1469 	}
1470 
1471 	/*
1472 	 * swapdev_strategy() manually sets b_vp and b_bufobj before calling
1473 	 * bstrategy(). Set them back to NULL now we're done with it, or we'll
1474 	 * trigger a KASSERT in relpbuf().
1475 	 */
1476 	if (bp->b_vp) {
1477 		    bp->b_vp = NULL;
1478 		    bp->b_bufobj = NULL;
1479 	}
1480 	/*
1481 	 * release the physical I/O buffer
1482 	 */
1483 	relpbuf(
1484 	    bp,
1485 	    ((bp->b_iocmd == BIO_READ) ? &nsw_rcount :
1486 		((bp->b_flags & B_ASYNC) ?
1487 		    &nsw_wcount_async :
1488 		    &nsw_wcount_sync
1489 		)
1490 	    )
1491 	);
1492 }
1493 
1494 /*
1495  *	swap_pager_isswapped:
1496  *
1497  *	Return 1 if at least one page in the given object is paged
1498  *	out to the given swap device.
1499  *
1500  *	This routine may not block.
1501  */
1502 int
1503 swap_pager_isswapped(vm_object_t object, struct swdevt *sp)
1504 {
1505 	daddr_t index = 0;
1506 	int bcount;
1507 	int i;
1508 
1509 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1510 	if (object->type != OBJT_SWAP)
1511 		return (0);
1512 
1513 	mtx_lock(&swhash_mtx);
1514 	for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) {
1515 		struct swblock *swap;
1516 
1517 		if ((swap = *swp_pager_hash(object, index)) != NULL) {
1518 			for (i = 0; i < SWAP_META_PAGES; ++i) {
1519 				if (swp_pager_isondev(swap->swb_pages[i], sp)) {
1520 					mtx_unlock(&swhash_mtx);
1521 					return (1);
1522 				}
1523 			}
1524 		}
1525 		index += SWAP_META_PAGES;
1526 		if (index > 0x20000000)
1527 			panic("swap_pager_isswapped: failed to locate all swap meta blocks");
1528 	}
1529 	mtx_unlock(&swhash_mtx);
1530 	return (0);
1531 }
1532 
1533 /*
1534  * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in
1535  *
1536  *	This routine dissociates the page at the given index within a
1537  *	swap block from its backing store, paging it in if necessary.
1538  *	If the page is paged in, it is placed in the inactive queue,
1539  *	since it had its backing store ripped out from under it.
1540  *	We also attempt to swap in all other pages in the swap block,
1541  *	we only guarantee that the one at the specified index is
1542  *	paged in.
1543  *
1544  *	XXX - The code to page the whole block in doesn't work, so we
1545  *	      revert to the one-by-one behavior for now.  Sigh.
1546  */
1547 static inline void
1548 swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex)
1549 {
1550 	vm_page_t m;
1551 
1552 	vm_object_pip_add(object, 1);
1553 	m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL|VM_ALLOC_RETRY);
1554 	if (m->valid == VM_PAGE_BITS_ALL) {
1555 		vm_object_pip_subtract(object, 1);
1556 		vm_page_lock_queues();
1557 		vm_page_activate(m);
1558 		vm_page_dirty(m);
1559 		vm_page_unlock_queues();
1560 		vm_page_wakeup(m);
1561 		vm_pager_page_unswapped(m);
1562 		return;
1563 	}
1564 
1565 	if (swap_pager_getpages(object, &m, 1, 0) != VM_PAGER_OK)
1566 		panic("swap_pager_force_pagein: read from swap failed");/*XXX*/
1567 	vm_object_pip_subtract(object, 1);
1568 	vm_page_lock_queues();
1569 	vm_page_dirty(m);
1570 	vm_page_dontneed(m);
1571 	vm_page_unlock_queues();
1572 	vm_page_wakeup(m);
1573 	vm_pager_page_unswapped(m);
1574 }
1575 
1576 /*
1577  *	swap_pager_swapoff:
1578  *
1579  *	Page in all of the pages that have been paged out to the
1580  *	given device.  The corresponding blocks in the bitmap must be
1581  *	marked as allocated and the device must be flagged SW_CLOSING.
1582  *	There may be no processes swapped out to the device.
1583  *
1584  *	This routine may block.
1585  */
1586 static void
1587 swap_pager_swapoff(struct swdevt *sp)
1588 {
1589 	struct swblock *swap;
1590 	int i, j, retries;
1591 
1592 	GIANT_REQUIRED;
1593 
1594 	retries = 0;
1595 full_rescan:
1596 	mtx_lock(&swhash_mtx);
1597 	for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */
1598 restart:
1599 		for (swap = swhash[i]; swap != NULL; swap = swap->swb_hnext) {
1600 			vm_object_t object = swap->swb_object;
1601 			vm_pindex_t pindex = swap->swb_index;
1602                         for (j = 0; j < SWAP_META_PAGES; ++j) {
1603                                 if (swp_pager_isondev(swap->swb_pages[j], sp)) {
1604 					/* avoid deadlock */
1605 					if (!VM_OBJECT_TRYLOCK(object)) {
1606 						break;
1607 					} else {
1608 						mtx_unlock(&swhash_mtx);
1609 						swp_pager_force_pagein(object,
1610 						    pindex + j);
1611 						VM_OBJECT_UNLOCK(object);
1612 						mtx_lock(&swhash_mtx);
1613 						goto restart;
1614 					}
1615 				}
1616                         }
1617 		}
1618 	}
1619 	mtx_unlock(&swhash_mtx);
1620 	if (sp->sw_used) {
1621 		/*
1622 		 * Objects may be locked or paging to the device being
1623 		 * removed, so we will miss their pages and need to
1624 		 * make another pass.  We have marked this device as
1625 		 * SW_CLOSING, so the activity should finish soon.
1626 		 */
1627 		retries++;
1628 		if (retries > 100) {
1629 			panic("swapoff: failed to locate %d swap blocks",
1630 			    sp->sw_used);
1631 		}
1632 		pause("swpoff", hz / 20);
1633 		goto full_rescan;
1634 	}
1635 }
1636 
1637 /************************************************************************
1638  *				SWAP META DATA 				*
1639  ************************************************************************
1640  *
1641  *	These routines manipulate the swap metadata stored in the
1642  *	OBJT_SWAP object.  All swp_*() routines must be called at
1643  *	splvm() because swap can be freed up by the low level vm_page
1644  *	code which might be called from interrupts beyond what splbio() covers.
1645  *
1646  *	Swap metadata is implemented with a global hash and not directly
1647  *	linked into the object.  Instead the object simply contains
1648  *	appropriate tracking counters.
1649  */
1650 
1651 /*
1652  * SWP_PAGER_META_BUILD() -	add swap block to swap meta data for object
1653  *
1654  *	We first convert the object to a swap object if it is a default
1655  *	object.
1656  *
1657  *	The specified swapblk is added to the object's swap metadata.  If
1658  *	the swapblk is not valid, it is freed instead.  Any previously
1659  *	assigned swapblk is freed.
1660  *
1661  *	This routine must be called at splvm(), except when used to convert
1662  *	an OBJT_DEFAULT object into an OBJT_SWAP object.
1663  */
1664 static void
1665 swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
1666 {
1667 	struct swblock *swap;
1668 	struct swblock **pswap;
1669 	int idx;
1670 
1671 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1672 	/*
1673 	 * Convert default object to swap object if necessary
1674 	 */
1675 	if (object->type != OBJT_SWAP) {
1676 		object->type = OBJT_SWAP;
1677 		object->un_pager.swp.swp_bcount = 0;
1678 
1679 		if (object->handle != NULL) {
1680 			mtx_lock(&sw_alloc_mtx);
1681 			TAILQ_INSERT_TAIL(
1682 			    NOBJLIST(object->handle),
1683 			    object,
1684 			    pager_object_list
1685 			);
1686 			mtx_unlock(&sw_alloc_mtx);
1687 		}
1688 	}
1689 
1690 	/*
1691 	 * Locate hash entry.  If not found create, but if we aren't adding
1692 	 * anything just return.  If we run out of space in the map we wait
1693 	 * and, since the hash table may have changed, retry.
1694 	 */
1695 retry:
1696 	mtx_lock(&swhash_mtx);
1697 	pswap = swp_pager_hash(object, pindex);
1698 
1699 	if ((swap = *pswap) == NULL) {
1700 		int i;
1701 
1702 		if (swapblk == SWAPBLK_NONE)
1703 			goto done;
1704 
1705 		swap = *pswap = uma_zalloc(swap_zone, M_NOWAIT);
1706 		if (swap == NULL) {
1707 			mtx_unlock(&swhash_mtx);
1708 			VM_OBJECT_UNLOCK(object);
1709 			if (uma_zone_exhausted(swap_zone)) {
1710 				printf("swap zone exhausted, increase kern.maxswzone\n");
1711 				vm_pageout_oom(VM_OOM_SWAPZ);
1712 				pause("swzonex", 10);
1713 			} else
1714 				VM_WAIT;
1715 			VM_OBJECT_LOCK(object);
1716 			goto retry;
1717 		}
1718 
1719 		swap->swb_hnext = NULL;
1720 		swap->swb_object = object;
1721 		swap->swb_index = pindex & ~(vm_pindex_t)SWAP_META_MASK;
1722 		swap->swb_count = 0;
1723 
1724 		++object->un_pager.swp.swp_bcount;
1725 
1726 		for (i = 0; i < SWAP_META_PAGES; ++i)
1727 			swap->swb_pages[i] = SWAPBLK_NONE;
1728 	}
1729 
1730 	/*
1731 	 * Delete prior contents of metadata
1732 	 */
1733 	idx = pindex & SWAP_META_MASK;
1734 
1735 	if (swap->swb_pages[idx] != SWAPBLK_NONE) {
1736 		swp_pager_freeswapspace(swap->swb_pages[idx], 1);
1737 		--swap->swb_count;
1738 	}
1739 
1740 	/*
1741 	 * Enter block into metadata
1742 	 */
1743 	swap->swb_pages[idx] = swapblk;
1744 	if (swapblk != SWAPBLK_NONE)
1745 		++swap->swb_count;
1746 done:
1747 	mtx_unlock(&swhash_mtx);
1748 }
1749 
1750 /*
1751  * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
1752  *
1753  *	The requested range of blocks is freed, with any associated swap
1754  *	returned to the swap bitmap.
1755  *
1756  *	This routine will free swap metadata structures as they are cleaned
1757  *	out.  This routine does *NOT* operate on swap metadata associated
1758  *	with resident pages.
1759  *
1760  *	This routine must be called at splvm()
1761  */
1762 static void
1763 swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count)
1764 {
1765 
1766 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1767 	if (object->type != OBJT_SWAP)
1768 		return;
1769 
1770 	while (count > 0) {
1771 		struct swblock **pswap;
1772 		struct swblock *swap;
1773 
1774 		mtx_lock(&swhash_mtx);
1775 		pswap = swp_pager_hash(object, index);
1776 
1777 		if ((swap = *pswap) != NULL) {
1778 			daddr_t v = swap->swb_pages[index & SWAP_META_MASK];
1779 
1780 			if (v != SWAPBLK_NONE) {
1781 				swp_pager_freeswapspace(v, 1);
1782 				swap->swb_pages[index & SWAP_META_MASK] =
1783 					SWAPBLK_NONE;
1784 				if (--swap->swb_count == 0) {
1785 					*pswap = swap->swb_hnext;
1786 					uma_zfree(swap_zone, swap);
1787 					--object->un_pager.swp.swp_bcount;
1788 				}
1789 			}
1790 			--count;
1791 			++index;
1792 		} else {
1793 			int n = SWAP_META_PAGES - (index & SWAP_META_MASK);
1794 			count -= n;
1795 			index += n;
1796 		}
1797 		mtx_unlock(&swhash_mtx);
1798 	}
1799 }
1800 
1801 /*
1802  * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
1803  *
1804  *	This routine locates and destroys all swap metadata associated with
1805  *	an object.
1806  *
1807  *	This routine must be called at splvm()
1808  */
1809 static void
1810 swp_pager_meta_free_all(vm_object_t object)
1811 {
1812 	daddr_t index = 0;
1813 
1814 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1815 	if (object->type != OBJT_SWAP)
1816 		return;
1817 
1818 	while (object->un_pager.swp.swp_bcount) {
1819 		struct swblock **pswap;
1820 		struct swblock *swap;
1821 
1822 		mtx_lock(&swhash_mtx);
1823 		pswap = swp_pager_hash(object, index);
1824 		if ((swap = *pswap) != NULL) {
1825 			int i;
1826 
1827 			for (i = 0; i < SWAP_META_PAGES; ++i) {
1828 				daddr_t v = swap->swb_pages[i];
1829 				if (v != SWAPBLK_NONE) {
1830 					--swap->swb_count;
1831 					swp_pager_freeswapspace(v, 1);
1832 				}
1833 			}
1834 			if (swap->swb_count != 0)
1835 				panic("swap_pager_meta_free_all: swb_count != 0");
1836 			*pswap = swap->swb_hnext;
1837 			uma_zfree(swap_zone, swap);
1838 			--object->un_pager.swp.swp_bcount;
1839 		}
1840 		mtx_unlock(&swhash_mtx);
1841 		index += SWAP_META_PAGES;
1842 		if (index > 0x20000000)
1843 			panic("swp_pager_meta_free_all: failed to locate all swap meta blocks");
1844 	}
1845 }
1846 
1847 /*
1848  * SWP_PAGER_METACTL() -  misc control of swap and vm_page_t meta data.
1849  *
1850  *	This routine is capable of looking up, popping, or freeing
1851  *	swapblk assignments in the swap meta data or in the vm_page_t.
1852  *	The routine typically returns the swapblk being looked-up, or popped,
1853  *	or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block
1854  *	was invalid.  This routine will automatically free any invalid
1855  *	meta-data swapblks.
1856  *
1857  *	It is not possible to store invalid swapblks in the swap meta data
1858  *	(other then a literal 'SWAPBLK_NONE'), so we don't bother checking.
1859  *
1860  *	When acting on a busy resident page and paging is in progress, we
1861  *	have to wait until paging is complete but otherwise can act on the
1862  *	busy page.
1863  *
1864  *	This routine must be called at splvm().
1865  *
1866  *	SWM_FREE	remove and free swap block from metadata
1867  *	SWM_POP		remove from meta data but do not free.. pop it out
1868  */
1869 static daddr_t
1870 swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
1871 {
1872 	struct swblock **pswap;
1873 	struct swblock *swap;
1874 	daddr_t r1;
1875 	int idx;
1876 
1877 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1878 	/*
1879 	 * The meta data only exists of the object is OBJT_SWAP
1880 	 * and even then might not be allocated yet.
1881 	 */
1882 	if (object->type != OBJT_SWAP)
1883 		return (SWAPBLK_NONE);
1884 
1885 	r1 = SWAPBLK_NONE;
1886 	mtx_lock(&swhash_mtx);
1887 	pswap = swp_pager_hash(object, pindex);
1888 
1889 	if ((swap = *pswap) != NULL) {
1890 		idx = pindex & SWAP_META_MASK;
1891 		r1 = swap->swb_pages[idx];
1892 
1893 		if (r1 != SWAPBLK_NONE) {
1894 			if (flags & SWM_FREE) {
1895 				swp_pager_freeswapspace(r1, 1);
1896 				r1 = SWAPBLK_NONE;
1897 			}
1898 			if (flags & (SWM_FREE|SWM_POP)) {
1899 				swap->swb_pages[idx] = SWAPBLK_NONE;
1900 				if (--swap->swb_count == 0) {
1901 					*pswap = swap->swb_hnext;
1902 					uma_zfree(swap_zone, swap);
1903 					--object->un_pager.swp.swp_bcount;
1904 				}
1905 			}
1906 		}
1907 	}
1908 	mtx_unlock(&swhash_mtx);
1909 	return (r1);
1910 }
1911 
1912 /*
1913  * System call swapon(name) enables swapping on device name,
1914  * which must be in the swdevsw.  Return EBUSY
1915  * if already swapping on this device.
1916  */
1917 #ifndef _SYS_SYSPROTO_H_
1918 struct swapon_args {
1919 	char *name;
1920 };
1921 #endif
1922 
1923 /*
1924  * MPSAFE
1925  */
1926 /* ARGSUSED */
1927 int
1928 swapon(struct thread *td, struct swapon_args *uap)
1929 {
1930 	struct vattr attr;
1931 	struct vnode *vp;
1932 	struct nameidata nd;
1933 	int error;
1934 
1935 	error = priv_check(td, PRIV_SWAPON);
1936 	if (error)
1937 		return (error);
1938 
1939 	mtx_lock(&Giant);
1940 	while (swdev_syscall_active)
1941 	    tsleep(&swdev_syscall_active, PUSER - 1, "swpon", 0);
1942 	swdev_syscall_active = 1;
1943 
1944 	/*
1945 	 * Swap metadata may not fit in the KVM if we have physical
1946 	 * memory of >1GB.
1947 	 */
1948 	if (swap_zone == NULL) {
1949 		error = ENOMEM;
1950 		goto done;
1951 	}
1952 
1953 	NDINIT(&nd, LOOKUP, ISOPEN | FOLLOW | AUDITVNODE1, UIO_USERSPACE,
1954 	    uap->name, td);
1955 	error = namei(&nd);
1956 	if (error)
1957 		goto done;
1958 
1959 	NDFREE(&nd, NDF_ONLY_PNBUF);
1960 	vp = nd.ni_vp;
1961 
1962 	if (vn_isdisk(vp, &error)) {
1963 		error = swapongeom(td, vp);
1964 	} else if (vp->v_type == VREG &&
1965 	    (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
1966 	    (error = VOP_GETATTR(vp, &attr, td->td_ucred)) == 0) {
1967 		/*
1968 		 * Allow direct swapping to NFS regular files in the same
1969 		 * way that nfs_mountroot() sets up diskless swapping.
1970 		 */
1971 		error = swaponvp(td, vp, attr.va_size / DEV_BSIZE);
1972 	}
1973 
1974 	if (error)
1975 		vrele(vp);
1976 done:
1977 	swdev_syscall_active = 0;
1978 	wakeup_one(&swdev_syscall_active);
1979 	mtx_unlock(&Giant);
1980 	return (error);
1981 }
1982 
1983 static void
1984 swaponsomething(struct vnode *vp, void *id, u_long nblks, sw_strategy_t *strategy, sw_close_t *close, dev_t dev)
1985 {
1986 	struct swdevt *sp, *tsp;
1987 	swblk_t dvbase;
1988 	u_long mblocks;
1989 
1990 	/*
1991 	 * If we go beyond this, we get overflows in the radix
1992 	 * tree bitmap code.
1993 	 */
1994 	mblocks = 0x40000000 / BLIST_META_RADIX;
1995 	if (nblks > mblocks) {
1996 		printf("WARNING: reducing size to maximum of %lu blocks per swap unit\n",
1997 			mblocks);
1998 		nblks = mblocks;
1999 	}
2000 	/*
2001 	 * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks.
2002 	 * First chop nblks off to page-align it, then convert.
2003 	 *
2004 	 * sw->sw_nblks is in page-sized chunks now too.
2005 	 */
2006 	nblks &= ~(ctodb(1) - 1);
2007 	nblks = dbtoc(nblks);
2008 
2009 	sp = malloc(sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO);
2010 	sp->sw_vp = vp;
2011 	sp->sw_id = id;
2012 	sp->sw_dev = dev;
2013 	sp->sw_flags = 0;
2014 	sp->sw_nblks = nblks;
2015 	sp->sw_used = 0;
2016 	sp->sw_strategy = strategy;
2017 	sp->sw_close = close;
2018 
2019 	sp->sw_blist = blist_create(nblks, M_WAITOK);
2020 	/*
2021 	 * Do not free the first two block in order to avoid overwriting
2022 	 * any bsd label at the front of the partition
2023 	 */
2024 	blist_free(sp->sw_blist, 2, nblks - 2);
2025 
2026 	dvbase = 0;
2027 	mtx_lock(&sw_dev_mtx);
2028 	TAILQ_FOREACH(tsp, &swtailq, sw_list) {
2029 		if (tsp->sw_end >= dvbase) {
2030 			/*
2031 			 * We put one uncovered page between the devices
2032 			 * in order to definitively prevent any cross-device
2033 			 * I/O requests
2034 			 */
2035 			dvbase = tsp->sw_end + 1;
2036 		}
2037 	}
2038 	sp->sw_first = dvbase;
2039 	sp->sw_end = dvbase + nblks;
2040 	TAILQ_INSERT_TAIL(&swtailq, sp, sw_list);
2041 	nswapdev++;
2042 	swap_pager_avail += nblks;
2043 	swp_sizecheck();
2044 	mtx_unlock(&sw_dev_mtx);
2045 }
2046 
2047 /*
2048  * SYSCALL: swapoff(devname)
2049  *
2050  * Disable swapping on the given device.
2051  *
2052  * XXX: Badly designed system call: it should use a device index
2053  * rather than filename as specification.  We keep sw_vp around
2054  * only to make this work.
2055  */
2056 #ifndef _SYS_SYSPROTO_H_
2057 struct swapoff_args {
2058 	char *name;
2059 };
2060 #endif
2061 
2062 /*
2063  * MPSAFE
2064  */
2065 /* ARGSUSED */
2066 int
2067 swapoff(struct thread *td, struct swapoff_args *uap)
2068 {
2069 	struct vnode *vp;
2070 	struct nameidata nd;
2071 	struct swdevt *sp;
2072 	int error;
2073 
2074 	error = priv_check(td, PRIV_SWAPOFF);
2075 	if (error)
2076 		return (error);
2077 
2078 	mtx_lock(&Giant);
2079 	while (swdev_syscall_active)
2080 	    tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
2081 	swdev_syscall_active = 1;
2082 
2083 	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, UIO_USERSPACE, uap->name,
2084 	    td);
2085 	error = namei(&nd);
2086 	if (error)
2087 		goto done;
2088 	NDFREE(&nd, NDF_ONLY_PNBUF);
2089 	vp = nd.ni_vp;
2090 
2091 	mtx_lock(&sw_dev_mtx);
2092 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2093 		if (sp->sw_vp == vp)
2094 			break;
2095 	}
2096 	mtx_unlock(&sw_dev_mtx);
2097 	if (sp == NULL) {
2098 		error = EINVAL;
2099 		goto done;
2100 	}
2101 	error = swapoff_one(sp, td->td_ucred);
2102 done:
2103 	swdev_syscall_active = 0;
2104 	wakeup_one(&swdev_syscall_active);
2105 	mtx_unlock(&Giant);
2106 	return (error);
2107 }
2108 
2109 static int
2110 swapoff_one(struct swdevt *sp, struct ucred *cred)
2111 {
2112 	u_long nblks, dvbase;
2113 #ifdef MAC
2114 	int error;
2115 #endif
2116 
2117 	mtx_assert(&Giant, MA_OWNED);
2118 #ifdef MAC
2119 	(void) vn_lock(sp->sw_vp, LK_EXCLUSIVE | LK_RETRY);
2120 	error = mac_system_check_swapoff(cred, sp->sw_vp);
2121 	(void) VOP_UNLOCK(sp->sw_vp, 0);
2122 	if (error != 0)
2123 		return (error);
2124 #endif
2125 	nblks = sp->sw_nblks;
2126 
2127 	/*
2128 	 * We can turn off this swap device safely only if the
2129 	 * available virtual memory in the system will fit the amount
2130 	 * of data we will have to page back in, plus an epsilon so
2131 	 * the system doesn't become critically low on swap space.
2132 	 */
2133 	if (cnt.v_free_count + cnt.v_cache_count + swap_pager_avail <
2134 	    nblks + nswap_lowat) {
2135 		return (ENOMEM);
2136 	}
2137 
2138 	/*
2139 	 * Prevent further allocations on this device.
2140 	 */
2141 	mtx_lock(&sw_dev_mtx);
2142 	sp->sw_flags |= SW_CLOSING;
2143 	for (dvbase = 0; dvbase < sp->sw_end; dvbase += dmmax) {
2144 		swap_pager_avail -= blist_fill(sp->sw_blist,
2145 		     dvbase, dmmax);
2146 	}
2147 	mtx_unlock(&sw_dev_mtx);
2148 
2149 	/*
2150 	 * Page in the contents of the device and close it.
2151 	 */
2152 	swap_pager_swapoff(sp);
2153 
2154 	sp->sw_close(curthread, sp);
2155 	sp->sw_id = NULL;
2156 	mtx_lock(&sw_dev_mtx);
2157 	TAILQ_REMOVE(&swtailq, sp, sw_list);
2158 	nswapdev--;
2159 	if (nswapdev == 0) {
2160 		swap_pager_full = 2;
2161 		swap_pager_almost_full = 1;
2162 	}
2163 	if (swdevhd == sp)
2164 		swdevhd = NULL;
2165 	mtx_unlock(&sw_dev_mtx);
2166 	blist_destroy(sp->sw_blist);
2167 	free(sp, M_VMPGDATA);
2168 	return (0);
2169 }
2170 
2171 void
2172 swapoff_all(void)
2173 {
2174 	struct swdevt *sp, *spt;
2175 	const char *devname;
2176 	int error;
2177 
2178 	mtx_lock(&Giant);
2179 	while (swdev_syscall_active)
2180 		tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
2181 	swdev_syscall_active = 1;
2182 
2183 	mtx_lock(&sw_dev_mtx);
2184 	TAILQ_FOREACH_SAFE(sp, &swtailq, sw_list, spt) {
2185 		mtx_unlock(&sw_dev_mtx);
2186 		if (vn_isdisk(sp->sw_vp, NULL))
2187 			devname = sp->sw_vp->v_rdev->si_name;
2188 		else
2189 			devname = "[file]";
2190 		error = swapoff_one(sp, thread0.td_ucred);
2191 		if (error != 0) {
2192 			printf("Cannot remove swap device %s (error=%d), "
2193 			    "skipping.\n", devname, error);
2194 		} else if (bootverbose) {
2195 			printf("Swap device %s removed.\n", devname);
2196 		}
2197 		mtx_lock(&sw_dev_mtx);
2198 	}
2199 	mtx_unlock(&sw_dev_mtx);
2200 
2201 	swdev_syscall_active = 0;
2202 	wakeup_one(&swdev_syscall_active);
2203 	mtx_unlock(&Giant);
2204 }
2205 
2206 void
2207 swap_pager_status(int *total, int *used)
2208 {
2209 	struct swdevt *sp;
2210 
2211 	*total = 0;
2212 	*used = 0;
2213 	mtx_lock(&sw_dev_mtx);
2214 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2215 		*total += sp->sw_nblks;
2216 		*used += sp->sw_used;
2217 	}
2218 	mtx_unlock(&sw_dev_mtx);
2219 }
2220 
2221 static int
2222 sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS)
2223 {
2224 	int	*name = (int *)arg1;
2225 	int	error, n;
2226 	struct xswdev xs;
2227 	struct swdevt *sp;
2228 
2229 	if (arg2 != 1) /* name length */
2230 		return (EINVAL);
2231 
2232 	n = 0;
2233 	mtx_lock(&sw_dev_mtx);
2234 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2235 		if (n == *name) {
2236 			mtx_unlock(&sw_dev_mtx);
2237 			xs.xsw_version = XSWDEV_VERSION;
2238 			xs.xsw_dev = sp->sw_dev;
2239 			xs.xsw_flags = sp->sw_flags;
2240 			xs.xsw_nblks = sp->sw_nblks;
2241 			xs.xsw_used = sp->sw_used;
2242 
2243 			error = SYSCTL_OUT(req, &xs, sizeof(xs));
2244 			return (error);
2245 		}
2246 		n++;
2247 	}
2248 	mtx_unlock(&sw_dev_mtx);
2249 	return (ENOENT);
2250 }
2251 
2252 SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, &nswapdev, 0,
2253     "Number of swap devices");
2254 SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD, sysctl_vm_swap_info,
2255     "Swap statistics by device");
2256 
2257 /*
2258  * vmspace_swap_count() - count the approximate swap usage in pages for a
2259  *			  vmspace.
2260  *
2261  *	The map must be locked.
2262  *
2263  *	Swap usage is determined by taking the proportional swap used by
2264  *	VM objects backing the VM map.  To make up for fractional losses,
2265  *	if the VM object has any swap use at all the associated map entries
2266  *	count for at least 1 swap page.
2267  */
2268 int
2269 vmspace_swap_count(struct vmspace *vmspace)
2270 {
2271 	vm_map_t map = &vmspace->vm_map;
2272 	vm_map_entry_t cur;
2273 	int count = 0;
2274 
2275 	for (cur = map->header.next; cur != &map->header; cur = cur->next) {
2276 		vm_object_t object;
2277 
2278 		if ((cur->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
2279 		    (object = cur->object.vm_object) != NULL) {
2280 			VM_OBJECT_LOCK(object);
2281 			if (object->type == OBJT_SWAP &&
2282 			    object->un_pager.swp.swp_bcount != 0) {
2283 				int n = (cur->end - cur->start) / PAGE_SIZE;
2284 
2285 				count += object->un_pager.swp.swp_bcount *
2286 				    SWAP_META_PAGES * n / object->size + 1;
2287 			}
2288 			VM_OBJECT_UNLOCK(object);
2289 		}
2290 	}
2291 	return (count);
2292 }
2293 
2294 /*
2295  * GEOM backend
2296  *
2297  * Swapping onto disk devices.
2298  *
2299  */
2300 
2301 static g_orphan_t swapgeom_orphan;
2302 
2303 static struct g_class g_swap_class = {
2304 	.name = "SWAP",
2305 	.version = G_VERSION,
2306 	.orphan = swapgeom_orphan,
2307 };
2308 
2309 DECLARE_GEOM_CLASS(g_swap_class, g_class);
2310 
2311 
2312 static void
2313 swapgeom_done(struct bio *bp2)
2314 {
2315 	struct buf *bp;
2316 
2317 	bp = bp2->bio_caller2;
2318 	bp->b_ioflags = bp2->bio_flags;
2319 	if (bp2->bio_error)
2320 		bp->b_ioflags |= BIO_ERROR;
2321 	bp->b_resid = bp->b_bcount - bp2->bio_completed;
2322 	bp->b_error = bp2->bio_error;
2323 	bufdone(bp);
2324 	g_destroy_bio(bp2);
2325 }
2326 
2327 static void
2328 swapgeom_strategy(struct buf *bp, struct swdevt *sp)
2329 {
2330 	struct bio *bio;
2331 	struct g_consumer *cp;
2332 
2333 	cp = sp->sw_id;
2334 	if (cp == NULL) {
2335 		bp->b_error = ENXIO;
2336 		bp->b_ioflags |= BIO_ERROR;
2337 		bufdone(bp);
2338 		return;
2339 	}
2340 	if (bp->b_iocmd == BIO_WRITE)
2341 		bio = g_new_bio();
2342 	else
2343 		bio = g_alloc_bio();
2344 	if (bio == NULL) {
2345 		bp->b_error = ENOMEM;
2346 		bp->b_ioflags |= BIO_ERROR;
2347 		bufdone(bp);
2348 		return;
2349 	}
2350 
2351 	bio->bio_caller2 = bp;
2352 	bio->bio_cmd = bp->b_iocmd;
2353 	bio->bio_data = bp->b_data;
2354 	bio->bio_offset = (bp->b_blkno - sp->sw_first) * PAGE_SIZE;
2355 	bio->bio_length = bp->b_bcount;
2356 	bio->bio_done = swapgeom_done;
2357 	g_io_request(bio, cp);
2358 	return;
2359 }
2360 
2361 static void
2362 swapgeom_orphan(struct g_consumer *cp)
2363 {
2364 	struct swdevt *sp;
2365 
2366 	mtx_lock(&sw_dev_mtx);
2367 	TAILQ_FOREACH(sp, &swtailq, sw_list)
2368 		if (sp->sw_id == cp)
2369 			sp->sw_id = NULL;
2370 	mtx_unlock(&sw_dev_mtx);
2371 }
2372 
2373 static void
2374 swapgeom_close_ev(void *arg, int flags)
2375 {
2376 	struct g_consumer *cp;
2377 
2378 	cp = arg;
2379 	g_access(cp, -1, -1, 0);
2380 	g_detach(cp);
2381 	g_destroy_consumer(cp);
2382 }
2383 
2384 static void
2385 swapgeom_close(struct thread *td, struct swdevt *sw)
2386 {
2387 
2388 	/* XXX: direct call when Giant untangled */
2389 	g_waitfor_event(swapgeom_close_ev, sw->sw_id, M_WAITOK, NULL);
2390 }
2391 
2392 
2393 struct swh0h0 {
2394 	struct cdev *dev;
2395 	struct vnode *vp;
2396 	int	error;
2397 };
2398 
2399 static void
2400 swapongeom_ev(void *arg, int flags)
2401 {
2402 	struct swh0h0 *swh;
2403 	struct g_provider *pp;
2404 	struct g_consumer *cp;
2405 	static struct g_geom *gp;
2406 	struct swdevt *sp;
2407 	u_long nblks;
2408 	int error;
2409 
2410 	swh = arg;
2411 	swh->error = 0;
2412 	pp = g_dev_getprovider(swh->dev);
2413 	if (pp == NULL) {
2414 		swh->error = ENODEV;
2415 		return;
2416 	}
2417 	mtx_lock(&sw_dev_mtx);
2418 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2419 		cp = sp->sw_id;
2420 		if (cp != NULL && cp->provider == pp) {
2421 			mtx_unlock(&sw_dev_mtx);
2422 			swh->error = EBUSY;
2423 			return;
2424 		}
2425 	}
2426 	mtx_unlock(&sw_dev_mtx);
2427 	if (gp == NULL)
2428 		gp = g_new_geomf(&g_swap_class, "swap", NULL);
2429 	cp = g_new_consumer(gp);
2430 	g_attach(cp, pp);
2431 	/*
2432 	 * XXX: Everytime you think you can improve the margin for
2433 	 * footshooting, somebody depends on the ability to do so:
2434 	 * savecore(8) wants to write to our swapdev so we cannot
2435 	 * set an exclusive count :-(
2436 	 */
2437 	error = g_access(cp, 1, 1, 0);
2438 	if (error) {
2439 		g_detach(cp);
2440 		g_destroy_consumer(cp);
2441 		swh->error = error;
2442 		return;
2443 	}
2444 	nblks = pp->mediasize / DEV_BSIZE;
2445 	swaponsomething(swh->vp, cp, nblks, swapgeom_strategy,
2446 	    swapgeom_close, dev2udev(swh->dev));
2447 	swh->error = 0;
2448 	return;
2449 }
2450 
2451 static int
2452 swapongeom(struct thread *td, struct vnode *vp)
2453 {
2454 	int error;
2455 	struct swh0h0 swh;
2456 
2457 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2458 
2459 	swh.dev = vp->v_rdev;
2460 	swh.vp = vp;
2461 	swh.error = 0;
2462 	/* XXX: direct call when Giant untangled */
2463 	error = g_waitfor_event(swapongeom_ev, &swh, M_WAITOK, NULL);
2464 	if (!error)
2465 		error = swh.error;
2466 	VOP_UNLOCK(vp, 0);
2467 	return (error);
2468 }
2469 
2470 /*
2471  * VNODE backend
2472  *
2473  * This is used mainly for network filesystem (read: probably only tested
2474  * with NFS) swapfiles.
2475  *
2476  */
2477 
2478 static void
2479 swapdev_strategy(struct buf *bp, struct swdevt *sp)
2480 {
2481 	struct vnode *vp2;
2482 
2483 	bp->b_blkno = ctodb(bp->b_blkno - sp->sw_first);
2484 
2485 	vp2 = sp->sw_id;
2486 	vhold(vp2);
2487 	if (bp->b_iocmd == BIO_WRITE) {
2488 		if (bp->b_bufobj)
2489 			bufobj_wdrop(bp->b_bufobj);
2490 		bufobj_wref(&vp2->v_bufobj);
2491 	}
2492 	if (bp->b_bufobj != &vp2->v_bufobj)
2493 		bp->b_bufobj = &vp2->v_bufobj;
2494 	bp->b_vp = vp2;
2495 	bp->b_iooffset = dbtob(bp->b_blkno);
2496 	bstrategy(bp);
2497 	return;
2498 }
2499 
2500 static void
2501 swapdev_close(struct thread *td, struct swdevt *sp)
2502 {
2503 
2504 	VOP_CLOSE(sp->sw_vp, FREAD | FWRITE, td->td_ucred, td);
2505 	vrele(sp->sw_vp);
2506 }
2507 
2508 
2509 static int
2510 swaponvp(struct thread *td, struct vnode *vp, u_long nblks)
2511 {
2512 	struct swdevt *sp;
2513 	int error;
2514 
2515 	if (nblks == 0)
2516 		return (ENXIO);
2517 	mtx_lock(&sw_dev_mtx);
2518 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2519 		if (sp->sw_id == vp) {
2520 			mtx_unlock(&sw_dev_mtx);
2521 			return (EBUSY);
2522 		}
2523 	}
2524 	mtx_unlock(&sw_dev_mtx);
2525 
2526 	(void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2527 #ifdef MAC
2528 	error = mac_system_check_swapon(td->td_ucred, vp);
2529 	if (error == 0)
2530 #endif
2531 		error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, NULL);
2532 	(void) VOP_UNLOCK(vp, 0);
2533 	if (error)
2534 		return (error);
2535 
2536 	swaponsomething(vp, vp, nblks, swapdev_strategy, swapdev_close,
2537 	    NODEV);
2538 	return (0);
2539 }
2540