xref: /freebsd/sys/vm/swap_pager.c (revision 36c855b9c0050566dc9864145e1a5c1262e9681d)
1 /*-
2  * SPDX-License-Identifier: BSD-4-Clause
3  *
4  * Copyright (c) 1998 Matthew Dillon,
5  * Copyright (c) 1994 John S. Dyson
6  * Copyright (c) 1990 University of Utah.
7  * Copyright (c) 1982, 1986, 1989, 1993
8  *	The Regents of the University of California.  All rights reserved.
9  *
10  * This code is derived from software contributed to Berkeley by
11  * the Systems Programming Group of the University of Utah Computer
12  * Science Department.
13  *
14  * Redistribution and use in source and binary forms, with or without
15  * modification, are permitted provided that the following conditions
16  * are met:
17  * 1. Redistributions of source code must retain the above copyright
18  *    notice, this list of conditions and the following disclaimer.
19  * 2. Redistributions in binary form must reproduce the above copyright
20  *    notice, this list of conditions and the following disclaimer in the
21  *    documentation and/or other materials provided with the distribution.
22  * 3. All advertising materials mentioning features or use of this software
23  *    must display the following acknowledgement:
24  *	This product includes software developed by the University of
25  *	California, Berkeley and its contributors.
26  * 4. Neither the name of the University nor the names of its contributors
27  *    may be used to endorse or promote products derived from this software
28  *    without specific prior written permission.
29  *
30  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
31  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
32  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
33  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
34  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
35  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
36  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
37  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
38  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
39  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
40  * SUCH DAMAGE.
41  *
42  *				New Swap System
43  *				Matthew Dillon
44  *
45  * Radix Bitmap 'blists'.
46  *
47  *	- The new swapper uses the new radix bitmap code.  This should scale
48  *	  to arbitrarily small or arbitrarily large swap spaces and an almost
49  *	  arbitrary degree of fragmentation.
50  *
51  * Features:
52  *
53  *	- on the fly reallocation of swap during putpages.  The new system
54  *	  does not try to keep previously allocated swap blocks for dirty
55  *	  pages.
56  *
57  *	- on the fly deallocation of swap
58  *
59  *	- No more garbage collection required.  Unnecessarily allocated swap
60  *	  blocks only exist for dirty vm_page_t's now and these are already
61  *	  cycled (in a high-load system) by the pager.  We also do on-the-fly
62  *	  removal of invalidated swap blocks when a page is destroyed
63  *	  or renamed.
64  *
65  * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
66  *
67  *	@(#)swap_pager.c	8.9 (Berkeley) 3/21/94
68  *	@(#)vm_swap.c	8.5 (Berkeley) 2/17/94
69  */
70 
71 #include <sys/cdefs.h>
72 __FBSDID("$FreeBSD$");
73 
74 #include "opt_swap.h"
75 #include "opt_vm.h"
76 
77 #include <sys/param.h>
78 #include <sys/systm.h>
79 #include <sys/conf.h>
80 #include <sys/kernel.h>
81 #include <sys/priv.h>
82 #include <sys/proc.h>
83 #include <sys/bio.h>
84 #include <sys/buf.h>
85 #include <sys/disk.h>
86 #include <sys/fcntl.h>
87 #include <sys/mount.h>
88 #include <sys/namei.h>
89 #include <sys/vnode.h>
90 #include <sys/malloc.h>
91 #include <sys/pctrie.h>
92 #include <sys/racct.h>
93 #include <sys/resource.h>
94 #include <sys/resourcevar.h>
95 #include <sys/rwlock.h>
96 #include <sys/sbuf.h>
97 #include <sys/sysctl.h>
98 #include <sys/sysproto.h>
99 #include <sys/blist.h>
100 #include <sys/lock.h>
101 #include <sys/sx.h>
102 #include <sys/vmmeter.h>
103 
104 #include <security/mac/mac_framework.h>
105 
106 #include <vm/vm.h>
107 #include <vm/pmap.h>
108 #include <vm/vm_map.h>
109 #include <vm/vm_kern.h>
110 #include <vm/vm_object.h>
111 #include <vm/vm_page.h>
112 #include <vm/vm_pager.h>
113 #include <vm/vm_pageout.h>
114 #include <vm/vm_param.h>
115 #include <vm/swap_pager.h>
116 #include <vm/vm_extern.h>
117 #include <vm/uma.h>
118 
119 #include <geom/geom.h>
120 
121 /*
122  * MAX_PAGEOUT_CLUSTER must be a power of 2 between 1 and 64.
123  * The 64-page limit is due to the radix code (kern/subr_blist.c).
124  */
125 #ifndef MAX_PAGEOUT_CLUSTER
126 #define	MAX_PAGEOUT_CLUSTER	32
127 #endif
128 
129 #if !defined(SWB_NPAGES)
130 #define SWB_NPAGES	MAX_PAGEOUT_CLUSTER
131 #endif
132 
133 #define	SWAP_META_PAGES		PCTRIE_COUNT
134 
135 /*
136  * A swblk structure maps each page index within a
137  * SWAP_META_PAGES-aligned and sized range to the address of an
138  * on-disk swap block (or SWAPBLK_NONE). The collection of these
139  * mappings for an entire vm object is implemented as a pc-trie.
140  */
141 struct swblk {
142 	vm_pindex_t	p;
143 	daddr_t		d[SWAP_META_PAGES];
144 };
145 
146 static MALLOC_DEFINE(M_VMPGDATA, "vm_pgdata", "swap pager private data");
147 static struct mtx sw_dev_mtx;
148 static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq);
149 static struct swdevt *swdevhd;	/* Allocate from here next */
150 static int nswapdev;		/* Number of swap devices */
151 int swap_pager_avail;
152 static struct sx swdev_syscall_lock;	/* serialize swap(on|off) */
153 
154 static vm_ooffset_t swap_total;
155 SYSCTL_QUAD(_vm, OID_AUTO, swap_total, CTLFLAG_RD, &swap_total, 0,
156     "Total amount of available swap storage.");
157 static vm_ooffset_t swap_reserved;
158 SYSCTL_QUAD(_vm, OID_AUTO, swap_reserved, CTLFLAG_RD, &swap_reserved, 0,
159     "Amount of swap storage needed to back all allocated anonymous memory.");
160 static int overcommit = 0;
161 SYSCTL_INT(_vm, VM_OVERCOMMIT, overcommit, CTLFLAG_RW, &overcommit, 0,
162     "Configure virtual memory overcommit behavior. See tuning(7) "
163     "for details.");
164 static unsigned long swzone;
165 SYSCTL_ULONG(_vm, OID_AUTO, swzone, CTLFLAG_RD, &swzone, 0,
166     "Actual size of swap metadata zone");
167 static unsigned long swap_maxpages;
168 SYSCTL_ULONG(_vm, OID_AUTO, swap_maxpages, CTLFLAG_RD, &swap_maxpages, 0,
169     "Maximum amount of swap supported");
170 
171 /* bits from overcommit */
172 #define	SWAP_RESERVE_FORCE_ON		(1 << 0)
173 #define	SWAP_RESERVE_RLIMIT_ON		(1 << 1)
174 #define	SWAP_RESERVE_ALLOW_NONWIRED	(1 << 2)
175 
176 int
177 swap_reserve(vm_ooffset_t incr)
178 {
179 
180 	return (swap_reserve_by_cred(incr, curthread->td_ucred));
181 }
182 
183 int
184 swap_reserve_by_cred(vm_ooffset_t incr, struct ucred *cred)
185 {
186 	vm_ooffset_t r, s;
187 	int res, error;
188 	static int curfail;
189 	static struct timeval lastfail;
190 	struct uidinfo *uip;
191 
192 	uip = cred->cr_ruidinfo;
193 
194 	if (incr & PAGE_MASK)
195 		panic("swap_reserve: & PAGE_MASK");
196 
197 #ifdef RACCT
198 	if (racct_enable) {
199 		PROC_LOCK(curproc);
200 		error = racct_add(curproc, RACCT_SWAP, incr);
201 		PROC_UNLOCK(curproc);
202 		if (error != 0)
203 			return (0);
204 	}
205 #endif
206 
207 	res = 0;
208 	mtx_lock(&sw_dev_mtx);
209 	r = swap_reserved + incr;
210 	if (overcommit & SWAP_RESERVE_ALLOW_NONWIRED) {
211 		s = vm_cnt.v_page_count - vm_cnt.v_free_reserved -
212 		    vm_wire_count();
213 		s *= PAGE_SIZE;
214 	} else
215 		s = 0;
216 	s += swap_total;
217 	if ((overcommit & SWAP_RESERVE_FORCE_ON) == 0 || r <= s ||
218 	    (error = priv_check(curthread, PRIV_VM_SWAP_NOQUOTA)) == 0) {
219 		res = 1;
220 		swap_reserved = r;
221 	}
222 	mtx_unlock(&sw_dev_mtx);
223 
224 	if (res) {
225 		UIDINFO_VMSIZE_LOCK(uip);
226 		if ((overcommit & SWAP_RESERVE_RLIMIT_ON) != 0 &&
227 		    uip->ui_vmsize + incr > lim_cur(curthread, RLIMIT_SWAP) &&
228 		    priv_check(curthread, PRIV_VM_SWAP_NORLIMIT))
229 			res = 0;
230 		else
231 			uip->ui_vmsize += incr;
232 		UIDINFO_VMSIZE_UNLOCK(uip);
233 		if (!res) {
234 			mtx_lock(&sw_dev_mtx);
235 			swap_reserved -= incr;
236 			mtx_unlock(&sw_dev_mtx);
237 		}
238 	}
239 	if (!res && ppsratecheck(&lastfail, &curfail, 1)) {
240 		printf("uid %d, pid %d: swap reservation for %jd bytes failed\n",
241 		    uip->ui_uid, curproc->p_pid, incr);
242 	}
243 
244 #ifdef RACCT
245 	if (!res) {
246 		PROC_LOCK(curproc);
247 		racct_sub(curproc, RACCT_SWAP, incr);
248 		PROC_UNLOCK(curproc);
249 	}
250 #endif
251 
252 	return (res);
253 }
254 
255 void
256 swap_reserve_force(vm_ooffset_t incr)
257 {
258 	struct uidinfo *uip;
259 
260 	mtx_lock(&sw_dev_mtx);
261 	swap_reserved += incr;
262 	mtx_unlock(&sw_dev_mtx);
263 
264 #ifdef RACCT
265 	PROC_LOCK(curproc);
266 	racct_add_force(curproc, RACCT_SWAP, incr);
267 	PROC_UNLOCK(curproc);
268 #endif
269 
270 	uip = curthread->td_ucred->cr_ruidinfo;
271 	PROC_LOCK(curproc);
272 	UIDINFO_VMSIZE_LOCK(uip);
273 	uip->ui_vmsize += incr;
274 	UIDINFO_VMSIZE_UNLOCK(uip);
275 	PROC_UNLOCK(curproc);
276 }
277 
278 void
279 swap_release(vm_ooffset_t decr)
280 {
281 	struct ucred *cred;
282 
283 	PROC_LOCK(curproc);
284 	cred = curthread->td_ucred;
285 	swap_release_by_cred(decr, cred);
286 	PROC_UNLOCK(curproc);
287 }
288 
289 void
290 swap_release_by_cred(vm_ooffset_t decr, struct ucred *cred)
291 {
292  	struct uidinfo *uip;
293 
294 	uip = cred->cr_ruidinfo;
295 
296 	if (decr & PAGE_MASK)
297 		panic("swap_release: & PAGE_MASK");
298 
299 	mtx_lock(&sw_dev_mtx);
300 	if (swap_reserved < decr)
301 		panic("swap_reserved < decr");
302 	swap_reserved -= decr;
303 	mtx_unlock(&sw_dev_mtx);
304 
305 	UIDINFO_VMSIZE_LOCK(uip);
306 	if (uip->ui_vmsize < decr)
307 		printf("negative vmsize for uid = %d\n", uip->ui_uid);
308 	uip->ui_vmsize -= decr;
309 	UIDINFO_VMSIZE_UNLOCK(uip);
310 
311 	racct_sub_cred(cred, RACCT_SWAP, decr);
312 }
313 
314 #define SWM_POP		0x01	/* pop out			*/
315 
316 static int swap_pager_full = 2;	/* swap space exhaustion (task killing) */
317 static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/
318 static int nsw_rcount;		/* free read buffers			*/
319 static int nsw_wcount_sync;	/* limit write buffers / synchronous	*/
320 static int nsw_wcount_async;	/* limit write buffers / asynchronous	*/
321 static int nsw_wcount_async_max;/* assigned maximum			*/
322 static int nsw_cluster_max;	/* maximum VOP I/O allowed		*/
323 
324 static int sysctl_swap_async_max(SYSCTL_HANDLER_ARGS);
325 SYSCTL_PROC(_vm, OID_AUTO, swap_async_max, CTLTYPE_INT | CTLFLAG_RW |
326     CTLFLAG_MPSAFE, NULL, 0, sysctl_swap_async_max, "I",
327     "Maximum running async swap ops");
328 static int sysctl_swap_fragmentation(SYSCTL_HANDLER_ARGS);
329 SYSCTL_PROC(_vm, OID_AUTO, swap_fragmentation, CTLTYPE_STRING | CTLFLAG_RD |
330     CTLFLAG_MPSAFE, NULL, 0, sysctl_swap_fragmentation, "A",
331     "Swap Fragmentation Info");
332 
333 static struct sx sw_alloc_sx;
334 
335 /*
336  * "named" and "unnamed" anon region objects.  Try to reduce the overhead
337  * of searching a named list by hashing it just a little.
338  */
339 
340 #define NOBJLISTS		8
341 
342 #define NOBJLIST(handle)	\
343 	(&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)])
344 
345 static struct pagerlst	swap_pager_object_list[NOBJLISTS];
346 static uma_zone_t swblk_zone;
347 static uma_zone_t swpctrie_zone;
348 
349 /*
350  * pagerops for OBJT_SWAP - "swap pager".  Some ops are also global procedure
351  * calls hooked from other parts of the VM system and do not appear here.
352  * (see vm/swap_pager.h).
353  */
354 static vm_object_t
355 		swap_pager_alloc(void *handle, vm_ooffset_t size,
356 		    vm_prot_t prot, vm_ooffset_t offset, struct ucred *);
357 static void	swap_pager_dealloc(vm_object_t object);
358 static int	swap_pager_getpages(vm_object_t, vm_page_t *, int, int *,
359     int *);
360 static int	swap_pager_getpages_async(vm_object_t, vm_page_t *, int, int *,
361     int *, pgo_getpages_iodone_t, void *);
362 static void	swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *);
363 static boolean_t
364 		swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
365 static void	swap_pager_init(void);
366 static void	swap_pager_unswapped(vm_page_t);
367 static void	swap_pager_swapoff(struct swdevt *sp);
368 
369 struct pagerops swappagerops = {
370 	.pgo_init =	swap_pager_init,	/* early system initialization of pager	*/
371 	.pgo_alloc =	swap_pager_alloc,	/* allocate an OBJT_SWAP object		*/
372 	.pgo_dealloc =	swap_pager_dealloc,	/* deallocate an OBJT_SWAP object	*/
373 	.pgo_getpages =	swap_pager_getpages,	/* pagein				*/
374 	.pgo_getpages_async = swap_pager_getpages_async, /* pagein (async)		*/
375 	.pgo_putpages =	swap_pager_putpages,	/* pageout				*/
376 	.pgo_haspage =	swap_pager_haspage,	/* get backing store status for page	*/
377 	.pgo_pageunswapped = swap_pager_unswapped,	/* remove swap related to page		*/
378 };
379 
380 /*
381  * swap_*() routines are externally accessible.  swp_*() routines are
382  * internal.
383  */
384 static int nswap_lowat = 128;	/* in pages, swap_pager_almost_full warn */
385 static int nswap_hiwat = 512;	/* in pages, swap_pager_almost_full warn */
386 
387 SYSCTL_INT(_vm, OID_AUTO, dmmax, CTLFLAG_RD, &nsw_cluster_max, 0,
388     "Maximum size of a swap block in pages");
389 
390 static void	swp_sizecheck(void);
391 static void	swp_pager_async_iodone(struct buf *bp);
392 static bool	swp_pager_swblk_empty(struct swblk *sb, int start, int limit);
393 static int	swapongeom(struct vnode *);
394 static int	swaponvp(struct thread *, struct vnode *, u_long);
395 static int	swapoff_one(struct swdevt *sp, struct ucred *cred);
396 
397 /*
398  * Swap bitmap functions
399  */
400 static void	swp_pager_freeswapspace(daddr_t blk, daddr_t npages);
401 static daddr_t	swp_pager_getswapspace(int npages);
402 
403 /*
404  * Metadata functions
405  */
406 static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
407 static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t);
408 static void swp_pager_meta_free_all(vm_object_t);
409 static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);
410 
411 static void
412 swp_pager_init_freerange(daddr_t *start, daddr_t *num)
413 {
414 
415 	*start = SWAPBLK_NONE;
416 	*num = 0;
417 }
418 
419 static void
420 swp_pager_update_freerange(daddr_t *start, daddr_t *num, daddr_t addr)
421 {
422 
423 	if (*start + *num == addr) {
424 		(*num)++;
425 	} else {
426 		swp_pager_freeswapspace(*start, *num);
427 		*start = addr;
428 		*num = 1;
429 	}
430 }
431 
432 static void *
433 swblk_trie_alloc(struct pctrie *ptree)
434 {
435 
436 	return (uma_zalloc(swpctrie_zone, M_NOWAIT | (curproc == pageproc ?
437 	    M_USE_RESERVE : 0)));
438 }
439 
440 static void
441 swblk_trie_free(struct pctrie *ptree, void *node)
442 {
443 
444 	uma_zfree(swpctrie_zone, node);
445 }
446 
447 PCTRIE_DEFINE(SWAP, swblk, p, swblk_trie_alloc, swblk_trie_free);
448 
449 /*
450  * SWP_SIZECHECK() -	update swap_pager_full indication
451  *
452  *	update the swap_pager_almost_full indication and warn when we are
453  *	about to run out of swap space, using lowat/hiwat hysteresis.
454  *
455  *	Clear swap_pager_full ( task killing ) indication when lowat is met.
456  *
457  *	No restrictions on call
458  *	This routine may not block.
459  */
460 static void
461 swp_sizecheck(void)
462 {
463 
464 	if (swap_pager_avail < nswap_lowat) {
465 		if (swap_pager_almost_full == 0) {
466 			printf("swap_pager: out of swap space\n");
467 			swap_pager_almost_full = 1;
468 		}
469 	} else {
470 		swap_pager_full = 0;
471 		if (swap_pager_avail > nswap_hiwat)
472 			swap_pager_almost_full = 0;
473 	}
474 }
475 
476 /*
477  * SWAP_PAGER_INIT() -	initialize the swap pager!
478  *
479  *	Expected to be started from system init.  NOTE:  This code is run
480  *	before much else so be careful what you depend on.  Most of the VM
481  *	system has yet to be initialized at this point.
482  */
483 static void
484 swap_pager_init(void)
485 {
486 	/*
487 	 * Initialize object lists
488 	 */
489 	int i;
490 
491 	for (i = 0; i < NOBJLISTS; ++i)
492 		TAILQ_INIT(&swap_pager_object_list[i]);
493 	mtx_init(&sw_dev_mtx, "swapdev", NULL, MTX_DEF);
494 	sx_init(&sw_alloc_sx, "swspsx");
495 	sx_init(&swdev_syscall_lock, "swsysc");
496 }
497 
498 /*
499  * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process
500  *
501  *	Expected to be started from pageout process once, prior to entering
502  *	its main loop.
503  */
504 void
505 swap_pager_swap_init(void)
506 {
507 	unsigned long n, n2;
508 
509 	/*
510 	 * Number of in-transit swap bp operations.  Don't
511 	 * exhaust the pbufs completely.  Make sure we
512 	 * initialize workable values (0 will work for hysteresis
513 	 * but it isn't very efficient).
514 	 *
515 	 * The nsw_cluster_max is constrained by the bp->b_pages[]
516 	 * array (MAXPHYS/PAGE_SIZE) and our locally defined
517 	 * MAX_PAGEOUT_CLUSTER.   Also be aware that swap ops are
518 	 * constrained by the swap device interleave stripe size.
519 	 *
520 	 * Currently we hardwire nsw_wcount_async to 4.  This limit is
521 	 * designed to prevent other I/O from having high latencies due to
522 	 * our pageout I/O.  The value 4 works well for one or two active swap
523 	 * devices but is probably a little low if you have more.  Even so,
524 	 * a higher value would probably generate only a limited improvement
525 	 * with three or four active swap devices since the system does not
526 	 * typically have to pageout at extreme bandwidths.   We will want
527 	 * at least 2 per swap devices, and 4 is a pretty good value if you
528 	 * have one NFS swap device due to the command/ack latency over NFS.
529 	 * So it all works out pretty well.
530 	 */
531 	nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER);
532 
533 	mtx_lock(&pbuf_mtx);
534 	nsw_rcount = (nswbuf + 1) / 2;
535 	nsw_wcount_sync = (nswbuf + 3) / 4;
536 	nsw_wcount_async = 4;
537 	nsw_wcount_async_max = nsw_wcount_async;
538 	mtx_unlock(&pbuf_mtx);
539 
540 	/*
541 	 * Initialize our zone, guessing on the number we need based
542 	 * on the number of pages in the system.
543 	 */
544 	n = vm_cnt.v_page_count / 2;
545 	if (maxswzone && n > maxswzone / sizeof(struct swblk))
546 		n = maxswzone / sizeof(struct swblk);
547 	swpctrie_zone = uma_zcreate("swpctrie", pctrie_node_size(), NULL, NULL,
548 	    pctrie_zone_init, NULL, UMA_ALIGN_PTR,
549 	    UMA_ZONE_NOFREE | UMA_ZONE_VM);
550 	if (swpctrie_zone == NULL)
551 		panic("failed to create swap pctrie zone.");
552 	swblk_zone = uma_zcreate("swblk", sizeof(struct swblk), NULL, NULL,
553 	    NULL, NULL, _Alignof(struct swblk) - 1,
554 	    UMA_ZONE_NOFREE | UMA_ZONE_VM);
555 	if (swblk_zone == NULL)
556 		panic("failed to create swap blk zone.");
557 	n2 = n;
558 	do {
559 		if (uma_zone_reserve_kva(swblk_zone, n))
560 			break;
561 		/*
562 		 * if the allocation failed, try a zone two thirds the
563 		 * size of the previous attempt.
564 		 */
565 		n -= ((n + 2) / 3);
566 	} while (n > 0);
567 
568 	/*
569 	 * Often uma_zone_reserve_kva() cannot reserve exactly the
570 	 * requested size.  Account for the difference when
571 	 * calculating swap_maxpages.
572 	 */
573 	n = uma_zone_get_max(swblk_zone);
574 
575 	if (n < n2)
576 		printf("Swap blk zone entries reduced from %lu to %lu.\n",
577 		    n2, n);
578 	swap_maxpages = n * SWAP_META_PAGES;
579 	swzone = n * sizeof(struct swblk);
580 	if (!uma_zone_reserve_kva(swpctrie_zone, n))
581 		printf("Cannot reserve swap pctrie zone, "
582 		    "reduce kern.maxswzone.\n");
583 }
584 
585 static vm_object_t
586 swap_pager_alloc_init(void *handle, struct ucred *cred, vm_ooffset_t size,
587     vm_ooffset_t offset)
588 {
589 	vm_object_t object;
590 
591 	if (cred != NULL) {
592 		if (!swap_reserve_by_cred(size, cred))
593 			return (NULL);
594 		crhold(cred);
595 	}
596 
597 	/*
598 	 * The un_pager.swp.swp_blks trie is initialized by
599 	 * vm_object_allocate() to ensure the correct order of
600 	 * visibility to other threads.
601 	 */
602 	object = vm_object_allocate(OBJT_SWAP, OFF_TO_IDX(offset +
603 	    PAGE_MASK + size));
604 
605 	object->handle = handle;
606 	if (cred != NULL) {
607 		object->cred = cred;
608 		object->charge = size;
609 	}
610 	return (object);
611 }
612 
613 /*
614  * SWAP_PAGER_ALLOC() -	allocate a new OBJT_SWAP VM object and instantiate
615  *			its metadata structures.
616  *
617  *	This routine is called from the mmap and fork code to create a new
618  *	OBJT_SWAP object.
619  *
620  *	This routine must ensure that no live duplicate is created for
621  *	the named object request, which is protected against by
622  *	holding the sw_alloc_sx lock in case handle != NULL.
623  */
624 static vm_object_t
625 swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
626     vm_ooffset_t offset, struct ucred *cred)
627 {
628 	vm_object_t object;
629 
630 	if (handle != NULL) {
631 		/*
632 		 * Reference existing named region or allocate new one.  There
633 		 * should not be a race here against swp_pager_meta_build()
634 		 * as called from vm_page_remove() in regards to the lookup
635 		 * of the handle.
636 		 */
637 		sx_xlock(&sw_alloc_sx);
638 		object = vm_pager_object_lookup(NOBJLIST(handle), handle);
639 		if (object == NULL) {
640 			object = swap_pager_alloc_init(handle, cred, size,
641 			    offset);
642 			if (object != NULL) {
643 				TAILQ_INSERT_TAIL(NOBJLIST(object->handle),
644 				    object, pager_object_list);
645 			}
646 		}
647 		sx_xunlock(&sw_alloc_sx);
648 	} else {
649 		object = swap_pager_alloc_init(handle, cred, size, offset);
650 	}
651 	return (object);
652 }
653 
654 /*
655  * SWAP_PAGER_DEALLOC() -	remove swap metadata from object
656  *
657  *	The swap backing for the object is destroyed.  The code is
658  *	designed such that we can reinstantiate it later, but this
659  *	routine is typically called only when the entire object is
660  *	about to be destroyed.
661  *
662  *	The object must be locked.
663  */
664 static void
665 swap_pager_dealloc(vm_object_t object)
666 {
667 
668 	VM_OBJECT_ASSERT_WLOCKED(object);
669 	KASSERT((object->flags & OBJ_DEAD) != 0, ("dealloc of reachable obj"));
670 
671 	/*
672 	 * Remove from list right away so lookups will fail if we block for
673 	 * pageout completion.
674 	 */
675 	if (object->handle != NULL) {
676 		VM_OBJECT_WUNLOCK(object);
677 		sx_xlock(&sw_alloc_sx);
678 		TAILQ_REMOVE(NOBJLIST(object->handle), object,
679 		    pager_object_list);
680 		sx_xunlock(&sw_alloc_sx);
681 		VM_OBJECT_WLOCK(object);
682 	}
683 
684 	vm_object_pip_wait(object, "swpdea");
685 
686 	/*
687 	 * Free all remaining metadata.  We only bother to free it from
688 	 * the swap meta data.  We do not attempt to free swapblk's still
689 	 * associated with vm_page_t's for this object.  We do not care
690 	 * if paging is still in progress on some objects.
691 	 */
692 	swp_pager_meta_free_all(object);
693 	object->handle = NULL;
694 	object->type = OBJT_DEAD;
695 }
696 
697 /************************************************************************
698  *			SWAP PAGER BITMAP ROUTINES			*
699  ************************************************************************/
700 
701 /*
702  * SWP_PAGER_GETSWAPSPACE() -	allocate raw swap space
703  *
704  *	Allocate swap for the requested number of pages.  The starting
705  *	swap block number (a page index) is returned or SWAPBLK_NONE
706  *	if the allocation failed.
707  *
708  *	Also has the side effect of advising that somebody made a mistake
709  *	when they configured swap and didn't configure enough.
710  *
711  *	This routine may not sleep.
712  *
713  *	We allocate in round-robin fashion from the configured devices.
714  */
715 static daddr_t
716 swp_pager_getswapspace(int npages)
717 {
718 	daddr_t blk;
719 	struct swdevt *sp;
720 	int i;
721 
722 	blk = SWAPBLK_NONE;
723 	mtx_lock(&sw_dev_mtx);
724 	sp = swdevhd;
725 	for (i = 0; i < nswapdev; i++) {
726 		if (sp == NULL)
727 			sp = TAILQ_FIRST(&swtailq);
728 		if (!(sp->sw_flags & SW_CLOSING)) {
729 			blk = blist_alloc(sp->sw_blist, npages);
730 			if (blk != SWAPBLK_NONE) {
731 				blk += sp->sw_first;
732 				sp->sw_used += npages;
733 				swap_pager_avail -= npages;
734 				swp_sizecheck();
735 				swdevhd = TAILQ_NEXT(sp, sw_list);
736 				goto done;
737 			}
738 		}
739 		sp = TAILQ_NEXT(sp, sw_list);
740 	}
741 	if (swap_pager_full != 2) {
742 		printf("swap_pager_getswapspace(%d): failed\n", npages);
743 		swap_pager_full = 2;
744 		swap_pager_almost_full = 1;
745 	}
746 	swdevhd = NULL;
747 done:
748 	mtx_unlock(&sw_dev_mtx);
749 	return (blk);
750 }
751 
752 static int
753 swp_pager_isondev(daddr_t blk, struct swdevt *sp)
754 {
755 
756 	return (blk >= sp->sw_first && blk < sp->sw_end);
757 }
758 
759 static void
760 swp_pager_strategy(struct buf *bp)
761 {
762 	struct swdevt *sp;
763 
764 	mtx_lock(&sw_dev_mtx);
765 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
766 		if (bp->b_blkno >= sp->sw_first && bp->b_blkno < sp->sw_end) {
767 			mtx_unlock(&sw_dev_mtx);
768 			if ((sp->sw_flags & SW_UNMAPPED) != 0 &&
769 			    unmapped_buf_allowed) {
770 				bp->b_data = unmapped_buf;
771 				bp->b_offset = 0;
772 			} else {
773 				pmap_qenter((vm_offset_t)bp->b_data,
774 				    &bp->b_pages[0], bp->b_bcount / PAGE_SIZE);
775 			}
776 			sp->sw_strategy(bp, sp);
777 			return;
778 		}
779 	}
780 	panic("Swapdev not found");
781 }
782 
783 
784 /*
785  * SWP_PAGER_FREESWAPSPACE() -	free raw swap space
786  *
787  *	This routine returns the specified swap blocks back to the bitmap.
788  *
789  *	This routine may not sleep.
790  */
791 static void
792 swp_pager_freeswapspace(daddr_t blk, daddr_t npages)
793 {
794 	struct swdevt *sp;
795 
796 	if (npages == 0)
797 		return;
798 	mtx_lock(&sw_dev_mtx);
799 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
800 		if (blk >= sp->sw_first && blk < sp->sw_end) {
801 			sp->sw_used -= npages;
802 			/*
803 			 * If we are attempting to stop swapping on
804 			 * this device, we don't want to mark any
805 			 * blocks free lest they be reused.
806 			 */
807 			if ((sp->sw_flags & SW_CLOSING) == 0) {
808 				blist_free(sp->sw_blist, blk - sp->sw_first,
809 				    npages);
810 				swap_pager_avail += npages;
811 				swp_sizecheck();
812 			}
813 			mtx_unlock(&sw_dev_mtx);
814 			return;
815 		}
816 	}
817 	panic("Swapdev not found");
818 }
819 
820 /*
821  * SYSCTL_SWAP_FRAGMENTATION() -	produce raw swap space stats
822  */
823 static int
824 sysctl_swap_fragmentation(SYSCTL_HANDLER_ARGS)
825 {
826 	struct sbuf sbuf;
827 	struct swdevt *sp;
828 	const char *devname;
829 	int error;
830 
831 	error = sysctl_wire_old_buffer(req, 0);
832 	if (error != 0)
833 		return (error);
834 	sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
835 	mtx_lock(&sw_dev_mtx);
836 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
837 		if (vn_isdisk(sp->sw_vp, NULL))
838 			devname = devtoname(sp->sw_vp->v_rdev);
839 		else
840 			devname = "[file]";
841 		sbuf_printf(&sbuf, "\nFree space on device %s:\n", devname);
842 		blist_stats(sp->sw_blist, &sbuf);
843 	}
844 	mtx_unlock(&sw_dev_mtx);
845 	error = sbuf_finish(&sbuf);
846 	sbuf_delete(&sbuf);
847 	return (error);
848 }
849 
850 /*
851  * SWAP_PAGER_FREESPACE() -	frees swap blocks associated with a page
852  *				range within an object.
853  *
854  *	This is a globally accessible routine.
855  *
856  *	This routine removes swapblk assignments from swap metadata.
857  *
858  *	The external callers of this routine typically have already destroyed
859  *	or renamed vm_page_t's associated with this range in the object so
860  *	we should be ok.
861  *
862  *	The object must be locked.
863  */
864 void
865 swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size)
866 {
867 
868 	swp_pager_meta_free(object, start, size);
869 }
870 
871 /*
872  * SWAP_PAGER_RESERVE() - reserve swap blocks in object
873  *
874  *	Assigns swap blocks to the specified range within the object.  The
875  *	swap blocks are not zeroed.  Any previous swap assignment is destroyed.
876  *
877  *	Returns 0 on success, -1 on failure.
878  */
879 int
880 swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
881 {
882 	int n = 0;
883 	daddr_t blk = SWAPBLK_NONE;
884 	vm_pindex_t beg = start;	/* save start index */
885 	daddr_t addr, n_free, s_free;
886 
887 	swp_pager_init_freerange(&s_free, &n_free);
888 	VM_OBJECT_WLOCK(object);
889 	while (size) {
890 		if (n == 0) {
891 			n = BLIST_MAX_ALLOC;
892 			while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
893 				n >>= 1;
894 				if (n == 0) {
895 					swp_pager_meta_free(object, beg, start - beg);
896 					VM_OBJECT_WUNLOCK(object);
897 					return (-1);
898 				}
899 			}
900 		}
901 		addr = swp_pager_meta_build(object, start, blk);
902 		if (addr != SWAPBLK_NONE)
903 			swp_pager_update_freerange(&s_free, &n_free, addr);
904 		--size;
905 		++start;
906 		++blk;
907 		--n;
908 	}
909 	swp_pager_freeswapspace(s_free, n_free);
910 	swp_pager_meta_free(object, start, n);
911 	VM_OBJECT_WUNLOCK(object);
912 	return (0);
913 }
914 
915 /*
916  * SWAP_PAGER_COPY() -  copy blocks from source pager to destination pager
917  *			and destroy the source.
918  *
919  *	Copy any valid swapblks from the source to the destination.  In
920  *	cases where both the source and destination have a valid swapblk,
921  *	we keep the destination's.
922  *
923  *	This routine is allowed to sleep.  It may sleep allocating metadata
924  *	indirectly through swp_pager_meta_build() or if paging is still in
925  *	progress on the source.
926  *
927  *	The source object contains no vm_page_t's (which is just as well)
928  *
929  *	The source object is of type OBJT_SWAP.
930  *
931  *	The source and destination objects must be locked.
932  *	Both object locks may temporarily be released.
933  */
934 void
935 swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
936     vm_pindex_t offset, int destroysource)
937 {
938 	vm_pindex_t i;
939 	daddr_t dstaddr, n_free, s_free, srcaddr;
940 
941 	VM_OBJECT_ASSERT_WLOCKED(srcobject);
942 	VM_OBJECT_ASSERT_WLOCKED(dstobject);
943 
944 	/*
945 	 * If destroysource is set, we remove the source object from the
946 	 * swap_pager internal queue now.
947 	 */
948 	if (destroysource && srcobject->handle != NULL) {
949 		vm_object_pip_add(srcobject, 1);
950 		VM_OBJECT_WUNLOCK(srcobject);
951 		vm_object_pip_add(dstobject, 1);
952 		VM_OBJECT_WUNLOCK(dstobject);
953 		sx_xlock(&sw_alloc_sx);
954 		TAILQ_REMOVE(NOBJLIST(srcobject->handle), srcobject,
955 		    pager_object_list);
956 		sx_xunlock(&sw_alloc_sx);
957 		VM_OBJECT_WLOCK(dstobject);
958 		vm_object_pip_wakeup(dstobject);
959 		VM_OBJECT_WLOCK(srcobject);
960 		vm_object_pip_wakeup(srcobject);
961 	}
962 
963 	/*
964 	 * Transfer source to destination.
965 	 */
966 	swp_pager_init_freerange(&s_free, &n_free);
967 	for (i = 0; i < dstobject->size; ++i) {
968 		srcaddr = swp_pager_meta_ctl(srcobject, i + offset, SWM_POP);
969 		if (srcaddr == SWAPBLK_NONE)
970 			continue;
971 		dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
972 		if (dstaddr != SWAPBLK_NONE) {
973 			/*
974 			 * Destination has valid swapblk or it is represented
975 			 * by a resident page.  We destroy the source block.
976 			 */
977 			swp_pager_update_freerange(&s_free, &n_free, srcaddr);
978 			continue;
979 		}
980 
981 		/*
982 		 * Destination has no swapblk and is not resident,
983 		 * copy source.
984 		 *
985 		 * swp_pager_meta_build() can sleep.
986 		 */
987 		vm_object_pip_add(srcobject, 1);
988 		VM_OBJECT_WUNLOCK(srcobject);
989 		vm_object_pip_add(dstobject, 1);
990 		dstaddr = swp_pager_meta_build(dstobject, i, srcaddr);
991 		KASSERT(dstaddr == SWAPBLK_NONE,
992 		    ("Unexpected destination swapblk"));
993 		vm_object_pip_wakeup(dstobject);
994 		VM_OBJECT_WLOCK(srcobject);
995 		vm_object_pip_wakeup(srcobject);
996 	}
997 	swp_pager_freeswapspace(s_free, n_free);
998 
999 	/*
1000 	 * Free left over swap blocks in source.
1001 	 *
1002 	 * We have to revert the type to OBJT_DEFAULT so we do not accidentally
1003 	 * double-remove the object from the swap queues.
1004 	 */
1005 	if (destroysource) {
1006 		swp_pager_meta_free_all(srcobject);
1007 		/*
1008 		 * Reverting the type is not necessary, the caller is going
1009 		 * to destroy srcobject directly, but I'm doing it here
1010 		 * for consistency since we've removed the object from its
1011 		 * queues.
1012 		 */
1013 		srcobject->type = OBJT_DEFAULT;
1014 	}
1015 }
1016 
1017 /*
1018  * SWAP_PAGER_HASPAGE() -	determine if we have good backing store for
1019  *				the requested page.
1020  *
1021  *	We determine whether good backing store exists for the requested
1022  *	page and return TRUE if it does, FALSE if it doesn't.
1023  *
1024  *	If TRUE, we also try to determine how much valid, contiguous backing
1025  *	store exists before and after the requested page.
1026  */
1027 static boolean_t
1028 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,
1029     int *after)
1030 {
1031 	daddr_t blk, blk0;
1032 	int i;
1033 
1034 	VM_OBJECT_ASSERT_LOCKED(object);
1035 
1036 	/*
1037 	 * do we have good backing store at the requested index ?
1038 	 */
1039 	blk0 = swp_pager_meta_ctl(object, pindex, 0);
1040 	if (blk0 == SWAPBLK_NONE) {
1041 		if (before)
1042 			*before = 0;
1043 		if (after)
1044 			*after = 0;
1045 		return (FALSE);
1046 	}
1047 
1048 	/*
1049 	 * find backwards-looking contiguous good backing store
1050 	 */
1051 	if (before != NULL) {
1052 		for (i = 1; i < SWB_NPAGES; i++) {
1053 			if (i > pindex)
1054 				break;
1055 			blk = swp_pager_meta_ctl(object, pindex - i, 0);
1056 			if (blk != blk0 - i)
1057 				break;
1058 		}
1059 		*before = i - 1;
1060 	}
1061 
1062 	/*
1063 	 * find forward-looking contiguous good backing store
1064 	 */
1065 	if (after != NULL) {
1066 		for (i = 1; i < SWB_NPAGES; i++) {
1067 			blk = swp_pager_meta_ctl(object, pindex + i, 0);
1068 			if (blk != blk0 + i)
1069 				break;
1070 		}
1071 		*after = i - 1;
1072 	}
1073 	return (TRUE);
1074 }
1075 
1076 /*
1077  * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page
1078  *
1079  *	This removes any associated swap backing store, whether valid or
1080  *	not, from the page.
1081  *
1082  *	This routine is typically called when a page is made dirty, at
1083  *	which point any associated swap can be freed.  MADV_FREE also
1084  *	calls us in a special-case situation
1085  *
1086  *	NOTE!!!  If the page is clean and the swap was valid, the caller
1087  *	should make the page dirty before calling this routine.  This routine
1088  *	does NOT change the m->dirty status of the page.  Also: MADV_FREE
1089  *	depends on it.
1090  *
1091  *	This routine may not sleep.
1092  *
1093  *	The object containing the page must be locked.
1094  */
1095 static void
1096 swap_pager_unswapped(vm_page_t m)
1097 {
1098 	daddr_t srcaddr;
1099 
1100 	srcaddr = swp_pager_meta_ctl(m->object, m->pindex, SWM_POP);
1101 	if (srcaddr != SWAPBLK_NONE)
1102 		swp_pager_freeswapspace(srcaddr, 1);
1103 }
1104 
1105 /*
1106  * swap_pager_getpages() - bring pages in from swap
1107  *
1108  *	Attempt to page in the pages in array "ma" of length "count".  The
1109  *	caller may optionally specify that additional pages preceding and
1110  *	succeeding the specified range be paged in.  The number of such pages
1111  *	is returned in the "rbehind" and "rahead" parameters, and they will
1112  *	be in the inactive queue upon return.
1113  *
1114  *	The pages in "ma" must be busied and will remain busied upon return.
1115  */
1116 static int
1117 swap_pager_getpages(vm_object_t object, vm_page_t *ma, int count, int *rbehind,
1118     int *rahead)
1119 {
1120 	struct buf *bp;
1121 	vm_page_t bm, mpred, msucc, p;
1122 	vm_pindex_t pindex;
1123 	daddr_t blk;
1124 	int i, maxahead, maxbehind, reqcount;
1125 
1126 	reqcount = count;
1127 
1128 	/*
1129 	 * Determine the final number of read-behind pages and
1130 	 * allocate them BEFORE releasing the object lock.  Otherwise,
1131 	 * there can be a problematic race with vm_object_split().
1132 	 * Specifically, vm_object_split() might first transfer pages
1133 	 * that precede ma[0] in the current object to a new object,
1134 	 * and then this function incorrectly recreates those pages as
1135 	 * read-behind pages in the current object.
1136 	 */
1137 	if (!swap_pager_haspage(object, ma[0]->pindex, &maxbehind, &maxahead))
1138 		return (VM_PAGER_FAIL);
1139 
1140 	/*
1141 	 * Clip the readahead and readbehind ranges to exclude resident pages.
1142 	 */
1143 	if (rahead != NULL) {
1144 		KASSERT(reqcount - 1 <= maxahead,
1145 		    ("page count %d extends beyond swap block", reqcount));
1146 		*rahead = imin(*rahead, maxahead - (reqcount - 1));
1147 		pindex = ma[reqcount - 1]->pindex;
1148 		msucc = TAILQ_NEXT(ma[reqcount - 1], listq);
1149 		if (msucc != NULL && msucc->pindex - pindex - 1 < *rahead)
1150 			*rahead = msucc->pindex - pindex - 1;
1151 	}
1152 	if (rbehind != NULL) {
1153 		*rbehind = imin(*rbehind, maxbehind);
1154 		pindex = ma[0]->pindex;
1155 		mpred = TAILQ_PREV(ma[0], pglist, listq);
1156 		if (mpred != NULL && pindex - mpred->pindex - 1 < *rbehind)
1157 			*rbehind = pindex - mpred->pindex - 1;
1158 	}
1159 
1160 	bm = ma[0];
1161 	for (i = 0; i < count; i++)
1162 		ma[i]->oflags |= VPO_SWAPINPROG;
1163 
1164 	/*
1165 	 * Allocate readahead and readbehind pages.
1166 	 */
1167 	if (rbehind != NULL) {
1168 		for (i = 1; i <= *rbehind; i++) {
1169 			p = vm_page_alloc(object, ma[0]->pindex - i,
1170 			    VM_ALLOC_NORMAL);
1171 			if (p == NULL)
1172 				break;
1173 			p->oflags |= VPO_SWAPINPROG;
1174 			bm = p;
1175 		}
1176 		*rbehind = i - 1;
1177 	}
1178 	if (rahead != NULL) {
1179 		for (i = 0; i < *rahead; i++) {
1180 			p = vm_page_alloc(object,
1181 			    ma[reqcount - 1]->pindex + i + 1, VM_ALLOC_NORMAL);
1182 			if (p == NULL)
1183 				break;
1184 			p->oflags |= VPO_SWAPINPROG;
1185 		}
1186 		*rahead = i;
1187 	}
1188 	if (rbehind != NULL)
1189 		count += *rbehind;
1190 	if (rahead != NULL)
1191 		count += *rahead;
1192 
1193 	vm_object_pip_add(object, count);
1194 
1195 	pindex = bm->pindex;
1196 	blk = swp_pager_meta_ctl(object, pindex, 0);
1197 	KASSERT(blk != SWAPBLK_NONE,
1198 	    ("no swap blocking containing %p(%jx)", object, (uintmax_t)pindex));
1199 
1200 	VM_OBJECT_WUNLOCK(object);
1201 	bp = getpbuf(&nsw_rcount);
1202 	/* Pages cannot leave the object while busy. */
1203 	for (i = 0, p = bm; i < count; i++, p = TAILQ_NEXT(p, listq)) {
1204 		MPASS(p->pindex == bm->pindex + i);
1205 		bp->b_pages[i] = p;
1206 	}
1207 
1208 	bp->b_flags |= B_PAGING;
1209 	bp->b_iocmd = BIO_READ;
1210 	bp->b_iodone = swp_pager_async_iodone;
1211 	bp->b_rcred = crhold(thread0.td_ucred);
1212 	bp->b_wcred = crhold(thread0.td_ucred);
1213 	bp->b_blkno = blk;
1214 	bp->b_bcount = PAGE_SIZE * count;
1215 	bp->b_bufsize = PAGE_SIZE * count;
1216 	bp->b_npages = count;
1217 	bp->b_pgbefore = rbehind != NULL ? *rbehind : 0;
1218 	bp->b_pgafter = rahead != NULL ? *rahead : 0;
1219 
1220 	VM_CNT_INC(v_swapin);
1221 	VM_CNT_ADD(v_swappgsin, count);
1222 
1223 	/*
1224 	 * perform the I/O.  NOTE!!!  bp cannot be considered valid after
1225 	 * this point because we automatically release it on completion.
1226 	 * Instead, we look at the one page we are interested in which we
1227 	 * still hold a lock on even through the I/O completion.
1228 	 *
1229 	 * The other pages in our ma[] array are also released on completion,
1230 	 * so we cannot assume they are valid anymore either.
1231 	 *
1232 	 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1233 	 */
1234 	BUF_KERNPROC(bp);
1235 	swp_pager_strategy(bp);
1236 
1237 	/*
1238 	 * Wait for the pages we want to complete.  VPO_SWAPINPROG is always
1239 	 * cleared on completion.  If an I/O error occurs, SWAPBLK_NONE
1240 	 * is set in the metadata for each page in the request.
1241 	 */
1242 	VM_OBJECT_WLOCK(object);
1243 	while ((ma[0]->oflags & VPO_SWAPINPROG) != 0) {
1244 		ma[0]->oflags |= VPO_SWAPSLEEP;
1245 		VM_CNT_INC(v_intrans);
1246 		if (VM_OBJECT_SLEEP(object, &object->paging_in_progress, PSWP,
1247 		    "swread", hz * 20)) {
1248 			printf(
1249 "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n",
1250 			    bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount);
1251 		}
1252 	}
1253 
1254 	/*
1255 	 * If we had an unrecoverable read error pages will not be valid.
1256 	 */
1257 	for (i = 0; i < reqcount; i++)
1258 		if (ma[i]->valid != VM_PAGE_BITS_ALL)
1259 			return (VM_PAGER_ERROR);
1260 
1261 	return (VM_PAGER_OK);
1262 
1263 	/*
1264 	 * A final note: in a low swap situation, we cannot deallocate swap
1265 	 * and mark a page dirty here because the caller is likely to mark
1266 	 * the page clean when we return, causing the page to possibly revert
1267 	 * to all-zero's later.
1268 	 */
1269 }
1270 
1271 /*
1272  * 	swap_pager_getpages_async():
1273  *
1274  *	Right now this is emulation of asynchronous operation on top of
1275  *	swap_pager_getpages().
1276  */
1277 static int
1278 swap_pager_getpages_async(vm_object_t object, vm_page_t *ma, int count,
1279     int *rbehind, int *rahead, pgo_getpages_iodone_t iodone, void *arg)
1280 {
1281 	int r, error;
1282 
1283 	r = swap_pager_getpages(object, ma, count, rbehind, rahead);
1284 	VM_OBJECT_WUNLOCK(object);
1285 	switch (r) {
1286 	case VM_PAGER_OK:
1287 		error = 0;
1288 		break;
1289 	case VM_PAGER_ERROR:
1290 		error = EIO;
1291 		break;
1292 	case VM_PAGER_FAIL:
1293 		error = EINVAL;
1294 		break;
1295 	default:
1296 		panic("unhandled swap_pager_getpages() error %d", r);
1297 	}
1298 	(iodone)(arg, ma, count, error);
1299 	VM_OBJECT_WLOCK(object);
1300 
1301 	return (r);
1302 }
1303 
1304 /*
1305  *	swap_pager_putpages:
1306  *
1307  *	Assign swap (if necessary) and initiate I/O on the specified pages.
1308  *
1309  *	We support both OBJT_DEFAULT and OBJT_SWAP objects.  DEFAULT objects
1310  *	are automatically converted to SWAP objects.
1311  *
1312  *	In a low memory situation we may block in VOP_STRATEGY(), but the new
1313  *	vm_page reservation system coupled with properly written VFS devices
1314  *	should ensure that no low-memory deadlock occurs.  This is an area
1315  *	which needs work.
1316  *
1317  *	The parent has N vm_object_pip_add() references prior to
1318  *	calling us and will remove references for rtvals[] that are
1319  *	not set to VM_PAGER_PEND.  We need to remove the rest on I/O
1320  *	completion.
1321  *
1322  *	The parent has soft-busy'd the pages it passes us and will unbusy
1323  *	those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
1324  *	We need to unbusy the rest on I/O completion.
1325  */
1326 static void
1327 swap_pager_putpages(vm_object_t object, vm_page_t *ma, int count,
1328     int flags, int *rtvals)
1329 {
1330 	int i, n;
1331 	boolean_t sync;
1332 	daddr_t addr, n_free, s_free;
1333 
1334 	swp_pager_init_freerange(&s_free, &n_free);
1335 	if (count && ma[0]->object != object) {
1336 		panic("swap_pager_putpages: object mismatch %p/%p",
1337 		    object,
1338 		    ma[0]->object
1339 		);
1340 	}
1341 
1342 	/*
1343 	 * Step 1
1344 	 *
1345 	 * Turn object into OBJT_SWAP
1346 	 * check for bogus sysops
1347 	 * force sync if not pageout process
1348 	 */
1349 	if (object->type != OBJT_SWAP) {
1350 		addr = swp_pager_meta_build(object, 0, SWAPBLK_NONE);
1351 		KASSERT(addr == SWAPBLK_NONE,
1352 		    ("unexpected object swap block"));
1353 	}
1354 	VM_OBJECT_WUNLOCK(object);
1355 
1356 	n = 0;
1357 	if (curproc != pageproc)
1358 		sync = TRUE;
1359 	else
1360 		sync = (flags & VM_PAGER_PUT_SYNC) != 0;
1361 
1362 	/*
1363 	 * Step 2
1364 	 *
1365 	 * Assign swap blocks and issue I/O.  We reallocate swap on the fly.
1366 	 * The page is left dirty until the pageout operation completes
1367 	 * successfully.
1368 	 */
1369 	for (i = 0; i < count; i += n) {
1370 		int j;
1371 		struct buf *bp;
1372 		daddr_t blk;
1373 
1374 		/*
1375 		 * Maximum I/O size is limited by a number of factors.
1376 		 */
1377 		n = min(BLIST_MAX_ALLOC, count - i);
1378 		n = min(n, nsw_cluster_max);
1379 
1380 		/*
1381 		 * Get biggest block of swap we can.  If we fail, fall
1382 		 * back and try to allocate a smaller block.  Don't go
1383 		 * overboard trying to allocate space if it would overly
1384 		 * fragment swap.
1385 		 */
1386 		while (
1387 		    (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE &&
1388 		    n > 4
1389 		) {
1390 			n >>= 1;
1391 		}
1392 		if (blk == SWAPBLK_NONE) {
1393 			for (j = 0; j < n; ++j)
1394 				rtvals[i+j] = VM_PAGER_FAIL;
1395 			continue;
1396 		}
1397 
1398 		/*
1399 		 * All I/O parameters have been satisfied, build the I/O
1400 		 * request and assign the swap space.
1401 		 */
1402 		if (sync == TRUE) {
1403 			bp = getpbuf(&nsw_wcount_sync);
1404 		} else {
1405 			bp = getpbuf(&nsw_wcount_async);
1406 			bp->b_flags = B_ASYNC;
1407 		}
1408 		bp->b_flags |= B_PAGING;
1409 		bp->b_iocmd = BIO_WRITE;
1410 
1411 		bp->b_rcred = crhold(thread0.td_ucred);
1412 		bp->b_wcred = crhold(thread0.td_ucred);
1413 		bp->b_bcount = PAGE_SIZE * n;
1414 		bp->b_bufsize = PAGE_SIZE * n;
1415 		bp->b_blkno = blk;
1416 
1417 		VM_OBJECT_WLOCK(object);
1418 		for (j = 0; j < n; ++j) {
1419 			vm_page_t mreq = ma[i+j];
1420 
1421 			addr = swp_pager_meta_build(mreq->object, mreq->pindex,
1422 			    blk + j);
1423 			if (addr != SWAPBLK_NONE)
1424 				swp_pager_update_freerange(&s_free, &n_free,
1425 				    addr);
1426 			MPASS(mreq->dirty == VM_PAGE_BITS_ALL);
1427 			mreq->oflags |= VPO_SWAPINPROG;
1428 			bp->b_pages[j] = mreq;
1429 		}
1430 		VM_OBJECT_WUNLOCK(object);
1431 		bp->b_npages = n;
1432 		/*
1433 		 * Must set dirty range for NFS to work.
1434 		 */
1435 		bp->b_dirtyoff = 0;
1436 		bp->b_dirtyend = bp->b_bcount;
1437 
1438 		VM_CNT_INC(v_swapout);
1439 		VM_CNT_ADD(v_swappgsout, bp->b_npages);
1440 
1441 		/*
1442 		 * We unconditionally set rtvals[] to VM_PAGER_PEND so that we
1443 		 * can call the async completion routine at the end of a
1444 		 * synchronous I/O operation.  Otherwise, our caller would
1445 		 * perform duplicate unbusy and wakeup operations on the page
1446 		 * and object, respectively.
1447 		 */
1448 		for (j = 0; j < n; j++)
1449 			rtvals[i + j] = VM_PAGER_PEND;
1450 
1451 		/*
1452 		 * asynchronous
1453 		 *
1454 		 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1455 		 */
1456 		if (sync == FALSE) {
1457 			bp->b_iodone = swp_pager_async_iodone;
1458 			BUF_KERNPROC(bp);
1459 			swp_pager_strategy(bp);
1460 			continue;
1461 		}
1462 
1463 		/*
1464 		 * synchronous
1465 		 *
1466 		 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1467 		 */
1468 		bp->b_iodone = bdone;
1469 		swp_pager_strategy(bp);
1470 
1471 		/*
1472 		 * Wait for the sync I/O to complete.
1473 		 */
1474 		bwait(bp, PVM, "swwrt");
1475 
1476 		/*
1477 		 * Now that we are through with the bp, we can call the
1478 		 * normal async completion, which frees everything up.
1479 		 */
1480 		swp_pager_async_iodone(bp);
1481 	}
1482 	VM_OBJECT_WLOCK(object);
1483 	swp_pager_freeswapspace(s_free, n_free);
1484 }
1485 
1486 /*
1487  *	swp_pager_async_iodone:
1488  *
1489  *	Completion routine for asynchronous reads and writes from/to swap.
1490  *	Also called manually by synchronous code to finish up a bp.
1491  *
1492  *	This routine may not sleep.
1493  */
1494 static void
1495 swp_pager_async_iodone(struct buf *bp)
1496 {
1497 	int i;
1498 	vm_object_t object = NULL;
1499 
1500 	/*
1501 	 * report error
1502 	 */
1503 	if (bp->b_ioflags & BIO_ERROR) {
1504 		printf(
1505 		    "swap_pager: I/O error - %s failed; blkno %ld,"
1506 			"size %ld, error %d\n",
1507 		    ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
1508 		    (long)bp->b_blkno,
1509 		    (long)bp->b_bcount,
1510 		    bp->b_error
1511 		);
1512 	}
1513 
1514 	/*
1515 	 * remove the mapping for kernel virtual
1516 	 */
1517 	if (buf_mapped(bp))
1518 		pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
1519 	else
1520 		bp->b_data = bp->b_kvabase;
1521 
1522 	if (bp->b_npages) {
1523 		object = bp->b_pages[0]->object;
1524 		VM_OBJECT_WLOCK(object);
1525 	}
1526 
1527 	/*
1528 	 * cleanup pages.  If an error occurs writing to swap, we are in
1529 	 * very serious trouble.  If it happens to be a disk error, though,
1530 	 * we may be able to recover by reassigning the swap later on.  So
1531 	 * in this case we remove the m->swapblk assignment for the page
1532 	 * but do not free it in the rlist.  The errornous block(s) are thus
1533 	 * never reallocated as swap.  Redirty the page and continue.
1534 	 */
1535 	for (i = 0; i < bp->b_npages; ++i) {
1536 		vm_page_t m = bp->b_pages[i];
1537 
1538 		m->oflags &= ~VPO_SWAPINPROG;
1539 		if (m->oflags & VPO_SWAPSLEEP) {
1540 			m->oflags &= ~VPO_SWAPSLEEP;
1541 			wakeup(&object->paging_in_progress);
1542 		}
1543 
1544 		if (bp->b_ioflags & BIO_ERROR) {
1545 			/*
1546 			 * If an error occurs I'd love to throw the swapblk
1547 			 * away without freeing it back to swapspace, so it
1548 			 * can never be used again.  But I can't from an
1549 			 * interrupt.
1550 			 */
1551 			if (bp->b_iocmd == BIO_READ) {
1552 				/*
1553 				 * NOTE: for reads, m->dirty will probably
1554 				 * be overridden by the original caller of
1555 				 * getpages so don't play cute tricks here.
1556 				 */
1557 				m->valid = 0;
1558 			} else {
1559 				/*
1560 				 * If a write error occurs, reactivate page
1561 				 * so it doesn't clog the inactive list,
1562 				 * then finish the I/O.
1563 				 */
1564 				MPASS(m->dirty == VM_PAGE_BITS_ALL);
1565 				vm_page_lock(m);
1566 				vm_page_activate(m);
1567 				vm_page_unlock(m);
1568 				vm_page_sunbusy(m);
1569 			}
1570 		} else if (bp->b_iocmd == BIO_READ) {
1571 			/*
1572 			 * NOTE: for reads, m->dirty will probably be
1573 			 * overridden by the original caller of getpages so
1574 			 * we cannot set them in order to free the underlying
1575 			 * swap in a low-swap situation.  I don't think we'd
1576 			 * want to do that anyway, but it was an optimization
1577 			 * that existed in the old swapper for a time before
1578 			 * it got ripped out due to precisely this problem.
1579 			 */
1580 			KASSERT(!pmap_page_is_mapped(m),
1581 			    ("swp_pager_async_iodone: page %p is mapped", m));
1582 			KASSERT(m->dirty == 0,
1583 			    ("swp_pager_async_iodone: page %p is dirty", m));
1584 
1585 			m->valid = VM_PAGE_BITS_ALL;
1586 			if (i < bp->b_pgbefore ||
1587 			    i >= bp->b_npages - bp->b_pgafter)
1588 				vm_page_readahead_finish(m);
1589 		} else {
1590 			/*
1591 			 * For write success, clear the dirty
1592 			 * status, then finish the I/O ( which decrements the
1593 			 * busy count and possibly wakes waiter's up ).
1594 			 * A page is only written to swap after a period of
1595 			 * inactivity.  Therefore, we do not expect it to be
1596 			 * reused.
1597 			 */
1598 			KASSERT(!pmap_page_is_write_mapped(m),
1599 			    ("swp_pager_async_iodone: page %p is not write"
1600 			    " protected", m));
1601 			vm_page_undirty(m);
1602 			vm_page_lock(m);
1603 			vm_page_deactivate_noreuse(m);
1604 			vm_page_unlock(m);
1605 			vm_page_sunbusy(m);
1606 		}
1607 	}
1608 
1609 	/*
1610 	 * adjust pip.  NOTE: the original parent may still have its own
1611 	 * pip refs on the object.
1612 	 */
1613 	if (object != NULL) {
1614 		vm_object_pip_wakeupn(object, bp->b_npages);
1615 		VM_OBJECT_WUNLOCK(object);
1616 	}
1617 
1618 	/*
1619 	 * swapdev_strategy() manually sets b_vp and b_bufobj before calling
1620 	 * bstrategy(). Set them back to NULL now we're done with it, or we'll
1621 	 * trigger a KASSERT in relpbuf().
1622 	 */
1623 	if (bp->b_vp) {
1624 		    bp->b_vp = NULL;
1625 		    bp->b_bufobj = NULL;
1626 	}
1627 	/*
1628 	 * release the physical I/O buffer
1629 	 */
1630 	relpbuf(
1631 	    bp,
1632 	    ((bp->b_iocmd == BIO_READ) ? &nsw_rcount :
1633 		((bp->b_flags & B_ASYNC) ?
1634 		    &nsw_wcount_async :
1635 		    &nsw_wcount_sync
1636 		)
1637 	    )
1638 	);
1639 }
1640 
1641 int
1642 swap_pager_nswapdev(void)
1643 {
1644 
1645 	return (nswapdev);
1646 }
1647 
1648 /*
1649  * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in
1650  *
1651  *	This routine dissociates the page at the given index within an object
1652  *	from its backing store, paging it in if it does not reside in memory.
1653  *	If the page is paged in, it is marked dirty and placed in the laundry
1654  *	queue.  The page is marked dirty because it no longer has backing
1655  *	store.  It is placed in the laundry queue because it has not been
1656  *	accessed recently.  Otherwise, it would already reside in memory.
1657  *
1658  *	We also attempt to swap in all other pages in the swap block.
1659  *	However, we only guarantee that the one at the specified index is
1660  *	paged in.
1661  *
1662  *	XXX - The code to page the whole block in doesn't work, so we
1663  *	      revert to the one-by-one behavior for now.  Sigh.
1664  */
1665 static inline void
1666 swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex)
1667 {
1668 	vm_page_t m;
1669 
1670 	vm_object_pip_add(object, 1);
1671 	m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL);
1672 	if (m->valid == VM_PAGE_BITS_ALL) {
1673 		vm_object_pip_wakeup(object);
1674 		vm_page_dirty(m);
1675 #ifdef INVARIANTS
1676 		vm_page_lock(m);
1677 		if (m->wire_count == 0 && m->queue == PQ_NONE)
1678 			panic("page %p is neither wired nor queued", m);
1679 		vm_page_unlock(m);
1680 #endif
1681 		vm_page_xunbusy(m);
1682 		vm_pager_page_unswapped(m);
1683 		return;
1684 	}
1685 
1686 	if (swap_pager_getpages(object, &m, 1, NULL, NULL) != VM_PAGER_OK)
1687 		panic("swap_pager_force_pagein: read from swap failed");/*XXX*/
1688 	vm_object_pip_wakeup(object);
1689 	vm_page_dirty(m);
1690 	vm_page_lock(m);
1691 	vm_page_launder(m);
1692 	vm_page_unlock(m);
1693 	vm_page_xunbusy(m);
1694 	vm_pager_page_unswapped(m);
1695 }
1696 
1697 /*
1698  *	swap_pager_swapoff:
1699  *
1700  *	Page in all of the pages that have been paged out to the
1701  *	given device.  The corresponding blocks in the bitmap must be
1702  *	marked as allocated and the device must be flagged SW_CLOSING.
1703  *	There may be no processes swapped out to the device.
1704  *
1705  *	This routine may block.
1706  */
1707 static void
1708 swap_pager_swapoff(struct swdevt *sp)
1709 {
1710 	struct swblk *sb;
1711 	vm_object_t object;
1712 	vm_pindex_t pi;
1713 	int i, retries;
1714 
1715 	sx_assert(&swdev_syscall_lock, SA_XLOCKED);
1716 
1717 	retries = 0;
1718 full_rescan:
1719 	mtx_lock(&vm_object_list_mtx);
1720 	TAILQ_FOREACH(object, &vm_object_list, object_list) {
1721 		if (object->type != OBJT_SWAP)
1722 			continue;
1723 		mtx_unlock(&vm_object_list_mtx);
1724 		/* Depends on type-stability. */
1725 		VM_OBJECT_WLOCK(object);
1726 
1727 		/*
1728 		 * Dead objects are eventually terminated on their own.
1729 		 */
1730 		if ((object->flags & OBJ_DEAD) != 0)
1731 			goto next_obj;
1732 
1733 		/*
1734 		 * Sync with fences placed after pctrie
1735 		 * initialization.  We must not access pctrie below
1736 		 * unless we checked that our object is swap and not
1737 		 * dead.
1738 		 */
1739 		atomic_thread_fence_acq();
1740 		if (object->type != OBJT_SWAP)
1741 			goto next_obj;
1742 
1743 		for (pi = 0; (sb = SWAP_PCTRIE_LOOKUP_GE(
1744 		    &object->un_pager.swp.swp_blks, pi)) != NULL; ) {
1745 			pi = sb->p + SWAP_META_PAGES;
1746 			for (i = 0; i < SWAP_META_PAGES; i++) {
1747 				if (sb->d[i] == SWAPBLK_NONE)
1748 					continue;
1749 				if (swp_pager_isondev(sb->d[i], sp))
1750 					swp_pager_force_pagein(object,
1751 					    sb->p + i);
1752 			}
1753 		}
1754 next_obj:
1755 		VM_OBJECT_WUNLOCK(object);
1756 		mtx_lock(&vm_object_list_mtx);
1757 	}
1758 	mtx_unlock(&vm_object_list_mtx);
1759 
1760 	if (sp->sw_used) {
1761 		/*
1762 		 * Objects may be locked or paging to the device being
1763 		 * removed, so we will miss their pages and need to
1764 		 * make another pass.  We have marked this device as
1765 		 * SW_CLOSING, so the activity should finish soon.
1766 		 */
1767 		retries++;
1768 		if (retries > 100) {
1769 			panic("swapoff: failed to locate %d swap blocks",
1770 			    sp->sw_used);
1771 		}
1772 		pause("swpoff", hz / 20);
1773 		goto full_rescan;
1774 	}
1775 	EVENTHANDLER_INVOKE(swapoff, sp);
1776 }
1777 
1778 /************************************************************************
1779  *				SWAP META DATA 				*
1780  ************************************************************************
1781  *
1782  *	These routines manipulate the swap metadata stored in the
1783  *	OBJT_SWAP object.
1784  *
1785  *	Swap metadata is implemented with a global hash and not directly
1786  *	linked into the object.  Instead the object simply contains
1787  *	appropriate tracking counters.
1788  */
1789 
1790 /*
1791  * SWP_PAGER_SWBLK_EMPTY() - is a range of blocks free?
1792  */
1793 static bool
1794 swp_pager_swblk_empty(struct swblk *sb, int start, int limit)
1795 {
1796 	int i;
1797 
1798 	MPASS(0 <= start && start <= limit && limit <= SWAP_META_PAGES);
1799 	for (i = start; i < limit; i++) {
1800 		if (sb->d[i] != SWAPBLK_NONE)
1801 			return (false);
1802 	}
1803 	return (true);
1804 }
1805 
1806 /*
1807  * SWP_PAGER_META_BUILD() -	add swap block to swap meta data for object
1808  *
1809  *	We first convert the object to a swap object if it is a default
1810  *	object.
1811  *
1812  *	The specified swapblk is added to the object's swap metadata.  If
1813  *	the swapblk is not valid, it is freed instead.  Any previously
1814  *	assigned swapblk is returned.
1815  */
1816 static daddr_t
1817 swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
1818 {
1819 	static volatile int swblk_zone_exhausted, swpctrie_zone_exhausted;
1820 	struct swblk *sb, *sb1;
1821 	vm_pindex_t modpi, rdpi;
1822 	daddr_t prev_swapblk;
1823 	int error, i;
1824 
1825 	VM_OBJECT_ASSERT_WLOCKED(object);
1826 
1827 	/*
1828 	 * Convert default object to swap object if necessary
1829 	 */
1830 	if (object->type != OBJT_SWAP) {
1831 		pctrie_init(&object->un_pager.swp.swp_blks);
1832 
1833 		/*
1834 		 * Ensure that swap_pager_swapoff()'s iteration over
1835 		 * object_list does not see a garbage pctrie.
1836 		 */
1837 		atomic_thread_fence_rel();
1838 
1839 		object->type = OBJT_SWAP;
1840 		KASSERT(object->handle == NULL, ("default pager with handle"));
1841 	}
1842 
1843 	rdpi = rounddown(pindex, SWAP_META_PAGES);
1844 	sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, rdpi);
1845 	if (sb == NULL) {
1846 		if (swapblk == SWAPBLK_NONE)
1847 			return (SWAPBLK_NONE);
1848 		for (;;) {
1849 			sb = uma_zalloc(swblk_zone, M_NOWAIT | (curproc ==
1850 			    pageproc ? M_USE_RESERVE : 0));
1851 			if (sb != NULL) {
1852 				sb->p = rdpi;
1853 				for (i = 0; i < SWAP_META_PAGES; i++)
1854 					sb->d[i] = SWAPBLK_NONE;
1855 				if (atomic_cmpset_int(&swblk_zone_exhausted,
1856 				    1, 0))
1857 					printf("swblk zone ok\n");
1858 				break;
1859 			}
1860 			VM_OBJECT_WUNLOCK(object);
1861 			if (uma_zone_exhausted(swblk_zone)) {
1862 				if (atomic_cmpset_int(&swblk_zone_exhausted,
1863 				    0, 1))
1864 					printf("swap blk zone exhausted, "
1865 					    "increase kern.maxswzone\n");
1866 				vm_pageout_oom(VM_OOM_SWAPZ);
1867 				pause("swzonxb", 10);
1868 			} else
1869 				uma_zwait(swblk_zone);
1870 			VM_OBJECT_WLOCK(object);
1871 			sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks,
1872 			    rdpi);
1873 			if (sb != NULL)
1874 				/*
1875 				 * Somebody swapped out a nearby page,
1876 				 * allocating swblk at the rdpi index,
1877 				 * while we dropped the object lock.
1878 				 */
1879 				goto allocated;
1880 		}
1881 		for (;;) {
1882 			error = SWAP_PCTRIE_INSERT(
1883 			    &object->un_pager.swp.swp_blks, sb);
1884 			if (error == 0) {
1885 				if (atomic_cmpset_int(&swpctrie_zone_exhausted,
1886 				    1, 0))
1887 					printf("swpctrie zone ok\n");
1888 				break;
1889 			}
1890 			VM_OBJECT_WUNLOCK(object);
1891 			if (uma_zone_exhausted(swpctrie_zone)) {
1892 				if (atomic_cmpset_int(&swpctrie_zone_exhausted,
1893 				    0, 1))
1894 					printf("swap pctrie zone exhausted, "
1895 					    "increase kern.maxswzone\n");
1896 				vm_pageout_oom(VM_OOM_SWAPZ);
1897 				pause("swzonxp", 10);
1898 			} else
1899 				uma_zwait(swpctrie_zone);
1900 			VM_OBJECT_WLOCK(object);
1901 			sb1 = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks,
1902 			    rdpi);
1903 			if (sb1 != NULL) {
1904 				uma_zfree(swblk_zone, sb);
1905 				sb = sb1;
1906 				goto allocated;
1907 			}
1908 		}
1909 	}
1910 allocated:
1911 	MPASS(sb->p == rdpi);
1912 
1913 	modpi = pindex % SWAP_META_PAGES;
1914 	/* Return prior contents of metadata. */
1915 	prev_swapblk = sb->d[modpi];
1916 	/* Enter block into metadata. */
1917 	sb->d[modpi] = swapblk;
1918 
1919 	/*
1920 	 * Free the swblk if we end up with the empty page run.
1921 	 */
1922 	if (swapblk == SWAPBLK_NONE &&
1923 	    swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {
1924 		SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, rdpi);
1925 		uma_zfree(swblk_zone, sb);
1926 	}
1927 	return (prev_swapblk);
1928 }
1929 
1930 /*
1931  * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
1932  *
1933  *	The requested range of blocks is freed, with any associated swap
1934  *	returned to the swap bitmap.
1935  *
1936  *	This routine will free swap metadata structures as they are cleaned
1937  *	out.  This routine does *NOT* operate on swap metadata associated
1938  *	with resident pages.
1939  */
1940 static void
1941 swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count)
1942 {
1943 	struct swblk *sb;
1944 	daddr_t n_free, s_free;
1945 	vm_pindex_t last;
1946 	int i, limit, start;
1947 
1948 	VM_OBJECT_ASSERT_WLOCKED(object);
1949 	if (object->type != OBJT_SWAP || count == 0)
1950 		return;
1951 
1952 	swp_pager_init_freerange(&s_free, &n_free);
1953 	last = pindex + count;
1954 	for (;;) {
1955 		sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks,
1956 		    rounddown(pindex, SWAP_META_PAGES));
1957 		if (sb == NULL || sb->p >= last)
1958 			break;
1959 		start = pindex > sb->p ? pindex - sb->p : 0;
1960 		limit = last - sb->p < SWAP_META_PAGES ? last - sb->p :
1961 		    SWAP_META_PAGES;
1962 		for (i = start; i < limit; i++) {
1963 			if (sb->d[i] == SWAPBLK_NONE)
1964 				continue;
1965 			swp_pager_update_freerange(&s_free, &n_free, sb->d[i]);
1966 			sb->d[i] = SWAPBLK_NONE;
1967 		}
1968 		if (swp_pager_swblk_empty(sb, 0, start) &&
1969 		    swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) {
1970 			SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,
1971 			    sb->p);
1972 			uma_zfree(swblk_zone, sb);
1973 		}
1974 		pindex = sb->p + SWAP_META_PAGES;
1975 	}
1976 	swp_pager_freeswapspace(s_free, n_free);
1977 }
1978 
1979 /*
1980  * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
1981  *
1982  *	This routine locates and destroys all swap metadata associated with
1983  *	an object.
1984  */
1985 static void
1986 swp_pager_meta_free_all(vm_object_t object)
1987 {
1988 	struct swblk *sb;
1989 	daddr_t n_free, s_free;
1990 	vm_pindex_t pindex;
1991 	int i;
1992 
1993 	VM_OBJECT_ASSERT_WLOCKED(object);
1994 	if (object->type != OBJT_SWAP)
1995 		return;
1996 
1997 	swp_pager_init_freerange(&s_free, &n_free);
1998 	for (pindex = 0; (sb = SWAP_PCTRIE_LOOKUP_GE(
1999 	    &object->un_pager.swp.swp_blks, pindex)) != NULL;) {
2000 		pindex = sb->p + SWAP_META_PAGES;
2001 		for (i = 0; i < SWAP_META_PAGES; i++) {
2002 			if (sb->d[i] == SWAPBLK_NONE)
2003 				continue;
2004 			swp_pager_update_freerange(&s_free, &n_free, sb->d[i]);
2005 		}
2006 		SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p);
2007 		uma_zfree(swblk_zone, sb);
2008 	}
2009 	swp_pager_freeswapspace(s_free, n_free);
2010 }
2011 
2012 /*
2013  * SWP_PAGER_METACTL() -  misc control of swap meta data.
2014  *
2015  *	This routine is capable of looking up, or removing swapblk
2016  *	assignments in the swap meta data.  It returns the swapblk being
2017  *	looked-up, popped, or SWAPBLK_NONE if the block was invalid.
2018  *
2019  *	When acting on a busy resident page and paging is in progress, we
2020  *	have to wait until paging is complete but otherwise can act on the
2021  *	busy page.
2022  *
2023  *	SWM_POP		remove from meta data but do not free it
2024  */
2025 static daddr_t
2026 swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
2027 {
2028 	struct swblk *sb;
2029 	daddr_t r1;
2030 
2031 	if ((flags & SWM_POP) != 0)
2032 		VM_OBJECT_ASSERT_WLOCKED(object);
2033 	else
2034 		VM_OBJECT_ASSERT_LOCKED(object);
2035 
2036 	/*
2037 	 * The meta data only exists if the object is OBJT_SWAP
2038 	 * and even then might not be allocated yet.
2039 	 */
2040 	if (object->type != OBJT_SWAP)
2041 		return (SWAPBLK_NONE);
2042 
2043 	sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks,
2044 	    rounddown(pindex, SWAP_META_PAGES));
2045 	if (sb == NULL)
2046 		return (SWAPBLK_NONE);
2047 	r1 = sb->d[pindex % SWAP_META_PAGES];
2048 	if (r1 == SWAPBLK_NONE)
2049 		return (SWAPBLK_NONE);
2050 	if ((flags & SWM_POP) != 0) {
2051 		sb->d[pindex % SWAP_META_PAGES] = SWAPBLK_NONE;
2052 		if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {
2053 			SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,
2054 			    rounddown(pindex, SWAP_META_PAGES));
2055 			uma_zfree(swblk_zone, sb);
2056 		}
2057 	}
2058 	return (r1);
2059 }
2060 
2061 /*
2062  * Returns the least page index which is greater than or equal to the
2063  * parameter pindex and for which there is a swap block allocated.
2064  * Returns object's size if the object's type is not swap or if there
2065  * are no allocated swap blocks for the object after the requested
2066  * pindex.
2067  */
2068 vm_pindex_t
2069 swap_pager_find_least(vm_object_t object, vm_pindex_t pindex)
2070 {
2071 	struct swblk *sb;
2072 	int i;
2073 
2074 	VM_OBJECT_ASSERT_LOCKED(object);
2075 	if (object->type != OBJT_SWAP)
2076 		return (object->size);
2077 
2078 	sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks,
2079 	    rounddown(pindex, SWAP_META_PAGES));
2080 	if (sb == NULL)
2081 		return (object->size);
2082 	if (sb->p < pindex) {
2083 		for (i = pindex % SWAP_META_PAGES; i < SWAP_META_PAGES; i++) {
2084 			if (sb->d[i] != SWAPBLK_NONE)
2085 				return (sb->p + i);
2086 		}
2087 		sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks,
2088 		    roundup(pindex, SWAP_META_PAGES));
2089 		if (sb == NULL)
2090 			return (object->size);
2091 	}
2092 	for (i = 0; i < SWAP_META_PAGES; i++) {
2093 		if (sb->d[i] != SWAPBLK_NONE)
2094 			return (sb->p + i);
2095 	}
2096 
2097 	/*
2098 	 * We get here if a swblk is present in the trie but it
2099 	 * doesn't map any blocks.
2100 	 */
2101 	MPASS(0);
2102 	return (object->size);
2103 }
2104 
2105 /*
2106  * System call swapon(name) enables swapping on device name,
2107  * which must be in the swdevsw.  Return EBUSY
2108  * if already swapping on this device.
2109  */
2110 #ifndef _SYS_SYSPROTO_H_
2111 struct swapon_args {
2112 	char *name;
2113 };
2114 #endif
2115 
2116 /*
2117  * MPSAFE
2118  */
2119 /* ARGSUSED */
2120 int
2121 sys_swapon(struct thread *td, struct swapon_args *uap)
2122 {
2123 	struct vattr attr;
2124 	struct vnode *vp;
2125 	struct nameidata nd;
2126 	int error;
2127 
2128 	error = priv_check(td, PRIV_SWAPON);
2129 	if (error)
2130 		return (error);
2131 
2132 	sx_xlock(&swdev_syscall_lock);
2133 
2134 	/*
2135 	 * Swap metadata may not fit in the KVM if we have physical
2136 	 * memory of >1GB.
2137 	 */
2138 	if (swblk_zone == NULL) {
2139 		error = ENOMEM;
2140 		goto done;
2141 	}
2142 
2143 	NDINIT(&nd, LOOKUP, ISOPEN | FOLLOW | AUDITVNODE1, UIO_USERSPACE,
2144 	    uap->name, td);
2145 	error = namei(&nd);
2146 	if (error)
2147 		goto done;
2148 
2149 	NDFREE(&nd, NDF_ONLY_PNBUF);
2150 	vp = nd.ni_vp;
2151 
2152 	if (vn_isdisk(vp, &error)) {
2153 		error = swapongeom(vp);
2154 	} else if (vp->v_type == VREG &&
2155 	    (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
2156 	    (error = VOP_GETATTR(vp, &attr, td->td_ucred)) == 0) {
2157 		/*
2158 		 * Allow direct swapping to NFS regular files in the same
2159 		 * way that nfs_mountroot() sets up diskless swapping.
2160 		 */
2161 		error = swaponvp(td, vp, attr.va_size / DEV_BSIZE);
2162 	}
2163 
2164 	if (error)
2165 		vrele(vp);
2166 done:
2167 	sx_xunlock(&swdev_syscall_lock);
2168 	return (error);
2169 }
2170 
2171 /*
2172  * Check that the total amount of swap currently configured does not
2173  * exceed half the theoretical maximum.  If it does, print a warning
2174  * message.
2175  */
2176 static void
2177 swapon_check_swzone(void)
2178 {
2179 	unsigned long maxpages, npages;
2180 
2181 	npages = swap_total / PAGE_SIZE;
2182 	/* absolute maximum we can handle assuming 100% efficiency */
2183 	maxpages = uma_zone_get_max(swblk_zone) * SWAP_META_PAGES;
2184 
2185 	/* recommend using no more than half that amount */
2186 	if (npages > maxpages / 2) {
2187 		printf("warning: total configured swap (%lu pages) "
2188 		    "exceeds maximum recommended amount (%lu pages).\n",
2189 		    npages, maxpages / 2);
2190 		printf("warning: increase kern.maxswzone "
2191 		    "or reduce amount of swap.\n");
2192 	}
2193 }
2194 
2195 static void
2196 swaponsomething(struct vnode *vp, void *id, u_long nblks,
2197     sw_strategy_t *strategy, sw_close_t *close, dev_t dev, int flags)
2198 {
2199 	struct swdevt *sp, *tsp;
2200 	swblk_t dvbase;
2201 	u_long mblocks;
2202 
2203 	/*
2204 	 * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks.
2205 	 * First chop nblks off to page-align it, then convert.
2206 	 *
2207 	 * sw->sw_nblks is in page-sized chunks now too.
2208 	 */
2209 	nblks &= ~(ctodb(1) - 1);
2210 	nblks = dbtoc(nblks);
2211 
2212 	/*
2213 	 * If we go beyond this, we get overflows in the radix
2214 	 * tree bitmap code.
2215 	 */
2216 	mblocks = 0x40000000 / BLIST_META_RADIX;
2217 	if (nblks > mblocks) {
2218 		printf(
2219     "WARNING: reducing swap size to maximum of %luMB per unit\n",
2220 		    mblocks / 1024 / 1024 * PAGE_SIZE);
2221 		nblks = mblocks;
2222 	}
2223 
2224 	sp = malloc(sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO);
2225 	sp->sw_vp = vp;
2226 	sp->sw_id = id;
2227 	sp->sw_dev = dev;
2228 	sp->sw_flags = 0;
2229 	sp->sw_nblks = nblks;
2230 	sp->sw_used = 0;
2231 	sp->sw_strategy = strategy;
2232 	sp->sw_close = close;
2233 	sp->sw_flags = flags;
2234 
2235 	sp->sw_blist = blist_create(nblks, M_WAITOK);
2236 	/*
2237 	 * Do not free the first two block in order to avoid overwriting
2238 	 * any bsd label at the front of the partition
2239 	 */
2240 	blist_free(sp->sw_blist, 2, nblks - 2);
2241 
2242 	dvbase = 0;
2243 	mtx_lock(&sw_dev_mtx);
2244 	TAILQ_FOREACH(tsp, &swtailq, sw_list) {
2245 		if (tsp->sw_end >= dvbase) {
2246 			/*
2247 			 * We put one uncovered page between the devices
2248 			 * in order to definitively prevent any cross-device
2249 			 * I/O requests
2250 			 */
2251 			dvbase = tsp->sw_end + 1;
2252 		}
2253 	}
2254 	sp->sw_first = dvbase;
2255 	sp->sw_end = dvbase + nblks;
2256 	TAILQ_INSERT_TAIL(&swtailq, sp, sw_list);
2257 	nswapdev++;
2258 	swap_pager_avail += nblks - 2;
2259 	swap_total += (vm_ooffset_t)nblks * PAGE_SIZE;
2260 	swapon_check_swzone();
2261 	swp_sizecheck();
2262 	mtx_unlock(&sw_dev_mtx);
2263 	EVENTHANDLER_INVOKE(swapon, sp);
2264 }
2265 
2266 /*
2267  * SYSCALL: swapoff(devname)
2268  *
2269  * Disable swapping on the given device.
2270  *
2271  * XXX: Badly designed system call: it should use a device index
2272  * rather than filename as specification.  We keep sw_vp around
2273  * only to make this work.
2274  */
2275 #ifndef _SYS_SYSPROTO_H_
2276 struct swapoff_args {
2277 	char *name;
2278 };
2279 #endif
2280 
2281 /*
2282  * MPSAFE
2283  */
2284 /* ARGSUSED */
2285 int
2286 sys_swapoff(struct thread *td, struct swapoff_args *uap)
2287 {
2288 	struct vnode *vp;
2289 	struct nameidata nd;
2290 	struct swdevt *sp;
2291 	int error;
2292 
2293 	error = priv_check(td, PRIV_SWAPOFF);
2294 	if (error)
2295 		return (error);
2296 
2297 	sx_xlock(&swdev_syscall_lock);
2298 
2299 	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, UIO_USERSPACE, uap->name,
2300 	    td);
2301 	error = namei(&nd);
2302 	if (error)
2303 		goto done;
2304 	NDFREE(&nd, NDF_ONLY_PNBUF);
2305 	vp = nd.ni_vp;
2306 
2307 	mtx_lock(&sw_dev_mtx);
2308 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2309 		if (sp->sw_vp == vp)
2310 			break;
2311 	}
2312 	mtx_unlock(&sw_dev_mtx);
2313 	if (sp == NULL) {
2314 		error = EINVAL;
2315 		goto done;
2316 	}
2317 	error = swapoff_one(sp, td->td_ucred);
2318 done:
2319 	sx_xunlock(&swdev_syscall_lock);
2320 	return (error);
2321 }
2322 
2323 static int
2324 swapoff_one(struct swdevt *sp, struct ucred *cred)
2325 {
2326 	u_long nblks;
2327 #ifdef MAC
2328 	int error;
2329 #endif
2330 
2331 	sx_assert(&swdev_syscall_lock, SA_XLOCKED);
2332 #ifdef MAC
2333 	(void) vn_lock(sp->sw_vp, LK_EXCLUSIVE | LK_RETRY);
2334 	error = mac_system_check_swapoff(cred, sp->sw_vp);
2335 	(void) VOP_UNLOCK(sp->sw_vp, 0);
2336 	if (error != 0)
2337 		return (error);
2338 #endif
2339 	nblks = sp->sw_nblks;
2340 
2341 	/*
2342 	 * We can turn off this swap device safely only if the
2343 	 * available virtual memory in the system will fit the amount
2344 	 * of data we will have to page back in, plus an epsilon so
2345 	 * the system doesn't become critically low on swap space.
2346 	 */
2347 	if (vm_free_count() + swap_pager_avail < nblks + nswap_lowat)
2348 		return (ENOMEM);
2349 
2350 	/*
2351 	 * Prevent further allocations on this device.
2352 	 */
2353 	mtx_lock(&sw_dev_mtx);
2354 	sp->sw_flags |= SW_CLOSING;
2355 	swap_pager_avail -= blist_fill(sp->sw_blist, 0, nblks);
2356 	swap_total -= (vm_ooffset_t)nblks * PAGE_SIZE;
2357 	mtx_unlock(&sw_dev_mtx);
2358 
2359 	/*
2360 	 * Page in the contents of the device and close it.
2361 	 */
2362 	swap_pager_swapoff(sp);
2363 
2364 	sp->sw_close(curthread, sp);
2365 	mtx_lock(&sw_dev_mtx);
2366 	sp->sw_id = NULL;
2367 	TAILQ_REMOVE(&swtailq, sp, sw_list);
2368 	nswapdev--;
2369 	if (nswapdev == 0) {
2370 		swap_pager_full = 2;
2371 		swap_pager_almost_full = 1;
2372 	}
2373 	if (swdevhd == sp)
2374 		swdevhd = NULL;
2375 	mtx_unlock(&sw_dev_mtx);
2376 	blist_destroy(sp->sw_blist);
2377 	free(sp, M_VMPGDATA);
2378 	return (0);
2379 }
2380 
2381 void
2382 swapoff_all(void)
2383 {
2384 	struct swdevt *sp, *spt;
2385 	const char *devname;
2386 	int error;
2387 
2388 	sx_xlock(&swdev_syscall_lock);
2389 
2390 	mtx_lock(&sw_dev_mtx);
2391 	TAILQ_FOREACH_SAFE(sp, &swtailq, sw_list, spt) {
2392 		mtx_unlock(&sw_dev_mtx);
2393 		if (vn_isdisk(sp->sw_vp, NULL))
2394 			devname = devtoname(sp->sw_vp->v_rdev);
2395 		else
2396 			devname = "[file]";
2397 		error = swapoff_one(sp, thread0.td_ucred);
2398 		if (error != 0) {
2399 			printf("Cannot remove swap device %s (error=%d), "
2400 			    "skipping.\n", devname, error);
2401 		} else if (bootverbose) {
2402 			printf("Swap device %s removed.\n", devname);
2403 		}
2404 		mtx_lock(&sw_dev_mtx);
2405 	}
2406 	mtx_unlock(&sw_dev_mtx);
2407 
2408 	sx_xunlock(&swdev_syscall_lock);
2409 }
2410 
2411 void
2412 swap_pager_status(int *total, int *used)
2413 {
2414 	struct swdevt *sp;
2415 
2416 	*total = 0;
2417 	*used = 0;
2418 	mtx_lock(&sw_dev_mtx);
2419 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2420 		*total += sp->sw_nblks;
2421 		*used += sp->sw_used;
2422 	}
2423 	mtx_unlock(&sw_dev_mtx);
2424 }
2425 
2426 int
2427 swap_dev_info(int name, struct xswdev *xs, char *devname, size_t len)
2428 {
2429 	struct swdevt *sp;
2430 	const char *tmp_devname;
2431 	int error, n;
2432 
2433 	n = 0;
2434 	error = ENOENT;
2435 	mtx_lock(&sw_dev_mtx);
2436 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2437 		if (n != name) {
2438 			n++;
2439 			continue;
2440 		}
2441 		xs->xsw_version = XSWDEV_VERSION;
2442 		xs->xsw_dev = sp->sw_dev;
2443 		xs->xsw_flags = sp->sw_flags;
2444 		xs->xsw_nblks = sp->sw_nblks;
2445 		xs->xsw_used = sp->sw_used;
2446 		if (devname != NULL) {
2447 			if (vn_isdisk(sp->sw_vp, NULL))
2448 				tmp_devname = devtoname(sp->sw_vp->v_rdev);
2449 			else
2450 				tmp_devname = "[file]";
2451 			strncpy(devname, tmp_devname, len);
2452 		}
2453 		error = 0;
2454 		break;
2455 	}
2456 	mtx_unlock(&sw_dev_mtx);
2457 	return (error);
2458 }
2459 
2460 #if defined(COMPAT_FREEBSD11)
2461 #define XSWDEV_VERSION_11	1
2462 struct xswdev11 {
2463 	u_int	xsw_version;
2464 	uint32_t xsw_dev;
2465 	int	xsw_flags;
2466 	int	xsw_nblks;
2467 	int     xsw_used;
2468 };
2469 #endif
2470 
2471 static int
2472 sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS)
2473 {
2474 	struct xswdev xs;
2475 #if defined(COMPAT_FREEBSD11)
2476 	struct xswdev11 xs11;
2477 #endif
2478 	int error;
2479 
2480 	if (arg2 != 1)			/* name length */
2481 		return (EINVAL);
2482 	error = swap_dev_info(*(int *)arg1, &xs, NULL, 0);
2483 	if (error != 0)
2484 		return (error);
2485 #if defined(COMPAT_FREEBSD11)
2486 	if (req->oldlen == sizeof(xs11)) {
2487 		xs11.xsw_version = XSWDEV_VERSION_11;
2488 		xs11.xsw_dev = xs.xsw_dev; /* truncation */
2489 		xs11.xsw_flags = xs.xsw_flags;
2490 		xs11.xsw_nblks = xs.xsw_nblks;
2491 		xs11.xsw_used = xs.xsw_used;
2492 		error = SYSCTL_OUT(req, &xs11, sizeof(xs11));
2493 	} else
2494 #endif
2495 		error = SYSCTL_OUT(req, &xs, sizeof(xs));
2496 	return (error);
2497 }
2498 
2499 SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, &nswapdev, 0,
2500     "Number of swap devices");
2501 SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD | CTLFLAG_MPSAFE,
2502     sysctl_vm_swap_info,
2503     "Swap statistics by device");
2504 
2505 /*
2506  * Count the approximate swap usage in pages for a vmspace.  The
2507  * shadowed or not yet copied on write swap blocks are not accounted.
2508  * The map must be locked.
2509  */
2510 long
2511 vmspace_swap_count(struct vmspace *vmspace)
2512 {
2513 	vm_map_t map;
2514 	vm_map_entry_t cur;
2515 	vm_object_t object;
2516 	struct swblk *sb;
2517 	vm_pindex_t e, pi;
2518 	long count;
2519 	int i;
2520 
2521 	map = &vmspace->vm_map;
2522 	count = 0;
2523 
2524 	for (cur = map->header.next; cur != &map->header; cur = cur->next) {
2525 		if ((cur->eflags & MAP_ENTRY_IS_SUB_MAP) != 0)
2526 			continue;
2527 		object = cur->object.vm_object;
2528 		if (object == NULL || object->type != OBJT_SWAP)
2529 			continue;
2530 		VM_OBJECT_RLOCK(object);
2531 		if (object->type != OBJT_SWAP)
2532 			goto unlock;
2533 		pi = OFF_TO_IDX(cur->offset);
2534 		e = pi + OFF_TO_IDX(cur->end - cur->start);
2535 		for (;; pi = sb->p + SWAP_META_PAGES) {
2536 			sb = SWAP_PCTRIE_LOOKUP_GE(
2537 			    &object->un_pager.swp.swp_blks, pi);
2538 			if (sb == NULL || sb->p >= e)
2539 				break;
2540 			for (i = 0; i < SWAP_META_PAGES; i++) {
2541 				if (sb->p + i < e &&
2542 				    sb->d[i] != SWAPBLK_NONE)
2543 					count++;
2544 			}
2545 		}
2546 unlock:
2547 		VM_OBJECT_RUNLOCK(object);
2548 	}
2549 	return (count);
2550 }
2551 
2552 /*
2553  * GEOM backend
2554  *
2555  * Swapping onto disk devices.
2556  *
2557  */
2558 
2559 static g_orphan_t swapgeom_orphan;
2560 
2561 static struct g_class g_swap_class = {
2562 	.name = "SWAP",
2563 	.version = G_VERSION,
2564 	.orphan = swapgeom_orphan,
2565 };
2566 
2567 DECLARE_GEOM_CLASS(g_swap_class, g_class);
2568 
2569 
2570 static void
2571 swapgeom_close_ev(void *arg, int flags)
2572 {
2573 	struct g_consumer *cp;
2574 
2575 	cp = arg;
2576 	g_access(cp, -1, -1, 0);
2577 	g_detach(cp);
2578 	g_destroy_consumer(cp);
2579 }
2580 
2581 /*
2582  * Add a reference to the g_consumer for an inflight transaction.
2583  */
2584 static void
2585 swapgeom_acquire(struct g_consumer *cp)
2586 {
2587 
2588 	mtx_assert(&sw_dev_mtx, MA_OWNED);
2589 	cp->index++;
2590 }
2591 
2592 /*
2593  * Remove a reference from the g_consumer.  Post a close event if all
2594  * references go away, since the function might be called from the
2595  * biodone context.
2596  */
2597 static void
2598 swapgeom_release(struct g_consumer *cp, struct swdevt *sp)
2599 {
2600 
2601 	mtx_assert(&sw_dev_mtx, MA_OWNED);
2602 	cp->index--;
2603 	if (cp->index == 0) {
2604 		if (g_post_event(swapgeom_close_ev, cp, M_NOWAIT, NULL) == 0)
2605 			sp->sw_id = NULL;
2606 	}
2607 }
2608 
2609 static void
2610 swapgeom_done(struct bio *bp2)
2611 {
2612 	struct swdevt *sp;
2613 	struct buf *bp;
2614 	struct g_consumer *cp;
2615 
2616 	bp = bp2->bio_caller2;
2617 	cp = bp2->bio_from;
2618 	bp->b_ioflags = bp2->bio_flags;
2619 	if (bp2->bio_error)
2620 		bp->b_ioflags |= BIO_ERROR;
2621 	bp->b_resid = bp->b_bcount - bp2->bio_completed;
2622 	bp->b_error = bp2->bio_error;
2623 	bufdone(bp);
2624 	sp = bp2->bio_caller1;
2625 	mtx_lock(&sw_dev_mtx);
2626 	swapgeom_release(cp, sp);
2627 	mtx_unlock(&sw_dev_mtx);
2628 	g_destroy_bio(bp2);
2629 }
2630 
2631 static void
2632 swapgeom_strategy(struct buf *bp, struct swdevt *sp)
2633 {
2634 	struct bio *bio;
2635 	struct g_consumer *cp;
2636 
2637 	mtx_lock(&sw_dev_mtx);
2638 	cp = sp->sw_id;
2639 	if (cp == NULL) {
2640 		mtx_unlock(&sw_dev_mtx);
2641 		bp->b_error = ENXIO;
2642 		bp->b_ioflags |= BIO_ERROR;
2643 		bufdone(bp);
2644 		return;
2645 	}
2646 	swapgeom_acquire(cp);
2647 	mtx_unlock(&sw_dev_mtx);
2648 	if (bp->b_iocmd == BIO_WRITE)
2649 		bio = g_new_bio();
2650 	else
2651 		bio = g_alloc_bio();
2652 	if (bio == NULL) {
2653 		mtx_lock(&sw_dev_mtx);
2654 		swapgeom_release(cp, sp);
2655 		mtx_unlock(&sw_dev_mtx);
2656 		bp->b_error = ENOMEM;
2657 		bp->b_ioflags |= BIO_ERROR;
2658 		bufdone(bp);
2659 		return;
2660 	}
2661 
2662 	bio->bio_caller1 = sp;
2663 	bio->bio_caller2 = bp;
2664 	bio->bio_cmd = bp->b_iocmd;
2665 	bio->bio_offset = (bp->b_blkno - sp->sw_first) * PAGE_SIZE;
2666 	bio->bio_length = bp->b_bcount;
2667 	bio->bio_done = swapgeom_done;
2668 	if (!buf_mapped(bp)) {
2669 		bio->bio_ma = bp->b_pages;
2670 		bio->bio_data = unmapped_buf;
2671 		bio->bio_ma_offset = (vm_offset_t)bp->b_offset & PAGE_MASK;
2672 		bio->bio_ma_n = bp->b_npages;
2673 		bio->bio_flags |= BIO_UNMAPPED;
2674 	} else {
2675 		bio->bio_data = bp->b_data;
2676 		bio->bio_ma = NULL;
2677 	}
2678 	g_io_request(bio, cp);
2679 	return;
2680 }
2681 
2682 static void
2683 swapgeom_orphan(struct g_consumer *cp)
2684 {
2685 	struct swdevt *sp;
2686 	int destroy;
2687 
2688 	mtx_lock(&sw_dev_mtx);
2689 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2690 		if (sp->sw_id == cp) {
2691 			sp->sw_flags |= SW_CLOSING;
2692 			break;
2693 		}
2694 	}
2695 	/*
2696 	 * Drop reference we were created with. Do directly since we're in a
2697 	 * special context where we don't have to queue the call to
2698 	 * swapgeom_close_ev().
2699 	 */
2700 	cp->index--;
2701 	destroy = ((sp != NULL) && (cp->index == 0));
2702 	if (destroy)
2703 		sp->sw_id = NULL;
2704 	mtx_unlock(&sw_dev_mtx);
2705 	if (destroy)
2706 		swapgeom_close_ev(cp, 0);
2707 }
2708 
2709 static void
2710 swapgeom_close(struct thread *td, struct swdevt *sw)
2711 {
2712 	struct g_consumer *cp;
2713 
2714 	mtx_lock(&sw_dev_mtx);
2715 	cp = sw->sw_id;
2716 	sw->sw_id = NULL;
2717 	mtx_unlock(&sw_dev_mtx);
2718 
2719 	/*
2720 	 * swapgeom_close() may be called from the biodone context,
2721 	 * where we cannot perform topology changes.  Delegate the
2722 	 * work to the events thread.
2723 	 */
2724 	if (cp != NULL)
2725 		g_waitfor_event(swapgeom_close_ev, cp, M_WAITOK, NULL);
2726 }
2727 
2728 static int
2729 swapongeom_locked(struct cdev *dev, struct vnode *vp)
2730 {
2731 	struct g_provider *pp;
2732 	struct g_consumer *cp;
2733 	static struct g_geom *gp;
2734 	struct swdevt *sp;
2735 	u_long nblks;
2736 	int error;
2737 
2738 	pp = g_dev_getprovider(dev);
2739 	if (pp == NULL)
2740 		return (ENODEV);
2741 	mtx_lock(&sw_dev_mtx);
2742 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2743 		cp = sp->sw_id;
2744 		if (cp != NULL && cp->provider == pp) {
2745 			mtx_unlock(&sw_dev_mtx);
2746 			return (EBUSY);
2747 		}
2748 	}
2749 	mtx_unlock(&sw_dev_mtx);
2750 	if (gp == NULL)
2751 		gp = g_new_geomf(&g_swap_class, "swap");
2752 	cp = g_new_consumer(gp);
2753 	cp->index = 1;	/* Number of active I/Os, plus one for being active. */
2754 	cp->flags |=  G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
2755 	g_attach(cp, pp);
2756 	/*
2757 	 * XXX: Every time you think you can improve the margin for
2758 	 * footshooting, somebody depends on the ability to do so:
2759 	 * savecore(8) wants to write to our swapdev so we cannot
2760 	 * set an exclusive count :-(
2761 	 */
2762 	error = g_access(cp, 1, 1, 0);
2763 	if (error != 0) {
2764 		g_detach(cp);
2765 		g_destroy_consumer(cp);
2766 		return (error);
2767 	}
2768 	nblks = pp->mediasize / DEV_BSIZE;
2769 	swaponsomething(vp, cp, nblks, swapgeom_strategy,
2770 	    swapgeom_close, dev2udev(dev),
2771 	    (pp->flags & G_PF_ACCEPT_UNMAPPED) != 0 ? SW_UNMAPPED : 0);
2772 	return (0);
2773 }
2774 
2775 static int
2776 swapongeom(struct vnode *vp)
2777 {
2778 	int error;
2779 
2780 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2781 	if (vp->v_type != VCHR || (vp->v_iflag & VI_DOOMED) != 0) {
2782 		error = ENOENT;
2783 	} else {
2784 		g_topology_lock();
2785 		error = swapongeom_locked(vp->v_rdev, vp);
2786 		g_topology_unlock();
2787 	}
2788 	VOP_UNLOCK(vp, 0);
2789 	return (error);
2790 }
2791 
2792 /*
2793  * VNODE backend
2794  *
2795  * This is used mainly for network filesystem (read: probably only tested
2796  * with NFS) swapfiles.
2797  *
2798  */
2799 
2800 static void
2801 swapdev_strategy(struct buf *bp, struct swdevt *sp)
2802 {
2803 	struct vnode *vp2;
2804 
2805 	bp->b_blkno = ctodb(bp->b_blkno - sp->sw_first);
2806 
2807 	vp2 = sp->sw_id;
2808 	vhold(vp2);
2809 	if (bp->b_iocmd == BIO_WRITE) {
2810 		if (bp->b_bufobj)
2811 			bufobj_wdrop(bp->b_bufobj);
2812 		bufobj_wref(&vp2->v_bufobj);
2813 	}
2814 	if (bp->b_bufobj != &vp2->v_bufobj)
2815 		bp->b_bufobj = &vp2->v_bufobj;
2816 	bp->b_vp = vp2;
2817 	bp->b_iooffset = dbtob(bp->b_blkno);
2818 	bstrategy(bp);
2819 	return;
2820 }
2821 
2822 static void
2823 swapdev_close(struct thread *td, struct swdevt *sp)
2824 {
2825 
2826 	VOP_CLOSE(sp->sw_vp, FREAD | FWRITE, td->td_ucred, td);
2827 	vrele(sp->sw_vp);
2828 }
2829 
2830 
2831 static int
2832 swaponvp(struct thread *td, struct vnode *vp, u_long nblks)
2833 {
2834 	struct swdevt *sp;
2835 	int error;
2836 
2837 	if (nblks == 0)
2838 		return (ENXIO);
2839 	mtx_lock(&sw_dev_mtx);
2840 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
2841 		if (sp->sw_id == vp) {
2842 			mtx_unlock(&sw_dev_mtx);
2843 			return (EBUSY);
2844 		}
2845 	}
2846 	mtx_unlock(&sw_dev_mtx);
2847 
2848 	(void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2849 #ifdef MAC
2850 	error = mac_system_check_swapon(td->td_ucred, vp);
2851 	if (error == 0)
2852 #endif
2853 		error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, NULL);
2854 	(void) VOP_UNLOCK(vp, 0);
2855 	if (error)
2856 		return (error);
2857 
2858 	swaponsomething(vp, vp, nblks, swapdev_strategy, swapdev_close,
2859 	    NODEV, 0);
2860 	return (0);
2861 }
2862 
2863 static int
2864 sysctl_swap_async_max(SYSCTL_HANDLER_ARGS)
2865 {
2866 	int error, new, n;
2867 
2868 	new = nsw_wcount_async_max;
2869 	error = sysctl_handle_int(oidp, &new, 0, req);
2870 	if (error != 0 || req->newptr == NULL)
2871 		return (error);
2872 
2873 	if (new > nswbuf / 2 || new < 1)
2874 		return (EINVAL);
2875 
2876 	mtx_lock(&pbuf_mtx);
2877 	while (nsw_wcount_async_max != new) {
2878 		/*
2879 		 * Adjust difference.  If the current async count is too low,
2880 		 * we will need to sqeeze our update slowly in.  Sleep with a
2881 		 * higher priority than getpbuf() to finish faster.
2882 		 */
2883 		n = new - nsw_wcount_async_max;
2884 		if (nsw_wcount_async + n >= 0) {
2885 			nsw_wcount_async += n;
2886 			nsw_wcount_async_max += n;
2887 			wakeup(&nsw_wcount_async);
2888 		} else {
2889 			nsw_wcount_async_max -= nsw_wcount_async;
2890 			nsw_wcount_async = 0;
2891 			msleep(&nsw_wcount_async, &pbuf_mtx, PSWP,
2892 			    "swpsysctl", 0);
2893 		}
2894 	}
2895 	mtx_unlock(&pbuf_mtx);
2896 
2897 	return (0);
2898 }
2899