xref: /freebsd/sys/vm/vm_pageout.c (revision aaa3852435aee8c1b5abfe38ed7d43f9ba57e6dd)
1 /*-
2  * SPDX-License-Identifier: (BSD-4-Clause AND MIT-CMU)
3  *
4  * Copyright (c) 1991 Regents of the University of California.
5  * All rights reserved.
6  * Copyright (c) 1994 John S. Dyson
7  * All rights reserved.
8  * Copyright (c) 1994 David Greenman
9  * All rights reserved.
10  * Copyright (c) 2005 Yahoo! Technologies Norway AS
11  * All rights reserved.
12  *
13  * This code is derived from software contributed to Berkeley by
14  * The Mach Operating System project at Carnegie-Mellon University.
15  *
16  * Redistribution and use in source and binary forms, with or without
17  * modification, are permitted provided that the following conditions
18  * are met:
19  * 1. Redistributions of source code must retain the above copyright
20  *    notice, this list of conditions and the following disclaimer.
21  * 2. Redistributions in binary form must reproduce the above copyright
22  *    notice, this list of conditions and the following disclaimer in the
23  *    documentation and/or other materials provided with the distribution.
24  * 3. All advertising materials mentioning features or use of this software
25  *    must display the following acknowledgement:
26  *	This product includes software developed by the University of
27  *	California, Berkeley and its contributors.
28  * 4. Neither the name of the University nor the names of its contributors
29  *    may be used to endorse or promote products derived from this software
30  *    without specific prior written permission.
31  *
32  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
33  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
34  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
35  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
36  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
37  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
38  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
39  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
40  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
41  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
42  * SUCH DAMAGE.
43  *
44  *	from: @(#)vm_pageout.c	7.4 (Berkeley) 5/7/91
45  *
46  *
47  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
48  * All rights reserved.
49  *
50  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
51  *
52  * Permission to use, copy, modify and distribute this software and
53  * its documentation is hereby granted, provided that both the copyright
54  * notice and this permission notice appear in all copies of the
55  * software, derivative works or modified versions, and any portions
56  * thereof, and that both notices appear in supporting documentation.
57  *
58  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
59  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
60  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
61  *
62  * Carnegie Mellon requests users of this software to return to
63  *
64  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
65  *  School of Computer Science
66  *  Carnegie Mellon University
67  *  Pittsburgh PA 15213-3890
68  *
69  * any improvements or extensions that they make and grant Carnegie the
70  * rights to redistribute these changes.
71  */
72 
73 /*
74  *	The proverbial page-out daemon.
75  */
76 
77 #include <sys/cdefs.h>
78 __FBSDID("$FreeBSD$");
79 
80 #include "opt_vm.h"
81 
82 #include <sys/param.h>
83 #include <sys/systm.h>
84 #include <sys/kernel.h>
85 #include <sys/eventhandler.h>
86 #include <sys/lock.h>
87 #include <sys/mutex.h>
88 #include <sys/proc.h>
89 #include <sys/kthread.h>
90 #include <sys/ktr.h>
91 #include <sys/mount.h>
92 #include <sys/racct.h>
93 #include <sys/resourcevar.h>
94 #include <sys/sched.h>
95 #include <sys/sdt.h>
96 #include <sys/signalvar.h>
97 #include <sys/smp.h>
98 #include <sys/time.h>
99 #include <sys/vnode.h>
100 #include <sys/vmmeter.h>
101 #include <sys/rwlock.h>
102 #include <sys/sx.h>
103 #include <sys/sysctl.h>
104 
105 #include <vm/vm.h>
106 #include <vm/vm_param.h>
107 #include <vm/vm_object.h>
108 #include <vm/vm_page.h>
109 #include <vm/vm_map.h>
110 #include <vm/vm_pageout.h>
111 #include <vm/vm_pager.h>
112 #include <vm/vm_phys.h>
113 #include <vm/vm_pagequeue.h>
114 #include <vm/swap_pager.h>
115 #include <vm/vm_extern.h>
116 #include <vm/uma.h>
117 
118 /*
119  * System initialization
120  */
121 
122 /* the kernel process "vm_pageout"*/
123 static void vm_pageout(void);
124 static void vm_pageout_init(void);
125 static int vm_pageout_clean(vm_page_t m, int *numpagedout);
126 static int vm_pageout_cluster(vm_page_t m);
127 static void vm_pageout_mightbe_oom(struct vm_domain *vmd, int page_shortage,
128     int starting_page_shortage);
129 
130 SYSINIT(pagedaemon_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_FIRST, vm_pageout_init,
131     NULL);
132 
133 struct proc *pageproc;
134 
135 static struct kproc_desc page_kp = {
136 	"pagedaemon",
137 	vm_pageout,
138 	&pageproc
139 };
140 SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_SECOND, kproc_start,
141     &page_kp);
142 
143 SDT_PROVIDER_DEFINE(vm);
144 SDT_PROBE_DEFINE(vm, , , vm__lowmem_scan);
145 
146 /* Pagedaemon activity rates, in subdivisions of one second. */
147 #define	VM_LAUNDER_RATE		10
148 #define	VM_INACT_SCAN_RATE	10
149 
150 static int vm_pageout_oom_seq = 12;
151 
152 static int vm_pageout_update_period;
153 static int disable_swap_pageouts;
154 static int lowmem_period = 10;
155 static int swapdev_enabled;
156 
157 static int vm_panic_on_oom = 0;
158 
159 SYSCTL_INT(_vm, OID_AUTO, panic_on_oom,
160 	CTLFLAG_RWTUN, &vm_panic_on_oom, 0,
161 	"panic on out of memory instead of killing the largest process");
162 
163 SYSCTL_INT(_vm, OID_AUTO, pageout_update_period,
164 	CTLFLAG_RWTUN, &vm_pageout_update_period, 0,
165 	"Maximum active LRU update period");
166 
167 SYSCTL_INT(_vm, OID_AUTO, lowmem_period, CTLFLAG_RWTUN, &lowmem_period, 0,
168 	"Low memory callback period");
169 
170 SYSCTL_INT(_vm, OID_AUTO, disable_swapspace_pageouts,
171 	CTLFLAG_RWTUN, &disable_swap_pageouts, 0, "Disallow swapout of dirty pages");
172 
173 static int pageout_lock_miss;
174 SYSCTL_INT(_vm, OID_AUTO, pageout_lock_miss,
175 	CTLFLAG_RD, &pageout_lock_miss, 0, "vget() lock misses during pageout");
176 
177 SYSCTL_INT(_vm, OID_AUTO, pageout_oom_seq,
178 	CTLFLAG_RWTUN, &vm_pageout_oom_seq, 0,
179 	"back-to-back calls to oom detector to start OOM");
180 
181 static int act_scan_laundry_weight = 3;
182 SYSCTL_INT(_vm, OID_AUTO, act_scan_laundry_weight, CTLFLAG_RWTUN,
183     &act_scan_laundry_weight, 0,
184     "weight given to clean vs. dirty pages in active queue scans");
185 
186 static u_int vm_background_launder_rate = 4096;
187 SYSCTL_UINT(_vm, OID_AUTO, background_launder_rate, CTLFLAG_RWTUN,
188     &vm_background_launder_rate, 0,
189     "background laundering rate, in kilobytes per second");
190 
191 static u_int vm_background_launder_max = 20 * 1024;
192 SYSCTL_UINT(_vm, OID_AUTO, background_launder_max, CTLFLAG_RWTUN,
193     &vm_background_launder_max, 0, "background laundering cap, in kilobytes");
194 
195 int vm_pageout_page_count = 32;
196 
197 u_long vm_page_max_user_wired;
198 SYSCTL_ULONG(_vm, OID_AUTO, max_user_wired, CTLFLAG_RW,
199     &vm_page_max_user_wired, 0,
200     "system-wide limit to user-wired page count");
201 
202 static u_int isqrt(u_int num);
203 static int vm_pageout_launder(struct vm_domain *vmd, int launder,
204     bool in_shortfall);
205 static void vm_pageout_laundry_worker(void *arg);
206 
207 struct scan_state {
208 	struct vm_batchqueue bq;
209 	struct vm_pagequeue *pq;
210 	vm_page_t	marker;
211 	int		maxscan;
212 	int		scanned;
213 };
214 
215 static void
216 vm_pageout_init_scan(struct scan_state *ss, struct vm_pagequeue *pq,
217     vm_page_t marker, vm_page_t after, int maxscan)
218 {
219 
220 	vm_pagequeue_assert_locked(pq);
221 	KASSERT((marker->aflags & PGA_ENQUEUED) == 0,
222 	    ("marker %p already enqueued", marker));
223 
224 	if (after == NULL)
225 		TAILQ_INSERT_HEAD(&pq->pq_pl, marker, plinks.q);
226 	else
227 		TAILQ_INSERT_AFTER(&pq->pq_pl, after, marker, plinks.q);
228 	vm_page_aflag_set(marker, PGA_ENQUEUED);
229 
230 	vm_batchqueue_init(&ss->bq);
231 	ss->pq = pq;
232 	ss->marker = marker;
233 	ss->maxscan = maxscan;
234 	ss->scanned = 0;
235 	vm_pagequeue_unlock(pq);
236 }
237 
238 static void
239 vm_pageout_end_scan(struct scan_state *ss)
240 {
241 	struct vm_pagequeue *pq;
242 
243 	pq = ss->pq;
244 	vm_pagequeue_assert_locked(pq);
245 	KASSERT((ss->marker->aflags & PGA_ENQUEUED) != 0,
246 	    ("marker %p not enqueued", ss->marker));
247 
248 	TAILQ_REMOVE(&pq->pq_pl, ss->marker, plinks.q);
249 	vm_page_aflag_clear(ss->marker, PGA_ENQUEUED);
250 	pq->pq_pdpages += ss->scanned;
251 }
252 
253 /*
254  * Add a small number of queued pages to a batch queue for later processing
255  * without the corresponding queue lock held.  The caller must have enqueued a
256  * marker page at the desired start point for the scan.  Pages will be
257  * physically dequeued if the caller so requests.  Otherwise, the returned
258  * batch may contain marker pages, and it is up to the caller to handle them.
259  *
260  * When processing the batch queue, vm_page_queue() must be used to
261  * determine whether the page has been logically dequeued by another thread.
262  * Once this check is performed, the page lock guarantees that the page will
263  * not be disassociated from the queue.
264  */
265 static __always_inline void
266 vm_pageout_collect_batch(struct scan_state *ss, const bool dequeue)
267 {
268 	struct vm_pagequeue *pq;
269 	vm_page_t m, marker, n;
270 
271 	marker = ss->marker;
272 	pq = ss->pq;
273 
274 	KASSERT((marker->aflags & PGA_ENQUEUED) != 0,
275 	    ("marker %p not enqueued", ss->marker));
276 
277 	vm_pagequeue_lock(pq);
278 	for (m = TAILQ_NEXT(marker, plinks.q); m != NULL &&
279 	    ss->scanned < ss->maxscan && ss->bq.bq_cnt < VM_BATCHQUEUE_SIZE;
280 	    m = n, ss->scanned++) {
281 		n = TAILQ_NEXT(m, plinks.q);
282 		if ((m->flags & PG_MARKER) == 0) {
283 			KASSERT((m->aflags & PGA_ENQUEUED) != 0,
284 			    ("page %p not enqueued", m));
285 			KASSERT((m->flags & PG_FICTITIOUS) == 0,
286 			    ("Fictitious page %p cannot be in page queue", m));
287 			KASSERT((m->oflags & VPO_UNMANAGED) == 0,
288 			    ("Unmanaged page %p cannot be in page queue", m));
289 		} else if (dequeue)
290 			continue;
291 
292 		(void)vm_batchqueue_insert(&ss->bq, m);
293 		if (dequeue) {
294 			TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
295 			vm_page_aflag_clear(m, PGA_ENQUEUED);
296 		}
297 	}
298 	TAILQ_REMOVE(&pq->pq_pl, marker, plinks.q);
299 	if (__predict_true(m != NULL))
300 		TAILQ_INSERT_BEFORE(m, marker, plinks.q);
301 	else
302 		TAILQ_INSERT_TAIL(&pq->pq_pl, marker, plinks.q);
303 	if (dequeue)
304 		vm_pagequeue_cnt_add(pq, -ss->bq.bq_cnt);
305 	vm_pagequeue_unlock(pq);
306 }
307 
308 /*
309  * Return the next page to be scanned, or NULL if the scan is complete.
310  */
311 static __always_inline vm_page_t
312 vm_pageout_next(struct scan_state *ss, const bool dequeue)
313 {
314 
315 	if (ss->bq.bq_cnt == 0)
316 		vm_pageout_collect_batch(ss, dequeue);
317 	return (vm_batchqueue_pop(&ss->bq));
318 }
319 
320 /*
321  * Scan for pages at adjacent offsets within the given page's object that are
322  * eligible for laundering, form a cluster of these pages and the given page,
323  * and launder that cluster.
324  */
325 static int
326 vm_pageout_cluster(vm_page_t m)
327 {
328 	vm_object_t object;
329 	vm_page_t mc[2 * vm_pageout_page_count], p, pb, ps;
330 	vm_pindex_t pindex;
331 	int ib, is, page_base, pageout_count;
332 
333 	object = m->object;
334 	VM_OBJECT_ASSERT_WLOCKED(object);
335 	pindex = m->pindex;
336 
337 	vm_page_assert_unbusied(m);
338 
339 	mc[vm_pageout_page_count] = pb = ps = m;
340 	pageout_count = 1;
341 	page_base = vm_pageout_page_count;
342 	ib = 1;
343 	is = 1;
344 
345 	/*
346 	 * We can cluster only if the page is not clean, busy, or held, and
347 	 * the page is in the laundry queue.
348 	 *
349 	 * During heavy mmap/modification loads the pageout
350 	 * daemon can really fragment the underlying file
351 	 * due to flushing pages out of order and not trying to
352 	 * align the clusters (which leaves sporadic out-of-order
353 	 * holes).  To solve this problem we do the reverse scan
354 	 * first and attempt to align our cluster, then do a
355 	 * forward scan if room remains.
356 	 */
357 more:
358 	while (ib != 0 && pageout_count < vm_pageout_page_count) {
359 		if (ib > pindex) {
360 			ib = 0;
361 			break;
362 		}
363 		if ((p = vm_page_prev(pb)) == NULL || vm_page_busied(p) ||
364 		    vm_page_wired(p)) {
365 			ib = 0;
366 			break;
367 		}
368 		vm_page_test_dirty(p);
369 		if (p->dirty == 0) {
370 			ib = 0;
371 			break;
372 		}
373 		vm_page_lock(p);
374 		if (!vm_page_in_laundry(p) || !vm_page_try_remove_write(p)) {
375 			vm_page_unlock(p);
376 			ib = 0;
377 			break;
378 		}
379 		vm_page_unlock(p);
380 		mc[--page_base] = pb = p;
381 		++pageout_count;
382 		++ib;
383 
384 		/*
385 		 * We are at an alignment boundary.  Stop here, and switch
386 		 * directions.  Do not clear ib.
387 		 */
388 		if ((pindex - (ib - 1)) % vm_pageout_page_count == 0)
389 			break;
390 	}
391 	while (pageout_count < vm_pageout_page_count &&
392 	    pindex + is < object->size) {
393 		if ((p = vm_page_next(ps)) == NULL || vm_page_busied(p) ||
394 		    vm_page_wired(p))
395 			break;
396 		vm_page_test_dirty(p);
397 		if (p->dirty == 0)
398 			break;
399 		vm_page_lock(p);
400 		if (!vm_page_in_laundry(p) || !vm_page_try_remove_write(p)) {
401 			vm_page_unlock(p);
402 			break;
403 		}
404 		vm_page_unlock(p);
405 		mc[page_base + pageout_count] = ps = p;
406 		++pageout_count;
407 		++is;
408 	}
409 
410 	/*
411 	 * If we exhausted our forward scan, continue with the reverse scan
412 	 * when possible, even past an alignment boundary.  This catches
413 	 * boundary conditions.
414 	 */
415 	if (ib != 0 && pageout_count < vm_pageout_page_count)
416 		goto more;
417 
418 	return (vm_pageout_flush(&mc[page_base], pageout_count,
419 	    VM_PAGER_PUT_NOREUSE, 0, NULL, NULL));
420 }
421 
422 /*
423  * vm_pageout_flush() - launder the given pages
424  *
425  *	The given pages are laundered.  Note that we setup for the start of
426  *	I/O ( i.e. busy the page ), mark it read-only, and bump the object
427  *	reference count all in here rather then in the parent.  If we want
428  *	the parent to do more sophisticated things we may have to change
429  *	the ordering.
430  *
431  *	Returned runlen is the count of pages between mreq and first
432  *	page after mreq with status VM_PAGER_AGAIN.
433  *	*eio is set to TRUE if pager returned VM_PAGER_ERROR or VM_PAGER_FAIL
434  *	for any page in runlen set.
435  */
436 int
437 vm_pageout_flush(vm_page_t *mc, int count, int flags, int mreq, int *prunlen,
438     boolean_t *eio)
439 {
440 	vm_object_t object = mc[0]->object;
441 	int pageout_status[count];
442 	int numpagedout = 0;
443 	int i, runlen;
444 
445 	VM_OBJECT_ASSERT_WLOCKED(object);
446 
447 	/*
448 	 * Initiate I/O.  Mark the pages busy and verify that they're valid
449 	 * and read-only.
450 	 *
451 	 * We do not have to fixup the clean/dirty bits here... we can
452 	 * allow the pager to do it after the I/O completes.
453 	 *
454 	 * NOTE! mc[i]->dirty may be partial or fragmented due to an
455 	 * edge case with file fragments.
456 	 */
457 	for (i = 0; i < count; i++) {
458 		KASSERT(mc[i]->valid == VM_PAGE_BITS_ALL,
459 		    ("vm_pageout_flush: partially invalid page %p index %d/%d",
460 			mc[i], i, count));
461 		KASSERT((mc[i]->aflags & PGA_WRITEABLE) == 0,
462 		    ("vm_pageout_flush: writeable page %p", mc[i]));
463 		vm_page_sbusy(mc[i]);
464 	}
465 	vm_object_pip_add(object, count);
466 
467 	vm_pager_put_pages(object, mc, count, flags, pageout_status);
468 
469 	runlen = count - mreq;
470 	if (eio != NULL)
471 		*eio = FALSE;
472 	for (i = 0; i < count; i++) {
473 		vm_page_t mt = mc[i];
474 
475 		KASSERT(pageout_status[i] == VM_PAGER_PEND ||
476 		    !pmap_page_is_write_mapped(mt),
477 		    ("vm_pageout_flush: page %p is not write protected", mt));
478 		switch (pageout_status[i]) {
479 		case VM_PAGER_OK:
480 			vm_page_lock(mt);
481 			if (vm_page_in_laundry(mt))
482 				vm_page_deactivate_noreuse(mt);
483 			vm_page_unlock(mt);
484 			/* FALLTHROUGH */
485 		case VM_PAGER_PEND:
486 			numpagedout++;
487 			break;
488 		case VM_PAGER_BAD:
489 			/*
490 			 * The page is outside the object's range.  We pretend
491 			 * that the page out worked and clean the page, so the
492 			 * changes will be lost if the page is reclaimed by
493 			 * the page daemon.
494 			 */
495 			vm_page_undirty(mt);
496 			vm_page_lock(mt);
497 			if (vm_page_in_laundry(mt))
498 				vm_page_deactivate_noreuse(mt);
499 			vm_page_unlock(mt);
500 			break;
501 		case VM_PAGER_ERROR:
502 		case VM_PAGER_FAIL:
503 			/*
504 			 * If the page couldn't be paged out to swap because the
505 			 * pager wasn't able to find space, place the page in
506 			 * the PQ_UNSWAPPABLE holding queue.  This is an
507 			 * optimization that prevents the page daemon from
508 			 * wasting CPU cycles on pages that cannot be reclaimed
509 			 * becase no swap device is configured.
510 			 *
511 			 * Otherwise, reactivate the page so that it doesn't
512 			 * clog the laundry and inactive queues.  (We will try
513 			 * paging it out again later.)
514 			 */
515 			vm_page_lock(mt);
516 			if (object->type == OBJT_SWAP &&
517 			    pageout_status[i] == VM_PAGER_FAIL) {
518 				vm_page_unswappable(mt);
519 				numpagedout++;
520 			} else
521 				vm_page_activate(mt);
522 			vm_page_unlock(mt);
523 			if (eio != NULL && i >= mreq && i - mreq < runlen)
524 				*eio = TRUE;
525 			break;
526 		case VM_PAGER_AGAIN:
527 			if (i >= mreq && i - mreq < runlen)
528 				runlen = i - mreq;
529 			break;
530 		}
531 
532 		/*
533 		 * If the operation is still going, leave the page busy to
534 		 * block all other accesses. Also, leave the paging in
535 		 * progress indicator set so that we don't attempt an object
536 		 * collapse.
537 		 */
538 		if (pageout_status[i] != VM_PAGER_PEND) {
539 			vm_object_pip_wakeup(object);
540 			vm_page_sunbusy(mt);
541 		}
542 	}
543 	if (prunlen != NULL)
544 		*prunlen = runlen;
545 	return (numpagedout);
546 }
547 
548 static void
549 vm_pageout_swapon(void *arg __unused, struct swdevt *sp __unused)
550 {
551 
552 	atomic_store_rel_int(&swapdev_enabled, 1);
553 }
554 
555 static void
556 vm_pageout_swapoff(void *arg __unused, struct swdevt *sp __unused)
557 {
558 
559 	if (swap_pager_nswapdev() == 1)
560 		atomic_store_rel_int(&swapdev_enabled, 0);
561 }
562 
563 /*
564  * Attempt to acquire all of the necessary locks to launder a page and
565  * then call through the clustering layer to PUTPAGES.  Wait a short
566  * time for a vnode lock.
567  *
568  * Requires the page and object lock on entry, releases both before return.
569  * Returns 0 on success and an errno otherwise.
570  */
571 static int
572 vm_pageout_clean(vm_page_t m, int *numpagedout)
573 {
574 	struct vnode *vp;
575 	struct mount *mp;
576 	vm_object_t object;
577 	vm_pindex_t pindex;
578 	int error, lockmode;
579 
580 	vm_page_assert_locked(m);
581 	object = m->object;
582 	VM_OBJECT_ASSERT_WLOCKED(object);
583 	error = 0;
584 	vp = NULL;
585 	mp = NULL;
586 
587 	/*
588 	 * The object is already known NOT to be dead.   It
589 	 * is possible for the vget() to block the whole
590 	 * pageout daemon, but the new low-memory handling
591 	 * code should prevent it.
592 	 *
593 	 * We can't wait forever for the vnode lock, we might
594 	 * deadlock due to a vn_read() getting stuck in
595 	 * vm_wait while holding this vnode.  We skip the
596 	 * vnode if we can't get it in a reasonable amount
597 	 * of time.
598 	 */
599 	if (object->type == OBJT_VNODE) {
600 		vm_page_unlock(m);
601 		vp = object->handle;
602 		if (vp->v_type == VREG &&
603 		    vn_start_write(vp, &mp, V_NOWAIT) != 0) {
604 			mp = NULL;
605 			error = EDEADLK;
606 			goto unlock_all;
607 		}
608 		KASSERT(mp != NULL,
609 		    ("vp %p with NULL v_mount", vp));
610 		vm_object_reference_locked(object);
611 		pindex = m->pindex;
612 		VM_OBJECT_WUNLOCK(object);
613 		lockmode = MNT_SHARED_WRITES(vp->v_mount) ?
614 		    LK_SHARED : LK_EXCLUSIVE;
615 		if (vget(vp, lockmode | LK_TIMELOCK, curthread)) {
616 			vp = NULL;
617 			error = EDEADLK;
618 			goto unlock_mp;
619 		}
620 		VM_OBJECT_WLOCK(object);
621 
622 		/*
623 		 * Ensure that the object and vnode were not disassociated
624 		 * while locks were dropped.
625 		 */
626 		if (vp->v_object != object) {
627 			error = ENOENT;
628 			goto unlock_all;
629 		}
630 		vm_page_lock(m);
631 
632 		/*
633 		 * While the object and page were unlocked, the page
634 		 * may have been:
635 		 * (1) moved to a different queue,
636 		 * (2) reallocated to a different object,
637 		 * (3) reallocated to a different offset, or
638 		 * (4) cleaned.
639 		 */
640 		if (!vm_page_in_laundry(m) || m->object != object ||
641 		    m->pindex != pindex || m->dirty == 0) {
642 			vm_page_unlock(m);
643 			error = ENXIO;
644 			goto unlock_all;
645 		}
646 
647 		/*
648 		 * The page may have been busied while the object and page
649 		 * locks were released.
650 		 */
651 		if (vm_page_busied(m)) {
652 			vm_page_unlock(m);
653 			error = EBUSY;
654 			goto unlock_all;
655 		}
656 	}
657 
658 	/*
659 	 * Remove all writeable mappings, failing if the page is wired.
660 	 */
661 	if (!vm_page_try_remove_write(m)) {
662 		vm_page_unlock(m);
663 		error = EBUSY;
664 		goto unlock_all;
665 	}
666 	vm_page_unlock(m);
667 
668 	/*
669 	 * If a page is dirty, then it is either being washed
670 	 * (but not yet cleaned) or it is still in the
671 	 * laundry.  If it is still in the laundry, then we
672 	 * start the cleaning operation.
673 	 */
674 	if ((*numpagedout = vm_pageout_cluster(m)) == 0)
675 		error = EIO;
676 
677 unlock_all:
678 	VM_OBJECT_WUNLOCK(object);
679 
680 unlock_mp:
681 	vm_page_lock_assert(m, MA_NOTOWNED);
682 	if (mp != NULL) {
683 		if (vp != NULL)
684 			vput(vp);
685 		vm_object_deallocate(object);
686 		vn_finished_write(mp);
687 	}
688 
689 	return (error);
690 }
691 
692 /*
693  * Attempt to launder the specified number of pages.
694  *
695  * Returns the number of pages successfully laundered.
696  */
697 static int
698 vm_pageout_launder(struct vm_domain *vmd, int launder, bool in_shortfall)
699 {
700 	struct scan_state ss;
701 	struct vm_pagequeue *pq;
702 	struct mtx *mtx;
703 	vm_object_t object;
704 	vm_page_t m, marker;
705 	int act_delta, error, numpagedout, queue, starting_target;
706 	int vnodes_skipped;
707 	bool pageout_ok;
708 
709 	mtx = NULL;
710 	object = NULL;
711 	starting_target = launder;
712 	vnodes_skipped = 0;
713 
714 	/*
715 	 * Scan the laundry queues for pages eligible to be laundered.  We stop
716 	 * once the target number of dirty pages have been laundered, or once
717 	 * we've reached the end of the queue.  A single iteration of this loop
718 	 * may cause more than one page to be laundered because of clustering.
719 	 *
720 	 * As an optimization, we avoid laundering from PQ_UNSWAPPABLE when no
721 	 * swap devices are configured.
722 	 */
723 	if (atomic_load_acq_int(&swapdev_enabled))
724 		queue = PQ_UNSWAPPABLE;
725 	else
726 		queue = PQ_LAUNDRY;
727 
728 scan:
729 	marker = &vmd->vmd_markers[queue];
730 	pq = &vmd->vmd_pagequeues[queue];
731 	vm_pagequeue_lock(pq);
732 	vm_pageout_init_scan(&ss, pq, marker, NULL, pq->pq_cnt);
733 	while (launder > 0 && (m = vm_pageout_next(&ss, false)) != NULL) {
734 		if (__predict_false((m->flags & PG_MARKER) != 0))
735 			continue;
736 
737 		vm_page_change_lock(m, &mtx);
738 
739 recheck:
740 		/*
741 		 * The page may have been disassociated from the queue
742 		 * or even freed while locks were dropped.  We thus must be
743 		 * careful whenever modifying page state.  Once the object lock
744 		 * has been acquired, we have a stable reference to the page.
745 		 */
746 		if (vm_page_queue(m) != queue)
747 			continue;
748 
749 		/*
750 		 * A requeue was requested, so this page gets a second
751 		 * chance.
752 		 */
753 		if ((m->aflags & PGA_REQUEUE) != 0) {
754 			vm_page_pqbatch_submit(m, queue);
755 			continue;
756 		}
757 
758 		/*
759 		 * Wired pages may not be freed.  Complete their removal
760 		 * from the queue now to avoid needless revisits during
761 		 * future scans.  This check is racy and must be reverified once
762 		 * we hold the object lock and have verified that the page
763 		 * is not busy.
764 		 */
765 		if (vm_page_wired(m)) {
766 			vm_page_dequeue_deferred(m);
767 			continue;
768 		}
769 
770 		if (object != m->object) {
771 			if (object != NULL)
772 				VM_OBJECT_WUNLOCK(object);
773 
774 			/*
775 			 * A page's object pointer may be set to NULL before
776 			 * the object lock is acquired.
777 			 */
778 			object = (vm_object_t)atomic_load_ptr(&m->object);
779 			if (object != NULL && !VM_OBJECT_TRYWLOCK(object)) {
780 				mtx_unlock(mtx);
781 				/* Depends on type-stability. */
782 				VM_OBJECT_WLOCK(object);
783 				mtx_lock(mtx);
784 				goto recheck;
785 			}
786 		}
787 		if (__predict_false(m->object == NULL))
788 			/*
789 			 * The page has been removed from its object.
790 			 */
791 			continue;
792 		KASSERT(m->object == object, ("page %p does not belong to %p",
793 		    m, object));
794 
795 		if (vm_page_busied(m))
796 			continue;
797 
798 		/*
799 		 * Re-check for wirings now that we hold the object lock and
800 		 * have verified that the page is unbusied.  If the page is
801 		 * mapped, it may still be wired by pmap lookups.  The call to
802 		 * vm_page_try_remove_all() below atomically checks for such
803 		 * wirings and removes mappings.  If the page is unmapped, the
804 		 * wire count is guaranteed not to increase.
805 		 */
806 		if (__predict_false(vm_page_wired(m))) {
807 			vm_page_dequeue_deferred(m);
808 			continue;
809 		}
810 
811 		/*
812 		 * Invalid pages can be easily freed.  They cannot be
813 		 * mapped; vm_page_free() asserts this.
814 		 */
815 		if (m->valid == 0)
816 			goto free_page;
817 
818 		/*
819 		 * If the page has been referenced and the object is not dead,
820 		 * reactivate or requeue the page depending on whether the
821 		 * object is mapped.
822 		 *
823 		 * Test PGA_REFERENCED after calling pmap_ts_referenced() so
824 		 * that a reference from a concurrently destroyed mapping is
825 		 * observed here and now.
826 		 */
827 		if (object->ref_count != 0)
828 			act_delta = pmap_ts_referenced(m);
829 		else {
830 			KASSERT(!pmap_page_is_mapped(m),
831 			    ("page %p is mapped", m));
832 			act_delta = 0;
833 		}
834 		if ((m->aflags & PGA_REFERENCED) != 0) {
835 			vm_page_aflag_clear(m, PGA_REFERENCED);
836 			act_delta++;
837 		}
838 		if (act_delta != 0) {
839 			if (object->ref_count != 0) {
840 				VM_CNT_INC(v_reactivated);
841 				vm_page_activate(m);
842 
843 				/*
844 				 * Increase the activation count if the page
845 				 * was referenced while in the laundry queue.
846 				 * This makes it less likely that the page will
847 				 * be returned prematurely to the inactive
848 				 * queue.
849  				 */
850 				m->act_count += act_delta + ACT_ADVANCE;
851 
852 				/*
853 				 * If this was a background laundering, count
854 				 * activated pages towards our target.  The
855 				 * purpose of background laundering is to ensure
856 				 * that pages are eventually cycled through the
857 				 * laundry queue, and an activation is a valid
858 				 * way out.
859 				 */
860 				if (!in_shortfall)
861 					launder--;
862 				continue;
863 			} else if ((object->flags & OBJ_DEAD) == 0) {
864 				vm_page_requeue(m);
865 				continue;
866 			}
867 		}
868 
869 		/*
870 		 * If the page appears to be clean at the machine-independent
871 		 * layer, then remove all of its mappings from the pmap in
872 		 * anticipation of freeing it.  If, however, any of the page's
873 		 * mappings allow write access, then the page may still be
874 		 * modified until the last of those mappings are removed.
875 		 */
876 		if (object->ref_count != 0) {
877 			vm_page_test_dirty(m);
878 			if (m->dirty == 0 && !vm_page_try_remove_all(m)) {
879 				vm_page_dequeue_deferred(m);
880 				continue;
881 			}
882 		}
883 
884 		/*
885 		 * Clean pages are freed, and dirty pages are paged out unless
886 		 * they belong to a dead object.  Requeueing dirty pages from
887 		 * dead objects is pointless, as they are being paged out and
888 		 * freed by the thread that destroyed the object.
889 		 */
890 		if (m->dirty == 0) {
891 free_page:
892 			vm_page_free(m);
893 			VM_CNT_INC(v_dfree);
894 		} else if ((object->flags & OBJ_DEAD) == 0) {
895 			if (object->type != OBJT_SWAP &&
896 			    object->type != OBJT_DEFAULT)
897 				pageout_ok = true;
898 			else if (disable_swap_pageouts)
899 				pageout_ok = false;
900 			else
901 				pageout_ok = true;
902 			if (!pageout_ok) {
903 				vm_page_requeue(m);
904 				continue;
905 			}
906 
907 			/*
908 			 * Form a cluster with adjacent, dirty pages from the
909 			 * same object, and page out that entire cluster.
910 			 *
911 			 * The adjacent, dirty pages must also be in the
912 			 * laundry.  However, their mappings are not checked
913 			 * for new references.  Consequently, a recently
914 			 * referenced page may be paged out.  However, that
915 			 * page will not be prematurely reclaimed.  After page
916 			 * out, the page will be placed in the inactive queue,
917 			 * where any new references will be detected and the
918 			 * page reactivated.
919 			 */
920 			error = vm_pageout_clean(m, &numpagedout);
921 			if (error == 0) {
922 				launder -= numpagedout;
923 				ss.scanned += numpagedout;
924 			} else if (error == EDEADLK) {
925 				pageout_lock_miss++;
926 				vnodes_skipped++;
927 			}
928 			mtx = NULL;
929 			object = NULL;
930 		}
931 	}
932 	if (mtx != NULL) {
933 		mtx_unlock(mtx);
934 		mtx = NULL;
935 	}
936 	if (object != NULL) {
937 		VM_OBJECT_WUNLOCK(object);
938 		object = NULL;
939 	}
940 	vm_pagequeue_lock(pq);
941 	vm_pageout_end_scan(&ss);
942 	vm_pagequeue_unlock(pq);
943 
944 	if (launder > 0 && queue == PQ_UNSWAPPABLE) {
945 		queue = PQ_LAUNDRY;
946 		goto scan;
947 	}
948 
949 	/*
950 	 * Wakeup the sync daemon if we skipped a vnode in a writeable object
951 	 * and we didn't launder enough pages.
952 	 */
953 	if (vnodes_skipped > 0 && launder > 0)
954 		(void)speedup_syncer();
955 
956 	return (starting_target - launder);
957 }
958 
959 /*
960  * Compute the integer square root.
961  */
962 static u_int
963 isqrt(u_int num)
964 {
965 	u_int bit, root, tmp;
966 
967 	bit = num != 0 ? (1u << ((fls(num) - 1) & ~1)) : 0;
968 	root = 0;
969 	while (bit != 0) {
970 		tmp = root + bit;
971 		root >>= 1;
972 		if (num >= tmp) {
973 			num -= tmp;
974 			root += bit;
975 		}
976 		bit >>= 2;
977 	}
978 	return (root);
979 }
980 
981 /*
982  * Perform the work of the laundry thread: periodically wake up and determine
983  * whether any pages need to be laundered.  If so, determine the number of pages
984  * that need to be laundered, and launder them.
985  */
986 static void
987 vm_pageout_laundry_worker(void *arg)
988 {
989 	struct vm_domain *vmd;
990 	struct vm_pagequeue *pq;
991 	uint64_t nclean, ndirty, nfreed;
992 	int domain, last_target, launder, shortfall, shortfall_cycle, target;
993 	bool in_shortfall;
994 
995 	domain = (uintptr_t)arg;
996 	vmd = VM_DOMAIN(domain);
997 	pq = &vmd->vmd_pagequeues[PQ_LAUNDRY];
998 	KASSERT(vmd->vmd_segs != 0, ("domain without segments"));
999 
1000 	shortfall = 0;
1001 	in_shortfall = false;
1002 	shortfall_cycle = 0;
1003 	last_target = target = 0;
1004 	nfreed = 0;
1005 
1006 	/*
1007 	 * Calls to these handlers are serialized by the swap syscall lock.
1008 	 */
1009 	(void)EVENTHANDLER_REGISTER(swapon, vm_pageout_swapon, vmd,
1010 	    EVENTHANDLER_PRI_ANY);
1011 	(void)EVENTHANDLER_REGISTER(swapoff, vm_pageout_swapoff, vmd,
1012 	    EVENTHANDLER_PRI_ANY);
1013 
1014 	/*
1015 	 * The pageout laundry worker is never done, so loop forever.
1016 	 */
1017 	for (;;) {
1018 		KASSERT(target >= 0, ("negative target %d", target));
1019 		KASSERT(shortfall_cycle >= 0,
1020 		    ("negative cycle %d", shortfall_cycle));
1021 		launder = 0;
1022 
1023 		/*
1024 		 * First determine whether we need to launder pages to meet a
1025 		 * shortage of free pages.
1026 		 */
1027 		if (shortfall > 0) {
1028 			in_shortfall = true;
1029 			shortfall_cycle = VM_LAUNDER_RATE / VM_INACT_SCAN_RATE;
1030 			target = shortfall;
1031 		} else if (!in_shortfall)
1032 			goto trybackground;
1033 		else if (shortfall_cycle == 0 || vm_laundry_target(vmd) <= 0) {
1034 			/*
1035 			 * We recently entered shortfall and began laundering
1036 			 * pages.  If we have completed that laundering run
1037 			 * (and we are no longer in shortfall) or we have met
1038 			 * our laundry target through other activity, then we
1039 			 * can stop laundering pages.
1040 			 */
1041 			in_shortfall = false;
1042 			target = 0;
1043 			goto trybackground;
1044 		}
1045 		launder = target / shortfall_cycle--;
1046 		goto dolaundry;
1047 
1048 		/*
1049 		 * There's no immediate need to launder any pages; see if we
1050 		 * meet the conditions to perform background laundering:
1051 		 *
1052 		 * 1. The ratio of dirty to clean inactive pages exceeds the
1053 		 *    background laundering threshold, or
1054 		 * 2. we haven't yet reached the target of the current
1055 		 *    background laundering run.
1056 		 *
1057 		 * The background laundering threshold is not a constant.
1058 		 * Instead, it is a slowly growing function of the number of
1059 		 * clean pages freed by the page daemon since the last
1060 		 * background laundering.  Thus, as the ratio of dirty to
1061 		 * clean inactive pages grows, the amount of memory pressure
1062 		 * required to trigger laundering decreases.  We ensure
1063 		 * that the threshold is non-zero after an inactive queue
1064 		 * scan, even if that scan failed to free a single clean page.
1065 		 */
1066 trybackground:
1067 		nclean = vmd->vmd_free_count +
1068 		    vmd->vmd_pagequeues[PQ_INACTIVE].pq_cnt;
1069 		ndirty = vmd->vmd_pagequeues[PQ_LAUNDRY].pq_cnt;
1070 		if (target == 0 && ndirty * isqrt(howmany(nfreed + 1,
1071 		    vmd->vmd_free_target - vmd->vmd_free_min)) >= nclean) {
1072 			target = vmd->vmd_background_launder_target;
1073 		}
1074 
1075 		/*
1076 		 * We have a non-zero background laundering target.  If we've
1077 		 * laundered up to our maximum without observing a page daemon
1078 		 * request, just stop.  This is a safety belt that ensures we
1079 		 * don't launder an excessive amount if memory pressure is low
1080 		 * and the ratio of dirty to clean pages is large.  Otherwise,
1081 		 * proceed at the background laundering rate.
1082 		 */
1083 		if (target > 0) {
1084 			if (nfreed > 0) {
1085 				nfreed = 0;
1086 				last_target = target;
1087 			} else if (last_target - target >=
1088 			    vm_background_launder_max * PAGE_SIZE / 1024) {
1089 				target = 0;
1090 			}
1091 			launder = vm_background_launder_rate * PAGE_SIZE / 1024;
1092 			launder /= VM_LAUNDER_RATE;
1093 			if (launder > target)
1094 				launder = target;
1095 		}
1096 
1097 dolaundry:
1098 		if (launder > 0) {
1099 			/*
1100 			 * Because of I/O clustering, the number of laundered
1101 			 * pages could exceed "target" by the maximum size of
1102 			 * a cluster minus one.
1103 			 */
1104 			target -= min(vm_pageout_launder(vmd, launder,
1105 			    in_shortfall), target);
1106 			pause("laundp", hz / VM_LAUNDER_RATE);
1107 		}
1108 
1109 		/*
1110 		 * If we're not currently laundering pages and the page daemon
1111 		 * hasn't posted a new request, sleep until the page daemon
1112 		 * kicks us.
1113 		 */
1114 		vm_pagequeue_lock(pq);
1115 		if (target == 0 && vmd->vmd_laundry_request == VM_LAUNDRY_IDLE)
1116 			(void)mtx_sleep(&vmd->vmd_laundry_request,
1117 			    vm_pagequeue_lockptr(pq), PVM, "launds", 0);
1118 
1119 		/*
1120 		 * If the pagedaemon has indicated that it's in shortfall, start
1121 		 * a shortfall laundering unless we're already in the middle of
1122 		 * one.  This may preempt a background laundering.
1123 		 */
1124 		if (vmd->vmd_laundry_request == VM_LAUNDRY_SHORTFALL &&
1125 		    (!in_shortfall || shortfall_cycle == 0)) {
1126 			shortfall = vm_laundry_target(vmd) +
1127 			    vmd->vmd_pageout_deficit;
1128 			target = 0;
1129 		} else
1130 			shortfall = 0;
1131 
1132 		if (target == 0)
1133 			vmd->vmd_laundry_request = VM_LAUNDRY_IDLE;
1134 		nfreed += vmd->vmd_clean_pages_freed;
1135 		vmd->vmd_clean_pages_freed = 0;
1136 		vm_pagequeue_unlock(pq);
1137 	}
1138 }
1139 
1140 /*
1141  * Compute the number of pages we want to try to move from the
1142  * active queue to either the inactive or laundry queue.
1143  *
1144  * When scanning active pages during a shortage, we make clean pages
1145  * count more heavily towards the page shortage than dirty pages.
1146  * This is because dirty pages must be laundered before they can be
1147  * reused and thus have less utility when attempting to quickly
1148  * alleviate a free page shortage.  However, this weighting also
1149  * causes the scan to deactivate dirty pages more aggressively,
1150  * improving the effectiveness of clustering.
1151  */
1152 static int
1153 vm_pageout_active_target(struct vm_domain *vmd)
1154 {
1155 	int shortage;
1156 
1157 	shortage = vmd->vmd_inactive_target + vm_paging_target(vmd) -
1158 	    (vmd->vmd_pagequeues[PQ_INACTIVE].pq_cnt +
1159 	    vmd->vmd_pagequeues[PQ_LAUNDRY].pq_cnt / act_scan_laundry_weight);
1160 	shortage *= act_scan_laundry_weight;
1161 	return (shortage);
1162 }
1163 
1164 /*
1165  * Scan the active queue.  If there is no shortage of inactive pages, scan a
1166  * small portion of the queue in order to maintain quasi-LRU.
1167  */
1168 static void
1169 vm_pageout_scan_active(struct vm_domain *vmd, int page_shortage)
1170 {
1171 	struct scan_state ss;
1172 	struct mtx *mtx;
1173 	vm_object_t object;
1174 	vm_page_t m, marker;
1175 	struct vm_pagequeue *pq;
1176 	long min_scan;
1177 	int act_delta, max_scan, scan_tick;
1178 
1179 	marker = &vmd->vmd_markers[PQ_ACTIVE];
1180 	pq = &vmd->vmd_pagequeues[PQ_ACTIVE];
1181 	vm_pagequeue_lock(pq);
1182 
1183 	/*
1184 	 * If we're just idle polling attempt to visit every
1185 	 * active page within 'update_period' seconds.
1186 	 */
1187 	scan_tick = ticks;
1188 	if (vm_pageout_update_period != 0) {
1189 		min_scan = pq->pq_cnt;
1190 		min_scan *= scan_tick - vmd->vmd_last_active_scan;
1191 		min_scan /= hz * vm_pageout_update_period;
1192 	} else
1193 		min_scan = 0;
1194 	if (min_scan > 0 || (page_shortage > 0 && pq->pq_cnt > 0))
1195 		vmd->vmd_last_active_scan = scan_tick;
1196 
1197 	/*
1198 	 * Scan the active queue for pages that can be deactivated.  Update
1199 	 * the per-page activity counter and use it to identify deactivation
1200 	 * candidates.  Held pages may be deactivated.
1201 	 *
1202 	 * To avoid requeuing each page that remains in the active queue, we
1203 	 * implement the CLOCK algorithm.  To keep the implementation of the
1204 	 * enqueue operation consistent for all page queues, we use two hands,
1205 	 * represented by marker pages. Scans begin at the first hand, which
1206 	 * precedes the second hand in the queue.  When the two hands meet,
1207 	 * they are moved back to the head and tail of the queue, respectively,
1208 	 * and scanning resumes.
1209 	 */
1210 	max_scan = page_shortage > 0 ? pq->pq_cnt : min_scan;
1211 	mtx = NULL;
1212 act_scan:
1213 	vm_pageout_init_scan(&ss, pq, marker, &vmd->vmd_clock[0], max_scan);
1214 	while ((m = vm_pageout_next(&ss, false)) != NULL) {
1215 		if (__predict_false(m == &vmd->vmd_clock[1])) {
1216 			vm_pagequeue_lock(pq);
1217 			TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_clock[0], plinks.q);
1218 			TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_clock[1], plinks.q);
1219 			TAILQ_INSERT_HEAD(&pq->pq_pl, &vmd->vmd_clock[0],
1220 			    plinks.q);
1221 			TAILQ_INSERT_TAIL(&pq->pq_pl, &vmd->vmd_clock[1],
1222 			    plinks.q);
1223 			max_scan -= ss.scanned;
1224 			vm_pageout_end_scan(&ss);
1225 			goto act_scan;
1226 		}
1227 		if (__predict_false((m->flags & PG_MARKER) != 0))
1228 			continue;
1229 
1230 		vm_page_change_lock(m, &mtx);
1231 
1232 		/*
1233 		 * The page may have been disassociated from the queue
1234 		 * or even freed while locks were dropped.  We thus must be
1235 		 * careful whenever modifying page state.  Once the object lock
1236 		 * has been acquired, we have a stable reference to the page.
1237 		 */
1238 		if (vm_page_queue(m) != PQ_ACTIVE)
1239 			continue;
1240 
1241 		/*
1242 		 * Wired pages are dequeued lazily.
1243 		 */
1244 		if (vm_page_wired(m)) {
1245 			vm_page_dequeue_deferred(m);
1246 			continue;
1247 		}
1248 
1249 		/*
1250 		 * A page's object pointer may be set to NULL before
1251 		 * the object lock is acquired.
1252 		 */
1253 		object = (vm_object_t)atomic_load_ptr(&m->object);
1254 		if (__predict_false(object == NULL))
1255 			/*
1256 			 * The page has been removed from its object.
1257 			 */
1258 			continue;
1259 
1260 		/*
1261 		 * Check to see "how much" the page has been used.
1262 		 *
1263 		 * Test PGA_REFERENCED after calling pmap_ts_referenced() so
1264 		 * that a reference from a concurrently destroyed mapping is
1265 		 * observed here and now.
1266 		 *
1267 		 * Perform an unsynchronized object ref count check.  While
1268 		 * the page lock ensures that the page is not reallocated to
1269 		 * another object, in particular, one with unmanaged mappings
1270 		 * that cannot support pmap_ts_referenced(), two races are,
1271 		 * nonetheless, possible:
1272 		 * 1) The count was transitioning to zero, but we saw a non-
1273 		 *    zero value.  pmap_ts_referenced() will return zero
1274 		 *    because the page is not mapped.
1275 		 * 2) The count was transitioning to one, but we saw zero.
1276 		 *    This race delays the detection of a new reference.  At
1277 		 *    worst, we will deactivate and reactivate the page.
1278 		 */
1279 		if (object->ref_count != 0)
1280 			act_delta = pmap_ts_referenced(m);
1281 		else
1282 			act_delta = 0;
1283 		if ((m->aflags & PGA_REFERENCED) != 0) {
1284 			vm_page_aflag_clear(m, PGA_REFERENCED);
1285 			act_delta++;
1286 		}
1287 
1288 		/*
1289 		 * Advance or decay the act_count based on recent usage.
1290 		 */
1291 		if (act_delta != 0) {
1292 			m->act_count += ACT_ADVANCE + act_delta;
1293 			if (m->act_count > ACT_MAX)
1294 				m->act_count = ACT_MAX;
1295 		} else
1296 			m->act_count -= min(m->act_count, ACT_DECLINE);
1297 
1298 		if (m->act_count == 0) {
1299 			/*
1300 			 * When not short for inactive pages, let dirty pages go
1301 			 * through the inactive queue before moving to the
1302 			 * laundry queues.  This gives them some extra time to
1303 			 * be reactivated, potentially avoiding an expensive
1304 			 * pageout.  However, during a page shortage, the
1305 			 * inactive queue is necessarily small, and so dirty
1306 			 * pages would only spend a trivial amount of time in
1307 			 * the inactive queue.  Therefore, we might as well
1308 			 * place them directly in the laundry queue to reduce
1309 			 * queuing overhead.
1310 			 */
1311 			if (page_shortage <= 0) {
1312 				vm_page_swapqueue(m, PQ_ACTIVE, PQ_INACTIVE);
1313 			} else {
1314 				/*
1315 				 * Calling vm_page_test_dirty() here would
1316 				 * require acquisition of the object's write
1317 				 * lock.  However, during a page shortage,
1318 				 * directing dirty pages into the laundry
1319 				 * queue is only an optimization and not a
1320 				 * requirement.  Therefore, we simply rely on
1321 				 * the opportunistic updates to the page's
1322 				 * dirty field by the pmap.
1323 				 */
1324 				if (m->dirty == 0) {
1325 					vm_page_swapqueue(m, PQ_ACTIVE,
1326 					    PQ_INACTIVE);
1327 					page_shortage -=
1328 					    act_scan_laundry_weight;
1329 				} else {
1330 					vm_page_swapqueue(m, PQ_ACTIVE,
1331 					    PQ_LAUNDRY);
1332 					page_shortage--;
1333 				}
1334 			}
1335 		}
1336 	}
1337 	if (mtx != NULL) {
1338 		mtx_unlock(mtx);
1339 		mtx = NULL;
1340 	}
1341 	vm_pagequeue_lock(pq);
1342 	TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_clock[0], plinks.q);
1343 	TAILQ_INSERT_AFTER(&pq->pq_pl, marker, &vmd->vmd_clock[0], plinks.q);
1344 	vm_pageout_end_scan(&ss);
1345 	vm_pagequeue_unlock(pq);
1346 }
1347 
1348 static int
1349 vm_pageout_reinsert_inactive_page(struct scan_state *ss, vm_page_t m)
1350 {
1351 	struct vm_domain *vmd;
1352 
1353 	if (m->queue != PQ_INACTIVE || (m->aflags & PGA_ENQUEUED) != 0)
1354 		return (0);
1355 	vm_page_aflag_set(m, PGA_ENQUEUED);
1356 	if ((m->aflags & PGA_REQUEUE_HEAD) != 0) {
1357 		vmd = vm_pagequeue_domain(m);
1358 		TAILQ_INSERT_BEFORE(&vmd->vmd_inacthead, m, plinks.q);
1359 		vm_page_aflag_clear(m, PGA_REQUEUE | PGA_REQUEUE_HEAD);
1360 	} else if ((m->aflags & PGA_REQUEUE) != 0) {
1361 		TAILQ_INSERT_TAIL(&ss->pq->pq_pl, m, plinks.q);
1362 		vm_page_aflag_clear(m, PGA_REQUEUE | PGA_REQUEUE_HEAD);
1363 	} else
1364 		TAILQ_INSERT_BEFORE(ss->marker, m, plinks.q);
1365 	return (1);
1366 }
1367 
1368 /*
1369  * Re-add stuck pages to the inactive queue.  We will examine them again
1370  * during the next scan.  If the queue state of a page has changed since
1371  * it was physically removed from the page queue in
1372  * vm_pageout_collect_batch(), don't do anything with that page.
1373  */
1374 static void
1375 vm_pageout_reinsert_inactive(struct scan_state *ss, struct vm_batchqueue *bq,
1376     vm_page_t m)
1377 {
1378 	struct vm_pagequeue *pq;
1379 	int delta;
1380 
1381 	delta = 0;
1382 	pq = ss->pq;
1383 
1384 	if (m != NULL) {
1385 		if (vm_batchqueue_insert(bq, m))
1386 			return;
1387 		vm_pagequeue_lock(pq);
1388 		delta += vm_pageout_reinsert_inactive_page(ss, m);
1389 	} else
1390 		vm_pagequeue_lock(pq);
1391 	while ((m = vm_batchqueue_pop(bq)) != NULL)
1392 		delta += vm_pageout_reinsert_inactive_page(ss, m);
1393 	vm_pagequeue_cnt_add(pq, delta);
1394 	vm_pagequeue_unlock(pq);
1395 	vm_batchqueue_init(bq);
1396 }
1397 
1398 /*
1399  * Attempt to reclaim the requested number of pages from the inactive queue.
1400  * Returns true if the shortage was addressed.
1401  */
1402 static int
1403 vm_pageout_scan_inactive(struct vm_domain *vmd, int shortage,
1404     int *addl_shortage)
1405 {
1406 	struct scan_state ss;
1407 	struct vm_batchqueue rq;
1408 	struct mtx *mtx;
1409 	vm_page_t m, marker;
1410 	struct vm_pagequeue *pq;
1411 	vm_object_t object;
1412 	int act_delta, addl_page_shortage, deficit, page_shortage;
1413 	int starting_page_shortage;
1414 
1415 	/*
1416 	 * The addl_page_shortage is an estimate of the number of temporarily
1417 	 * stuck pages in the inactive queue.  In other words, the
1418 	 * number of pages from the inactive count that should be
1419 	 * discounted in setting the target for the active queue scan.
1420 	 */
1421 	addl_page_shortage = 0;
1422 
1423 	/*
1424 	 * vmd_pageout_deficit counts the number of pages requested in
1425 	 * allocations that failed because of a free page shortage.  We assume
1426 	 * that the allocations will be reattempted and thus include the deficit
1427 	 * in our scan target.
1428 	 */
1429 	deficit = atomic_readandclear_int(&vmd->vmd_pageout_deficit);
1430 	starting_page_shortage = page_shortage = shortage + deficit;
1431 
1432 	mtx = NULL;
1433 	object = NULL;
1434 	vm_batchqueue_init(&rq);
1435 
1436 	/*
1437 	 * Start scanning the inactive queue for pages that we can free.  The
1438 	 * scan will stop when we reach the target or we have scanned the
1439 	 * entire queue.  (Note that m->act_count is not used to make
1440 	 * decisions for the inactive queue, only for the active queue.)
1441 	 */
1442 	marker = &vmd->vmd_markers[PQ_INACTIVE];
1443 	pq = &vmd->vmd_pagequeues[PQ_INACTIVE];
1444 	vm_pagequeue_lock(pq);
1445 	vm_pageout_init_scan(&ss, pq, marker, NULL, pq->pq_cnt);
1446 	while (page_shortage > 0 && (m = vm_pageout_next(&ss, true)) != NULL) {
1447 		KASSERT((m->flags & PG_MARKER) == 0,
1448 		    ("marker page %p was dequeued", m));
1449 
1450 		vm_page_change_lock(m, &mtx);
1451 
1452 recheck:
1453 		/*
1454 		 * The page may have been disassociated from the queue
1455 		 * or even freed while locks were dropped.  We thus must be
1456 		 * careful whenever modifying page state.  Once the object lock
1457 		 * has been acquired, we have a stable reference to the page.
1458 		 */
1459 		if (vm_page_queue(m) != PQ_INACTIVE) {
1460 			addl_page_shortage++;
1461 			continue;
1462 		}
1463 
1464 		/*
1465 		 * The page was re-enqueued after the page queue lock was
1466 		 * dropped, or a requeue was requested.  This page gets a second
1467 		 * chance.
1468 		 */
1469 		if ((m->aflags & (PGA_ENQUEUED | PGA_REQUEUE |
1470 		    PGA_REQUEUE_HEAD)) != 0)
1471 			goto reinsert;
1472 
1473 		/*
1474 		 * Wired pages may not be freed.  Complete their removal
1475 		 * from the queue now to avoid needless revisits during
1476 		 * future scans.  This check is racy and must be reverified once
1477 		 * we hold the object lock and have verified that the page
1478 		 * is not busy.
1479 		 */
1480 		if (vm_page_wired(m)) {
1481 			vm_page_dequeue_deferred(m);
1482 			continue;
1483 		}
1484 
1485 		if (object != m->object) {
1486 			if (object != NULL)
1487 				VM_OBJECT_WUNLOCK(object);
1488 
1489 			/*
1490 			 * A page's object pointer may be set to NULL before
1491 			 * the object lock is acquired.
1492 			 */
1493 			object = (vm_object_t)atomic_load_ptr(&m->object);
1494 			if (object != NULL && !VM_OBJECT_TRYWLOCK(object)) {
1495 				mtx_unlock(mtx);
1496 				/* Depends on type-stability. */
1497 				VM_OBJECT_WLOCK(object);
1498 				mtx_lock(mtx);
1499 				goto recheck;
1500 			}
1501 		}
1502 		if (__predict_false(m->object == NULL))
1503 			/*
1504 			 * The page has been removed from its object.
1505 			 */
1506 			continue;
1507 		KASSERT(m->object == object, ("page %p does not belong to %p",
1508 		    m, object));
1509 
1510 		if (vm_page_busied(m)) {
1511 			/*
1512 			 * Don't mess with busy pages.  Leave them at
1513 			 * the front of the queue.  Most likely, they
1514 			 * are being paged out and will leave the
1515 			 * queue shortly after the scan finishes.  So,
1516 			 * they ought to be discounted from the
1517 			 * inactive count.
1518 			 */
1519 			addl_page_shortage++;
1520 			goto reinsert;
1521 		}
1522 
1523 		/*
1524 		 * Re-check for wirings now that we hold the object lock and
1525 		 * have verified that the page is unbusied.  If the page is
1526 		 * mapped, it may still be wired by pmap lookups.  The call to
1527 		 * vm_page_try_remove_all() below atomically checks for such
1528 		 * wirings and removes mappings.  If the page is unmapped, the
1529 		 * wire count is guaranteed not to increase.
1530 		 */
1531 		if (__predict_false(vm_page_wired(m))) {
1532 			vm_page_dequeue_deferred(m);
1533 			continue;
1534 		}
1535 
1536 		/*
1537 		 * Invalid pages can be easily freed. They cannot be
1538 		 * mapped, vm_page_free() asserts this.
1539 		 */
1540 		if (m->valid == 0)
1541 			goto free_page;
1542 
1543 		/*
1544 		 * If the page has been referenced and the object is not dead,
1545 		 * reactivate or requeue the page depending on whether the
1546 		 * object is mapped.
1547 		 *
1548 		 * Test PGA_REFERENCED after calling pmap_ts_referenced() so
1549 		 * that a reference from a concurrently destroyed mapping is
1550 		 * observed here and now.
1551 		 */
1552 		if (object->ref_count != 0)
1553 			act_delta = pmap_ts_referenced(m);
1554 		else {
1555 			KASSERT(!pmap_page_is_mapped(m),
1556 			    ("page %p is mapped", m));
1557 			act_delta = 0;
1558 		}
1559 		if ((m->aflags & PGA_REFERENCED) != 0) {
1560 			vm_page_aflag_clear(m, PGA_REFERENCED);
1561 			act_delta++;
1562 		}
1563 		if (act_delta != 0) {
1564 			if (object->ref_count != 0) {
1565 				VM_CNT_INC(v_reactivated);
1566 				vm_page_activate(m);
1567 
1568 				/*
1569 				 * Increase the activation count if the page
1570 				 * was referenced while in the inactive queue.
1571 				 * This makes it less likely that the page will
1572 				 * be returned prematurely to the inactive
1573 				 * queue.
1574  				 */
1575 				m->act_count += act_delta + ACT_ADVANCE;
1576 				continue;
1577 			} else if ((object->flags & OBJ_DEAD) == 0) {
1578 				vm_page_aflag_set(m, PGA_REQUEUE);
1579 				goto reinsert;
1580 			}
1581 		}
1582 
1583 		/*
1584 		 * If the page appears to be clean at the machine-independent
1585 		 * layer, then remove all of its mappings from the pmap in
1586 		 * anticipation of freeing it.  If, however, any of the page's
1587 		 * mappings allow write access, then the page may still be
1588 		 * modified until the last of those mappings are removed.
1589 		 */
1590 		if (object->ref_count != 0) {
1591 			vm_page_test_dirty(m);
1592 			if (m->dirty == 0 && !vm_page_try_remove_all(m)) {
1593 				vm_page_dequeue_deferred(m);
1594 				continue;
1595 			}
1596 		}
1597 
1598 		/*
1599 		 * Clean pages can be freed, but dirty pages must be sent back
1600 		 * to the laundry, unless they belong to a dead object.
1601 		 * Requeueing dirty pages from dead objects is pointless, as
1602 		 * they are being paged out and freed by the thread that
1603 		 * destroyed the object.
1604 		 */
1605 		if (m->dirty == 0) {
1606 free_page:
1607 			/*
1608 			 * Because we dequeued the page and have already
1609 			 * checked for concurrent dequeue and enqueue
1610 			 * requests, we can safely disassociate the page
1611 			 * from the inactive queue.
1612 			 */
1613 			KASSERT((m->aflags & PGA_QUEUE_STATE_MASK) == 0,
1614 			    ("page %p has queue state", m));
1615 			m->queue = PQ_NONE;
1616 			vm_page_free(m);
1617 			page_shortage--;
1618 		} else if ((object->flags & OBJ_DEAD) == 0)
1619 			vm_page_launder(m);
1620 		continue;
1621 reinsert:
1622 		vm_pageout_reinsert_inactive(&ss, &rq, m);
1623 	}
1624 	if (mtx != NULL)
1625 		mtx_unlock(mtx);
1626 	if (object != NULL)
1627 		VM_OBJECT_WUNLOCK(object);
1628 	vm_pageout_reinsert_inactive(&ss, &rq, NULL);
1629 	vm_pageout_reinsert_inactive(&ss, &ss.bq, NULL);
1630 	vm_pagequeue_lock(pq);
1631 	vm_pageout_end_scan(&ss);
1632 	vm_pagequeue_unlock(pq);
1633 
1634 	VM_CNT_ADD(v_dfree, starting_page_shortage - page_shortage);
1635 
1636 	/*
1637 	 * Wake up the laundry thread so that it can perform any needed
1638 	 * laundering.  If we didn't meet our target, we're in shortfall and
1639 	 * need to launder more aggressively.  If PQ_LAUNDRY is empty and no
1640 	 * swap devices are configured, the laundry thread has no work to do, so
1641 	 * don't bother waking it up.
1642 	 *
1643 	 * The laundry thread uses the number of inactive queue scans elapsed
1644 	 * since the last laundering to determine whether to launder again, so
1645 	 * keep count.
1646 	 */
1647 	if (starting_page_shortage > 0) {
1648 		pq = &vmd->vmd_pagequeues[PQ_LAUNDRY];
1649 		vm_pagequeue_lock(pq);
1650 		if (vmd->vmd_laundry_request == VM_LAUNDRY_IDLE &&
1651 		    (pq->pq_cnt > 0 || atomic_load_acq_int(&swapdev_enabled))) {
1652 			if (page_shortage > 0) {
1653 				vmd->vmd_laundry_request = VM_LAUNDRY_SHORTFALL;
1654 				VM_CNT_INC(v_pdshortfalls);
1655 			} else if (vmd->vmd_laundry_request !=
1656 			    VM_LAUNDRY_SHORTFALL)
1657 				vmd->vmd_laundry_request =
1658 				    VM_LAUNDRY_BACKGROUND;
1659 			wakeup(&vmd->vmd_laundry_request);
1660 		}
1661 		vmd->vmd_clean_pages_freed +=
1662 		    starting_page_shortage - page_shortage;
1663 		vm_pagequeue_unlock(pq);
1664 	}
1665 
1666 	/*
1667 	 * Wakeup the swapout daemon if we didn't free the targeted number of
1668 	 * pages.
1669 	 */
1670 	if (page_shortage > 0)
1671 		vm_swapout_run();
1672 
1673 	/*
1674 	 * If the inactive queue scan fails repeatedly to meet its
1675 	 * target, kill the largest process.
1676 	 */
1677 	vm_pageout_mightbe_oom(vmd, page_shortage, starting_page_shortage);
1678 
1679 	/*
1680 	 * Reclaim pages by swapping out idle processes, if configured to do so.
1681 	 */
1682 	vm_swapout_run_idle();
1683 
1684 	/*
1685 	 * See the description of addl_page_shortage above.
1686 	 */
1687 	*addl_shortage = addl_page_shortage + deficit;
1688 
1689 	return (page_shortage <= 0);
1690 }
1691 
1692 static int vm_pageout_oom_vote;
1693 
1694 /*
1695  * The pagedaemon threads randlomly select one to perform the
1696  * OOM.  Trying to kill processes before all pagedaemons
1697  * failed to reach free target is premature.
1698  */
1699 static void
1700 vm_pageout_mightbe_oom(struct vm_domain *vmd, int page_shortage,
1701     int starting_page_shortage)
1702 {
1703 	int old_vote;
1704 
1705 	if (starting_page_shortage <= 0 || starting_page_shortage !=
1706 	    page_shortage)
1707 		vmd->vmd_oom_seq = 0;
1708 	else
1709 		vmd->vmd_oom_seq++;
1710 	if (vmd->vmd_oom_seq < vm_pageout_oom_seq) {
1711 		if (vmd->vmd_oom) {
1712 			vmd->vmd_oom = FALSE;
1713 			atomic_subtract_int(&vm_pageout_oom_vote, 1);
1714 		}
1715 		return;
1716 	}
1717 
1718 	/*
1719 	 * Do not follow the call sequence until OOM condition is
1720 	 * cleared.
1721 	 */
1722 	vmd->vmd_oom_seq = 0;
1723 
1724 	if (vmd->vmd_oom)
1725 		return;
1726 
1727 	vmd->vmd_oom = TRUE;
1728 	old_vote = atomic_fetchadd_int(&vm_pageout_oom_vote, 1);
1729 	if (old_vote != vm_ndomains - 1)
1730 		return;
1731 
1732 	/*
1733 	 * The current pagedaemon thread is the last in the quorum to
1734 	 * start OOM.  Initiate the selection and signaling of the
1735 	 * victim.
1736 	 */
1737 	vm_pageout_oom(VM_OOM_MEM);
1738 
1739 	/*
1740 	 * After one round of OOM terror, recall our vote.  On the
1741 	 * next pass, current pagedaemon would vote again if the low
1742 	 * memory condition is still there, due to vmd_oom being
1743 	 * false.
1744 	 */
1745 	vmd->vmd_oom = FALSE;
1746 	atomic_subtract_int(&vm_pageout_oom_vote, 1);
1747 }
1748 
1749 /*
1750  * The OOM killer is the page daemon's action of last resort when
1751  * memory allocation requests have been stalled for a prolonged period
1752  * of time because it cannot reclaim memory.  This function computes
1753  * the approximate number of physical pages that could be reclaimed if
1754  * the specified address space is destroyed.
1755  *
1756  * Private, anonymous memory owned by the address space is the
1757  * principal resource that we expect to recover after an OOM kill.
1758  * Since the physical pages mapped by the address space's COW entries
1759  * are typically shared pages, they are unlikely to be released and so
1760  * they are not counted.
1761  *
1762  * To get to the point where the page daemon runs the OOM killer, its
1763  * efforts to write-back vnode-backed pages may have stalled.  This
1764  * could be caused by a memory allocation deadlock in the write path
1765  * that might be resolved by an OOM kill.  Therefore, physical pages
1766  * belonging to vnode-backed objects are counted, because they might
1767  * be freed without being written out first if the address space holds
1768  * the last reference to an unlinked vnode.
1769  *
1770  * Similarly, physical pages belonging to OBJT_PHYS objects are
1771  * counted because the address space might hold the last reference to
1772  * the object.
1773  */
1774 static long
1775 vm_pageout_oom_pagecount(struct vmspace *vmspace)
1776 {
1777 	vm_map_t map;
1778 	vm_map_entry_t entry;
1779 	vm_object_t obj;
1780 	long res;
1781 
1782 	map = &vmspace->vm_map;
1783 	KASSERT(!map->system_map, ("system map"));
1784 	sx_assert(&map->lock, SA_LOCKED);
1785 	res = 0;
1786 	for (entry = map->header.next; entry != &map->header;
1787 	    entry = entry->next) {
1788 		if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0)
1789 			continue;
1790 		obj = entry->object.vm_object;
1791 		if (obj == NULL)
1792 			continue;
1793 		if ((entry->eflags & MAP_ENTRY_NEEDS_COPY) != 0 &&
1794 		    obj->ref_count != 1)
1795 			continue;
1796 		switch (obj->type) {
1797 		case OBJT_DEFAULT:
1798 		case OBJT_SWAP:
1799 		case OBJT_PHYS:
1800 		case OBJT_VNODE:
1801 			res += obj->resident_page_count;
1802 			break;
1803 		}
1804 	}
1805 	return (res);
1806 }
1807 
1808 static int vm_oom_ratelim_last;
1809 static int vm_oom_pf_secs = 10;
1810 SYSCTL_INT(_vm, OID_AUTO, oom_pf_secs, CTLFLAG_RWTUN, &vm_oom_pf_secs, 0,
1811     "");
1812 static struct mtx vm_oom_ratelim_mtx;
1813 
1814 void
1815 vm_pageout_oom(int shortage)
1816 {
1817 	struct proc *p, *bigproc;
1818 	vm_offset_t size, bigsize;
1819 	struct thread *td;
1820 	struct vmspace *vm;
1821 	int now;
1822 	bool breakout;
1823 
1824 	/*
1825 	 * For OOM requests originating from vm_fault(), there is a high
1826 	 * chance that a single large process faults simultaneously in
1827 	 * several threads.  Also, on an active system running many
1828 	 * processes of middle-size, like buildworld, all of them
1829 	 * could fault almost simultaneously as well.
1830 	 *
1831 	 * To avoid killing too many processes, rate-limit OOMs
1832 	 * initiated by vm_fault() time-outs on the waits for free
1833 	 * pages.
1834 	 */
1835 	mtx_lock(&vm_oom_ratelim_mtx);
1836 	now = ticks;
1837 	if (shortage == VM_OOM_MEM_PF &&
1838 	    (u_int)(now - vm_oom_ratelim_last) < hz * vm_oom_pf_secs) {
1839 		mtx_unlock(&vm_oom_ratelim_mtx);
1840 		return;
1841 	}
1842 	vm_oom_ratelim_last = now;
1843 	mtx_unlock(&vm_oom_ratelim_mtx);
1844 
1845 	/*
1846 	 * We keep the process bigproc locked once we find it to keep anyone
1847 	 * from messing with it; however, there is a possibility of
1848 	 * deadlock if process B is bigproc and one of its child processes
1849 	 * attempts to propagate a signal to B while we are waiting for A's
1850 	 * lock while walking this list.  To avoid this, we don't block on
1851 	 * the process lock but just skip a process if it is already locked.
1852 	 */
1853 	bigproc = NULL;
1854 	bigsize = 0;
1855 	sx_slock(&allproc_lock);
1856 	FOREACH_PROC_IN_SYSTEM(p) {
1857 		PROC_LOCK(p);
1858 
1859 		/*
1860 		 * If this is a system, protected or killed process, skip it.
1861 		 */
1862 		if (p->p_state != PRS_NORMAL || (p->p_flag & (P_INEXEC |
1863 		    P_PROTECTED | P_SYSTEM | P_WEXIT)) != 0 ||
1864 		    p->p_pid == 1 || P_KILLED(p) ||
1865 		    (p->p_pid < 48 && swap_pager_avail != 0)) {
1866 			PROC_UNLOCK(p);
1867 			continue;
1868 		}
1869 		/*
1870 		 * If the process is in a non-running type state,
1871 		 * don't touch it.  Check all the threads individually.
1872 		 */
1873 		breakout = false;
1874 		FOREACH_THREAD_IN_PROC(p, td) {
1875 			thread_lock(td);
1876 			if (!TD_ON_RUNQ(td) &&
1877 			    !TD_IS_RUNNING(td) &&
1878 			    !TD_IS_SLEEPING(td) &&
1879 			    !TD_IS_SUSPENDED(td) &&
1880 			    !TD_IS_SWAPPED(td)) {
1881 				thread_unlock(td);
1882 				breakout = true;
1883 				break;
1884 			}
1885 			thread_unlock(td);
1886 		}
1887 		if (breakout) {
1888 			PROC_UNLOCK(p);
1889 			continue;
1890 		}
1891 		/*
1892 		 * get the process size
1893 		 */
1894 		vm = vmspace_acquire_ref(p);
1895 		if (vm == NULL) {
1896 			PROC_UNLOCK(p);
1897 			continue;
1898 		}
1899 		_PHOLD_LITE(p);
1900 		PROC_UNLOCK(p);
1901 		sx_sunlock(&allproc_lock);
1902 		if (!vm_map_trylock_read(&vm->vm_map)) {
1903 			vmspace_free(vm);
1904 			sx_slock(&allproc_lock);
1905 			PRELE(p);
1906 			continue;
1907 		}
1908 		size = vmspace_swap_count(vm);
1909 		if (shortage == VM_OOM_MEM || shortage == VM_OOM_MEM_PF)
1910 			size += vm_pageout_oom_pagecount(vm);
1911 		vm_map_unlock_read(&vm->vm_map);
1912 		vmspace_free(vm);
1913 		sx_slock(&allproc_lock);
1914 
1915 		/*
1916 		 * If this process is bigger than the biggest one,
1917 		 * remember it.
1918 		 */
1919 		if (size > bigsize) {
1920 			if (bigproc != NULL)
1921 				PRELE(bigproc);
1922 			bigproc = p;
1923 			bigsize = size;
1924 		} else {
1925 			PRELE(p);
1926 		}
1927 	}
1928 	sx_sunlock(&allproc_lock);
1929 	if (bigproc != NULL) {
1930 		if (vm_panic_on_oom != 0)
1931 			panic("out of swap space");
1932 		PROC_LOCK(bigproc);
1933 		killproc(bigproc, "out of swap space");
1934 		sched_nice(bigproc, PRIO_MIN);
1935 		_PRELE(bigproc);
1936 		PROC_UNLOCK(bigproc);
1937 	}
1938 }
1939 
1940 static bool
1941 vm_pageout_lowmem(void)
1942 {
1943 	static int lowmem_ticks = 0;
1944 	int last;
1945 
1946 	last = atomic_load_int(&lowmem_ticks);
1947 	while ((u_int)(ticks - last) / hz >= lowmem_period) {
1948 		if (atomic_fcmpset_int(&lowmem_ticks, &last, ticks) == 0)
1949 			continue;
1950 
1951 		/*
1952 		 * Decrease registered cache sizes.
1953 		 */
1954 		SDT_PROBE0(vm, , , vm__lowmem_scan);
1955 		EVENTHANDLER_INVOKE(vm_lowmem, VM_LOW_PAGES);
1956 
1957 		/*
1958 		 * We do this explicitly after the caches have been
1959 		 * drained above.  If we have a severe page shortage on
1960 		 * our hands, completely drain all UMA zones.  Otherwise,
1961 		 * just prune the caches.
1962 		 */
1963 		uma_reclaim(vm_page_count_min() ? UMA_RECLAIM_DRAIN_CPU :
1964 		    UMA_RECLAIM_TRIM);
1965 		return (true);
1966 	}
1967 	return (false);
1968 }
1969 
1970 static void
1971 vm_pageout_worker(void *arg)
1972 {
1973 	struct vm_domain *vmd;
1974 	u_int ofree;
1975 	int addl_shortage, domain, shortage;
1976 	bool target_met;
1977 
1978 	domain = (uintptr_t)arg;
1979 	vmd = VM_DOMAIN(domain);
1980 	shortage = 0;
1981 	target_met = true;
1982 
1983 	/*
1984 	 * XXXKIB It could be useful to bind pageout daemon threads to
1985 	 * the cores belonging to the domain, from which vm_page_array
1986 	 * is allocated.
1987 	 */
1988 
1989 	KASSERT(vmd->vmd_segs != 0, ("domain without segments"));
1990 	vmd->vmd_last_active_scan = ticks;
1991 
1992 	/*
1993 	 * The pageout daemon worker is never done, so loop forever.
1994 	 */
1995 	while (TRUE) {
1996 		vm_domain_pageout_lock(vmd);
1997 
1998 		/*
1999 		 * We need to clear wanted before we check the limits.  This
2000 		 * prevents races with wakers who will check wanted after they
2001 		 * reach the limit.
2002 		 */
2003 		atomic_store_int(&vmd->vmd_pageout_wanted, 0);
2004 
2005 		/*
2006 		 * Might the page daemon need to run again?
2007 		 */
2008 		if (vm_paging_needed(vmd, vmd->vmd_free_count)) {
2009 			/*
2010 			 * Yes.  If the scan failed to produce enough free
2011 			 * pages, sleep uninterruptibly for some time in the
2012 			 * hope that the laundry thread will clean some pages.
2013 			 */
2014 			vm_domain_pageout_unlock(vmd);
2015 			if (!target_met)
2016 				pause("pwait", hz / VM_INACT_SCAN_RATE);
2017 		} else {
2018 			/*
2019 			 * No, sleep until the next wakeup or until pages
2020 			 * need to have their reference stats updated.
2021 			 */
2022 			if (mtx_sleep(&vmd->vmd_pageout_wanted,
2023 			    vm_domain_pageout_lockptr(vmd), PDROP | PVM,
2024 			    "psleep", hz / VM_INACT_SCAN_RATE) == 0)
2025 				VM_CNT_INC(v_pdwakeups);
2026 		}
2027 
2028 		/* Prevent spurious wakeups by ensuring that wanted is set. */
2029 		atomic_store_int(&vmd->vmd_pageout_wanted, 1);
2030 
2031 		/*
2032 		 * Use the controller to calculate how many pages to free in
2033 		 * this interval, and scan the inactive queue.  If the lowmem
2034 		 * handlers appear to have freed up some pages, subtract the
2035 		 * difference from the inactive queue scan target.
2036 		 */
2037 		shortage = pidctrl_daemon(&vmd->vmd_pid, vmd->vmd_free_count);
2038 		if (shortage > 0) {
2039 			ofree = vmd->vmd_free_count;
2040 			if (vm_pageout_lowmem() && vmd->vmd_free_count > ofree)
2041 				shortage -= min(vmd->vmd_free_count - ofree,
2042 				    (u_int)shortage);
2043 			target_met = vm_pageout_scan_inactive(vmd, shortage,
2044 			    &addl_shortage);
2045 		} else
2046 			addl_shortage = 0;
2047 
2048 		/*
2049 		 * Scan the active queue.  A positive value for shortage
2050 		 * indicates that we must aggressively deactivate pages to avoid
2051 		 * a shortfall.
2052 		 */
2053 		shortage = vm_pageout_active_target(vmd) + addl_shortage;
2054 		vm_pageout_scan_active(vmd, shortage);
2055 	}
2056 }
2057 
2058 /*
2059  *	vm_pageout_init initialises basic pageout daemon settings.
2060  */
2061 static void
2062 vm_pageout_init_domain(int domain)
2063 {
2064 	struct vm_domain *vmd;
2065 	struct sysctl_oid *oid;
2066 
2067 	vmd = VM_DOMAIN(domain);
2068 	vmd->vmd_interrupt_free_min = 2;
2069 
2070 	/*
2071 	 * v_free_reserved needs to include enough for the largest
2072 	 * swap pager structures plus enough for any pv_entry structs
2073 	 * when paging.
2074 	 */
2075 	if (vmd->vmd_page_count > 1024)
2076 		vmd->vmd_free_min = 4 + (vmd->vmd_page_count - 1024) / 200;
2077 	else
2078 		vmd->vmd_free_min = 4;
2079 	vmd->vmd_pageout_free_min = 2 * MAXBSIZE / PAGE_SIZE +
2080 	    vmd->vmd_interrupt_free_min;
2081 	vmd->vmd_free_reserved = vm_pageout_page_count +
2082 	    vmd->vmd_pageout_free_min + (vmd->vmd_page_count / 768);
2083 	vmd->vmd_free_severe = vmd->vmd_free_min / 2;
2084 	vmd->vmd_free_target = 4 * vmd->vmd_free_min + vmd->vmd_free_reserved;
2085 	vmd->vmd_free_min += vmd->vmd_free_reserved;
2086 	vmd->vmd_free_severe += vmd->vmd_free_reserved;
2087 	vmd->vmd_inactive_target = (3 * vmd->vmd_free_target) / 2;
2088 	if (vmd->vmd_inactive_target > vmd->vmd_free_count / 3)
2089 		vmd->vmd_inactive_target = vmd->vmd_free_count / 3;
2090 
2091 	/*
2092 	 * Set the default wakeup threshold to be 10% below the paging
2093 	 * target.  This keeps the steady state out of shortfall.
2094 	 */
2095 	vmd->vmd_pageout_wakeup_thresh = (vmd->vmd_free_target / 10) * 9;
2096 
2097 	/*
2098 	 * Target amount of memory to move out of the laundry queue during a
2099 	 * background laundering.  This is proportional to the amount of system
2100 	 * memory.
2101 	 */
2102 	vmd->vmd_background_launder_target = (vmd->vmd_free_target -
2103 	    vmd->vmd_free_min) / 10;
2104 
2105 	/* Initialize the pageout daemon pid controller. */
2106 	pidctrl_init(&vmd->vmd_pid, hz / VM_INACT_SCAN_RATE,
2107 	    vmd->vmd_free_target, PIDCTRL_BOUND,
2108 	    PIDCTRL_KPD, PIDCTRL_KID, PIDCTRL_KDD);
2109 	oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(vmd->vmd_oid), OID_AUTO,
2110 	    "pidctrl", CTLFLAG_RD, NULL, "");
2111 	pidctrl_init_sysctl(&vmd->vmd_pid, SYSCTL_CHILDREN(oid));
2112 }
2113 
2114 static void
2115 vm_pageout_init(void)
2116 {
2117 	u_int freecount;
2118 	int i;
2119 
2120 	/*
2121 	 * Initialize some paging parameters.
2122 	 */
2123 	if (vm_cnt.v_page_count < 2000)
2124 		vm_pageout_page_count = 8;
2125 
2126 	freecount = 0;
2127 	for (i = 0; i < vm_ndomains; i++) {
2128 		struct vm_domain *vmd;
2129 
2130 		vm_pageout_init_domain(i);
2131 		vmd = VM_DOMAIN(i);
2132 		vm_cnt.v_free_reserved += vmd->vmd_free_reserved;
2133 		vm_cnt.v_free_target += vmd->vmd_free_target;
2134 		vm_cnt.v_free_min += vmd->vmd_free_min;
2135 		vm_cnt.v_inactive_target += vmd->vmd_inactive_target;
2136 		vm_cnt.v_pageout_free_min += vmd->vmd_pageout_free_min;
2137 		vm_cnt.v_interrupt_free_min += vmd->vmd_interrupt_free_min;
2138 		vm_cnt.v_free_severe += vmd->vmd_free_severe;
2139 		freecount += vmd->vmd_free_count;
2140 	}
2141 
2142 	/*
2143 	 * Set interval in seconds for active scan.  We want to visit each
2144 	 * page at least once every ten minutes.  This is to prevent worst
2145 	 * case paging behaviors with stale active LRU.
2146 	 */
2147 	if (vm_pageout_update_period == 0)
2148 		vm_pageout_update_period = 600;
2149 
2150 	if (vm_page_max_user_wired == 0)
2151 		vm_page_max_user_wired = freecount / 3;
2152 }
2153 
2154 /*
2155  *     vm_pageout is the high level pageout daemon.
2156  */
2157 static void
2158 vm_pageout(void)
2159 {
2160 	struct proc *p;
2161 	struct thread *td;
2162 	int error, first, i;
2163 
2164 	p = curproc;
2165 	td = curthread;
2166 
2167 	mtx_init(&vm_oom_ratelim_mtx, "vmoomr", NULL, MTX_DEF);
2168 	swap_pager_swap_init();
2169 	for (first = -1, i = 0; i < vm_ndomains; i++) {
2170 		if (VM_DOMAIN_EMPTY(i)) {
2171 			if (bootverbose)
2172 				printf("domain %d empty; skipping pageout\n",
2173 				    i);
2174 			continue;
2175 		}
2176 		if (first == -1)
2177 			first = i;
2178 		else {
2179 			error = kthread_add(vm_pageout_worker,
2180 			    (void *)(uintptr_t)i, p, NULL, 0, 0, "dom%d", i);
2181 			if (error != 0)
2182 				panic("starting pageout for domain %d: %d\n",
2183 				    i, error);
2184 		}
2185 		error = kthread_add(vm_pageout_laundry_worker,
2186 		    (void *)(uintptr_t)i, p, NULL, 0, 0, "laundry: dom%d", i);
2187 		if (error != 0)
2188 			panic("starting laundry for domain %d: %d", i, error);
2189 	}
2190 	error = kthread_add(uma_reclaim_worker, NULL, p, NULL, 0, 0, "uma");
2191 	if (error != 0)
2192 		panic("starting uma_reclaim helper, error %d\n", error);
2193 
2194 	snprintf(td->td_name, sizeof(td->td_name), "dom%d", first);
2195 	vm_pageout_worker((void *)(uintptr_t)first);
2196 }
2197 
2198 /*
2199  * Perform an advisory wakeup of the page daemon.
2200  */
2201 void
2202 pagedaemon_wakeup(int domain)
2203 {
2204 	struct vm_domain *vmd;
2205 
2206 	vmd = VM_DOMAIN(domain);
2207 	vm_domain_pageout_assert_unlocked(vmd);
2208 	if (curproc == pageproc)
2209 		return;
2210 
2211 	if (atomic_fetchadd_int(&vmd->vmd_pageout_wanted, 1) == 0) {
2212 		vm_domain_pageout_lock(vmd);
2213 		atomic_store_int(&vmd->vmd_pageout_wanted, 1);
2214 		wakeup(&vmd->vmd_pageout_wanted);
2215 		vm_domain_pageout_unlock(vmd);
2216 	}
2217 }
2218