xref: /freebsd/sys/vm/vm_page.c (revision 61da91207d30f84533b0943ca66be8e1b51d5112)
1 /*-
2  * SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU)
3  *
4  * Copyright (c) 1991 Regents of the University of California.
5  * All rights reserved.
6  * Copyright (c) 1998 Matthew Dillon.  All Rights Reserved.
7  *
8  * This code is derived from software contributed to Berkeley by
9  * The Mach Operating System project at Carnegie-Mellon University.
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  * 3. Neither the name of the University nor the names of its contributors
20  *    may be used to endorse or promote products derived from this software
21  *    without specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
24  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
27  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33  * SUCH DAMAGE.
34  *
35  *	from: @(#)vm_page.c	7.4 (Berkeley) 5/7/91
36  */
37 
38 /*-
39  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
40  * All rights reserved.
41  *
42  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
43  *
44  * Permission to use, copy, modify and distribute this software and
45  * its documentation is hereby granted, provided that both the copyright
46  * notice and this permission notice appear in all copies of the
47  * software, derivative works or modified versions, and any portions
48  * thereof, and that both notices appear in supporting documentation.
49  *
50  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
51  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
52  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
53  *
54  * Carnegie Mellon requests users of this software to return to
55  *
56  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
57  *  School of Computer Science
58  *  Carnegie Mellon University
59  *  Pittsburgh PA 15213-3890
60  *
61  * any improvements or extensions that they make and grant Carnegie the
62  * rights to redistribute these changes.
63  */
64 
65 /*
66  *			GENERAL RULES ON VM_PAGE MANIPULATION
67  *
68  *	- A page queue lock is required when adding or removing a page from a
69  *	  page queue regardless of other locks or the busy state of a page.
70  *
71  *		* In general, no thread besides the page daemon can acquire or
72  *		  hold more than one page queue lock at a time.
73  *
74  *		* The page daemon can acquire and hold any pair of page queue
75  *		  locks in any order.
76  *
77  *	- The object lock is required when inserting or removing
78  *	  pages from an object (vm_page_insert() or vm_page_remove()).
79  *
80  */
81 
82 /*
83  *	Resident memory management module.
84  */
85 
86 #include <sys/cdefs.h>
87 __FBSDID("$FreeBSD$");
88 
89 #include "opt_vm.h"
90 
91 #include <sys/param.h>
92 #include <sys/systm.h>
93 #include <sys/lock.h>
94 #include <sys/kernel.h>
95 #include <sys/limits.h>
96 #include <sys/linker.h>
97 #include <sys/malloc.h>
98 #include <sys/mman.h>
99 #include <sys/msgbuf.h>
100 #include <sys/mutex.h>
101 #include <sys/proc.h>
102 #include <sys/rwlock.h>
103 #include <sys/sbuf.h>
104 #include <sys/smp.h>
105 #include <sys/sysctl.h>
106 #include <sys/vmmeter.h>
107 #include <sys/vnode.h>
108 
109 #include <vm/vm.h>
110 #include <vm/pmap.h>
111 #include <vm/vm_param.h>
112 #include <vm/vm_domain.h>
113 #include <vm/vm_kern.h>
114 #include <vm/vm_object.h>
115 #include <vm/vm_page.h>
116 #include <vm/vm_pageout.h>
117 #include <vm/vm_pager.h>
118 #include <vm/vm_phys.h>
119 #include <vm/vm_radix.h>
120 #include <vm/vm_reserv.h>
121 #include <vm/vm_extern.h>
122 #include <vm/uma.h>
123 #include <vm/uma_int.h>
124 
125 #include <machine/md_var.h>
126 
127 /*
128  *	Associated with page of user-allocatable memory is a
129  *	page structure.
130  */
131 
132 struct vm_domain vm_dom[MAXMEMDOM];
133 struct mtx_padalign __exclusive_cache_line vm_page_queue_free_mtx;
134 
135 struct mtx_padalign __exclusive_cache_line pa_lock[PA_LOCK_COUNT];
136 
137 /*
138  * bogus page -- for I/O to/from partially complete buffers,
139  * or for paging into sparsely invalid regions.
140  */
141 vm_page_t bogus_page;
142 
143 vm_page_t vm_page_array;
144 long vm_page_array_size;
145 long first_page;
146 
147 static int boot_pages = UMA_BOOT_PAGES;
148 SYSCTL_INT(_vm, OID_AUTO, boot_pages, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
149     &boot_pages, 0,
150     "number of pages allocated for bootstrapping the VM system");
151 
152 static int pa_tryrelock_restart;
153 SYSCTL_INT(_vm, OID_AUTO, tryrelock_restart, CTLFLAG_RD,
154     &pa_tryrelock_restart, 0, "Number of tryrelock restarts");
155 
156 static TAILQ_HEAD(, vm_page) blacklist_head;
157 static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS);
158 SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING | CTLFLAG_RD |
159     CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_page_blacklist, "A", "Blacklist pages");
160 
161 /* Is the page daemon waiting for free pages? */
162 static int vm_pageout_pages_needed;
163 
164 static uma_zone_t fakepg_zone;
165 
166 static void vm_page_alloc_check(vm_page_t m);
167 static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits);
168 static void vm_page_enqueue(uint8_t queue, vm_page_t m);
169 static void vm_page_free_phys(vm_page_t m);
170 static void vm_page_free_wakeup(void);
171 static void vm_page_init(void *dummy);
172 static int vm_page_insert_after(vm_page_t m, vm_object_t object,
173     vm_pindex_t pindex, vm_page_t mpred);
174 static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object,
175     vm_page_t mpred);
176 static int vm_page_reclaim_run(int req_class, u_long npages, vm_page_t m_run,
177     vm_paddr_t high);
178 static int vm_page_alloc_fail(vm_object_t object, int req);
179 
180 SYSINIT(vm_page, SI_SUB_VM, SI_ORDER_SECOND, vm_page_init, NULL);
181 
182 static void
183 vm_page_init(void *dummy)
184 {
185 
186 	fakepg_zone = uma_zcreate("fakepg", sizeof(struct vm_page), NULL, NULL,
187 	    NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM);
188 	bogus_page = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ |
189 	    VM_ALLOC_NORMAL | VM_ALLOC_WIRED);
190 }
191 
192 /* Make sure that u_long is at least 64 bits when PAGE_SIZE is 32K. */
193 #if PAGE_SIZE == 32768
194 #ifdef CTASSERT
195 CTASSERT(sizeof(u_long) >= 8);
196 #endif
197 #endif
198 
199 /*
200  * Try to acquire a physical address lock while a pmap is locked.  If we
201  * fail to trylock we unlock and lock the pmap directly and cache the
202  * locked pa in *locked.  The caller should then restart their loop in case
203  * the virtual to physical mapping has changed.
204  */
205 int
206 vm_page_pa_tryrelock(pmap_t pmap, vm_paddr_t pa, vm_paddr_t *locked)
207 {
208 	vm_paddr_t lockpa;
209 
210 	lockpa = *locked;
211 	*locked = pa;
212 	if (lockpa) {
213 		PA_LOCK_ASSERT(lockpa, MA_OWNED);
214 		if (PA_LOCKPTR(pa) == PA_LOCKPTR(lockpa))
215 			return (0);
216 		PA_UNLOCK(lockpa);
217 	}
218 	if (PA_TRYLOCK(pa))
219 		return (0);
220 	PMAP_UNLOCK(pmap);
221 	atomic_add_int(&pa_tryrelock_restart, 1);
222 	PA_LOCK(pa);
223 	PMAP_LOCK(pmap);
224 	return (EAGAIN);
225 }
226 
227 /*
228  *	vm_set_page_size:
229  *
230  *	Sets the page size, perhaps based upon the memory
231  *	size.  Must be called before any use of page-size
232  *	dependent functions.
233  */
234 void
235 vm_set_page_size(void)
236 {
237 	if (vm_cnt.v_page_size == 0)
238 		vm_cnt.v_page_size = PAGE_SIZE;
239 	if (((vm_cnt.v_page_size - 1) & vm_cnt.v_page_size) != 0)
240 		panic("vm_set_page_size: page size not a power of two");
241 }
242 
243 /*
244  *	vm_page_blacklist_next:
245  *
246  *	Find the next entry in the provided string of blacklist
247  *	addresses.  Entries are separated by space, comma, or newline.
248  *	If an invalid integer is encountered then the rest of the
249  *	string is skipped.  Updates the list pointer to the next
250  *	character, or NULL if the string is exhausted or invalid.
251  */
252 static vm_paddr_t
253 vm_page_blacklist_next(char **list, char *end)
254 {
255 	vm_paddr_t bad;
256 	char *cp, *pos;
257 
258 	if (list == NULL || *list == NULL)
259 		return (0);
260 	if (**list =='\0') {
261 		*list = NULL;
262 		return (0);
263 	}
264 
265 	/*
266 	 * If there's no end pointer then the buffer is coming from
267 	 * the kenv and we know it's null-terminated.
268 	 */
269 	if (end == NULL)
270 		end = *list + strlen(*list);
271 
272 	/* Ensure that strtoq() won't walk off the end */
273 	if (*end != '\0') {
274 		if (*end == '\n' || *end == ' ' || *end  == ',')
275 			*end = '\0';
276 		else {
277 			printf("Blacklist not terminated, skipping\n");
278 			*list = NULL;
279 			return (0);
280 		}
281 	}
282 
283 	for (pos = *list; *pos != '\0'; pos = cp) {
284 		bad = strtoq(pos, &cp, 0);
285 		if (*cp == '\0' || *cp == ' ' || *cp == ',' || *cp == '\n') {
286 			if (bad == 0) {
287 				if (++cp < end)
288 					continue;
289 				else
290 					break;
291 			}
292 		} else
293 			break;
294 		if (*cp == '\0' || ++cp >= end)
295 			*list = NULL;
296 		else
297 			*list = cp;
298 		return (trunc_page(bad));
299 	}
300 	printf("Garbage in RAM blacklist, skipping\n");
301 	*list = NULL;
302 	return (0);
303 }
304 
305 /*
306  *	vm_page_blacklist_check:
307  *
308  *	Iterate through the provided string of blacklist addresses, pulling
309  *	each entry out of the physical allocator free list and putting it
310  *	onto a list for reporting via the vm.page_blacklist sysctl.
311  */
312 static void
313 vm_page_blacklist_check(char *list, char *end)
314 {
315 	vm_paddr_t pa;
316 	vm_page_t m;
317 	char *next;
318 	int ret;
319 
320 	next = list;
321 	while (next != NULL) {
322 		if ((pa = vm_page_blacklist_next(&next, end)) == 0)
323 			continue;
324 		m = vm_phys_paddr_to_vm_page(pa);
325 		if (m == NULL)
326 			continue;
327 		mtx_lock(&vm_page_queue_free_mtx);
328 		ret = vm_phys_unfree_page(m);
329 		mtx_unlock(&vm_page_queue_free_mtx);
330 		if (ret == TRUE) {
331 			TAILQ_INSERT_TAIL(&blacklist_head, m, listq);
332 			if (bootverbose)
333 				printf("Skipping page with pa 0x%jx\n",
334 				    (uintmax_t)pa);
335 		}
336 	}
337 }
338 
339 /*
340  *	vm_page_blacklist_load:
341  *
342  *	Search for a special module named "ram_blacklist".  It'll be a
343  *	plain text file provided by the user via the loader directive
344  *	of the same name.
345  */
346 static void
347 vm_page_blacklist_load(char **list, char **end)
348 {
349 	void *mod;
350 	u_char *ptr;
351 	u_int len;
352 
353 	mod = NULL;
354 	ptr = NULL;
355 
356 	mod = preload_search_by_type("ram_blacklist");
357 	if (mod != NULL) {
358 		ptr = preload_fetch_addr(mod);
359 		len = preload_fetch_size(mod);
360         }
361 	*list = ptr;
362 	if (ptr != NULL)
363 		*end = ptr + len;
364 	else
365 		*end = NULL;
366 	return;
367 }
368 
369 static int
370 sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS)
371 {
372 	vm_page_t m;
373 	struct sbuf sbuf;
374 	int error, first;
375 
376 	first = 1;
377 	error = sysctl_wire_old_buffer(req, 0);
378 	if (error != 0)
379 		return (error);
380 	sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
381 	TAILQ_FOREACH(m, &blacklist_head, listq) {
382 		sbuf_printf(&sbuf, "%s%#jx", first ? "" : ",",
383 		    (uintmax_t)m->phys_addr);
384 		first = 0;
385 	}
386 	error = sbuf_finish(&sbuf);
387 	sbuf_delete(&sbuf);
388 	return (error);
389 }
390 
391 static void
392 vm_page_domain_init(struct vm_domain *vmd)
393 {
394 	struct vm_pagequeue *pq;
395 	int i;
396 
397 	*__DECONST(char **, &vmd->vmd_pagequeues[PQ_INACTIVE].pq_name) =
398 	    "vm inactive pagequeue";
399 	*__DECONST(u_int **, &vmd->vmd_pagequeues[PQ_INACTIVE].pq_vcnt) =
400 	    &vm_cnt.v_inactive_count;
401 	*__DECONST(char **, &vmd->vmd_pagequeues[PQ_ACTIVE].pq_name) =
402 	    "vm active pagequeue";
403 	*__DECONST(u_int **, &vmd->vmd_pagequeues[PQ_ACTIVE].pq_vcnt) =
404 	    &vm_cnt.v_active_count;
405 	*__DECONST(char **, &vmd->vmd_pagequeues[PQ_LAUNDRY].pq_name) =
406 	    "vm laundry pagequeue";
407 	*__DECONST(int **, &vmd->vmd_pagequeues[PQ_LAUNDRY].pq_vcnt) =
408 	    &vm_cnt.v_laundry_count;
409 	*__DECONST(char **, &vmd->vmd_pagequeues[PQ_UNSWAPPABLE].pq_name) =
410 	    "vm unswappable pagequeue";
411 	/* Unswappable dirty pages are counted as being in the laundry. */
412 	*__DECONST(int **, &vmd->vmd_pagequeues[PQ_UNSWAPPABLE].pq_vcnt) =
413 	    &vm_cnt.v_laundry_count;
414 	vmd->vmd_page_count = 0;
415 	vmd->vmd_free_count = 0;
416 	vmd->vmd_segs = 0;
417 	vmd->vmd_oom = FALSE;
418 	for (i = 0; i < PQ_COUNT; i++) {
419 		pq = &vmd->vmd_pagequeues[i];
420 		TAILQ_INIT(&pq->pq_pl);
421 		mtx_init(&pq->pq_mutex, pq->pq_name, "vm pagequeue",
422 		    MTX_DEF | MTX_DUPOK);
423 	}
424 }
425 
426 /*
427  * Initialize a physical page in preparation for adding it to the free
428  * lists.
429  */
430 static void
431 vm_page_init_page(vm_page_t m, vm_paddr_t pa, int segind)
432 {
433 
434 	m->object = NULL;
435 	m->wire_count = 0;
436 	m->busy_lock = VPB_UNBUSIED;
437 	m->hold_count = 0;
438 	m->flags = 0;
439 	m->phys_addr = pa;
440 	m->queue = PQ_NONE;
441 	m->psind = 0;
442 	m->segind = segind;
443 	m->order = VM_NFREEORDER;
444 	m->pool = VM_FREEPOOL_DEFAULT;
445 	m->valid = m->dirty = 0;
446 	pmap_page_init(m);
447 }
448 
449 /*
450  *	vm_page_startup:
451  *
452  *	Initializes the resident memory module.  Allocates physical memory for
453  *	bootstrapping UMA and some data structures that are used to manage
454  *	physical pages.  Initializes these structures, and populates the free
455  *	page queues.
456  */
457 vm_offset_t
458 vm_page_startup(vm_offset_t vaddr)
459 {
460 	struct vm_domain *vmd;
461 	struct vm_phys_seg *seg;
462 	vm_page_t m;
463 	char *list, *listend;
464 	vm_offset_t mapped;
465 	vm_paddr_t end, high_avail, low_avail, new_end, page_range, size;
466 	vm_paddr_t biggestsize, last_pa, pa;
467 	u_long pagecount;
468 	int biggestone, i, pages_per_zone, segind;
469 
470 	biggestsize = 0;
471 	biggestone = 0;
472 	vaddr = round_page(vaddr);
473 
474 	for (i = 0; phys_avail[i + 1]; i += 2) {
475 		phys_avail[i] = round_page(phys_avail[i]);
476 		phys_avail[i + 1] = trunc_page(phys_avail[i + 1]);
477 	}
478 	for (i = 0; phys_avail[i + 1]; i += 2) {
479 		size = phys_avail[i + 1] - phys_avail[i];
480 		if (size > biggestsize) {
481 			biggestone = i;
482 			biggestsize = size;
483 		}
484 	}
485 
486 	end = phys_avail[biggestone+1];
487 
488 	/*
489 	 * Initialize the page and queue locks.
490 	 */
491 	mtx_init(&vm_page_queue_free_mtx, "vm page free queue", NULL, MTX_DEF);
492 	for (i = 0; i < PA_LOCK_COUNT; i++)
493 		mtx_init(&pa_lock[i], "vm page", NULL, MTX_DEF);
494 	for (i = 0; i < vm_ndomains; i++)
495 		vm_page_domain_init(&vm_dom[i]);
496 
497 	/*
498 	 * Almost all of the pages needed for bootstrapping UMA are used
499 	 * for zone structures, so if the number of CPUs results in those
500 	 * structures taking more than one page each, we set aside more pages
501 	 * in proportion to the zone structure size.
502 	 */
503 	pages_per_zone = howmany(sizeof(struct uma_zone) +
504 	    sizeof(struct uma_cache) * (mp_maxid + 1) +
505 	    roundup2(sizeof(struct uma_slab), sizeof(void *)), UMA_SLAB_SIZE);
506 	if (pages_per_zone > 1) {
507 		/* Reserve more pages so that we don't run out. */
508 		boot_pages = UMA_BOOT_PAGES_ZONES * pages_per_zone;
509 	}
510 
511 	/*
512 	 * Allocate memory for use when boot strapping the kernel memory
513 	 * allocator.
514 	 *
515 	 * CTFLAG_RDTUN doesn't work during the early boot process, so we must
516 	 * manually fetch the value.
517 	 */
518 	TUNABLE_INT_FETCH("vm.boot_pages", &boot_pages);
519 	new_end = end - (boot_pages * UMA_SLAB_SIZE);
520 	new_end = trunc_page(new_end);
521 	mapped = pmap_map(&vaddr, new_end, end,
522 	    VM_PROT_READ | VM_PROT_WRITE);
523 	bzero((void *)mapped, end - new_end);
524 	uma_startup((void *)mapped, boot_pages);
525 
526 #if defined(__aarch64__) || defined(__amd64__) || defined(__arm__) || \
527     defined(__i386__) || defined(__mips__)
528 	/*
529 	 * Allocate a bitmap to indicate that a random physical page
530 	 * needs to be included in a minidump.
531 	 *
532 	 * The amd64 port needs this to indicate which direct map pages
533 	 * need to be dumped, via calls to dump_add_page()/dump_drop_page().
534 	 *
535 	 * However, i386 still needs this workspace internally within the
536 	 * minidump code.  In theory, they are not needed on i386, but are
537 	 * included should the sf_buf code decide to use them.
538 	 */
539 	last_pa = 0;
540 	for (i = 0; dump_avail[i + 1] != 0; i += 2)
541 		if (dump_avail[i + 1] > last_pa)
542 			last_pa = dump_avail[i + 1];
543 	page_range = last_pa / PAGE_SIZE;
544 	vm_page_dump_size = round_page(roundup2(page_range, NBBY) / NBBY);
545 	new_end -= vm_page_dump_size;
546 	vm_page_dump = (void *)(uintptr_t)pmap_map(&vaddr, new_end,
547 	    new_end + vm_page_dump_size, VM_PROT_READ | VM_PROT_WRITE);
548 	bzero((void *)vm_page_dump, vm_page_dump_size);
549 #else
550 	(void)last_pa;
551 #endif
552 #if defined(__aarch64__) || defined(__amd64__) || defined(__mips__)
553 	/*
554 	 * Include the UMA bootstrap pages and vm_page_dump in a crash dump.
555 	 * When pmap_map() uses the direct map, they are not automatically
556 	 * included.
557 	 */
558 	for (pa = new_end; pa < end; pa += PAGE_SIZE)
559 		dump_add_page(pa);
560 #endif
561 	phys_avail[biggestone + 1] = new_end;
562 #ifdef __amd64__
563 	/*
564 	 * Request that the physical pages underlying the message buffer be
565 	 * included in a crash dump.  Since the message buffer is accessed
566 	 * through the direct map, they are not automatically included.
567 	 */
568 	pa = DMAP_TO_PHYS((vm_offset_t)msgbufp->msg_ptr);
569 	last_pa = pa + round_page(msgbufsize);
570 	while (pa < last_pa) {
571 		dump_add_page(pa);
572 		pa += PAGE_SIZE;
573 	}
574 #endif
575 	/*
576 	 * Compute the number of pages of memory that will be available for
577 	 * use, taking into account the overhead of a page structure per page.
578 	 * In other words, solve
579 	 *	"available physical memory" - round_page(page_range *
580 	 *	    sizeof(struct vm_page)) = page_range * PAGE_SIZE
581 	 * for page_range.
582 	 */
583 	low_avail = phys_avail[0];
584 	high_avail = phys_avail[1];
585 	for (i = 0; i < vm_phys_nsegs; i++) {
586 		if (vm_phys_segs[i].start < low_avail)
587 			low_avail = vm_phys_segs[i].start;
588 		if (vm_phys_segs[i].end > high_avail)
589 			high_avail = vm_phys_segs[i].end;
590 	}
591 	/* Skip the first chunk.  It is already accounted for. */
592 	for (i = 2; phys_avail[i + 1] != 0; i += 2) {
593 		if (phys_avail[i] < low_avail)
594 			low_avail = phys_avail[i];
595 		if (phys_avail[i + 1] > high_avail)
596 			high_avail = phys_avail[i + 1];
597 	}
598 	first_page = low_avail / PAGE_SIZE;
599 #ifdef VM_PHYSSEG_SPARSE
600 	size = 0;
601 	for (i = 0; i < vm_phys_nsegs; i++)
602 		size += vm_phys_segs[i].end - vm_phys_segs[i].start;
603 	for (i = 0; phys_avail[i + 1] != 0; i += 2)
604 		size += phys_avail[i + 1] - phys_avail[i];
605 #elif defined(VM_PHYSSEG_DENSE)
606 	size = high_avail - low_avail;
607 #else
608 #error "Either VM_PHYSSEG_DENSE or VM_PHYSSEG_SPARSE must be defined."
609 #endif
610 
611 #ifdef VM_PHYSSEG_DENSE
612 	/*
613 	 * In the VM_PHYSSEG_DENSE case, the number of pages can account for
614 	 * the overhead of a page structure per page only if vm_page_array is
615 	 * allocated from the last physical memory chunk.  Otherwise, we must
616 	 * allocate page structures representing the physical memory
617 	 * underlying vm_page_array, even though they will not be used.
618 	 */
619 	if (new_end != high_avail)
620 		page_range = size / PAGE_SIZE;
621 	else
622 #endif
623 	{
624 		page_range = size / (PAGE_SIZE + sizeof(struct vm_page));
625 
626 		/*
627 		 * If the partial bytes remaining are large enough for
628 		 * a page (PAGE_SIZE) without a corresponding
629 		 * 'struct vm_page', then new_end will contain an
630 		 * extra page after subtracting the length of the VM
631 		 * page array.  Compensate by subtracting an extra
632 		 * page from new_end.
633 		 */
634 		if (size % (PAGE_SIZE + sizeof(struct vm_page)) >= PAGE_SIZE) {
635 			if (new_end == high_avail)
636 				high_avail -= PAGE_SIZE;
637 			new_end -= PAGE_SIZE;
638 		}
639 	}
640 	end = new_end;
641 
642 	/*
643 	 * Reserve an unmapped guard page to trap access to vm_page_array[-1].
644 	 * However, because this page is allocated from KVM, out-of-bounds
645 	 * accesses using the direct map will not be trapped.
646 	 */
647 	vaddr += PAGE_SIZE;
648 
649 	/*
650 	 * Allocate physical memory for the page structures, and map it.
651 	 */
652 	new_end = trunc_page(end - page_range * sizeof(struct vm_page));
653 	mapped = pmap_map(&vaddr, new_end, end,
654 	    VM_PROT_READ | VM_PROT_WRITE);
655 	vm_page_array = (vm_page_t)mapped;
656 	vm_page_array_size = page_range;
657 
658 #if VM_NRESERVLEVEL > 0
659 	/*
660 	 * Allocate physical memory for the reservation management system's
661 	 * data structures, and map it.
662 	 */
663 	if (high_avail == end)
664 		high_avail = new_end;
665 	new_end = vm_reserv_startup(&vaddr, new_end, high_avail);
666 #endif
667 #if defined(__aarch64__) || defined(__amd64__) || defined(__mips__)
668 	/*
669 	 * Include vm_page_array and vm_reserv_array in a crash dump.
670 	 */
671 	for (pa = new_end; pa < end; pa += PAGE_SIZE)
672 		dump_add_page(pa);
673 #endif
674 	phys_avail[biggestone + 1] = new_end;
675 
676 	/*
677 	 * Add physical memory segments corresponding to the available
678 	 * physical pages.
679 	 */
680 	for (i = 0; phys_avail[i + 1] != 0; i += 2)
681 		vm_phys_add_seg(phys_avail[i], phys_avail[i + 1]);
682 
683 	/*
684 	 * Initialize the physical memory allocator.
685 	 */
686 	vm_phys_init();
687 
688 	/*
689 	 * Initialize the page structures and add every available page to the
690 	 * physical memory allocator's free lists.
691 	 */
692 	vm_cnt.v_page_count = 0;
693 	vm_cnt.v_free_count = 0;
694 	for (segind = 0; segind < vm_phys_nsegs; segind++) {
695 		seg = &vm_phys_segs[segind];
696 		for (m = seg->first_page, pa = seg->start; pa < seg->end;
697 		    m++, pa += PAGE_SIZE)
698 			vm_page_init_page(m, pa, segind);
699 
700 		/*
701 		 * Add the segment to the free lists only if it is covered by
702 		 * one of the ranges in phys_avail.  Because we've added the
703 		 * ranges to the vm_phys_segs array, we can assume that each
704 		 * segment is either entirely contained in one of the ranges,
705 		 * or doesn't overlap any of them.
706 		 */
707 		for (i = 0; phys_avail[i + 1] != 0; i += 2) {
708 			if (seg->start < phys_avail[i] ||
709 			    seg->end > phys_avail[i + 1])
710 				continue;
711 
712 			m = seg->first_page;
713 			pagecount = (u_long)atop(seg->end - seg->start);
714 
715 			mtx_lock(&vm_page_queue_free_mtx);
716 			vm_phys_free_contig(m, pagecount);
717 			vm_phys_freecnt_adj(m, (int)pagecount);
718 			mtx_unlock(&vm_page_queue_free_mtx);
719 			vm_cnt.v_page_count += (u_int)pagecount;
720 
721 			vmd = &vm_dom[seg->domain];
722 			vmd->vmd_page_count += (u_int)pagecount;
723 			vmd->vmd_segs |= 1UL << m->segind;
724 			break;
725 		}
726 	}
727 
728 	/*
729 	 * Remove blacklisted pages from the physical memory allocator.
730 	 */
731 	TAILQ_INIT(&blacklist_head);
732 	vm_page_blacklist_load(&list, &listend);
733 	vm_page_blacklist_check(list, listend);
734 
735 	list = kern_getenv("vm.blacklist");
736 	vm_page_blacklist_check(list, NULL);
737 
738 	freeenv(list);
739 #if VM_NRESERVLEVEL > 0
740 	/*
741 	 * Initialize the reservation management system.
742 	 */
743 	vm_reserv_init();
744 #endif
745 	return (vaddr);
746 }
747 
748 void
749 vm_page_reference(vm_page_t m)
750 {
751 
752 	vm_page_aflag_set(m, PGA_REFERENCED);
753 }
754 
755 /*
756  *	vm_page_busy_downgrade:
757  *
758  *	Downgrade an exclusive busy page into a single shared busy page.
759  */
760 void
761 vm_page_busy_downgrade(vm_page_t m)
762 {
763 	u_int x;
764 	bool locked;
765 
766 	vm_page_assert_xbusied(m);
767 	locked = mtx_owned(vm_page_lockptr(m));
768 
769 	for (;;) {
770 		x = m->busy_lock;
771 		x &= VPB_BIT_WAITERS;
772 		if (x != 0 && !locked)
773 			vm_page_lock(m);
774 		if (atomic_cmpset_rel_int(&m->busy_lock,
775 		    VPB_SINGLE_EXCLUSIVER | x, VPB_SHARERS_WORD(1)))
776 			break;
777 		if (x != 0 && !locked)
778 			vm_page_unlock(m);
779 	}
780 	if (x != 0) {
781 		wakeup(m);
782 		if (!locked)
783 			vm_page_unlock(m);
784 	}
785 }
786 
787 /*
788  *	vm_page_sbusied:
789  *
790  *	Return a positive value if the page is shared busied, 0 otherwise.
791  */
792 int
793 vm_page_sbusied(vm_page_t m)
794 {
795 	u_int x;
796 
797 	x = m->busy_lock;
798 	return ((x & VPB_BIT_SHARED) != 0 && x != VPB_UNBUSIED);
799 }
800 
801 /*
802  *	vm_page_sunbusy:
803  *
804  *	Shared unbusy a page.
805  */
806 void
807 vm_page_sunbusy(vm_page_t m)
808 {
809 	u_int x;
810 
811 	vm_page_lock_assert(m, MA_NOTOWNED);
812 	vm_page_assert_sbusied(m);
813 
814 	for (;;) {
815 		x = m->busy_lock;
816 		if (VPB_SHARERS(x) > 1) {
817 			if (atomic_cmpset_int(&m->busy_lock, x,
818 			    x - VPB_ONE_SHARER))
819 				break;
820 			continue;
821 		}
822 		if ((x & VPB_BIT_WAITERS) == 0) {
823 			KASSERT(x == VPB_SHARERS_WORD(1),
824 			    ("vm_page_sunbusy: invalid lock state"));
825 			if (atomic_cmpset_int(&m->busy_lock,
826 			    VPB_SHARERS_WORD(1), VPB_UNBUSIED))
827 				break;
828 			continue;
829 		}
830 		KASSERT(x == (VPB_SHARERS_WORD(1) | VPB_BIT_WAITERS),
831 		    ("vm_page_sunbusy: invalid lock state for waiters"));
832 
833 		vm_page_lock(m);
834 		if (!atomic_cmpset_int(&m->busy_lock, x, VPB_UNBUSIED)) {
835 			vm_page_unlock(m);
836 			continue;
837 		}
838 		wakeup(m);
839 		vm_page_unlock(m);
840 		break;
841 	}
842 }
843 
844 /*
845  *	vm_page_busy_sleep:
846  *
847  *	Sleep and release the page lock, using the page pointer as wchan.
848  *	This is used to implement the hard-path of busying mechanism.
849  *
850  *	The given page must be locked.
851  *
852  *	If nonshared is true, sleep only if the page is xbusy.
853  */
854 void
855 vm_page_busy_sleep(vm_page_t m, const char *wmesg, bool nonshared)
856 {
857 	u_int x;
858 
859 	vm_page_assert_locked(m);
860 
861 	x = m->busy_lock;
862 	if (x == VPB_UNBUSIED || (nonshared && (x & VPB_BIT_SHARED) != 0) ||
863 	    ((x & VPB_BIT_WAITERS) == 0 &&
864 	    !atomic_cmpset_int(&m->busy_lock, x, x | VPB_BIT_WAITERS))) {
865 		vm_page_unlock(m);
866 		return;
867 	}
868 	msleep(m, vm_page_lockptr(m), PVM | PDROP, wmesg, 0);
869 }
870 
871 /*
872  *	vm_page_trysbusy:
873  *
874  *	Try to shared busy a page.
875  *	If the operation succeeds 1 is returned otherwise 0.
876  *	The operation never sleeps.
877  */
878 int
879 vm_page_trysbusy(vm_page_t m)
880 {
881 	u_int x;
882 
883 	for (;;) {
884 		x = m->busy_lock;
885 		if ((x & VPB_BIT_SHARED) == 0)
886 			return (0);
887 		if (atomic_cmpset_acq_int(&m->busy_lock, x, x + VPB_ONE_SHARER))
888 			return (1);
889 	}
890 }
891 
892 static void
893 vm_page_xunbusy_locked(vm_page_t m)
894 {
895 
896 	vm_page_assert_xbusied(m);
897 	vm_page_assert_locked(m);
898 
899 	atomic_store_rel_int(&m->busy_lock, VPB_UNBUSIED);
900 	/* There is a waiter, do wakeup() instead of vm_page_flash(). */
901 	wakeup(m);
902 }
903 
904 void
905 vm_page_xunbusy_maybelocked(vm_page_t m)
906 {
907 	bool lockacq;
908 
909 	vm_page_assert_xbusied(m);
910 
911 	/*
912 	 * Fast path for unbusy.  If it succeeds, we know that there
913 	 * are no waiters, so we do not need a wakeup.
914 	 */
915 	if (atomic_cmpset_rel_int(&m->busy_lock, VPB_SINGLE_EXCLUSIVER,
916 	    VPB_UNBUSIED))
917 		return;
918 
919 	lockacq = !mtx_owned(vm_page_lockptr(m));
920 	if (lockacq)
921 		vm_page_lock(m);
922 	vm_page_xunbusy_locked(m);
923 	if (lockacq)
924 		vm_page_unlock(m);
925 }
926 
927 /*
928  *	vm_page_xunbusy_hard:
929  *
930  *	Called after the first try the exclusive unbusy of a page failed.
931  *	It is assumed that the waiters bit is on.
932  */
933 void
934 vm_page_xunbusy_hard(vm_page_t m)
935 {
936 
937 	vm_page_assert_xbusied(m);
938 
939 	vm_page_lock(m);
940 	vm_page_xunbusy_locked(m);
941 	vm_page_unlock(m);
942 }
943 
944 /*
945  *	vm_page_flash:
946  *
947  *	Wakeup anyone waiting for the page.
948  *	The ownership bits do not change.
949  *
950  *	The given page must be locked.
951  */
952 void
953 vm_page_flash(vm_page_t m)
954 {
955 	u_int x;
956 
957 	vm_page_lock_assert(m, MA_OWNED);
958 
959 	for (;;) {
960 		x = m->busy_lock;
961 		if ((x & VPB_BIT_WAITERS) == 0)
962 			return;
963 		if (atomic_cmpset_int(&m->busy_lock, x,
964 		    x & (~VPB_BIT_WAITERS)))
965 			break;
966 	}
967 	wakeup(m);
968 }
969 
970 /*
971  * Avoid releasing and reacquiring the same page lock.
972  */
973 void
974 vm_page_change_lock(vm_page_t m, struct mtx **mtx)
975 {
976 	struct mtx *mtx1;
977 
978 	mtx1 = vm_page_lockptr(m);
979 	if (*mtx == mtx1)
980 		return;
981 	if (*mtx != NULL)
982 		mtx_unlock(*mtx);
983 	*mtx = mtx1;
984 	mtx_lock(mtx1);
985 }
986 
987 /*
988  * Keep page from being freed by the page daemon
989  * much of the same effect as wiring, except much lower
990  * overhead and should be used only for *very* temporary
991  * holding ("wiring").
992  */
993 void
994 vm_page_hold(vm_page_t mem)
995 {
996 
997 	vm_page_lock_assert(mem, MA_OWNED);
998         mem->hold_count++;
999 }
1000 
1001 void
1002 vm_page_unhold(vm_page_t mem)
1003 {
1004 
1005 	vm_page_lock_assert(mem, MA_OWNED);
1006 	KASSERT(mem->hold_count >= 1, ("vm_page_unhold: hold count < 0!!!"));
1007 	--mem->hold_count;
1008 	if (mem->hold_count == 0 && (mem->flags & PG_UNHOLDFREE) != 0)
1009 		vm_page_free_toq(mem);
1010 }
1011 
1012 /*
1013  *	vm_page_unhold_pages:
1014  *
1015  *	Unhold each of the pages that is referenced by the given array.
1016  */
1017 void
1018 vm_page_unhold_pages(vm_page_t *ma, int count)
1019 {
1020 	struct mtx *mtx;
1021 
1022 	mtx = NULL;
1023 	for (; count != 0; count--) {
1024 		vm_page_change_lock(*ma, &mtx);
1025 		vm_page_unhold(*ma);
1026 		ma++;
1027 	}
1028 	if (mtx != NULL)
1029 		mtx_unlock(mtx);
1030 }
1031 
1032 vm_page_t
1033 PHYS_TO_VM_PAGE(vm_paddr_t pa)
1034 {
1035 	vm_page_t m;
1036 
1037 #ifdef VM_PHYSSEG_SPARSE
1038 	m = vm_phys_paddr_to_vm_page(pa);
1039 	if (m == NULL)
1040 		m = vm_phys_fictitious_to_vm_page(pa);
1041 	return (m);
1042 #elif defined(VM_PHYSSEG_DENSE)
1043 	long pi;
1044 
1045 	pi = atop(pa);
1046 	if (pi >= first_page && (pi - first_page) < vm_page_array_size) {
1047 		m = &vm_page_array[pi - first_page];
1048 		return (m);
1049 	}
1050 	return (vm_phys_fictitious_to_vm_page(pa));
1051 #else
1052 #error "Either VM_PHYSSEG_DENSE or VM_PHYSSEG_SPARSE must be defined."
1053 #endif
1054 }
1055 
1056 /*
1057  *	vm_page_getfake:
1058  *
1059  *	Create a fictitious page with the specified physical address and
1060  *	memory attribute.  The memory attribute is the only the machine-
1061  *	dependent aspect of a fictitious page that must be initialized.
1062  */
1063 vm_page_t
1064 vm_page_getfake(vm_paddr_t paddr, vm_memattr_t memattr)
1065 {
1066 	vm_page_t m;
1067 
1068 	m = uma_zalloc(fakepg_zone, M_WAITOK | M_ZERO);
1069 	vm_page_initfake(m, paddr, memattr);
1070 	return (m);
1071 }
1072 
1073 void
1074 vm_page_initfake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr)
1075 {
1076 
1077 	if ((m->flags & PG_FICTITIOUS) != 0) {
1078 		/*
1079 		 * The page's memattr might have changed since the
1080 		 * previous initialization.  Update the pmap to the
1081 		 * new memattr.
1082 		 */
1083 		goto memattr;
1084 	}
1085 	m->phys_addr = paddr;
1086 	m->queue = PQ_NONE;
1087 	/* Fictitious pages don't use "segind". */
1088 	m->flags = PG_FICTITIOUS;
1089 	/* Fictitious pages don't use "order" or "pool". */
1090 	m->oflags = VPO_UNMANAGED;
1091 	m->busy_lock = VPB_SINGLE_EXCLUSIVER;
1092 	m->wire_count = 1;
1093 	pmap_page_init(m);
1094 memattr:
1095 	pmap_page_set_memattr(m, memattr);
1096 }
1097 
1098 /*
1099  *	vm_page_putfake:
1100  *
1101  *	Release a fictitious page.
1102  */
1103 void
1104 vm_page_putfake(vm_page_t m)
1105 {
1106 
1107 	KASSERT((m->oflags & VPO_UNMANAGED) != 0, ("managed %p", m));
1108 	KASSERT((m->flags & PG_FICTITIOUS) != 0,
1109 	    ("vm_page_putfake: bad page %p", m));
1110 	uma_zfree(fakepg_zone, m);
1111 }
1112 
1113 /*
1114  *	vm_page_updatefake:
1115  *
1116  *	Update the given fictitious page to the specified physical address and
1117  *	memory attribute.
1118  */
1119 void
1120 vm_page_updatefake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr)
1121 {
1122 
1123 	KASSERT((m->flags & PG_FICTITIOUS) != 0,
1124 	    ("vm_page_updatefake: bad page %p", m));
1125 	m->phys_addr = paddr;
1126 	pmap_page_set_memattr(m, memattr);
1127 }
1128 
1129 /*
1130  *	vm_page_free:
1131  *
1132  *	Free a page.
1133  */
1134 void
1135 vm_page_free(vm_page_t m)
1136 {
1137 
1138 	m->flags &= ~PG_ZERO;
1139 	vm_page_free_toq(m);
1140 }
1141 
1142 /*
1143  *	vm_page_free_zero:
1144  *
1145  *	Free a page to the zerod-pages queue
1146  */
1147 void
1148 vm_page_free_zero(vm_page_t m)
1149 {
1150 
1151 	m->flags |= PG_ZERO;
1152 	vm_page_free_toq(m);
1153 }
1154 
1155 /*
1156  * Unbusy and handle the page queueing for a page from a getpages request that
1157  * was optionally read ahead or behind.
1158  */
1159 void
1160 vm_page_readahead_finish(vm_page_t m)
1161 {
1162 
1163 	/* We shouldn't put invalid pages on queues. */
1164 	KASSERT(m->valid != 0, ("%s: %p is invalid", __func__, m));
1165 
1166 	/*
1167 	 * Since the page is not the actually needed one, whether it should
1168 	 * be activated or deactivated is not obvious.  Empirical results
1169 	 * have shown that deactivating the page is usually the best choice,
1170 	 * unless the page is wanted by another thread.
1171 	 */
1172 	vm_page_lock(m);
1173 	if ((m->busy_lock & VPB_BIT_WAITERS) != 0)
1174 		vm_page_activate(m);
1175 	else
1176 		vm_page_deactivate(m);
1177 	vm_page_unlock(m);
1178 	vm_page_xunbusy(m);
1179 }
1180 
1181 /*
1182  *	vm_page_sleep_if_busy:
1183  *
1184  *	Sleep and release the page queues lock if the page is busied.
1185  *	Returns TRUE if the thread slept.
1186  *
1187  *	The given page must be unlocked and object containing it must
1188  *	be locked.
1189  */
1190 int
1191 vm_page_sleep_if_busy(vm_page_t m, const char *msg)
1192 {
1193 	vm_object_t obj;
1194 
1195 	vm_page_lock_assert(m, MA_NOTOWNED);
1196 	VM_OBJECT_ASSERT_WLOCKED(m->object);
1197 
1198 	if (vm_page_busied(m)) {
1199 		/*
1200 		 * The page-specific object must be cached because page
1201 		 * identity can change during the sleep, causing the
1202 		 * re-lock of a different object.
1203 		 * It is assumed that a reference to the object is already
1204 		 * held by the callers.
1205 		 */
1206 		obj = m->object;
1207 		vm_page_lock(m);
1208 		VM_OBJECT_WUNLOCK(obj);
1209 		vm_page_busy_sleep(m, msg, false);
1210 		VM_OBJECT_WLOCK(obj);
1211 		return (TRUE);
1212 	}
1213 	return (FALSE);
1214 }
1215 
1216 /*
1217  *	vm_page_dirty_KBI:		[ internal use only ]
1218  *
1219  *	Set all bits in the page's dirty field.
1220  *
1221  *	The object containing the specified page must be locked if the
1222  *	call is made from the machine-independent layer.
1223  *
1224  *	See vm_page_clear_dirty_mask().
1225  *
1226  *	This function should only be called by vm_page_dirty().
1227  */
1228 void
1229 vm_page_dirty_KBI(vm_page_t m)
1230 {
1231 
1232 	/* Refer to this operation by its public name. */
1233 	KASSERT(m->valid == VM_PAGE_BITS_ALL,
1234 	    ("vm_page_dirty: page is invalid!"));
1235 	m->dirty = VM_PAGE_BITS_ALL;
1236 }
1237 
1238 /*
1239  *	vm_page_insert:		[ internal use only ]
1240  *
1241  *	Inserts the given mem entry into the object and object list.
1242  *
1243  *	The object must be locked.
1244  */
1245 int
1246 vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)
1247 {
1248 	vm_page_t mpred;
1249 
1250 	VM_OBJECT_ASSERT_WLOCKED(object);
1251 	mpred = vm_radix_lookup_le(&object->rtree, pindex);
1252 	return (vm_page_insert_after(m, object, pindex, mpred));
1253 }
1254 
1255 /*
1256  *	vm_page_insert_after:
1257  *
1258  *	Inserts the page "m" into the specified object at offset "pindex".
1259  *
1260  *	The page "mpred" must immediately precede the offset "pindex" within
1261  *	the specified object.
1262  *
1263  *	The object must be locked.
1264  */
1265 static int
1266 vm_page_insert_after(vm_page_t m, vm_object_t object, vm_pindex_t pindex,
1267     vm_page_t mpred)
1268 {
1269 	vm_page_t msucc;
1270 
1271 	VM_OBJECT_ASSERT_WLOCKED(object);
1272 	KASSERT(m->object == NULL,
1273 	    ("vm_page_insert_after: page already inserted"));
1274 	if (mpred != NULL) {
1275 		KASSERT(mpred->object == object,
1276 		    ("vm_page_insert_after: object doesn't contain mpred"));
1277 		KASSERT(mpred->pindex < pindex,
1278 		    ("vm_page_insert_after: mpred doesn't precede pindex"));
1279 		msucc = TAILQ_NEXT(mpred, listq);
1280 	} else
1281 		msucc = TAILQ_FIRST(&object->memq);
1282 	if (msucc != NULL)
1283 		KASSERT(msucc->pindex > pindex,
1284 		    ("vm_page_insert_after: msucc doesn't succeed pindex"));
1285 
1286 	/*
1287 	 * Record the object/offset pair in this page
1288 	 */
1289 	m->object = object;
1290 	m->pindex = pindex;
1291 
1292 	/*
1293 	 * Now link into the object's ordered list of backed pages.
1294 	 */
1295 	if (vm_radix_insert(&object->rtree, m)) {
1296 		m->object = NULL;
1297 		m->pindex = 0;
1298 		return (1);
1299 	}
1300 	vm_page_insert_radixdone(m, object, mpred);
1301 	return (0);
1302 }
1303 
1304 /*
1305  *	vm_page_insert_radixdone:
1306  *
1307  *	Complete page "m" insertion into the specified object after the
1308  *	radix trie hooking.
1309  *
1310  *	The page "mpred" must precede the offset "m->pindex" within the
1311  *	specified object.
1312  *
1313  *	The object must be locked.
1314  */
1315 static void
1316 vm_page_insert_radixdone(vm_page_t m, vm_object_t object, vm_page_t mpred)
1317 {
1318 
1319 	VM_OBJECT_ASSERT_WLOCKED(object);
1320 	KASSERT(object != NULL && m->object == object,
1321 	    ("vm_page_insert_radixdone: page %p has inconsistent object", m));
1322 	if (mpred != NULL) {
1323 		KASSERT(mpred->object == object,
1324 		    ("vm_page_insert_after: object doesn't contain mpred"));
1325 		KASSERT(mpred->pindex < m->pindex,
1326 		    ("vm_page_insert_after: mpred doesn't precede pindex"));
1327 	}
1328 
1329 	if (mpred != NULL)
1330 		TAILQ_INSERT_AFTER(&object->memq, mpred, m, listq);
1331 	else
1332 		TAILQ_INSERT_HEAD(&object->memq, m, listq);
1333 
1334 	/*
1335 	 * Show that the object has one more resident page.
1336 	 */
1337 	object->resident_page_count++;
1338 
1339 	/*
1340 	 * Hold the vnode until the last page is released.
1341 	 */
1342 	if (object->resident_page_count == 1 && object->type == OBJT_VNODE)
1343 		vhold(object->handle);
1344 
1345 	/*
1346 	 * Since we are inserting a new and possibly dirty page,
1347 	 * update the object's OBJ_MIGHTBEDIRTY flag.
1348 	 */
1349 	if (pmap_page_is_write_mapped(m))
1350 		vm_object_set_writeable_dirty(object);
1351 }
1352 
1353 /*
1354  *	vm_page_remove:
1355  *
1356  *	Removes the specified page from its containing object, but does not
1357  *	invalidate any backing storage.
1358  *
1359  *	The object must be locked.  The page must be locked if it is managed.
1360  */
1361 void
1362 vm_page_remove(vm_page_t m)
1363 {
1364 	vm_object_t object;
1365 	vm_page_t mrem;
1366 
1367 	if ((m->oflags & VPO_UNMANAGED) == 0)
1368 		vm_page_assert_locked(m);
1369 	if ((object = m->object) == NULL)
1370 		return;
1371 	VM_OBJECT_ASSERT_WLOCKED(object);
1372 	if (vm_page_xbusied(m))
1373 		vm_page_xunbusy_maybelocked(m);
1374 	mrem = vm_radix_remove(&object->rtree, m->pindex);
1375 	KASSERT(mrem == m, ("removed page %p, expected page %p", mrem, m));
1376 
1377 	/*
1378 	 * Now remove from the object's list of backed pages.
1379 	 */
1380 	TAILQ_REMOVE(&object->memq, m, listq);
1381 
1382 	/*
1383 	 * And show that the object has one fewer resident page.
1384 	 */
1385 	object->resident_page_count--;
1386 
1387 	/*
1388 	 * The vnode may now be recycled.
1389 	 */
1390 	if (object->resident_page_count == 0 && object->type == OBJT_VNODE)
1391 		vdrop(object->handle);
1392 
1393 	m->object = NULL;
1394 }
1395 
1396 /*
1397  *	vm_page_lookup:
1398  *
1399  *	Returns the page associated with the object/offset
1400  *	pair specified; if none is found, NULL is returned.
1401  *
1402  *	The object must be locked.
1403  */
1404 vm_page_t
1405 vm_page_lookup(vm_object_t object, vm_pindex_t pindex)
1406 {
1407 
1408 	VM_OBJECT_ASSERT_LOCKED(object);
1409 	return (vm_radix_lookup(&object->rtree, pindex));
1410 }
1411 
1412 /*
1413  *	vm_page_find_least:
1414  *
1415  *	Returns the page associated with the object with least pindex
1416  *	greater than or equal to the parameter pindex, or NULL.
1417  *
1418  *	The object must be locked.
1419  */
1420 vm_page_t
1421 vm_page_find_least(vm_object_t object, vm_pindex_t pindex)
1422 {
1423 	vm_page_t m;
1424 
1425 	VM_OBJECT_ASSERT_LOCKED(object);
1426 	if ((m = TAILQ_FIRST(&object->memq)) != NULL && m->pindex < pindex)
1427 		m = vm_radix_lookup_ge(&object->rtree, pindex);
1428 	return (m);
1429 }
1430 
1431 /*
1432  * Returns the given page's successor (by pindex) within the object if it is
1433  * resident; if none is found, NULL is returned.
1434  *
1435  * The object must be locked.
1436  */
1437 vm_page_t
1438 vm_page_next(vm_page_t m)
1439 {
1440 	vm_page_t next;
1441 
1442 	VM_OBJECT_ASSERT_LOCKED(m->object);
1443 	if ((next = TAILQ_NEXT(m, listq)) != NULL) {
1444 		MPASS(next->object == m->object);
1445 		if (next->pindex != m->pindex + 1)
1446 			next = NULL;
1447 	}
1448 	return (next);
1449 }
1450 
1451 /*
1452  * Returns the given page's predecessor (by pindex) within the object if it is
1453  * resident; if none is found, NULL is returned.
1454  *
1455  * The object must be locked.
1456  */
1457 vm_page_t
1458 vm_page_prev(vm_page_t m)
1459 {
1460 	vm_page_t prev;
1461 
1462 	VM_OBJECT_ASSERT_LOCKED(m->object);
1463 	if ((prev = TAILQ_PREV(m, pglist, listq)) != NULL) {
1464 		MPASS(prev->object == m->object);
1465 		if (prev->pindex != m->pindex - 1)
1466 			prev = NULL;
1467 	}
1468 	return (prev);
1469 }
1470 
1471 /*
1472  * Uses the page mnew as a replacement for an existing page at index
1473  * pindex which must be already present in the object.
1474  *
1475  * The existing page must not be on a paging queue.
1476  */
1477 vm_page_t
1478 vm_page_replace(vm_page_t mnew, vm_object_t object, vm_pindex_t pindex)
1479 {
1480 	vm_page_t mold;
1481 
1482 	VM_OBJECT_ASSERT_WLOCKED(object);
1483 	KASSERT(mnew->object == NULL,
1484 	    ("vm_page_replace: page already in object"));
1485 
1486 	/*
1487 	 * This function mostly follows vm_page_insert() and
1488 	 * vm_page_remove() without the radix, object count and vnode
1489 	 * dance.  Double check such functions for more comments.
1490 	 */
1491 
1492 	mnew->object = object;
1493 	mnew->pindex = pindex;
1494 	mold = vm_radix_replace(&object->rtree, mnew);
1495 	KASSERT(mold->queue == PQ_NONE,
1496 	    ("vm_page_replace: mold is on a paging queue"));
1497 
1498 	/* Keep the resident page list in sorted order. */
1499 	TAILQ_INSERT_AFTER(&object->memq, mold, mnew, listq);
1500 	TAILQ_REMOVE(&object->memq, mold, listq);
1501 
1502 	mold->object = NULL;
1503 	vm_page_xunbusy_maybelocked(mold);
1504 
1505 	/*
1506 	 * The object's resident_page_count does not change because we have
1507 	 * swapped one page for another, but OBJ_MIGHTBEDIRTY.
1508 	 */
1509 	if (pmap_page_is_write_mapped(mnew))
1510 		vm_object_set_writeable_dirty(object);
1511 	return (mold);
1512 }
1513 
1514 /*
1515  *	vm_page_rename:
1516  *
1517  *	Move the given memory entry from its
1518  *	current object to the specified target object/offset.
1519  *
1520  *	Note: swap associated with the page must be invalidated by the move.  We
1521  *	      have to do this for several reasons:  (1) we aren't freeing the
1522  *	      page, (2) we are dirtying the page, (3) the VM system is probably
1523  *	      moving the page from object A to B, and will then later move
1524  *	      the backing store from A to B and we can't have a conflict.
1525  *
1526  *	Note: we *always* dirty the page.  It is necessary both for the
1527  *	      fact that we moved it, and because we may be invalidating
1528  *	      swap.
1529  *
1530  *	The objects must be locked.
1531  */
1532 int
1533 vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex)
1534 {
1535 	vm_page_t mpred;
1536 	vm_pindex_t opidx;
1537 
1538 	VM_OBJECT_ASSERT_WLOCKED(new_object);
1539 
1540 	mpred = vm_radix_lookup_le(&new_object->rtree, new_pindex);
1541 	KASSERT(mpred == NULL || mpred->pindex != new_pindex,
1542 	    ("vm_page_rename: pindex already renamed"));
1543 
1544 	/*
1545 	 * Create a custom version of vm_page_insert() which does not depend
1546 	 * by m_prev and can cheat on the implementation aspects of the
1547 	 * function.
1548 	 */
1549 	opidx = m->pindex;
1550 	m->pindex = new_pindex;
1551 	if (vm_radix_insert(&new_object->rtree, m)) {
1552 		m->pindex = opidx;
1553 		return (1);
1554 	}
1555 
1556 	/*
1557 	 * The operation cannot fail anymore.  The removal must happen before
1558 	 * the listq iterator is tainted.
1559 	 */
1560 	m->pindex = opidx;
1561 	vm_page_lock(m);
1562 	vm_page_remove(m);
1563 
1564 	/* Return back to the new pindex to complete vm_page_insert(). */
1565 	m->pindex = new_pindex;
1566 	m->object = new_object;
1567 	vm_page_unlock(m);
1568 	vm_page_insert_radixdone(m, new_object, mpred);
1569 	vm_page_dirty(m);
1570 	return (0);
1571 }
1572 
1573 /*
1574  *	vm_page_alloc:
1575  *
1576  *	Allocate and return a page that is associated with the specified
1577  *	object and offset pair.  By default, this page is exclusive busied.
1578  *
1579  *	The caller must always specify an allocation class.
1580  *
1581  *	allocation classes:
1582  *	VM_ALLOC_NORMAL		normal process request
1583  *	VM_ALLOC_SYSTEM		system *really* needs a page
1584  *	VM_ALLOC_INTERRUPT	interrupt time request
1585  *
1586  *	optional allocation flags:
1587  *	VM_ALLOC_COUNT(number)	the number of additional pages that the caller
1588  *				intends to allocate
1589  *	VM_ALLOC_NOBUSY		do not exclusive busy the page
1590  *	VM_ALLOC_NODUMP		do not include the page in a kernel core dump
1591  *	VM_ALLOC_NOOBJ		page is not associated with an object and
1592  *				should not be exclusive busy
1593  *	VM_ALLOC_SBUSY		shared busy the allocated page
1594  *	VM_ALLOC_WIRED		wire the allocated page
1595  *	VM_ALLOC_ZERO		prefer a zeroed page
1596  */
1597 vm_page_t
1598 vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
1599 {
1600 
1601 	return (vm_page_alloc_after(object, pindex, req, object != NULL ?
1602 	    vm_radix_lookup_le(&object->rtree, pindex) : NULL));
1603 }
1604 
1605 vm_page_t
1606 vm_page_alloc_domain(vm_object_t object, vm_pindex_t pindex, int domain,
1607     int req)
1608 {
1609 
1610 	return (vm_page_alloc_domain_after(object, pindex, domain, req,
1611 	    object != NULL ? vm_radix_lookup_le(&object->rtree, pindex) :
1612 	    NULL));
1613 }
1614 
1615 /*
1616  * Allocate a page in the specified object with the given page index.  To
1617  * optimize insertion of the page into the object, the caller must also specifiy
1618  * the resident page in the object with largest index smaller than the given
1619  * page index, or NULL if no such page exists.
1620  */
1621 vm_page_t
1622 vm_page_alloc_after(vm_object_t object, vm_pindex_t pindex,
1623     int req, vm_page_t mpred)
1624 {
1625 	struct vm_domain_iterator vi;
1626 	vm_page_t m;
1627 	int domain, wait;
1628 
1629 	m = NULL;
1630 	vm_policy_iterator_init(&vi);
1631 	wait = req & (VM_ALLOC_WAITFAIL | VM_ALLOC_WAITOK);
1632 	req &= ~wait;
1633 	while (vm_domain_iterator_run(&vi, &domain) == 0) {
1634 		if (vm_domain_iterator_isdone(&vi))
1635 			req |= wait;
1636 		m = vm_page_alloc_domain_after(object, pindex, domain, req,
1637 		    mpred);
1638 		if (m != NULL)
1639 			break;
1640 	}
1641 	vm_policy_iterator_finish(&vi);
1642 
1643 	return (m);
1644 }
1645 
1646 vm_page_t
1647 vm_page_alloc_domain_after(vm_object_t object, vm_pindex_t pindex, int domain,
1648     int req, vm_page_t mpred)
1649 {
1650 	vm_page_t m;
1651 	int flags, req_class;
1652 	u_int free_count;
1653 
1654 	KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) &&
1655 	    (object != NULL || (req & VM_ALLOC_SBUSY) == 0) &&
1656 	    ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) !=
1657 	    (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)),
1658 	    ("inconsistent object(%p)/req(%x)", object, req));
1659 	KASSERT(object == NULL || (req & VM_ALLOC_WAITOK) == 0,
1660 	    ("Can't sleep and retry object insertion."));
1661 	KASSERT(mpred == NULL || mpred->pindex < pindex,
1662 	    ("mpred %p doesn't precede pindex 0x%jx", mpred,
1663 	    (uintmax_t)pindex));
1664 	if (object != NULL)
1665 		VM_OBJECT_ASSERT_WLOCKED(object);
1666 
1667 	req_class = req & VM_ALLOC_CLASS_MASK;
1668 
1669 	/*
1670 	 * The page daemon is allowed to dig deeper into the free page list.
1671 	 */
1672 	if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT)
1673 		req_class = VM_ALLOC_SYSTEM;
1674 
1675 	/*
1676 	 * Allocate a page if the number of free pages exceeds the minimum
1677 	 * for the request class.
1678 	 */
1679 again:
1680 	m = NULL;
1681 	mtx_lock(&vm_page_queue_free_mtx);
1682 	if (vm_cnt.v_free_count > vm_cnt.v_free_reserved ||
1683 	    (req_class == VM_ALLOC_SYSTEM &&
1684 	    vm_cnt.v_free_count > vm_cnt.v_interrupt_free_min) ||
1685 	    (req_class == VM_ALLOC_INTERRUPT &&
1686 	    vm_cnt.v_free_count > 0)) {
1687 		/*
1688 		 * Can we allocate the page from a reservation?
1689 		 */
1690 #if VM_NRESERVLEVEL > 0
1691 		if (object == NULL || (object->flags & (OBJ_COLORED |
1692 		    OBJ_FICTITIOUS)) != OBJ_COLORED || (m =
1693 		    vm_reserv_alloc_page(object, pindex, domain,
1694 		    mpred)) == NULL)
1695 #endif
1696 		{
1697 			/*
1698 			 * If not, allocate it from the free page queues.
1699 			 */
1700 			m = vm_phys_alloc_pages(domain, object != NULL ?
1701 			    VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, 0);
1702 #if VM_NRESERVLEVEL > 0
1703 			if (m == NULL && vm_reserv_reclaim_inactive(domain)) {
1704 				m = vm_phys_alloc_pages(domain,
1705 				    object != NULL ?
1706 				    VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT,
1707 				    0);
1708 			}
1709 #endif
1710 		}
1711 	}
1712 	if (m == NULL) {
1713 		/*
1714 		 * Not allocatable, give up.
1715 		 */
1716 		if (vm_page_alloc_fail(object, req))
1717 			goto again;
1718 		return (NULL);
1719 	}
1720 
1721 	/*
1722 	 *  At this point we had better have found a good page.
1723 	 */
1724 	KASSERT(m != NULL, ("missing page"));
1725 	free_count = vm_phys_freecnt_adj(m, -1);
1726 	mtx_unlock(&vm_page_queue_free_mtx);
1727 	vm_page_alloc_check(m);
1728 
1729 	/*
1730 	 * Initialize the page.  Only the PG_ZERO flag is inherited.
1731 	 */
1732 	flags = 0;
1733 	if ((req & VM_ALLOC_ZERO) != 0)
1734 		flags = PG_ZERO;
1735 	flags &= m->flags;
1736 	if ((req & VM_ALLOC_NODUMP) != 0)
1737 		flags |= PG_NODUMP;
1738 	m->flags = flags;
1739 	m->aflags = 0;
1740 	m->oflags = object == NULL || (object->flags & OBJ_UNMANAGED) != 0 ?
1741 	    VPO_UNMANAGED : 0;
1742 	m->busy_lock = VPB_UNBUSIED;
1743 	if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0)
1744 		m->busy_lock = VPB_SINGLE_EXCLUSIVER;
1745 	if ((req & VM_ALLOC_SBUSY) != 0)
1746 		m->busy_lock = VPB_SHARERS_WORD(1);
1747 	if (req & VM_ALLOC_WIRED) {
1748 		/*
1749 		 * The page lock is not required for wiring a page until that
1750 		 * page is inserted into the object.
1751 		 */
1752 		atomic_add_int(&vm_cnt.v_wire_count, 1);
1753 		m->wire_count = 1;
1754 	}
1755 	m->act_count = 0;
1756 
1757 	if (object != NULL) {
1758 		if (vm_page_insert_after(m, object, pindex, mpred)) {
1759 			pagedaemon_wakeup();
1760 			if (req & VM_ALLOC_WIRED) {
1761 				atomic_subtract_int(&vm_cnt.v_wire_count, 1);
1762 				m->wire_count = 0;
1763 			}
1764 			KASSERT(m->object == NULL, ("page %p has object", m));
1765 			m->oflags = VPO_UNMANAGED;
1766 			m->busy_lock = VPB_UNBUSIED;
1767 			/* Don't change PG_ZERO. */
1768 			vm_page_free_toq(m);
1769 			if (req & VM_ALLOC_WAITFAIL) {
1770 				VM_OBJECT_WUNLOCK(object);
1771 				vm_radix_wait();
1772 				VM_OBJECT_WLOCK(object);
1773 			}
1774 			return (NULL);
1775 		}
1776 
1777 		/* Ignore device objects; the pager sets "memattr" for them. */
1778 		if (object->memattr != VM_MEMATTR_DEFAULT &&
1779 		    (object->flags & OBJ_FICTITIOUS) == 0)
1780 			pmap_page_set_memattr(m, object->memattr);
1781 	} else
1782 		m->pindex = pindex;
1783 
1784 	/*
1785 	 * Don't wakeup too often - wakeup the pageout daemon when
1786 	 * we would be nearly out of memory.
1787 	 */
1788 	if (vm_paging_needed(free_count))
1789 		pagedaemon_wakeup();
1790 
1791 	return (m);
1792 }
1793 
1794 /*
1795  *	vm_page_alloc_contig:
1796  *
1797  *	Allocate a contiguous set of physical pages of the given size "npages"
1798  *	from the free lists.  All of the physical pages must be at or above
1799  *	the given physical address "low" and below the given physical address
1800  *	"high".  The given value "alignment" determines the alignment of the
1801  *	first physical page in the set.  If the given value "boundary" is
1802  *	non-zero, then the set of physical pages cannot cross any physical
1803  *	address boundary that is a multiple of that value.  Both "alignment"
1804  *	and "boundary" must be a power of two.
1805  *
1806  *	If the specified memory attribute, "memattr", is VM_MEMATTR_DEFAULT,
1807  *	then the memory attribute setting for the physical pages is configured
1808  *	to the object's memory attribute setting.  Otherwise, the memory
1809  *	attribute setting for the physical pages is configured to "memattr",
1810  *	overriding the object's memory attribute setting.  However, if the
1811  *	object's memory attribute setting is not VM_MEMATTR_DEFAULT, then the
1812  *	memory attribute setting for the physical pages cannot be configured
1813  *	to VM_MEMATTR_DEFAULT.
1814  *
1815  *	The specified object may not contain fictitious pages.
1816  *
1817  *	The caller must always specify an allocation class.
1818  *
1819  *	allocation classes:
1820  *	VM_ALLOC_NORMAL		normal process request
1821  *	VM_ALLOC_SYSTEM		system *really* needs a page
1822  *	VM_ALLOC_INTERRUPT	interrupt time request
1823  *
1824  *	optional allocation flags:
1825  *	VM_ALLOC_NOBUSY		do not exclusive busy the page
1826  *	VM_ALLOC_NODUMP		do not include the page in a kernel core dump
1827  *	VM_ALLOC_NOOBJ		page is not associated with an object and
1828  *				should not be exclusive busy
1829  *	VM_ALLOC_SBUSY		shared busy the allocated page
1830  *	VM_ALLOC_WIRED		wire the allocated page
1831  *	VM_ALLOC_ZERO		prefer a zeroed page
1832  */
1833 vm_page_t
1834 vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
1835     u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment,
1836     vm_paddr_t boundary, vm_memattr_t memattr)
1837 {
1838 	struct vm_domain_iterator vi;
1839 	vm_page_t m;
1840 	int domain, wait;
1841 
1842 	m = NULL;
1843 	vm_policy_iterator_init(&vi);
1844 	wait = req & (VM_ALLOC_WAITFAIL | VM_ALLOC_WAITOK);
1845 	req &= ~wait;
1846 	while (vm_domain_iterator_run(&vi, &domain) == 0) {
1847 		if (vm_domain_iterator_isdone(&vi))
1848 			req |= wait;
1849 		m = vm_page_alloc_contig_domain(object, pindex, domain, req,
1850 		    npages, low, high, alignment, boundary, memattr);
1851 		if (m != NULL)
1852 			break;
1853 	}
1854 	vm_policy_iterator_finish(&vi);
1855 
1856 	return (m);
1857 }
1858 
1859 vm_page_t
1860 vm_page_alloc_contig_domain(vm_object_t object, vm_pindex_t pindex, int domain,
1861     int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment,
1862     vm_paddr_t boundary, vm_memattr_t memattr)
1863 {
1864 	vm_page_t m, m_ret, mpred;
1865 	u_int busy_lock, flags, oflags;
1866 	int req_class;
1867 
1868 	mpred = NULL;	/* XXX: pacify gcc */
1869 	KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) &&
1870 	    (object != NULL || (req & VM_ALLOC_SBUSY) == 0) &&
1871 	    ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) !=
1872 	    (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)),
1873 	    ("vm_page_alloc_contig: inconsistent object(%p)/req(%x)", object,
1874 	    req));
1875 	KASSERT(object == NULL || (req & VM_ALLOC_WAITOK) == 0,
1876 	    ("Can't sleep and retry object insertion."));
1877 	if (object != NULL) {
1878 		VM_OBJECT_ASSERT_WLOCKED(object);
1879 		KASSERT((object->flags & OBJ_FICTITIOUS) == 0,
1880 		    ("vm_page_alloc_contig: object %p has fictitious pages",
1881 		    object));
1882 	}
1883 	KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero"));
1884 	req_class = req & VM_ALLOC_CLASS_MASK;
1885 
1886 	/*
1887 	 * The page daemon is allowed to dig deeper into the free page list.
1888 	 */
1889 	if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT)
1890 		req_class = VM_ALLOC_SYSTEM;
1891 
1892 	if (object != NULL) {
1893 		mpred = vm_radix_lookup_le(&object->rtree, pindex);
1894 		KASSERT(mpred == NULL || mpred->pindex != pindex,
1895 		    ("vm_page_alloc_contig: pindex already allocated"));
1896 	}
1897 
1898 	/*
1899 	 * Can we allocate the pages without the number of free pages falling
1900 	 * below the lower bound for the allocation class?
1901 	 */
1902 again:
1903 	m_ret = NULL;
1904 	mtx_lock(&vm_page_queue_free_mtx);
1905 	if (vm_cnt.v_free_count >= npages + vm_cnt.v_free_reserved ||
1906 	    (req_class == VM_ALLOC_SYSTEM &&
1907 	    vm_cnt.v_free_count >= npages + vm_cnt.v_interrupt_free_min) ||
1908 	    (req_class == VM_ALLOC_INTERRUPT &&
1909 	    vm_cnt.v_free_count >= npages)) {
1910 		/*
1911 		 * Can we allocate the pages from a reservation?
1912 		 */
1913 #if VM_NRESERVLEVEL > 0
1914 retry:
1915 		if (object == NULL || (object->flags & OBJ_COLORED) == 0 ||
1916 		    (m_ret = vm_reserv_alloc_contig(object, pindex, domain,
1917 		    npages, low, high, alignment, boundary, mpred)) == NULL)
1918 #endif
1919 			/*
1920 			 * If not, allocate them from the free page queues.
1921 			 */
1922 			m_ret = vm_phys_alloc_contig(domain, npages, low, high,
1923 			    alignment, boundary);
1924 #if VM_NRESERVLEVEL > 0
1925 		if (m_ret == NULL && vm_reserv_reclaim_contig(
1926 		    domain, npages, low, high, alignment, boundary))
1927 			goto retry;
1928 #endif
1929 	}
1930 	if (m_ret == NULL) {
1931 		if (vm_page_alloc_fail(object, req))
1932 			goto again;
1933 		return (NULL);
1934 	}
1935 	vm_phys_freecnt_adj(m_ret, -npages);
1936 	mtx_unlock(&vm_page_queue_free_mtx);
1937 	for (m = m_ret; m < &m_ret[npages]; m++)
1938 		vm_page_alloc_check(m);
1939 
1940 	/*
1941 	 * Initialize the pages.  Only the PG_ZERO flag is inherited.
1942 	 */
1943 	flags = 0;
1944 	if ((req & VM_ALLOC_ZERO) != 0)
1945 		flags = PG_ZERO;
1946 	if ((req & VM_ALLOC_NODUMP) != 0)
1947 		flags |= PG_NODUMP;
1948 	oflags = object == NULL || (object->flags & OBJ_UNMANAGED) != 0 ?
1949 	    VPO_UNMANAGED : 0;
1950 	busy_lock = VPB_UNBUSIED;
1951 	if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0)
1952 		busy_lock = VPB_SINGLE_EXCLUSIVER;
1953 	if ((req & VM_ALLOC_SBUSY) != 0)
1954 		busy_lock = VPB_SHARERS_WORD(1);
1955 	if ((req & VM_ALLOC_WIRED) != 0)
1956 		atomic_add_int(&vm_cnt.v_wire_count, npages);
1957 	if (object != NULL) {
1958 		if (object->memattr != VM_MEMATTR_DEFAULT &&
1959 		    memattr == VM_MEMATTR_DEFAULT)
1960 			memattr = object->memattr;
1961 	}
1962 	for (m = m_ret; m < &m_ret[npages]; m++) {
1963 		m->aflags = 0;
1964 		m->flags = (m->flags | PG_NODUMP) & flags;
1965 		m->busy_lock = busy_lock;
1966 		if ((req & VM_ALLOC_WIRED) != 0)
1967 			m->wire_count = 1;
1968 		m->act_count = 0;
1969 		m->oflags = oflags;
1970 		if (object != NULL) {
1971 			if (vm_page_insert_after(m, object, pindex, mpred)) {
1972 				pagedaemon_wakeup();
1973 				if ((req & VM_ALLOC_WIRED) != 0)
1974 					atomic_subtract_int(
1975 					    &vm_cnt.v_wire_count, npages);
1976 				KASSERT(m->object == NULL,
1977 				    ("page %p has object", m));
1978 				mpred = m;
1979 				for (m = m_ret; m < &m_ret[npages]; m++) {
1980 					if (m <= mpred &&
1981 					    (req & VM_ALLOC_WIRED) != 0)
1982 						m->wire_count = 0;
1983 					m->oflags = VPO_UNMANAGED;
1984 					m->busy_lock = VPB_UNBUSIED;
1985 					/* Don't change PG_ZERO. */
1986 					vm_page_free_toq(m);
1987 				}
1988 				if (req & VM_ALLOC_WAITFAIL) {
1989 					VM_OBJECT_WUNLOCK(object);
1990 					vm_radix_wait();
1991 					VM_OBJECT_WLOCK(object);
1992 				}
1993 				return (NULL);
1994 			}
1995 			mpred = m;
1996 		} else
1997 			m->pindex = pindex;
1998 		if (memattr != VM_MEMATTR_DEFAULT)
1999 			pmap_page_set_memattr(m, memattr);
2000 		pindex++;
2001 	}
2002 	if (vm_paging_needed(vm_cnt.v_free_count))
2003 		pagedaemon_wakeup();
2004 	return (m_ret);
2005 }
2006 
2007 /*
2008  * Check a page that has been freshly dequeued from a freelist.
2009  */
2010 static void
2011 vm_page_alloc_check(vm_page_t m)
2012 {
2013 
2014 	KASSERT(m->object == NULL, ("page %p has object", m));
2015 	KASSERT(m->queue == PQ_NONE,
2016 	    ("page %p has unexpected queue %d", m, m->queue));
2017 	KASSERT(m->wire_count == 0, ("page %p is wired", m));
2018 	KASSERT(m->hold_count == 0, ("page %p is held", m));
2019 	KASSERT(!vm_page_busied(m), ("page %p is busy", m));
2020 	KASSERT(m->dirty == 0, ("page %p is dirty", m));
2021 	KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT,
2022 	    ("page %p has unexpected memattr %d",
2023 	    m, pmap_page_get_memattr(m)));
2024 	KASSERT(m->valid == 0, ("free page %p is valid", m));
2025 }
2026 
2027 /*
2028  * 	vm_page_alloc_freelist:
2029  *
2030  *	Allocate a physical page from the specified free page list.
2031  *
2032  *	The caller must always specify an allocation class.
2033  *
2034  *	allocation classes:
2035  *	VM_ALLOC_NORMAL		normal process request
2036  *	VM_ALLOC_SYSTEM		system *really* needs a page
2037  *	VM_ALLOC_INTERRUPT	interrupt time request
2038  *
2039  *	optional allocation flags:
2040  *	VM_ALLOC_COUNT(number)	the number of additional pages that the caller
2041  *				intends to allocate
2042  *	VM_ALLOC_WIRED		wire the allocated page
2043  *	VM_ALLOC_ZERO		prefer a zeroed page
2044  */
2045 vm_page_t
2046 vm_page_alloc_freelist(int freelist, int req)
2047 {
2048 	struct vm_domain_iterator vi;
2049 	vm_page_t m;
2050 	int domain, wait;
2051 
2052 	m = NULL;
2053 	vm_policy_iterator_init(&vi);
2054 	wait = req & (VM_ALLOC_WAITFAIL | VM_ALLOC_WAITOK);
2055 	req &= ~wait;
2056 	while (vm_domain_iterator_run(&vi, &domain) == 0) {
2057 		if (vm_domain_iterator_isdone(&vi))
2058 			req |= wait;
2059 		m = vm_page_alloc_freelist_domain(domain, freelist, req);
2060 		if (m != NULL)
2061 			break;
2062 	}
2063 	vm_policy_iterator_finish(&vi);
2064 
2065 	return (m);
2066 }
2067 
2068 vm_page_t
2069 vm_page_alloc_freelist_domain(int domain, int freelist, int req)
2070 {
2071 	vm_page_t m;
2072 	u_int flags, free_count;
2073 	int req_class;
2074 
2075 	req_class = req & VM_ALLOC_CLASS_MASK;
2076 
2077 	/*
2078 	 * The page daemon is allowed to dig deeper into the free page list.
2079 	 */
2080 	if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT)
2081 		req_class = VM_ALLOC_SYSTEM;
2082 
2083 	/*
2084 	 * Do not allocate reserved pages unless the req has asked for it.
2085 	 */
2086 again:
2087 	mtx_lock(&vm_page_queue_free_mtx);
2088 	if (vm_cnt.v_free_count > vm_cnt.v_free_reserved ||
2089 	    (req_class == VM_ALLOC_SYSTEM &&
2090 	    vm_cnt.v_free_count > vm_cnt.v_interrupt_free_min) ||
2091 	    (req_class == VM_ALLOC_INTERRUPT &&
2092 	    vm_cnt.v_free_count > 0))
2093 		m = vm_phys_alloc_freelist_pages(domain, freelist,
2094 		    VM_FREEPOOL_DIRECT, 0);
2095 	if (m == NULL) {
2096 		if (vm_page_alloc_fail(NULL, req))
2097 			goto again;
2098 		return (NULL);
2099 	}
2100 	free_count = vm_phys_freecnt_adj(m, -1);
2101 	mtx_unlock(&vm_page_queue_free_mtx);
2102 	vm_page_alloc_check(m);
2103 
2104 	/*
2105 	 * Initialize the page.  Only the PG_ZERO flag is inherited.
2106 	 */
2107 	m->aflags = 0;
2108 	flags = 0;
2109 	if ((req & VM_ALLOC_ZERO) != 0)
2110 		flags = PG_ZERO;
2111 	m->flags &= flags;
2112 	if ((req & VM_ALLOC_WIRED) != 0) {
2113 		/*
2114 		 * The page lock is not required for wiring a page that does
2115 		 * not belong to an object.
2116 		 */
2117 		atomic_add_int(&vm_cnt.v_wire_count, 1);
2118 		m->wire_count = 1;
2119 	}
2120 	/* Unmanaged pages don't use "act_count". */
2121 	m->oflags = VPO_UNMANAGED;
2122 	if (vm_paging_needed(free_count))
2123 		pagedaemon_wakeup();
2124 	return (m);
2125 }
2126 
2127 #define	VPSC_ANY	0	/* No restrictions. */
2128 #define	VPSC_NORESERV	1	/* Skip reservations; implies VPSC_NOSUPER. */
2129 #define	VPSC_NOSUPER	2	/* Skip superpages. */
2130 
2131 /*
2132  *	vm_page_scan_contig:
2133  *
2134  *	Scan vm_page_array[] between the specified entries "m_start" and
2135  *	"m_end" for a run of contiguous physical pages that satisfy the
2136  *	specified conditions, and return the lowest page in the run.  The
2137  *	specified "alignment" determines the alignment of the lowest physical
2138  *	page in the run.  If the specified "boundary" is non-zero, then the
2139  *	run of physical pages cannot span a physical address that is a
2140  *	multiple of "boundary".
2141  *
2142  *	"m_end" is never dereferenced, so it need not point to a vm_page
2143  *	structure within vm_page_array[].
2144  *
2145  *	"npages" must be greater than zero.  "m_start" and "m_end" must not
2146  *	span a hole (or discontiguity) in the physical address space.  Both
2147  *	"alignment" and "boundary" must be a power of two.
2148  */
2149 vm_page_t
2150 vm_page_scan_contig(u_long npages, vm_page_t m_start, vm_page_t m_end,
2151     u_long alignment, vm_paddr_t boundary, int options)
2152 {
2153 	struct mtx *m_mtx;
2154 	vm_object_t object;
2155 	vm_paddr_t pa;
2156 	vm_page_t m, m_run;
2157 #if VM_NRESERVLEVEL > 0
2158 	int level;
2159 #endif
2160 	int m_inc, order, run_ext, run_len;
2161 
2162 	KASSERT(npages > 0, ("npages is 0"));
2163 	KASSERT(powerof2(alignment), ("alignment is not a power of 2"));
2164 	KASSERT(powerof2(boundary), ("boundary is not a power of 2"));
2165 	m_run = NULL;
2166 	run_len = 0;
2167 	m_mtx = NULL;
2168 	for (m = m_start; m < m_end && run_len < npages; m += m_inc) {
2169 		KASSERT((m->flags & PG_MARKER) == 0,
2170 		    ("page %p is PG_MARKER", m));
2171 		KASSERT((m->flags & PG_FICTITIOUS) == 0 || m->wire_count == 1,
2172 		    ("fictitious page %p has invalid wire count", m));
2173 
2174 		/*
2175 		 * If the current page would be the start of a run, check its
2176 		 * physical address against the end, alignment, and boundary
2177 		 * conditions.  If it doesn't satisfy these conditions, either
2178 		 * terminate the scan or advance to the next page that
2179 		 * satisfies the failed condition.
2180 		 */
2181 		if (run_len == 0) {
2182 			KASSERT(m_run == NULL, ("m_run != NULL"));
2183 			if (m + npages > m_end)
2184 				break;
2185 			pa = VM_PAGE_TO_PHYS(m);
2186 			if ((pa & (alignment - 1)) != 0) {
2187 				m_inc = atop(roundup2(pa, alignment) - pa);
2188 				continue;
2189 			}
2190 			if (rounddown2(pa ^ (pa + ptoa(npages) - 1),
2191 			    boundary) != 0) {
2192 				m_inc = atop(roundup2(pa, boundary) - pa);
2193 				continue;
2194 			}
2195 		} else
2196 			KASSERT(m_run != NULL, ("m_run == NULL"));
2197 
2198 		vm_page_change_lock(m, &m_mtx);
2199 		m_inc = 1;
2200 retry:
2201 		if (m->wire_count != 0 || m->hold_count != 0)
2202 			run_ext = 0;
2203 #if VM_NRESERVLEVEL > 0
2204 		else if ((level = vm_reserv_level(m)) >= 0 &&
2205 		    (options & VPSC_NORESERV) != 0) {
2206 			run_ext = 0;
2207 			/* Advance to the end of the reservation. */
2208 			pa = VM_PAGE_TO_PHYS(m);
2209 			m_inc = atop(roundup2(pa + 1, vm_reserv_size(level)) -
2210 			    pa);
2211 		}
2212 #endif
2213 		else if ((object = m->object) != NULL) {
2214 			/*
2215 			 * The page is considered eligible for relocation if
2216 			 * and only if it could be laundered or reclaimed by
2217 			 * the page daemon.
2218 			 */
2219 			if (!VM_OBJECT_TRYRLOCK(object)) {
2220 				mtx_unlock(m_mtx);
2221 				VM_OBJECT_RLOCK(object);
2222 				mtx_lock(m_mtx);
2223 				if (m->object != object) {
2224 					/*
2225 					 * The page may have been freed.
2226 					 */
2227 					VM_OBJECT_RUNLOCK(object);
2228 					goto retry;
2229 				} else if (m->wire_count != 0 ||
2230 				    m->hold_count != 0) {
2231 					run_ext = 0;
2232 					goto unlock;
2233 				}
2234 			}
2235 			KASSERT((m->flags & PG_UNHOLDFREE) == 0,
2236 			    ("page %p is PG_UNHOLDFREE", m));
2237 			/* Don't care: PG_NODUMP, PG_ZERO. */
2238 			if (object->type != OBJT_DEFAULT &&
2239 			    object->type != OBJT_SWAP &&
2240 			    object->type != OBJT_VNODE) {
2241 				run_ext = 0;
2242 #if VM_NRESERVLEVEL > 0
2243 			} else if ((options & VPSC_NOSUPER) != 0 &&
2244 			    (level = vm_reserv_level_iffullpop(m)) >= 0) {
2245 				run_ext = 0;
2246 				/* Advance to the end of the superpage. */
2247 				pa = VM_PAGE_TO_PHYS(m);
2248 				m_inc = atop(roundup2(pa + 1,
2249 				    vm_reserv_size(level)) - pa);
2250 #endif
2251 			} else if (object->memattr == VM_MEMATTR_DEFAULT &&
2252 			    m->queue != PQ_NONE && !vm_page_busied(m)) {
2253 				/*
2254 				 * The page is allocated but eligible for
2255 				 * relocation.  Extend the current run by one
2256 				 * page.
2257 				 */
2258 				KASSERT(pmap_page_get_memattr(m) ==
2259 				    VM_MEMATTR_DEFAULT,
2260 				    ("page %p has an unexpected memattr", m));
2261 				KASSERT((m->oflags & (VPO_SWAPINPROG |
2262 				    VPO_SWAPSLEEP | VPO_UNMANAGED)) == 0,
2263 				    ("page %p has unexpected oflags", m));
2264 				/* Don't care: VPO_NOSYNC. */
2265 				run_ext = 1;
2266 			} else
2267 				run_ext = 0;
2268 unlock:
2269 			VM_OBJECT_RUNLOCK(object);
2270 #if VM_NRESERVLEVEL > 0
2271 		} else if (level >= 0) {
2272 			/*
2273 			 * The page is reserved but not yet allocated.  In
2274 			 * other words, it is still free.  Extend the current
2275 			 * run by one page.
2276 			 */
2277 			run_ext = 1;
2278 #endif
2279 		} else if ((order = m->order) < VM_NFREEORDER) {
2280 			/*
2281 			 * The page is enqueued in the physical memory
2282 			 * allocator's free page queues.  Moreover, it is the
2283 			 * first page in a power-of-two-sized run of
2284 			 * contiguous free pages.  Add these pages to the end
2285 			 * of the current run, and jump ahead.
2286 			 */
2287 			run_ext = 1 << order;
2288 			m_inc = 1 << order;
2289 		} else {
2290 			/*
2291 			 * Skip the page for one of the following reasons: (1)
2292 			 * It is enqueued in the physical memory allocator's
2293 			 * free page queues.  However, it is not the first
2294 			 * page in a run of contiguous free pages.  (This case
2295 			 * rarely occurs because the scan is performed in
2296 			 * ascending order.) (2) It is not reserved, and it is
2297 			 * transitioning from free to allocated.  (Conversely,
2298 			 * the transition from allocated to free for managed
2299 			 * pages is blocked by the page lock.) (3) It is
2300 			 * allocated but not contained by an object and not
2301 			 * wired, e.g., allocated by Xen's balloon driver.
2302 			 */
2303 			run_ext = 0;
2304 		}
2305 
2306 		/*
2307 		 * Extend or reset the current run of pages.
2308 		 */
2309 		if (run_ext > 0) {
2310 			if (run_len == 0)
2311 				m_run = m;
2312 			run_len += run_ext;
2313 		} else {
2314 			if (run_len > 0) {
2315 				m_run = NULL;
2316 				run_len = 0;
2317 			}
2318 		}
2319 	}
2320 	if (m_mtx != NULL)
2321 		mtx_unlock(m_mtx);
2322 	if (run_len >= npages)
2323 		return (m_run);
2324 	return (NULL);
2325 }
2326 
2327 /*
2328  *	vm_page_reclaim_run:
2329  *
2330  *	Try to relocate each of the allocated virtual pages within the
2331  *	specified run of physical pages to a new physical address.  Free the
2332  *	physical pages underlying the relocated virtual pages.  A virtual page
2333  *	is relocatable if and only if it could be laundered or reclaimed by
2334  *	the page daemon.  Whenever possible, a virtual page is relocated to a
2335  *	physical address above "high".
2336  *
2337  *	Returns 0 if every physical page within the run was already free or
2338  *	just freed by a successful relocation.  Otherwise, returns a non-zero
2339  *	value indicating why the last attempt to relocate a virtual page was
2340  *	unsuccessful.
2341  *
2342  *	"req_class" must be an allocation class.
2343  */
2344 static int
2345 vm_page_reclaim_run(int req_class, u_long npages, vm_page_t m_run,
2346     vm_paddr_t high)
2347 {
2348 	struct mtx *m_mtx;
2349 	struct spglist free;
2350 	vm_object_t object;
2351 	vm_paddr_t pa;
2352 	vm_page_t m, m_end, m_new;
2353 	int error, order, req;
2354 
2355 	KASSERT((req_class & VM_ALLOC_CLASS_MASK) == req_class,
2356 	    ("req_class is not an allocation class"));
2357 	SLIST_INIT(&free);
2358 	error = 0;
2359 	m = m_run;
2360 	m_end = m_run + npages;
2361 	m_mtx = NULL;
2362 	for (; error == 0 && m < m_end; m++) {
2363 		KASSERT((m->flags & (PG_FICTITIOUS | PG_MARKER)) == 0,
2364 		    ("page %p is PG_FICTITIOUS or PG_MARKER", m));
2365 
2366 		/*
2367 		 * Avoid releasing and reacquiring the same page lock.
2368 		 */
2369 		vm_page_change_lock(m, &m_mtx);
2370 retry:
2371 		if (m->wire_count != 0 || m->hold_count != 0)
2372 			error = EBUSY;
2373 		else if ((object = m->object) != NULL) {
2374 			/*
2375 			 * The page is relocated if and only if it could be
2376 			 * laundered or reclaimed by the page daemon.
2377 			 */
2378 			if (!VM_OBJECT_TRYWLOCK(object)) {
2379 				mtx_unlock(m_mtx);
2380 				VM_OBJECT_WLOCK(object);
2381 				mtx_lock(m_mtx);
2382 				if (m->object != object) {
2383 					/*
2384 					 * The page may have been freed.
2385 					 */
2386 					VM_OBJECT_WUNLOCK(object);
2387 					goto retry;
2388 				} else if (m->wire_count != 0 ||
2389 				    m->hold_count != 0) {
2390 					error = EBUSY;
2391 					goto unlock;
2392 				}
2393 			}
2394 			KASSERT((m->flags & PG_UNHOLDFREE) == 0,
2395 			    ("page %p is PG_UNHOLDFREE", m));
2396 			/* Don't care: PG_NODUMP, PG_ZERO. */
2397 			if (object->type != OBJT_DEFAULT &&
2398 			    object->type != OBJT_SWAP &&
2399 			    object->type != OBJT_VNODE)
2400 				error = EINVAL;
2401 			else if (object->memattr != VM_MEMATTR_DEFAULT)
2402 				error = EINVAL;
2403 			else if (m->queue != PQ_NONE && !vm_page_busied(m)) {
2404 				KASSERT(pmap_page_get_memattr(m) ==
2405 				    VM_MEMATTR_DEFAULT,
2406 				    ("page %p has an unexpected memattr", m));
2407 				KASSERT((m->oflags & (VPO_SWAPINPROG |
2408 				    VPO_SWAPSLEEP | VPO_UNMANAGED)) == 0,
2409 				    ("page %p has unexpected oflags", m));
2410 				/* Don't care: VPO_NOSYNC. */
2411 				if (m->valid != 0) {
2412 					/*
2413 					 * First, try to allocate a new page
2414 					 * that is above "high".  Failing
2415 					 * that, try to allocate a new page
2416 					 * that is below "m_run".  Allocate
2417 					 * the new page between the end of
2418 					 * "m_run" and "high" only as a last
2419 					 * resort.
2420 					 */
2421 					req = req_class | VM_ALLOC_NOOBJ;
2422 					if ((m->flags & PG_NODUMP) != 0)
2423 						req |= VM_ALLOC_NODUMP;
2424 					if (trunc_page(high) !=
2425 					    ~(vm_paddr_t)PAGE_MASK) {
2426 						m_new = vm_page_alloc_contig(
2427 						    NULL, 0, req, 1,
2428 						    round_page(high),
2429 						    ~(vm_paddr_t)0,
2430 						    PAGE_SIZE, 0,
2431 						    VM_MEMATTR_DEFAULT);
2432 					} else
2433 						m_new = NULL;
2434 					if (m_new == NULL) {
2435 						pa = VM_PAGE_TO_PHYS(m_run);
2436 						m_new = vm_page_alloc_contig(
2437 						    NULL, 0, req, 1,
2438 						    0, pa - 1, PAGE_SIZE, 0,
2439 						    VM_MEMATTR_DEFAULT);
2440 					}
2441 					if (m_new == NULL) {
2442 						pa += ptoa(npages);
2443 						m_new = vm_page_alloc_contig(
2444 						    NULL, 0, req, 1,
2445 						    pa, high, PAGE_SIZE, 0,
2446 						    VM_MEMATTR_DEFAULT);
2447 					}
2448 					if (m_new == NULL) {
2449 						error = ENOMEM;
2450 						goto unlock;
2451 					}
2452 					KASSERT(m_new->wire_count == 0,
2453 					    ("page %p is wired", m));
2454 
2455 					/*
2456 					 * Replace "m" with the new page.  For
2457 					 * vm_page_replace(), "m" must be busy
2458 					 * and dequeued.  Finally, change "m"
2459 					 * as if vm_page_free() was called.
2460 					 */
2461 					if (object->ref_count != 0)
2462 						pmap_remove_all(m);
2463 					m_new->aflags = m->aflags;
2464 					KASSERT(m_new->oflags == VPO_UNMANAGED,
2465 					    ("page %p is managed", m));
2466 					m_new->oflags = m->oflags & VPO_NOSYNC;
2467 					pmap_copy_page(m, m_new);
2468 					m_new->valid = m->valid;
2469 					m_new->dirty = m->dirty;
2470 					m->flags &= ~PG_ZERO;
2471 					vm_page_xbusy(m);
2472 					vm_page_remque(m);
2473 					vm_page_replace_checked(m_new, object,
2474 					    m->pindex, m);
2475 					m->valid = 0;
2476 					vm_page_undirty(m);
2477 
2478 					/*
2479 					 * The new page must be deactivated
2480 					 * before the object is unlocked.
2481 					 */
2482 					vm_page_change_lock(m_new, &m_mtx);
2483 					vm_page_deactivate(m_new);
2484 				} else {
2485 					m->flags &= ~PG_ZERO;
2486 					vm_page_remque(m);
2487 					vm_page_remove(m);
2488 					KASSERT(m->dirty == 0,
2489 					    ("page %p is dirty", m));
2490 				}
2491 				SLIST_INSERT_HEAD(&free, m, plinks.s.ss);
2492 			} else
2493 				error = EBUSY;
2494 unlock:
2495 			VM_OBJECT_WUNLOCK(object);
2496 		} else {
2497 			mtx_lock(&vm_page_queue_free_mtx);
2498 			order = m->order;
2499 			if (order < VM_NFREEORDER) {
2500 				/*
2501 				 * The page is enqueued in the physical memory
2502 				 * allocator's free page queues.  Moreover, it
2503 				 * is the first page in a power-of-two-sized
2504 				 * run of contiguous free pages.  Jump ahead
2505 				 * to the last page within that run, and
2506 				 * continue from there.
2507 				 */
2508 				m += (1 << order) - 1;
2509 			}
2510 #if VM_NRESERVLEVEL > 0
2511 			else if (vm_reserv_is_page_free(m))
2512 				order = 0;
2513 #endif
2514 			mtx_unlock(&vm_page_queue_free_mtx);
2515 			if (order == VM_NFREEORDER)
2516 				error = EINVAL;
2517 		}
2518 	}
2519 	if (m_mtx != NULL)
2520 		mtx_unlock(m_mtx);
2521 	if ((m = SLIST_FIRST(&free)) != NULL) {
2522 		mtx_lock(&vm_page_queue_free_mtx);
2523 		do {
2524 			SLIST_REMOVE_HEAD(&free, plinks.s.ss);
2525 			vm_page_free_phys(m);
2526 		} while ((m = SLIST_FIRST(&free)) != NULL);
2527 		vm_page_free_wakeup();
2528 		mtx_unlock(&vm_page_queue_free_mtx);
2529 	}
2530 	return (error);
2531 }
2532 
2533 #define	NRUNS	16
2534 
2535 CTASSERT(powerof2(NRUNS));
2536 
2537 #define	RUN_INDEX(count)	((count) & (NRUNS - 1))
2538 
2539 #define	MIN_RECLAIM	8
2540 
2541 /*
2542  *	vm_page_reclaim_contig:
2543  *
2544  *	Reclaim allocated, contiguous physical memory satisfying the specified
2545  *	conditions by relocating the virtual pages using that physical memory.
2546  *	Returns true if reclamation is successful and false otherwise.  Since
2547  *	relocation requires the allocation of physical pages, reclamation may
2548  *	fail due to a shortage of free pages.  When reclamation fails, callers
2549  *	are expected to perform VM_WAIT before retrying a failed allocation
2550  *	operation, e.g., vm_page_alloc_contig().
2551  *
2552  *	The caller must always specify an allocation class through "req".
2553  *
2554  *	allocation classes:
2555  *	VM_ALLOC_NORMAL		normal process request
2556  *	VM_ALLOC_SYSTEM		system *really* needs a page
2557  *	VM_ALLOC_INTERRUPT	interrupt time request
2558  *
2559  *	The optional allocation flags are ignored.
2560  *
2561  *	"npages" must be greater than zero.  Both "alignment" and "boundary"
2562  *	must be a power of two.
2563  */
2564 bool
2565 vm_page_reclaim_contig(int req, u_long npages, vm_paddr_t low, vm_paddr_t high,
2566     u_long alignment, vm_paddr_t boundary)
2567 {
2568 	vm_paddr_t curr_low;
2569 	vm_page_t m_run, m_runs[NRUNS];
2570 	u_long count, reclaimed;
2571 	int error, i, options, req_class;
2572 
2573 	KASSERT(npages > 0, ("npages is 0"));
2574 	KASSERT(powerof2(alignment), ("alignment is not a power of 2"));
2575 	KASSERT(powerof2(boundary), ("boundary is not a power of 2"));
2576 	req_class = req & VM_ALLOC_CLASS_MASK;
2577 
2578 	/*
2579 	 * The page daemon is allowed to dig deeper into the free page list.
2580 	 */
2581 	if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT)
2582 		req_class = VM_ALLOC_SYSTEM;
2583 
2584 	/*
2585 	 * Return if the number of free pages cannot satisfy the requested
2586 	 * allocation.
2587 	 */
2588 	count = vm_cnt.v_free_count;
2589 	if (count < npages + vm_cnt.v_free_reserved || (count < npages +
2590 	    vm_cnt.v_interrupt_free_min && req_class == VM_ALLOC_SYSTEM) ||
2591 	    (count < npages && req_class == VM_ALLOC_INTERRUPT))
2592 		return (false);
2593 
2594 	/*
2595 	 * Scan up to three times, relaxing the restrictions ("options") on
2596 	 * the reclamation of reservations and superpages each time.
2597 	 */
2598 	for (options = VPSC_NORESERV;;) {
2599 		/*
2600 		 * Find the highest runs that satisfy the given constraints
2601 		 * and restrictions, and record them in "m_runs".
2602 		 */
2603 		curr_low = low;
2604 		count = 0;
2605 		for (;;) {
2606 			m_run = vm_phys_scan_contig(npages, curr_low, high,
2607 			    alignment, boundary, options);
2608 			if (m_run == NULL)
2609 				break;
2610 			curr_low = VM_PAGE_TO_PHYS(m_run) + ptoa(npages);
2611 			m_runs[RUN_INDEX(count)] = m_run;
2612 			count++;
2613 		}
2614 
2615 		/*
2616 		 * Reclaim the highest runs in LIFO (descending) order until
2617 		 * the number of reclaimed pages, "reclaimed", is at least
2618 		 * MIN_RECLAIM.  Reset "reclaimed" each time because each
2619 		 * reclamation is idempotent, and runs will (likely) recur
2620 		 * from one scan to the next as restrictions are relaxed.
2621 		 */
2622 		reclaimed = 0;
2623 		for (i = 0; count > 0 && i < NRUNS; i++) {
2624 			count--;
2625 			m_run = m_runs[RUN_INDEX(count)];
2626 			error = vm_page_reclaim_run(req_class, npages, m_run,
2627 			    high);
2628 			if (error == 0) {
2629 				reclaimed += npages;
2630 				if (reclaimed >= MIN_RECLAIM)
2631 					return (true);
2632 			}
2633 		}
2634 
2635 		/*
2636 		 * Either relax the restrictions on the next scan or return if
2637 		 * the last scan had no restrictions.
2638 		 */
2639 		if (options == VPSC_NORESERV)
2640 			options = VPSC_NOSUPER;
2641 		else if (options == VPSC_NOSUPER)
2642 			options = VPSC_ANY;
2643 		else if (options == VPSC_ANY)
2644 			return (reclaimed != 0);
2645 	}
2646 }
2647 
2648 /*
2649  *	vm_wait:	(also see VM_WAIT macro)
2650  *
2651  *	Sleep until free pages are available for allocation.
2652  *	- Called in various places before memory allocations.
2653  */
2654 static void
2655 _vm_wait(void)
2656 {
2657 
2658 	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
2659 	if (curproc == pageproc) {
2660 		vm_pageout_pages_needed = 1;
2661 		msleep(&vm_pageout_pages_needed, &vm_page_queue_free_mtx,
2662 		    PDROP | PSWP, "VMWait", 0);
2663 	} else {
2664 		if (__predict_false(pageproc == NULL))
2665 			panic("vm_wait in early boot");
2666 		if (!vm_pageout_wanted) {
2667 			vm_pageout_wanted = true;
2668 			wakeup(&vm_pageout_wanted);
2669 		}
2670 		vm_pages_needed = true;
2671 		msleep(&vm_cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | PVM,
2672 		    "vmwait", 0);
2673 	}
2674 }
2675 
2676 void
2677 vm_wait(void)
2678 {
2679 
2680 	mtx_lock(&vm_page_queue_free_mtx);
2681 	_vm_wait();
2682 }
2683 
2684 /*
2685  *	vm_page_alloc_fail:
2686  *
2687  *	Called when a page allocation function fails.  Informs the
2688  *	pagedaemon and performs the requested wait.  Requires the
2689  *	page_queue_free and object lock on entry.  Returns with the
2690  *	object lock held and free lock released.  Returns an error when
2691  *	retry is necessary.
2692  *
2693  */
2694 static int
2695 vm_page_alloc_fail(vm_object_t object, int req)
2696 {
2697 
2698 	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
2699 
2700 	atomic_add_int(&vm_pageout_deficit,
2701 	    max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1));
2702 	pagedaemon_wakeup();
2703 	if (req & (VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) {
2704 		if (object != NULL)
2705 			VM_OBJECT_WUNLOCK(object);
2706 		_vm_wait();
2707 		if (object != NULL)
2708 			VM_OBJECT_WLOCK(object);
2709 		if (req & VM_ALLOC_WAITOK)
2710 			return (EAGAIN);
2711 	} else
2712 		mtx_unlock(&vm_page_queue_free_mtx);
2713 	return (0);
2714 }
2715 
2716 /*
2717  *	vm_waitpfault:	(also see VM_WAITPFAULT macro)
2718  *
2719  *	Sleep until free pages are available for allocation.
2720  *	- Called only in vm_fault so that processes page faulting
2721  *	  can be easily tracked.
2722  *	- Sleeps at a lower priority than vm_wait() so that vm_wait()ing
2723  *	  processes will be able to grab memory first.  Do not change
2724  *	  this balance without careful testing first.
2725  */
2726 void
2727 vm_waitpfault(void)
2728 {
2729 
2730 	mtx_lock(&vm_page_queue_free_mtx);
2731 	if (!vm_pageout_wanted) {
2732 		vm_pageout_wanted = true;
2733 		wakeup(&vm_pageout_wanted);
2734 	}
2735 	vm_pages_needed = true;
2736 	msleep(&vm_cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | PUSER,
2737 	    "pfault", 0);
2738 }
2739 
2740 struct vm_pagequeue *
2741 vm_page_pagequeue(vm_page_t m)
2742 {
2743 
2744 	if (vm_page_in_laundry(m))
2745 		return (&vm_dom[0].vmd_pagequeues[m->queue]);
2746 	else
2747 		return (&vm_phys_domain(m)->vmd_pagequeues[m->queue]);
2748 }
2749 
2750 /*
2751  *	vm_page_dequeue:
2752  *
2753  *	Remove the given page from its current page queue.
2754  *
2755  *	The page must be locked.
2756  */
2757 void
2758 vm_page_dequeue(vm_page_t m)
2759 {
2760 	struct vm_pagequeue *pq;
2761 
2762 	vm_page_assert_locked(m);
2763 	KASSERT(m->queue < PQ_COUNT, ("vm_page_dequeue: page %p is not queued",
2764 	    m));
2765 	pq = vm_page_pagequeue(m);
2766 	vm_pagequeue_lock(pq);
2767 	m->queue = PQ_NONE;
2768 	TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
2769 	vm_pagequeue_cnt_dec(pq);
2770 	vm_pagequeue_unlock(pq);
2771 }
2772 
2773 /*
2774  *	vm_page_dequeue_locked:
2775  *
2776  *	Remove the given page from its current page queue.
2777  *
2778  *	The page and page queue must be locked.
2779  */
2780 void
2781 vm_page_dequeue_locked(vm_page_t m)
2782 {
2783 	struct vm_pagequeue *pq;
2784 
2785 	vm_page_lock_assert(m, MA_OWNED);
2786 	pq = vm_page_pagequeue(m);
2787 	vm_pagequeue_assert_locked(pq);
2788 	m->queue = PQ_NONE;
2789 	TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
2790 	vm_pagequeue_cnt_dec(pq);
2791 }
2792 
2793 /*
2794  *	vm_page_enqueue:
2795  *
2796  *	Add the given page to the specified page queue.
2797  *
2798  *	The page must be locked.
2799  */
2800 static void
2801 vm_page_enqueue(uint8_t queue, vm_page_t m)
2802 {
2803 	struct vm_pagequeue *pq;
2804 
2805 	vm_page_lock_assert(m, MA_OWNED);
2806 	KASSERT(queue < PQ_COUNT,
2807 	    ("vm_page_enqueue: invalid queue %u request for page %p",
2808 	    queue, m));
2809 	if (queue == PQ_LAUNDRY || queue == PQ_UNSWAPPABLE)
2810 		pq = &vm_dom[0].vmd_pagequeues[queue];
2811 	else
2812 		pq = &vm_phys_domain(m)->vmd_pagequeues[queue];
2813 	vm_pagequeue_lock(pq);
2814 	m->queue = queue;
2815 	TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
2816 	vm_pagequeue_cnt_inc(pq);
2817 	vm_pagequeue_unlock(pq);
2818 }
2819 
2820 /*
2821  *	vm_page_requeue:
2822  *
2823  *	Move the given page to the tail of its current page queue.
2824  *
2825  *	The page must be locked.
2826  */
2827 void
2828 vm_page_requeue(vm_page_t m)
2829 {
2830 	struct vm_pagequeue *pq;
2831 
2832 	vm_page_lock_assert(m, MA_OWNED);
2833 	KASSERT(m->queue != PQ_NONE,
2834 	    ("vm_page_requeue: page %p is not queued", m));
2835 	pq = vm_page_pagequeue(m);
2836 	vm_pagequeue_lock(pq);
2837 	TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
2838 	TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
2839 	vm_pagequeue_unlock(pq);
2840 }
2841 
2842 /*
2843  *	vm_page_requeue_locked:
2844  *
2845  *	Move the given page to the tail of its current page queue.
2846  *
2847  *	The page queue must be locked.
2848  */
2849 void
2850 vm_page_requeue_locked(vm_page_t m)
2851 {
2852 	struct vm_pagequeue *pq;
2853 
2854 	KASSERT(m->queue != PQ_NONE,
2855 	    ("vm_page_requeue_locked: page %p is not queued", m));
2856 	pq = vm_page_pagequeue(m);
2857 	vm_pagequeue_assert_locked(pq);
2858 	TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
2859 	TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
2860 }
2861 
2862 /*
2863  *	vm_page_activate:
2864  *
2865  *	Put the specified page on the active list (if appropriate).
2866  *	Ensure that act_count is at least ACT_INIT but do not otherwise
2867  *	mess with it.
2868  *
2869  *	The page must be locked.
2870  */
2871 void
2872 vm_page_activate(vm_page_t m)
2873 {
2874 	int queue;
2875 
2876 	vm_page_lock_assert(m, MA_OWNED);
2877 	if ((queue = m->queue) != PQ_ACTIVE) {
2878 		if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) {
2879 			if (m->act_count < ACT_INIT)
2880 				m->act_count = ACT_INIT;
2881 			if (queue != PQ_NONE)
2882 				vm_page_dequeue(m);
2883 			vm_page_enqueue(PQ_ACTIVE, m);
2884 		} else
2885 			KASSERT(queue == PQ_NONE,
2886 			    ("vm_page_activate: wired page %p is queued", m));
2887 	} else {
2888 		if (m->act_count < ACT_INIT)
2889 			m->act_count = ACT_INIT;
2890 	}
2891 }
2892 
2893 /*
2894  *	vm_page_free_wakeup:
2895  *
2896  *	Helper routine for vm_page_free_toq().  This routine is called
2897  *	when a page is added to the free queues.
2898  *
2899  *	The page queues must be locked.
2900  */
2901 static void
2902 vm_page_free_wakeup(void)
2903 {
2904 
2905 	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
2906 	/*
2907 	 * if pageout daemon needs pages, then tell it that there are
2908 	 * some free.
2909 	 */
2910 	if (vm_pageout_pages_needed &&
2911 	    vm_cnt.v_free_count >= vm_cnt.v_pageout_free_min) {
2912 		wakeup(&vm_pageout_pages_needed);
2913 		vm_pageout_pages_needed = 0;
2914 	}
2915 	/*
2916 	 * wakeup processes that are waiting on memory if we hit a
2917 	 * high water mark. And wakeup scheduler process if we have
2918 	 * lots of memory. this process will swapin processes.
2919 	 */
2920 	if (vm_pages_needed && !vm_page_count_min()) {
2921 		vm_pages_needed = false;
2922 		wakeup(&vm_cnt.v_free_count);
2923 	}
2924 }
2925 
2926 /*
2927  *	vm_page_free_prep:
2928  *
2929  *	Prepares the given page to be put on the free list,
2930  *	disassociating it from any VM object. The caller may return
2931  *	the page to the free list only if this function returns true.
2932  *
2933  *	The object must be locked.  The page must be locked if it is
2934  *	managed.  For a queued managed page, the pagequeue_locked
2935  *	argument specifies whether the page queue is already locked.
2936  */
2937 bool
2938 vm_page_free_prep(vm_page_t m, bool pagequeue_locked)
2939 {
2940 
2941 #if defined(DIAGNOSTIC) && defined(PHYS_TO_DMAP)
2942 	if ((m->flags & PG_ZERO) != 0) {
2943 		uint64_t *p;
2944 		int i;
2945 		p = (uint64_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
2946 		for (i = 0; i < PAGE_SIZE / sizeof(uint64_t); i++, p++)
2947 			KASSERT(*p == 0, ("vm_page_free_prep %p PG_ZERO %d %jx",
2948 			    m, i, (uintmax_t)*p));
2949 	}
2950 #endif
2951 	if ((m->oflags & VPO_UNMANAGED) == 0) {
2952 		vm_page_lock_assert(m, MA_OWNED);
2953 		KASSERT(!pmap_page_is_mapped(m),
2954 		    ("vm_page_free_toq: freeing mapped page %p", m));
2955 	} else
2956 		KASSERT(m->queue == PQ_NONE,
2957 		    ("vm_page_free_toq: unmanaged page %p is queued", m));
2958 	VM_CNT_INC(v_tfree);
2959 
2960 	if (vm_page_sbusied(m))
2961 		panic("vm_page_free: freeing busy page %p", m);
2962 
2963 	vm_page_remove(m);
2964 
2965 	/*
2966 	 * If fictitious remove object association and
2967 	 * return.
2968 	 */
2969 	if ((m->flags & PG_FICTITIOUS) != 0) {
2970 		KASSERT(m->wire_count == 1,
2971 		    ("fictitious page %p is not wired", m));
2972 		KASSERT(m->queue == PQ_NONE,
2973 		    ("fictitious page %p is queued", m));
2974 		return (false);
2975 	}
2976 
2977 	if (m->queue != PQ_NONE) {
2978 		if (pagequeue_locked)
2979 			vm_page_dequeue_locked(m);
2980 		else
2981 			vm_page_dequeue(m);
2982 	}
2983 	m->valid = 0;
2984 	vm_page_undirty(m);
2985 
2986 	if (m->wire_count != 0)
2987 		panic("vm_page_free: freeing wired page %p", m);
2988 	if (m->hold_count != 0) {
2989 		m->flags &= ~PG_ZERO;
2990 		KASSERT((m->flags & PG_UNHOLDFREE) == 0,
2991 		    ("vm_page_free: freeing PG_UNHOLDFREE page %p", m));
2992 		m->flags |= PG_UNHOLDFREE;
2993 		return (false);
2994 	}
2995 
2996 	/*
2997 	 * Restore the default memory attribute to the page.
2998 	 */
2999 	if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT)
3000 		pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT);
3001 
3002 	return (true);
3003 }
3004 
3005 /*
3006  * Insert the page into the physical memory allocator's free page
3007  * queues.  This is the last step to free a page.
3008  */
3009 static void
3010 vm_page_free_phys(vm_page_t m)
3011 {
3012 
3013 	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
3014 
3015 	vm_phys_freecnt_adj(m, 1);
3016 #if VM_NRESERVLEVEL > 0
3017 	if (!vm_reserv_free_page(m))
3018 #endif
3019 		vm_phys_free_pages(m, 0);
3020 }
3021 
3022 void
3023 vm_page_free_phys_pglist(struct pglist *tq)
3024 {
3025 	vm_page_t m;
3026 
3027 	if (TAILQ_EMPTY(tq))
3028 		return;
3029 	mtx_lock(&vm_page_queue_free_mtx);
3030 	TAILQ_FOREACH(m, tq, listq)
3031 		vm_page_free_phys(m);
3032 	vm_page_free_wakeup();
3033 	mtx_unlock(&vm_page_queue_free_mtx);
3034 }
3035 
3036 /*
3037  *	vm_page_free_toq:
3038  *
3039  *	Returns the given page to the free list, disassociating it
3040  *	from any VM object.
3041  *
3042  *	The object must be locked.  The page must be locked if it is
3043  *	managed.
3044  */
3045 void
3046 vm_page_free_toq(vm_page_t m)
3047 {
3048 
3049 	if (!vm_page_free_prep(m, false))
3050 		return;
3051 	mtx_lock(&vm_page_queue_free_mtx);
3052 	vm_page_free_phys(m);
3053 	vm_page_free_wakeup();
3054 	mtx_unlock(&vm_page_queue_free_mtx);
3055 }
3056 
3057 /*
3058  *	vm_page_wire:
3059  *
3060  *	Mark this page as wired down by yet
3061  *	another map, removing it from paging queues
3062  *	as necessary.
3063  *
3064  *	If the page is fictitious, then its wire count must remain one.
3065  *
3066  *	The page must be locked.
3067  */
3068 void
3069 vm_page_wire(vm_page_t m)
3070 {
3071 
3072 	/*
3073 	 * Only bump the wire statistics if the page is not already wired,
3074 	 * and only unqueue the page if it is on some queue (if it is unmanaged
3075 	 * it is already off the queues).
3076 	 */
3077 	vm_page_lock_assert(m, MA_OWNED);
3078 	if ((m->flags & PG_FICTITIOUS) != 0) {
3079 		KASSERT(m->wire_count == 1,
3080 		    ("vm_page_wire: fictitious page %p's wire count isn't one",
3081 		    m));
3082 		return;
3083 	}
3084 	if (m->wire_count == 0) {
3085 		KASSERT((m->oflags & VPO_UNMANAGED) == 0 ||
3086 		    m->queue == PQ_NONE,
3087 		    ("vm_page_wire: unmanaged page %p is queued", m));
3088 		vm_page_remque(m);
3089 		atomic_add_int(&vm_cnt.v_wire_count, 1);
3090 	}
3091 	m->wire_count++;
3092 	KASSERT(m->wire_count != 0, ("vm_page_wire: wire_count overflow m=%p", m));
3093 }
3094 
3095 /*
3096  * vm_page_unwire:
3097  *
3098  * Release one wiring of the specified page, potentially allowing it to be
3099  * paged out.  Returns TRUE if the number of wirings transitions to zero and
3100  * FALSE otherwise.
3101  *
3102  * Only managed pages belonging to an object can be paged out.  If the number
3103  * of wirings transitions to zero and the page is eligible for page out, then
3104  * the page is added to the specified paging queue (unless PQ_NONE is
3105  * specified).
3106  *
3107  * If a page is fictitious, then its wire count must always be one.
3108  *
3109  * A managed page must be locked.
3110  */
3111 boolean_t
3112 vm_page_unwire(vm_page_t m, uint8_t queue)
3113 {
3114 
3115 	KASSERT(queue < PQ_COUNT || queue == PQ_NONE,
3116 	    ("vm_page_unwire: invalid queue %u request for page %p",
3117 	    queue, m));
3118 	if ((m->oflags & VPO_UNMANAGED) == 0)
3119 		vm_page_assert_locked(m);
3120 	if ((m->flags & PG_FICTITIOUS) != 0) {
3121 		KASSERT(m->wire_count == 1,
3122 	    ("vm_page_unwire: fictitious page %p's wire count isn't one", m));
3123 		return (FALSE);
3124 	}
3125 	if (m->wire_count > 0) {
3126 		m->wire_count--;
3127 		if (m->wire_count == 0) {
3128 			atomic_subtract_int(&vm_cnt.v_wire_count, 1);
3129 			if ((m->oflags & VPO_UNMANAGED) == 0 &&
3130 			    m->object != NULL && queue != PQ_NONE)
3131 				vm_page_enqueue(queue, m);
3132 			return (TRUE);
3133 		} else
3134 			return (FALSE);
3135 	} else
3136 		panic("vm_page_unwire: page %p's wire count is zero", m);
3137 }
3138 
3139 /*
3140  * Move the specified page to the inactive queue.
3141  *
3142  * Normally, "noreuse" is FALSE, resulting in LRU ordering of the inactive
3143  * queue.  However, setting "noreuse" to TRUE will accelerate the specified
3144  * page's reclamation, but it will not unmap the page from any address space.
3145  * This is implemented by inserting the page near the head of the inactive
3146  * queue, using a marker page to guide FIFO insertion ordering.
3147  *
3148  * The page must be locked.
3149  */
3150 static inline void
3151 _vm_page_deactivate(vm_page_t m, boolean_t noreuse)
3152 {
3153 	struct vm_pagequeue *pq;
3154 	int queue;
3155 
3156 	vm_page_assert_locked(m);
3157 
3158 	/*
3159 	 * Ignore if the page is already inactive, unless it is unlikely to be
3160 	 * reactivated.
3161 	 */
3162 	if ((queue = m->queue) == PQ_INACTIVE && !noreuse)
3163 		return;
3164 	if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) {
3165 		pq = &vm_phys_domain(m)->vmd_pagequeues[PQ_INACTIVE];
3166 		/* Avoid multiple acquisitions of the inactive queue lock. */
3167 		if (queue == PQ_INACTIVE) {
3168 			vm_pagequeue_lock(pq);
3169 			vm_page_dequeue_locked(m);
3170 		} else {
3171 			if (queue != PQ_NONE)
3172 				vm_page_dequeue(m);
3173 			vm_pagequeue_lock(pq);
3174 		}
3175 		m->queue = PQ_INACTIVE;
3176 		if (noreuse)
3177 			TAILQ_INSERT_BEFORE(&vm_phys_domain(m)->vmd_inacthead,
3178 			    m, plinks.q);
3179 		else
3180 			TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
3181 		vm_pagequeue_cnt_inc(pq);
3182 		vm_pagequeue_unlock(pq);
3183 	}
3184 }
3185 
3186 /*
3187  * Move the specified page to the inactive queue.
3188  *
3189  * The page must be locked.
3190  */
3191 void
3192 vm_page_deactivate(vm_page_t m)
3193 {
3194 
3195 	_vm_page_deactivate(m, FALSE);
3196 }
3197 
3198 /*
3199  * Move the specified page to the inactive queue with the expectation
3200  * that it is unlikely to be reused.
3201  *
3202  * The page must be locked.
3203  */
3204 void
3205 vm_page_deactivate_noreuse(vm_page_t m)
3206 {
3207 
3208 	_vm_page_deactivate(m, TRUE);
3209 }
3210 
3211 /*
3212  * vm_page_launder
3213  *
3214  * 	Put a page in the laundry.
3215  */
3216 void
3217 vm_page_launder(vm_page_t m)
3218 {
3219 	int queue;
3220 
3221 	vm_page_assert_locked(m);
3222 	if ((queue = m->queue) != PQ_LAUNDRY) {
3223 		if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) {
3224 			if (queue != PQ_NONE)
3225 				vm_page_dequeue(m);
3226 			vm_page_enqueue(PQ_LAUNDRY, m);
3227 		} else
3228 			KASSERT(queue == PQ_NONE,
3229 			    ("wired page %p is queued", m));
3230 	}
3231 }
3232 
3233 /*
3234  * vm_page_unswappable
3235  *
3236  *	Put a page in the PQ_UNSWAPPABLE holding queue.
3237  */
3238 void
3239 vm_page_unswappable(vm_page_t m)
3240 {
3241 
3242 	vm_page_assert_locked(m);
3243 	KASSERT(m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0,
3244 	    ("page %p already unswappable", m));
3245 	if (m->queue != PQ_NONE)
3246 		vm_page_dequeue(m);
3247 	vm_page_enqueue(PQ_UNSWAPPABLE, m);
3248 }
3249 
3250 /*
3251  * Attempt to free the page.  If it cannot be freed, do nothing.  Returns true
3252  * if the page is freed and false otherwise.
3253  *
3254  * The page must be managed.  The page and its containing object must be
3255  * locked.
3256  */
3257 bool
3258 vm_page_try_to_free(vm_page_t m)
3259 {
3260 
3261 	vm_page_assert_locked(m);
3262 	VM_OBJECT_ASSERT_WLOCKED(m->object);
3263 	KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("page %p is unmanaged", m));
3264 	if (m->dirty != 0 || m->hold_count != 0 || m->wire_count != 0 ||
3265 	    vm_page_busied(m))
3266 		return (false);
3267 	if (m->object->ref_count != 0) {
3268 		pmap_remove_all(m);
3269 		if (m->dirty != 0)
3270 			return (false);
3271 	}
3272 	vm_page_free(m);
3273 	return (true);
3274 }
3275 
3276 /*
3277  * vm_page_advise
3278  *
3279  * 	Apply the specified advice to the given page.
3280  *
3281  *	The object and page must be locked.
3282  */
3283 void
3284 vm_page_advise(vm_page_t m, int advice)
3285 {
3286 
3287 	vm_page_assert_locked(m);
3288 	VM_OBJECT_ASSERT_WLOCKED(m->object);
3289 	if (advice == MADV_FREE)
3290 		/*
3291 		 * Mark the page clean.  This will allow the page to be freed
3292 		 * without first paging it out.  MADV_FREE pages are often
3293 		 * quickly reused by malloc(3), so we do not do anything that
3294 		 * would result in a page fault on a later access.
3295 		 */
3296 		vm_page_undirty(m);
3297 	else if (advice != MADV_DONTNEED) {
3298 		if (advice == MADV_WILLNEED)
3299 			vm_page_activate(m);
3300 		return;
3301 	}
3302 
3303 	/*
3304 	 * Clear any references to the page.  Otherwise, the page daemon will
3305 	 * immediately reactivate the page.
3306 	 */
3307 	vm_page_aflag_clear(m, PGA_REFERENCED);
3308 
3309 	if (advice != MADV_FREE && m->dirty == 0 && pmap_is_modified(m))
3310 		vm_page_dirty(m);
3311 
3312 	/*
3313 	 * Place clean pages near the head of the inactive queue rather than
3314 	 * the tail, thus defeating the queue's LRU operation and ensuring that
3315 	 * the page will be reused quickly.  Dirty pages not already in the
3316 	 * laundry are moved there.
3317 	 */
3318 	if (m->dirty == 0)
3319 		vm_page_deactivate_noreuse(m);
3320 	else
3321 		vm_page_launder(m);
3322 }
3323 
3324 /*
3325  * Grab a page, waiting until we are waken up due to the page
3326  * changing state.  We keep on waiting, if the page continues
3327  * to be in the object.  If the page doesn't exist, first allocate it
3328  * and then conditionally zero it.
3329  *
3330  * This routine may sleep.
3331  *
3332  * The object must be locked on entry.  The lock will, however, be released
3333  * and reacquired if the routine sleeps.
3334  */
3335 vm_page_t
3336 vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
3337 {
3338 	vm_page_t m;
3339 	int sleep;
3340 	int pflags;
3341 
3342 	VM_OBJECT_ASSERT_WLOCKED(object);
3343 	KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 ||
3344 	    (allocflags & VM_ALLOC_IGN_SBUSY) != 0,
3345 	    ("vm_page_grab: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY mismatch"));
3346 	pflags = allocflags &
3347 	    ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL);
3348 	if ((allocflags & VM_ALLOC_NOWAIT) == 0)
3349 		pflags |= VM_ALLOC_WAITFAIL;
3350 retrylookup:
3351 	if ((m = vm_page_lookup(object, pindex)) != NULL) {
3352 		sleep = (allocflags & VM_ALLOC_IGN_SBUSY) != 0 ?
3353 		    vm_page_xbusied(m) : vm_page_busied(m);
3354 		if (sleep) {
3355 			if ((allocflags & VM_ALLOC_NOWAIT) != 0)
3356 				return (NULL);
3357 			/*
3358 			 * Reference the page before unlocking and
3359 			 * sleeping so that the page daemon is less
3360 			 * likely to reclaim it.
3361 			 */
3362 			vm_page_aflag_set(m, PGA_REFERENCED);
3363 			vm_page_lock(m);
3364 			VM_OBJECT_WUNLOCK(object);
3365 			vm_page_busy_sleep(m, "pgrbwt", (allocflags &
3366 			    VM_ALLOC_IGN_SBUSY) != 0);
3367 			VM_OBJECT_WLOCK(object);
3368 			goto retrylookup;
3369 		} else {
3370 			if ((allocflags & VM_ALLOC_WIRED) != 0) {
3371 				vm_page_lock(m);
3372 				vm_page_wire(m);
3373 				vm_page_unlock(m);
3374 			}
3375 			if ((allocflags &
3376 			    (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) == 0)
3377 				vm_page_xbusy(m);
3378 			if ((allocflags & VM_ALLOC_SBUSY) != 0)
3379 				vm_page_sbusy(m);
3380 			return (m);
3381 		}
3382 	}
3383 	m = vm_page_alloc(object, pindex, pflags);
3384 	if (m == NULL) {
3385 		if ((allocflags & VM_ALLOC_NOWAIT) != 0)
3386 			return (NULL);
3387 		goto retrylookup;
3388 	}
3389 	if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0)
3390 		pmap_zero_page(m);
3391 	return (m);
3392 }
3393 
3394 /*
3395  * Return the specified range of pages from the given object.  For each
3396  * page offset within the range, if a page already exists within the object
3397  * at that offset and it is busy, then wait for it to change state.  If,
3398  * instead, the page doesn't exist, then allocate it.
3399  *
3400  * The caller must always specify an allocation class.
3401  *
3402  * allocation classes:
3403  *	VM_ALLOC_NORMAL		normal process request
3404  *	VM_ALLOC_SYSTEM		system *really* needs the pages
3405  *
3406  * The caller must always specify that the pages are to be busied and/or
3407  * wired.
3408  *
3409  * optional allocation flags:
3410  *	VM_ALLOC_IGN_SBUSY	do not sleep on soft busy pages
3411  *	VM_ALLOC_NOBUSY		do not exclusive busy the page
3412  *	VM_ALLOC_NOWAIT		do not sleep
3413  *	VM_ALLOC_SBUSY		set page to sbusy state
3414  *	VM_ALLOC_WIRED		wire the pages
3415  *	VM_ALLOC_ZERO		zero and validate any invalid pages
3416  *
3417  * If VM_ALLOC_NOWAIT is not specified, this routine may sleep.  Otherwise, it
3418  * may return a partial prefix of the requested range.
3419  */
3420 int
3421 vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags,
3422     vm_page_t *ma, int count)
3423 {
3424 	vm_page_t m, mpred;
3425 	int pflags;
3426 	int i;
3427 	bool sleep;
3428 
3429 	VM_OBJECT_ASSERT_WLOCKED(object);
3430 	KASSERT(((u_int)allocflags >> VM_ALLOC_COUNT_SHIFT) == 0,
3431 	    ("vm_page_grap_pages: VM_ALLOC_COUNT() is not allowed"));
3432 	KASSERT((allocflags & VM_ALLOC_NOBUSY) == 0 ||
3433 	    (allocflags & VM_ALLOC_WIRED) != 0,
3434 	    ("vm_page_grab_pages: the pages must be busied or wired"));
3435 	KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 ||
3436 	    (allocflags & VM_ALLOC_IGN_SBUSY) != 0,
3437 	    ("vm_page_grab_pages: VM_ALLOC_SBUSY/IGN_SBUSY mismatch"));
3438 	if (count == 0)
3439 		return (0);
3440 	pflags = allocflags & ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK |
3441 	    VM_ALLOC_WAITFAIL | VM_ALLOC_IGN_SBUSY);
3442 	if ((allocflags & VM_ALLOC_NOWAIT) == 0)
3443 		pflags |= VM_ALLOC_WAITFAIL;
3444 	i = 0;
3445 retrylookup:
3446 	m = vm_radix_lookup_le(&object->rtree, pindex + i);
3447 	if (m == NULL || m->pindex != pindex + i) {
3448 		mpred = m;
3449 		m = NULL;
3450 	} else
3451 		mpred = TAILQ_PREV(m, pglist, listq);
3452 	for (; i < count; i++) {
3453 		if (m != NULL) {
3454 			sleep = (allocflags & VM_ALLOC_IGN_SBUSY) != 0 ?
3455 			    vm_page_xbusied(m) : vm_page_busied(m);
3456 			if (sleep) {
3457 				if ((allocflags & VM_ALLOC_NOWAIT) != 0)
3458 					break;
3459 				/*
3460 				 * Reference the page before unlocking and
3461 				 * sleeping so that the page daemon is less
3462 				 * likely to reclaim it.
3463 				 */
3464 				vm_page_aflag_set(m, PGA_REFERENCED);
3465 				vm_page_lock(m);
3466 				VM_OBJECT_WUNLOCK(object);
3467 				vm_page_busy_sleep(m, "grbmaw", (allocflags &
3468 				    VM_ALLOC_IGN_SBUSY) != 0);
3469 				VM_OBJECT_WLOCK(object);
3470 				goto retrylookup;
3471 			}
3472 			if ((allocflags & VM_ALLOC_WIRED) != 0) {
3473 				vm_page_lock(m);
3474 				vm_page_wire(m);
3475 				vm_page_unlock(m);
3476 			}
3477 			if ((allocflags & (VM_ALLOC_NOBUSY |
3478 			    VM_ALLOC_SBUSY)) == 0)
3479 				vm_page_xbusy(m);
3480 			if ((allocflags & VM_ALLOC_SBUSY) != 0)
3481 				vm_page_sbusy(m);
3482 		} else {
3483 			m = vm_page_alloc_after(object, pindex + i,
3484 			    pflags | VM_ALLOC_COUNT(count - i), mpred);
3485 			if (m == NULL) {
3486 				if ((allocflags & VM_ALLOC_NOWAIT) != 0)
3487 					break;
3488 				goto retrylookup;
3489 			}
3490 		}
3491 		if (m->valid == 0 && (allocflags & VM_ALLOC_ZERO) != 0) {
3492 			if ((m->flags & PG_ZERO) == 0)
3493 				pmap_zero_page(m);
3494 			m->valid = VM_PAGE_BITS_ALL;
3495 		}
3496 		ma[i] = mpred = m;
3497 		m = vm_page_next(m);
3498 	}
3499 	return (i);
3500 }
3501 
3502 /*
3503  * Mapping function for valid or dirty bits in a page.
3504  *
3505  * Inputs are required to range within a page.
3506  */
3507 vm_page_bits_t
3508 vm_page_bits(int base, int size)
3509 {
3510 	int first_bit;
3511 	int last_bit;
3512 
3513 	KASSERT(
3514 	    base + size <= PAGE_SIZE,
3515 	    ("vm_page_bits: illegal base/size %d/%d", base, size)
3516 	);
3517 
3518 	if (size == 0)		/* handle degenerate case */
3519 		return (0);
3520 
3521 	first_bit = base >> DEV_BSHIFT;
3522 	last_bit = (base + size - 1) >> DEV_BSHIFT;
3523 
3524 	return (((vm_page_bits_t)2 << last_bit) -
3525 	    ((vm_page_bits_t)1 << first_bit));
3526 }
3527 
3528 /*
3529  *	vm_page_set_valid_range:
3530  *
3531  *	Sets portions of a page valid.  The arguments are expected
3532  *	to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive
3533  *	of any partial chunks touched by the range.  The invalid portion of
3534  *	such chunks will be zeroed.
3535  *
3536  *	(base + size) must be less then or equal to PAGE_SIZE.
3537  */
3538 void
3539 vm_page_set_valid_range(vm_page_t m, int base, int size)
3540 {
3541 	int endoff, frag;
3542 
3543 	VM_OBJECT_ASSERT_WLOCKED(m->object);
3544 	if (size == 0)	/* handle degenerate case */
3545 		return;
3546 
3547 	/*
3548 	 * If the base is not DEV_BSIZE aligned and the valid
3549 	 * bit is clear, we have to zero out a portion of the
3550 	 * first block.
3551 	 */
3552 	if ((frag = rounddown2(base, DEV_BSIZE)) != base &&
3553 	    (m->valid & (1 << (base >> DEV_BSHIFT))) == 0)
3554 		pmap_zero_page_area(m, frag, base - frag);
3555 
3556 	/*
3557 	 * If the ending offset is not DEV_BSIZE aligned and the
3558 	 * valid bit is clear, we have to zero out a portion of
3559 	 * the last block.
3560 	 */
3561 	endoff = base + size;
3562 	if ((frag = rounddown2(endoff, DEV_BSIZE)) != endoff &&
3563 	    (m->valid & (1 << (endoff >> DEV_BSHIFT))) == 0)
3564 		pmap_zero_page_area(m, endoff,
3565 		    DEV_BSIZE - (endoff & (DEV_BSIZE - 1)));
3566 
3567 	/*
3568 	 * Assert that no previously invalid block that is now being validated
3569 	 * is already dirty.
3570 	 */
3571 	KASSERT((~m->valid & vm_page_bits(base, size) & m->dirty) == 0,
3572 	    ("vm_page_set_valid_range: page %p is dirty", m));
3573 
3574 	/*
3575 	 * Set valid bits inclusive of any overlap.
3576 	 */
3577 	m->valid |= vm_page_bits(base, size);
3578 }
3579 
3580 /*
3581  * Clear the given bits from the specified page's dirty field.
3582  */
3583 static __inline void
3584 vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits)
3585 {
3586 	uintptr_t addr;
3587 #if PAGE_SIZE < 16384
3588 	int shift;
3589 #endif
3590 
3591 	/*
3592 	 * If the object is locked and the page is neither exclusive busy nor
3593 	 * write mapped, then the page's dirty field cannot possibly be
3594 	 * set by a concurrent pmap operation.
3595 	 */
3596 	VM_OBJECT_ASSERT_WLOCKED(m->object);
3597 	if (!vm_page_xbusied(m) && !pmap_page_is_write_mapped(m))
3598 		m->dirty &= ~pagebits;
3599 	else {
3600 		/*
3601 		 * The pmap layer can call vm_page_dirty() without
3602 		 * holding a distinguished lock.  The combination of
3603 		 * the object's lock and an atomic operation suffice
3604 		 * to guarantee consistency of the page dirty field.
3605 		 *
3606 		 * For PAGE_SIZE == 32768 case, compiler already
3607 		 * properly aligns the dirty field, so no forcible
3608 		 * alignment is needed. Only require existence of
3609 		 * atomic_clear_64 when page size is 32768.
3610 		 */
3611 		addr = (uintptr_t)&m->dirty;
3612 #if PAGE_SIZE == 32768
3613 		atomic_clear_64((uint64_t *)addr, pagebits);
3614 #elif PAGE_SIZE == 16384
3615 		atomic_clear_32((uint32_t *)addr, pagebits);
3616 #else		/* PAGE_SIZE <= 8192 */
3617 		/*
3618 		 * Use a trick to perform a 32-bit atomic on the
3619 		 * containing aligned word, to not depend on the existence
3620 		 * of atomic_clear_{8, 16}.
3621 		 */
3622 		shift = addr & (sizeof(uint32_t) - 1);
3623 #if BYTE_ORDER == BIG_ENDIAN
3624 		shift = (sizeof(uint32_t) - sizeof(m->dirty) - shift) * NBBY;
3625 #else
3626 		shift *= NBBY;
3627 #endif
3628 		addr &= ~(sizeof(uint32_t) - 1);
3629 		atomic_clear_32((uint32_t *)addr, pagebits << shift);
3630 #endif		/* PAGE_SIZE */
3631 	}
3632 }
3633 
3634 /*
3635  *	vm_page_set_validclean:
3636  *
3637  *	Sets portions of a page valid and clean.  The arguments are expected
3638  *	to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive
3639  *	of any partial chunks touched by the range.  The invalid portion of
3640  *	such chunks will be zero'd.
3641  *
3642  *	(base + size) must be less then or equal to PAGE_SIZE.
3643  */
3644 void
3645 vm_page_set_validclean(vm_page_t m, int base, int size)
3646 {
3647 	vm_page_bits_t oldvalid, pagebits;
3648 	int endoff, frag;
3649 
3650 	VM_OBJECT_ASSERT_WLOCKED(m->object);
3651 	if (size == 0)	/* handle degenerate case */
3652 		return;
3653 
3654 	/*
3655 	 * If the base is not DEV_BSIZE aligned and the valid
3656 	 * bit is clear, we have to zero out a portion of the
3657 	 * first block.
3658 	 */
3659 	if ((frag = rounddown2(base, DEV_BSIZE)) != base &&
3660 	    (m->valid & ((vm_page_bits_t)1 << (base >> DEV_BSHIFT))) == 0)
3661 		pmap_zero_page_area(m, frag, base - frag);
3662 
3663 	/*
3664 	 * If the ending offset is not DEV_BSIZE aligned and the
3665 	 * valid bit is clear, we have to zero out a portion of
3666 	 * the last block.
3667 	 */
3668 	endoff = base + size;
3669 	if ((frag = rounddown2(endoff, DEV_BSIZE)) != endoff &&
3670 	    (m->valid & ((vm_page_bits_t)1 << (endoff >> DEV_BSHIFT))) == 0)
3671 		pmap_zero_page_area(m, endoff,
3672 		    DEV_BSIZE - (endoff & (DEV_BSIZE - 1)));
3673 
3674 	/*
3675 	 * Set valid, clear dirty bits.  If validating the entire
3676 	 * page we can safely clear the pmap modify bit.  We also
3677 	 * use this opportunity to clear the VPO_NOSYNC flag.  If a process
3678 	 * takes a write fault on a MAP_NOSYNC memory area the flag will
3679 	 * be set again.
3680 	 *
3681 	 * We set valid bits inclusive of any overlap, but we can only
3682 	 * clear dirty bits for DEV_BSIZE chunks that are fully within
3683 	 * the range.
3684 	 */
3685 	oldvalid = m->valid;
3686 	pagebits = vm_page_bits(base, size);
3687 	m->valid |= pagebits;
3688 #if 0	/* NOT YET */
3689 	if ((frag = base & (DEV_BSIZE - 1)) != 0) {
3690 		frag = DEV_BSIZE - frag;
3691 		base += frag;
3692 		size -= frag;
3693 		if (size < 0)
3694 			size = 0;
3695 	}
3696 	pagebits = vm_page_bits(base, size & (DEV_BSIZE - 1));
3697 #endif
3698 	if (base == 0 && size == PAGE_SIZE) {
3699 		/*
3700 		 * The page can only be modified within the pmap if it is
3701 		 * mapped, and it can only be mapped if it was previously
3702 		 * fully valid.
3703 		 */
3704 		if (oldvalid == VM_PAGE_BITS_ALL)
3705 			/*
3706 			 * Perform the pmap_clear_modify() first.  Otherwise,
3707 			 * a concurrent pmap operation, such as
3708 			 * pmap_protect(), could clear a modification in the
3709 			 * pmap and set the dirty field on the page before
3710 			 * pmap_clear_modify() had begun and after the dirty
3711 			 * field was cleared here.
3712 			 */
3713 			pmap_clear_modify(m);
3714 		m->dirty = 0;
3715 		m->oflags &= ~VPO_NOSYNC;
3716 	} else if (oldvalid != VM_PAGE_BITS_ALL)
3717 		m->dirty &= ~pagebits;
3718 	else
3719 		vm_page_clear_dirty_mask(m, pagebits);
3720 }
3721 
3722 void
3723 vm_page_clear_dirty(vm_page_t m, int base, int size)
3724 {
3725 
3726 	vm_page_clear_dirty_mask(m, vm_page_bits(base, size));
3727 }
3728 
3729 /*
3730  *	vm_page_set_invalid:
3731  *
3732  *	Invalidates DEV_BSIZE'd chunks within a page.  Both the
3733  *	valid and dirty bits for the effected areas are cleared.
3734  */
3735 void
3736 vm_page_set_invalid(vm_page_t m, int base, int size)
3737 {
3738 	vm_page_bits_t bits;
3739 	vm_object_t object;
3740 
3741 	object = m->object;
3742 	VM_OBJECT_ASSERT_WLOCKED(object);
3743 	if (object->type == OBJT_VNODE && base == 0 && IDX_TO_OFF(m->pindex) +
3744 	    size >= object->un_pager.vnp.vnp_size)
3745 		bits = VM_PAGE_BITS_ALL;
3746 	else
3747 		bits = vm_page_bits(base, size);
3748 	if (object->ref_count != 0 && m->valid == VM_PAGE_BITS_ALL &&
3749 	    bits != 0)
3750 		pmap_remove_all(m);
3751 	KASSERT((bits == 0 && m->valid == VM_PAGE_BITS_ALL) ||
3752 	    !pmap_page_is_mapped(m),
3753 	    ("vm_page_set_invalid: page %p is mapped", m));
3754 	m->valid &= ~bits;
3755 	m->dirty &= ~bits;
3756 }
3757 
3758 /*
3759  * vm_page_zero_invalid()
3760  *
3761  *	The kernel assumes that the invalid portions of a page contain
3762  *	garbage, but such pages can be mapped into memory by user code.
3763  *	When this occurs, we must zero out the non-valid portions of the
3764  *	page so user code sees what it expects.
3765  *
3766  *	Pages are most often semi-valid when the end of a file is mapped
3767  *	into memory and the file's size is not page aligned.
3768  */
3769 void
3770 vm_page_zero_invalid(vm_page_t m, boolean_t setvalid)
3771 {
3772 	int b;
3773 	int i;
3774 
3775 	VM_OBJECT_ASSERT_WLOCKED(m->object);
3776 	/*
3777 	 * Scan the valid bits looking for invalid sections that
3778 	 * must be zeroed.  Invalid sub-DEV_BSIZE'd areas ( where the
3779 	 * valid bit may be set ) have already been zeroed by
3780 	 * vm_page_set_validclean().
3781 	 */
3782 	for (b = i = 0; i <= PAGE_SIZE / DEV_BSIZE; ++i) {
3783 		if (i == (PAGE_SIZE / DEV_BSIZE) ||
3784 		    (m->valid & ((vm_page_bits_t)1 << i))) {
3785 			if (i > b) {
3786 				pmap_zero_page_area(m,
3787 				    b << DEV_BSHIFT, (i - b) << DEV_BSHIFT);
3788 			}
3789 			b = i + 1;
3790 		}
3791 	}
3792 
3793 	/*
3794 	 * setvalid is TRUE when we can safely set the zero'd areas
3795 	 * as being valid.  We can do this if there are no cache consistancy
3796 	 * issues.  e.g. it is ok to do with UFS, but not ok to do with NFS.
3797 	 */
3798 	if (setvalid)
3799 		m->valid = VM_PAGE_BITS_ALL;
3800 }
3801 
3802 /*
3803  *	vm_page_is_valid:
3804  *
3805  *	Is (partial) page valid?  Note that the case where size == 0
3806  *	will return FALSE in the degenerate case where the page is
3807  *	entirely invalid, and TRUE otherwise.
3808  */
3809 int
3810 vm_page_is_valid(vm_page_t m, int base, int size)
3811 {
3812 	vm_page_bits_t bits;
3813 
3814 	VM_OBJECT_ASSERT_LOCKED(m->object);
3815 	bits = vm_page_bits(base, size);
3816 	return (m->valid != 0 && (m->valid & bits) == bits);
3817 }
3818 
3819 /*
3820  * Returns true if all of the specified predicates are true for the entire
3821  * (super)page and false otherwise.
3822  */
3823 bool
3824 vm_page_ps_test(vm_page_t m, int flags, vm_page_t skip_m)
3825 {
3826 	vm_object_t object;
3827 	int i, npages;
3828 
3829 	object = m->object;
3830 	VM_OBJECT_ASSERT_LOCKED(object);
3831 	npages = atop(pagesizes[m->psind]);
3832 
3833 	/*
3834 	 * The physically contiguous pages that make up a superpage, i.e., a
3835 	 * page with a page size index ("psind") greater than zero, will
3836 	 * occupy adjacent entries in vm_page_array[].
3837 	 */
3838 	for (i = 0; i < npages; i++) {
3839 		/* Always test object consistency, including "skip_m". */
3840 		if (m[i].object != object)
3841 			return (false);
3842 		if (&m[i] == skip_m)
3843 			continue;
3844 		if ((flags & PS_NONE_BUSY) != 0 && vm_page_busied(&m[i]))
3845 			return (false);
3846 		if ((flags & PS_ALL_DIRTY) != 0) {
3847 			/*
3848 			 * Calling vm_page_test_dirty() or pmap_is_modified()
3849 			 * might stop this case from spuriously returning
3850 			 * "false".  However, that would require a write lock
3851 			 * on the object containing "m[i]".
3852 			 */
3853 			if (m[i].dirty != VM_PAGE_BITS_ALL)
3854 				return (false);
3855 		}
3856 		if ((flags & PS_ALL_VALID) != 0 &&
3857 		    m[i].valid != VM_PAGE_BITS_ALL)
3858 			return (false);
3859 	}
3860 	return (true);
3861 }
3862 
3863 /*
3864  * Set the page's dirty bits if the page is modified.
3865  */
3866 void
3867 vm_page_test_dirty(vm_page_t m)
3868 {
3869 
3870 	VM_OBJECT_ASSERT_WLOCKED(m->object);
3871 	if (m->dirty != VM_PAGE_BITS_ALL && pmap_is_modified(m))
3872 		vm_page_dirty(m);
3873 }
3874 
3875 void
3876 vm_page_lock_KBI(vm_page_t m, const char *file, int line)
3877 {
3878 
3879 	mtx_lock_flags_(vm_page_lockptr(m), 0, file, line);
3880 }
3881 
3882 void
3883 vm_page_unlock_KBI(vm_page_t m, const char *file, int line)
3884 {
3885 
3886 	mtx_unlock_flags_(vm_page_lockptr(m), 0, file, line);
3887 }
3888 
3889 int
3890 vm_page_trylock_KBI(vm_page_t m, const char *file, int line)
3891 {
3892 
3893 	return (mtx_trylock_flags_(vm_page_lockptr(m), 0, file, line));
3894 }
3895 
3896 #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
3897 void
3898 vm_page_assert_locked_KBI(vm_page_t m, const char *file, int line)
3899 {
3900 
3901 	vm_page_lock_assert_KBI(m, MA_OWNED, file, line);
3902 }
3903 
3904 void
3905 vm_page_lock_assert_KBI(vm_page_t m, int a, const char *file, int line)
3906 {
3907 
3908 	mtx_assert_(vm_page_lockptr(m), a, file, line);
3909 }
3910 #endif
3911 
3912 #ifdef INVARIANTS
3913 void
3914 vm_page_object_lock_assert(vm_page_t m)
3915 {
3916 
3917 	/*
3918 	 * Certain of the page's fields may only be modified by the
3919 	 * holder of the containing object's lock or the exclusive busy.
3920 	 * holder.  Unfortunately, the holder of the write busy is
3921 	 * not recorded, and thus cannot be checked here.
3922 	 */
3923 	if (m->object != NULL && !vm_page_xbusied(m))
3924 		VM_OBJECT_ASSERT_WLOCKED(m->object);
3925 }
3926 
3927 void
3928 vm_page_assert_pga_writeable(vm_page_t m, uint8_t bits)
3929 {
3930 
3931 	if ((bits & PGA_WRITEABLE) == 0)
3932 		return;
3933 
3934 	/*
3935 	 * The PGA_WRITEABLE flag can only be set if the page is
3936 	 * managed, is exclusively busied or the object is locked.
3937 	 * Currently, this flag is only set by pmap_enter().
3938 	 */
3939 	KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3940 	    ("PGA_WRITEABLE on unmanaged page"));
3941 	if (!vm_page_xbusied(m))
3942 		VM_OBJECT_ASSERT_LOCKED(m->object);
3943 }
3944 #endif
3945 
3946 #include "opt_ddb.h"
3947 #ifdef DDB
3948 #include <sys/kernel.h>
3949 
3950 #include <ddb/ddb.h>
3951 
3952 DB_SHOW_COMMAND(page, vm_page_print_page_info)
3953 {
3954 
3955 	db_printf("vm_cnt.v_free_count: %d\n", vm_cnt.v_free_count);
3956 	db_printf("vm_cnt.v_inactive_count: %d\n", vm_cnt.v_inactive_count);
3957 	db_printf("vm_cnt.v_active_count: %d\n", vm_cnt.v_active_count);
3958 	db_printf("vm_cnt.v_laundry_count: %d\n", vm_cnt.v_laundry_count);
3959 	db_printf("vm_cnt.v_wire_count: %d\n", vm_cnt.v_wire_count);
3960 	db_printf("vm_cnt.v_free_reserved: %d\n", vm_cnt.v_free_reserved);
3961 	db_printf("vm_cnt.v_free_min: %d\n", vm_cnt.v_free_min);
3962 	db_printf("vm_cnt.v_free_target: %d\n", vm_cnt.v_free_target);
3963 	db_printf("vm_cnt.v_inactive_target: %d\n", vm_cnt.v_inactive_target);
3964 }
3965 
3966 DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info)
3967 {
3968 	int dom;
3969 
3970 	db_printf("pq_free %d\n", vm_cnt.v_free_count);
3971 	for (dom = 0; dom < vm_ndomains; dom++) {
3972 		db_printf(
3973     "dom %d page_cnt %d free %d pq_act %d pq_inact %d pq_laund %d pq_unsw %d\n",
3974 		    dom,
3975 		    vm_dom[dom].vmd_page_count,
3976 		    vm_dom[dom].vmd_free_count,
3977 		    vm_dom[dom].vmd_pagequeues[PQ_ACTIVE].pq_cnt,
3978 		    vm_dom[dom].vmd_pagequeues[PQ_INACTIVE].pq_cnt,
3979 		    vm_dom[dom].vmd_pagequeues[PQ_LAUNDRY].pq_cnt,
3980 		    vm_dom[dom].vmd_pagequeues[PQ_UNSWAPPABLE].pq_cnt);
3981 	}
3982 }
3983 
3984 DB_SHOW_COMMAND(pginfo, vm_page_print_pginfo)
3985 {
3986 	vm_page_t m;
3987 	boolean_t phys;
3988 
3989 	if (!have_addr) {
3990 		db_printf("show pginfo addr\n");
3991 		return;
3992 	}
3993 
3994 	phys = strchr(modif, 'p') != NULL;
3995 	if (phys)
3996 		m = PHYS_TO_VM_PAGE(addr);
3997 	else
3998 		m = (vm_page_t)addr;
3999 	db_printf(
4000     "page %p obj %p pidx 0x%jx phys 0x%jx q %d hold %d wire %d\n"
4001     "  af 0x%x of 0x%x f 0x%x act %d busy %x valid 0x%x dirty 0x%x\n",
4002 	    m, m->object, (uintmax_t)m->pindex, (uintmax_t)m->phys_addr,
4003 	    m->queue, m->hold_count, m->wire_count, m->aflags, m->oflags,
4004 	    m->flags, m->act_count, m->busy_lock, m->valid, m->dirty);
4005 }
4006 #endif /* DDB */
4007