xref: /freebsd/sys/vm/vm_page.c (revision 1e413cf93298b5b97441a21d9a50fdcd0ee9945e)
1 /*-
2  * Copyright (c) 1991 Regents of the University of California.
3  * All rights reserved.
4  *
5  * This code is derived from software contributed to Berkeley by
6  * The Mach Operating System project at Carnegie-Mellon University.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  * 4. Neither the name of the University nor the names of its contributors
17  *    may be used to endorse or promote products derived from this software
18  *    without specific prior written permission.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30  * SUCH DAMAGE.
31  *
32  *	from: @(#)vm_page.c	7.4 (Berkeley) 5/7/91
33  */
34 
35 /*-
36  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
37  * All rights reserved.
38  *
39  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
40  *
41  * Permission to use, copy, modify and distribute this software and
42  * its documentation is hereby granted, provided that both the copyright
43  * notice and this permission notice appear in all copies of the
44  * software, derivative works or modified versions, and any portions
45  * thereof, and that both notices appear in supporting documentation.
46  *
47  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
48  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
49  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
50  *
51  * Carnegie Mellon requests users of this software to return to
52  *
53  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
54  *  School of Computer Science
55  *  Carnegie Mellon University
56  *  Pittsburgh PA 15213-3890
57  *
58  * any improvements or extensions that they make and grant Carnegie the
59  * rights to redistribute these changes.
60  */
61 
62 /*
63  *			GENERAL RULES ON VM_PAGE MANIPULATION
64  *
65  *	- a pageq mutex is required when adding or removing a page from a
66  *	  page queue (vm_page_queue[]), regardless of other mutexes or the
67  *	  busy state of a page.
68  *
69  *	- a hash chain mutex is required when associating or disassociating
70  *	  a page from the VM PAGE CACHE hash table (vm_page_buckets),
71  *	  regardless of other mutexes or the busy state of a page.
72  *
73  *	- either a hash chain mutex OR a busied page is required in order
74  *	  to modify the page flags.  A hash chain mutex must be obtained in
75  *	  order to busy a page.  A page's flags cannot be modified by a
76  *	  hash chain mutex if the page is marked busy.
77  *
78  *	- The object memq mutex is held when inserting or removing
79  *	  pages from an object (vm_page_insert() or vm_page_remove()).  This
80  *	  is different from the object's main mutex.
81  *
82  *	Generally speaking, you have to be aware of side effects when running
83  *	vm_page ops.  A vm_page_lookup() will return with the hash chain
84  *	locked, whether it was able to lookup the page or not.  vm_page_free(),
85  *	vm_page_cache(), vm_page_activate(), and a number of other routines
86  *	will release the hash chain mutex for you.  Intermediate manipulation
87  *	routines such as vm_page_flag_set() expect the hash chain to be held
88  *	on entry and the hash chain will remain held on return.
89  *
90  *	pageq scanning can only occur with the pageq in question locked.
91  *	We have a known bottleneck with the active queue, but the cache
92  *	and free queues are actually arrays already.
93  */
94 
95 /*
96  *	Resident memory management module.
97  */
98 
99 #include <sys/cdefs.h>
100 __FBSDID("$FreeBSD$");
101 
102 #include "opt_vm.h"
103 
104 #include <sys/param.h>
105 #include <sys/systm.h>
106 #include <sys/lock.h>
107 #include <sys/kernel.h>
108 #include <sys/malloc.h>
109 #include <sys/mutex.h>
110 #include <sys/proc.h>
111 #include <sys/sysctl.h>
112 #include <sys/vmmeter.h>
113 #include <sys/vnode.h>
114 
115 #include <vm/vm.h>
116 #include <vm/vm_param.h>
117 #include <vm/vm_kern.h>
118 #include <vm/vm_object.h>
119 #include <vm/vm_page.h>
120 #include <vm/vm_pageout.h>
121 #include <vm/vm_pager.h>
122 #include <vm/vm_phys.h>
123 #include <vm/vm_reserv.h>
124 #include <vm/vm_extern.h>
125 #include <vm/uma.h>
126 #include <vm/uma_int.h>
127 
128 #include <machine/md_var.h>
129 
130 /*
131  *	Associated with page of user-allocatable memory is a
132  *	page structure.
133  */
134 
135 struct mtx vm_page_queue_mtx;
136 struct mtx vm_page_queue_free_mtx;
137 
138 vm_page_t vm_page_array = 0;
139 int vm_page_array_size = 0;
140 long first_page = 0;
141 int vm_page_zero_count = 0;
142 
143 static int boot_pages = UMA_BOOT_PAGES;
144 TUNABLE_INT("vm.boot_pages", &boot_pages);
145 SYSCTL_INT(_vm, OID_AUTO, boot_pages, CTLFLAG_RD, &boot_pages, 0,
146 	"number of pages allocated for bootstrapping the VM system");
147 
148 /*
149  *	vm_set_page_size:
150  *
151  *	Sets the page size, perhaps based upon the memory
152  *	size.  Must be called before any use of page-size
153  *	dependent functions.
154  */
155 void
156 vm_set_page_size(void)
157 {
158 	if (cnt.v_page_size == 0)
159 		cnt.v_page_size = PAGE_SIZE;
160 	if (((cnt.v_page_size - 1) & cnt.v_page_size) != 0)
161 		panic("vm_set_page_size: page size not a power of two");
162 }
163 
164 /*
165  *	vm_page_blacklist_lookup:
166  *
167  *	See if a physical address in this page has been listed
168  *	in the blacklist tunable.  Entries in the tunable are
169  *	separated by spaces or commas.  If an invalid integer is
170  *	encountered then the rest of the string is skipped.
171  */
172 static int
173 vm_page_blacklist_lookup(char *list, vm_paddr_t pa)
174 {
175 	vm_paddr_t bad;
176 	char *cp, *pos;
177 
178 	for (pos = list; *pos != '\0'; pos = cp) {
179 		bad = strtoq(pos, &cp, 0);
180 		if (*cp != '\0') {
181 			if (*cp == ' ' || *cp == ',') {
182 				cp++;
183 				if (cp == pos)
184 					continue;
185 			} else
186 				break;
187 		}
188 		if (pa == trunc_page(bad))
189 			return (1);
190 	}
191 	return (0);
192 }
193 
194 /*
195  *	vm_page_startup:
196  *
197  *	Initializes the resident memory module.
198  *
199  *	Allocates memory for the page cells, and
200  *	for the object/offset-to-page hash table headers.
201  *	Each page cell is initialized and placed on the free list.
202  */
203 vm_offset_t
204 vm_page_startup(vm_offset_t vaddr)
205 {
206 	vm_offset_t mapped;
207 	vm_paddr_t page_range;
208 	vm_paddr_t new_end;
209 	int i;
210 	vm_paddr_t pa;
211 	int nblocks;
212 	vm_paddr_t last_pa;
213 	char *list;
214 
215 	/* the biggest memory array is the second group of pages */
216 	vm_paddr_t end;
217 	vm_paddr_t biggestsize;
218 	vm_paddr_t low_water, high_water;
219 	int biggestone;
220 
221 	biggestsize = 0;
222 	biggestone = 0;
223 	nblocks = 0;
224 	vaddr = round_page(vaddr);
225 
226 	for (i = 0; phys_avail[i + 1]; i += 2) {
227 		phys_avail[i] = round_page(phys_avail[i]);
228 		phys_avail[i + 1] = trunc_page(phys_avail[i + 1]);
229 	}
230 
231 	low_water = phys_avail[0];
232 	high_water = phys_avail[1];
233 
234 	for (i = 0; phys_avail[i + 1]; i += 2) {
235 		vm_paddr_t size = phys_avail[i + 1] - phys_avail[i];
236 
237 		if (size > biggestsize) {
238 			biggestone = i;
239 			biggestsize = size;
240 		}
241 		if (phys_avail[i] < low_water)
242 			low_water = phys_avail[i];
243 		if (phys_avail[i + 1] > high_water)
244 			high_water = phys_avail[i + 1];
245 		++nblocks;
246 	}
247 
248 	end = phys_avail[biggestone+1];
249 
250 	/*
251 	 * Initialize the locks.
252 	 */
253 	mtx_init(&vm_page_queue_mtx, "vm page queue mutex", NULL, MTX_DEF |
254 	    MTX_RECURSE);
255 	mtx_init(&vm_page_queue_free_mtx, "vm page queue free mutex", NULL,
256 	    MTX_DEF);
257 
258 	/*
259 	 * Initialize the queue headers for the hold queue, the active queue,
260 	 * and the inactive queue.
261 	 */
262 	vm_pageq_init();
263 
264 	/*
265 	 * Allocate memory for use when boot strapping the kernel memory
266 	 * allocator.
267 	 */
268 	new_end = end - (boot_pages * UMA_SLAB_SIZE);
269 	new_end = trunc_page(new_end);
270 	mapped = pmap_map(&vaddr, new_end, end,
271 	    VM_PROT_READ | VM_PROT_WRITE);
272 	bzero((void *)mapped, end - new_end);
273 	uma_startup((void *)mapped, boot_pages);
274 
275 #if defined(__amd64__) || defined(__i386__)
276 	/*
277 	 * Allocate a bitmap to indicate that a random physical page
278 	 * needs to be included in a minidump.
279 	 *
280 	 * The amd64 port needs this to indicate which direct map pages
281 	 * need to be dumped, via calls to dump_add_page()/dump_drop_page().
282 	 *
283 	 * However, i386 still needs this workspace internally within the
284 	 * minidump code.  In theory, they are not needed on i386, but are
285 	 * included should the sf_buf code decide to use them.
286 	 */
287 	page_range = phys_avail[(nblocks - 1) * 2 + 1] / PAGE_SIZE;
288 	vm_page_dump_size = round_page(roundup2(page_range, NBBY) / NBBY);
289 	new_end -= vm_page_dump_size;
290 	vm_page_dump = (void *)(uintptr_t)pmap_map(&vaddr, new_end,
291 	    new_end + vm_page_dump_size, VM_PROT_READ | VM_PROT_WRITE);
292 	bzero((void *)vm_page_dump, vm_page_dump_size);
293 #endif
294 	/*
295 	 * Compute the number of pages of memory that will be available for
296 	 * use (taking into account the overhead of a page structure per
297 	 * page).
298 	 */
299 	first_page = low_water / PAGE_SIZE;
300 #ifdef VM_PHYSSEG_SPARSE
301 	page_range = 0;
302 	for (i = 0; phys_avail[i + 1] != 0; i += 2)
303 		page_range += atop(phys_avail[i + 1] - phys_avail[i]);
304 #elif defined(VM_PHYSSEG_DENSE)
305 	page_range = high_water / PAGE_SIZE - first_page;
306 #else
307 #error "Either VM_PHYSSEG_DENSE or VM_PHYSSEG_SPARSE must be defined."
308 #endif
309 	end = new_end;
310 
311 	/*
312 	 * Reserve an unmapped guard page to trap access to vm_page_array[-1].
313 	 */
314 	vaddr += PAGE_SIZE;
315 
316 	/*
317 	 * Initialize the mem entry structures now, and put them in the free
318 	 * queue.
319 	 */
320 	new_end = trunc_page(end - page_range * sizeof(struct vm_page));
321 	mapped = pmap_map(&vaddr, new_end, end,
322 	    VM_PROT_READ | VM_PROT_WRITE);
323 	vm_page_array = (vm_page_t) mapped;
324 #if VM_NRESERVLEVEL > 0
325 	/*
326 	 * Allocate memory for the reservation management system's data
327 	 * structures.
328 	 */
329 	new_end = vm_reserv_startup(&vaddr, new_end, high_water);
330 #endif
331 #ifdef __amd64__
332 	/*
333 	 * pmap_map on amd64 comes out of the direct-map, not kvm like i386,
334 	 * so the pages must be tracked for a crashdump to include this data.
335 	 * This includes the vm_page_array and the early UMA bootstrap pages.
336 	 */
337 	for (pa = new_end; pa < phys_avail[biggestone + 1]; pa += PAGE_SIZE)
338 		dump_add_page(pa);
339 #endif
340 	phys_avail[biggestone + 1] = new_end;
341 
342 	/*
343 	 * Clear all of the page structures
344 	 */
345 	bzero((caddr_t) vm_page_array, page_range * sizeof(struct vm_page));
346 	for (i = 0; i < page_range; i++)
347 		vm_page_array[i].order = VM_NFREEORDER;
348 	vm_page_array_size = page_range;
349 
350 	/*
351 	 * Initialize the physical memory allocator.
352 	 */
353 	vm_phys_init();
354 
355 	/*
356 	 * Add every available physical page that is not blacklisted to
357 	 * the free lists.
358 	 */
359 	cnt.v_page_count = 0;
360 	cnt.v_free_count = 0;
361 	list = getenv("vm.blacklist");
362 	for (i = 0; phys_avail[i + 1] != 0; i += 2) {
363 		pa = phys_avail[i];
364 		last_pa = phys_avail[i + 1];
365 		while (pa < last_pa) {
366 			if (list != NULL &&
367 			    vm_page_blacklist_lookup(list, pa))
368 				printf("Skipping page with pa 0x%jx\n",
369 				    (uintmax_t)pa);
370 			else
371 				vm_phys_add_page(pa);
372 			pa += PAGE_SIZE;
373 		}
374 	}
375 	freeenv(list);
376 #if VM_NRESERVLEVEL > 0
377 	/*
378 	 * Initialize the reservation management system.
379 	 */
380 	vm_reserv_init();
381 #endif
382 	return (vaddr);
383 }
384 
385 void
386 vm_page_flag_set(vm_page_t m, unsigned short bits)
387 {
388 
389 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
390 	m->flags |= bits;
391 }
392 
393 void
394 vm_page_flag_clear(vm_page_t m, unsigned short bits)
395 {
396 
397 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
398 	m->flags &= ~bits;
399 }
400 
401 void
402 vm_page_busy(vm_page_t m)
403 {
404 
405 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
406 	KASSERT((m->oflags & VPO_BUSY) == 0,
407 	    ("vm_page_busy: page already busy!!!"));
408 	m->oflags |= VPO_BUSY;
409 }
410 
411 /*
412  *      vm_page_flash:
413  *
414  *      wakeup anyone waiting for the page.
415  */
416 void
417 vm_page_flash(vm_page_t m)
418 {
419 
420 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
421 	if (m->oflags & VPO_WANTED) {
422 		m->oflags &= ~VPO_WANTED;
423 		wakeup(m);
424 	}
425 }
426 
427 /*
428  *      vm_page_wakeup:
429  *
430  *      clear the VPO_BUSY flag and wakeup anyone waiting for the
431  *      page.
432  *
433  */
434 void
435 vm_page_wakeup(vm_page_t m)
436 {
437 
438 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
439 	KASSERT(m->oflags & VPO_BUSY, ("vm_page_wakeup: page not busy!!!"));
440 	m->oflags &= ~VPO_BUSY;
441 	vm_page_flash(m);
442 }
443 
444 void
445 vm_page_io_start(vm_page_t m)
446 {
447 
448 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
449 	m->busy++;
450 }
451 
452 void
453 vm_page_io_finish(vm_page_t m)
454 {
455 
456 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
457 	m->busy--;
458 	if (m->busy == 0)
459 		vm_page_flash(m);
460 }
461 
462 /*
463  * Keep page from being freed by the page daemon
464  * much of the same effect as wiring, except much lower
465  * overhead and should be used only for *very* temporary
466  * holding ("wiring").
467  */
468 void
469 vm_page_hold(vm_page_t mem)
470 {
471 
472 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
473         mem->hold_count++;
474 }
475 
476 void
477 vm_page_unhold(vm_page_t mem)
478 {
479 
480 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
481 	--mem->hold_count;
482 	KASSERT(mem->hold_count >= 0, ("vm_page_unhold: hold count < 0!!!"));
483 	if (mem->hold_count == 0 && VM_PAGE_INQUEUE2(mem, PQ_HOLD))
484 		vm_page_free_toq(mem);
485 }
486 
487 /*
488  *	vm_page_free:
489  *
490  *	Free a page.
491  */
492 void
493 vm_page_free(vm_page_t m)
494 {
495 
496 	m->flags &= ~PG_ZERO;
497 	vm_page_free_toq(m);
498 }
499 
500 /*
501  *	vm_page_free_zero:
502  *
503  *	Free a page to the zerod-pages queue
504  */
505 void
506 vm_page_free_zero(vm_page_t m)
507 {
508 
509 	m->flags |= PG_ZERO;
510 	vm_page_free_toq(m);
511 }
512 
513 /*
514  *	vm_page_sleep:
515  *
516  *	Sleep and release the page queues lock.
517  *
518  *	The object containing the given page must be locked.
519  */
520 void
521 vm_page_sleep(vm_page_t m, const char *msg)
522 {
523 
524 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
525 	if (!mtx_owned(&vm_page_queue_mtx))
526 		vm_page_lock_queues();
527 	vm_page_flag_set(m, PG_REFERENCED);
528 	vm_page_unlock_queues();
529 
530 	/*
531 	 * It's possible that while we sleep, the page will get
532 	 * unbusied and freed.  If we are holding the object
533 	 * lock, we will assume we hold a reference to the object
534 	 * such that even if m->object changes, we can re-lock
535 	 * it.
536 	 */
537 	m->oflags |= VPO_WANTED;
538 	msleep(m, VM_OBJECT_MTX(m->object), PVM, msg, 0);
539 }
540 
541 /*
542  *	vm_page_dirty:
543  *
544  *	make page all dirty
545  */
546 void
547 vm_page_dirty(vm_page_t m)
548 {
549 	KASSERT((m->flags & PG_CACHED) == 0,
550 	    ("vm_page_dirty: page in cache!"));
551 	KASSERT(!VM_PAGE_IS_FREE(m),
552 	    ("vm_page_dirty: page is free!"));
553 	m->dirty = VM_PAGE_BITS_ALL;
554 }
555 
556 /*
557  *	vm_page_splay:
558  *
559  *	Implements Sleator and Tarjan's top-down splay algorithm.  Returns
560  *	the vm_page containing the given pindex.  If, however, that
561  *	pindex is not found in the vm_object, returns a vm_page that is
562  *	adjacent to the pindex, coming before or after it.
563  */
564 vm_page_t
565 vm_page_splay(vm_pindex_t pindex, vm_page_t root)
566 {
567 	struct vm_page dummy;
568 	vm_page_t lefttreemax, righttreemin, y;
569 
570 	if (root == NULL)
571 		return (root);
572 	lefttreemax = righttreemin = &dummy;
573 	for (;; root = y) {
574 		if (pindex < root->pindex) {
575 			if ((y = root->left) == NULL)
576 				break;
577 			if (pindex < y->pindex) {
578 				/* Rotate right. */
579 				root->left = y->right;
580 				y->right = root;
581 				root = y;
582 				if ((y = root->left) == NULL)
583 					break;
584 			}
585 			/* Link into the new root's right tree. */
586 			righttreemin->left = root;
587 			righttreemin = root;
588 		} else if (pindex > root->pindex) {
589 			if ((y = root->right) == NULL)
590 				break;
591 			if (pindex > y->pindex) {
592 				/* Rotate left. */
593 				root->right = y->left;
594 				y->left = root;
595 				root = y;
596 				if ((y = root->right) == NULL)
597 					break;
598 			}
599 			/* Link into the new root's left tree. */
600 			lefttreemax->right = root;
601 			lefttreemax = root;
602 		} else
603 			break;
604 	}
605 	/* Assemble the new root. */
606 	lefttreemax->right = root->left;
607 	righttreemin->left = root->right;
608 	root->left = dummy.right;
609 	root->right = dummy.left;
610 	return (root);
611 }
612 
613 /*
614  *	vm_page_insert:		[ internal use only ]
615  *
616  *	Inserts the given mem entry into the object and object list.
617  *
618  *	The pagetables are not updated but will presumably fault the page
619  *	in if necessary, or if a kernel page the caller will at some point
620  *	enter the page into the kernel's pmap.  We are not allowed to block
621  *	here so we *can't* do this anyway.
622  *
623  *	The object and page must be locked.
624  *	This routine may not block.
625  */
626 void
627 vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)
628 {
629 	vm_page_t root;
630 
631 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
632 	if (m->object != NULL)
633 		panic("vm_page_insert: page already inserted");
634 
635 	/*
636 	 * Record the object/offset pair in this page
637 	 */
638 	m->object = object;
639 	m->pindex = pindex;
640 
641 	/*
642 	 * Now link into the object's ordered list of backed pages.
643 	 */
644 	root = object->root;
645 	if (root == NULL) {
646 		m->left = NULL;
647 		m->right = NULL;
648 		TAILQ_INSERT_TAIL(&object->memq, m, listq);
649 	} else {
650 		root = vm_page_splay(pindex, root);
651 		if (pindex < root->pindex) {
652 			m->left = root->left;
653 			m->right = root;
654 			root->left = NULL;
655 			TAILQ_INSERT_BEFORE(root, m, listq);
656 		} else if (pindex == root->pindex)
657 			panic("vm_page_insert: offset already allocated");
658 		else {
659 			m->right = root->right;
660 			m->left = root;
661 			root->right = NULL;
662 			TAILQ_INSERT_AFTER(&object->memq, root, m, listq);
663 		}
664 	}
665 	object->root = m;
666 	object->generation++;
667 
668 	/*
669 	 * show that the object has one more resident page.
670 	 */
671 	object->resident_page_count++;
672 	/*
673 	 * Hold the vnode until the last page is released.
674 	 */
675 	if (object->resident_page_count == 1 && object->type == OBJT_VNODE)
676 		vhold((struct vnode *)object->handle);
677 
678 	/*
679 	 * Since we are inserting a new and possibly dirty page,
680 	 * update the object's OBJ_MIGHTBEDIRTY flag.
681 	 */
682 	if (m->flags & PG_WRITEABLE)
683 		vm_object_set_writeable_dirty(object);
684 }
685 
686 /*
687  *	vm_page_remove:
688  *				NOTE: used by device pager as well -wfj
689  *
690  *	Removes the given mem entry from the object/offset-page
691  *	table and the object page list, but do not invalidate/terminate
692  *	the backing store.
693  *
694  *	The object and page must be locked.
695  *	The underlying pmap entry (if any) is NOT removed here.
696  *	This routine may not block.
697  */
698 void
699 vm_page_remove(vm_page_t m)
700 {
701 	vm_object_t object;
702 	vm_page_t root;
703 
704 	if ((object = m->object) == NULL)
705 		return;
706 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
707 	if (m->oflags & VPO_BUSY) {
708 		m->oflags &= ~VPO_BUSY;
709 		vm_page_flash(m);
710 	}
711 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
712 
713 	/*
714 	 * Now remove from the object's list of backed pages.
715 	 */
716 	if (m != object->root)
717 		vm_page_splay(m->pindex, object->root);
718 	if (m->left == NULL)
719 		root = m->right;
720 	else {
721 		root = vm_page_splay(m->pindex, m->left);
722 		root->right = m->right;
723 	}
724 	object->root = root;
725 	TAILQ_REMOVE(&object->memq, m, listq);
726 
727 	/*
728 	 * And show that the object has one fewer resident page.
729 	 */
730 	object->resident_page_count--;
731 	object->generation++;
732 	/*
733 	 * The vnode may now be recycled.
734 	 */
735 	if (object->resident_page_count == 0 && object->type == OBJT_VNODE)
736 		vdrop((struct vnode *)object->handle);
737 
738 	m->object = NULL;
739 }
740 
741 /*
742  *	vm_page_lookup:
743  *
744  *	Returns the page associated with the object/offset
745  *	pair specified; if none is found, NULL is returned.
746  *
747  *	The object must be locked.
748  *	This routine may not block.
749  *	This is a critical path routine
750  */
751 vm_page_t
752 vm_page_lookup(vm_object_t object, vm_pindex_t pindex)
753 {
754 	vm_page_t m;
755 
756 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
757 	if ((m = object->root) != NULL && m->pindex != pindex) {
758 		m = vm_page_splay(pindex, m);
759 		if ((object->root = m)->pindex != pindex)
760 			m = NULL;
761 	}
762 	return (m);
763 }
764 
765 /*
766  *	vm_page_rename:
767  *
768  *	Move the given memory entry from its
769  *	current object to the specified target object/offset.
770  *
771  *	The object must be locked.
772  *	This routine may not block.
773  *
774  *	Note: swap associated with the page must be invalidated by the move.  We
775  *	      have to do this for several reasons:  (1) we aren't freeing the
776  *	      page, (2) we are dirtying the page, (3) the VM system is probably
777  *	      moving the page from object A to B, and will then later move
778  *	      the backing store from A to B and we can't have a conflict.
779  *
780  *	Note: we *always* dirty the page.  It is necessary both for the
781  *	      fact that we moved it, and because we may be invalidating
782  *	      swap.  If the page is on the cache, we have to deactivate it
783  *	      or vm_page_dirty() will panic.  Dirty pages are not allowed
784  *	      on the cache.
785  */
786 void
787 vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex)
788 {
789 
790 	vm_page_remove(m);
791 	vm_page_insert(m, new_object, new_pindex);
792 	vm_page_dirty(m);
793 }
794 
795 /*
796  *	Convert all of the given object's cached pages that have a
797  *	pindex within the given range into free pages.  If the value
798  *	zero is given for "end", then the range's upper bound is
799  *	infinity.  If the given object is backed by a vnode and it
800  *	transitions from having one or more cached pages to none, the
801  *	vnode's hold count is reduced.
802  */
803 void
804 vm_page_cache_free(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
805 {
806 	vm_page_t m, m_next;
807 	boolean_t empty;
808 
809 	mtx_lock(&vm_page_queue_free_mtx);
810 	if (__predict_false(object->cache == NULL)) {
811 		mtx_unlock(&vm_page_queue_free_mtx);
812 		return;
813 	}
814 	m = object->cache = vm_page_splay(start, object->cache);
815 	if (m->pindex < start) {
816 		if (m->right == NULL)
817 			m = NULL;
818 		else {
819 			m_next = vm_page_splay(start, m->right);
820 			m_next->left = m;
821 			m->right = NULL;
822 			m = object->cache = m_next;
823 		}
824 	}
825 
826 	/*
827 	 * At this point, "m" is either (1) a reference to the page
828 	 * with the least pindex that is greater than or equal to
829 	 * "start" or (2) NULL.
830 	 */
831 	for (; m != NULL && (m->pindex < end || end == 0); m = m_next) {
832 		/*
833 		 * Find "m"'s successor and remove "m" from the
834 		 * object's cache.
835 		 */
836 		if (m->right == NULL) {
837 			object->cache = m->left;
838 			m_next = NULL;
839 		} else {
840 			m_next = vm_page_splay(start, m->right);
841 			m_next->left = m->left;
842 			object->cache = m_next;
843 		}
844 		/* Convert "m" to a free page. */
845 		m->object = NULL;
846 		m->valid = 0;
847 		/* Clear PG_CACHED and set PG_FREE. */
848 		m->flags ^= PG_CACHED | PG_FREE;
849 		KASSERT((m->flags & (PG_CACHED | PG_FREE)) == PG_FREE,
850 		    ("vm_page_cache_free: page %p has inconsistent flags", m));
851 		cnt.v_cache_count--;
852 		cnt.v_free_count++;
853 	}
854 	empty = object->cache == NULL;
855 	mtx_unlock(&vm_page_queue_free_mtx);
856 	if (object->type == OBJT_VNODE && empty)
857 		vdrop(object->handle);
858 }
859 
860 /*
861  *	Returns the cached page that is associated with the given
862  *	object and offset.  If, however, none exists, returns NULL.
863  *
864  *	The free page queue must be locked.
865  */
866 static inline vm_page_t
867 vm_page_cache_lookup(vm_object_t object, vm_pindex_t pindex)
868 {
869 	vm_page_t m;
870 
871 	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
872 	if ((m = object->cache) != NULL && m->pindex != pindex) {
873 		m = vm_page_splay(pindex, m);
874 		if ((object->cache = m)->pindex != pindex)
875 			m = NULL;
876 	}
877 	return (m);
878 }
879 
880 /*
881  *	Remove the given cached page from its containing object's
882  *	collection of cached pages.
883  *
884  *	The free page queue must be locked.
885  */
886 void
887 vm_page_cache_remove(vm_page_t m)
888 {
889 	vm_object_t object;
890 	vm_page_t root;
891 
892 	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
893 	KASSERT((m->flags & PG_CACHED) != 0,
894 	    ("vm_page_cache_remove: page %p is not cached", m));
895 	object = m->object;
896 	if (m != object->cache) {
897 		root = vm_page_splay(m->pindex, object->cache);
898 		KASSERT(root == m,
899 		    ("vm_page_cache_remove: page %p is not cached in object %p",
900 		    m, object));
901 	}
902 	if (m->left == NULL)
903 		root = m->right;
904 	else if (m->right == NULL)
905 		root = m->left;
906 	else {
907 		root = vm_page_splay(m->pindex, m->left);
908 		root->right = m->right;
909 	}
910 	object->cache = root;
911 	m->object = NULL;
912 	cnt.v_cache_count--;
913 }
914 
915 /*
916  *	Transfer all of the cached pages with offset greater than or
917  *	equal to 'offidxstart' from the original object's cache to the
918  *	new object's cache.  However, any cached pages with offset
919  *	greater than or equal to the new object's size are kept in the
920  *	original object.  Initially, the new object's cache must be
921  *	empty.  Offset 'offidxstart' in the original object must
922  *	correspond to offset zero in the new object.
923  *
924  *	The new object must be locked.
925  */
926 void
927 vm_page_cache_transfer(vm_object_t orig_object, vm_pindex_t offidxstart,
928     vm_object_t new_object)
929 {
930 	vm_page_t m, m_next;
931 
932 	/*
933 	 * Insertion into an object's collection of cached pages
934 	 * requires the object to be locked.  In contrast, removal does
935 	 * not.
936 	 */
937 	VM_OBJECT_LOCK_ASSERT(new_object, MA_OWNED);
938 	KASSERT(new_object->cache == NULL,
939 	    ("vm_page_cache_transfer: object %p has cached pages",
940 	    new_object));
941 	mtx_lock(&vm_page_queue_free_mtx);
942 	if ((m = orig_object->cache) != NULL) {
943 		/*
944 		 * Transfer all of the pages with offset greater than or
945 		 * equal to 'offidxstart' from the original object's
946 		 * cache to the new object's cache.
947 		 */
948 		m = vm_page_splay(offidxstart, m);
949 		if (m->pindex < offidxstart) {
950 			orig_object->cache = m;
951 			new_object->cache = m->right;
952 			m->right = NULL;
953 		} else {
954 			orig_object->cache = m->left;
955 			new_object->cache = m;
956 			m->left = NULL;
957 		}
958 		while ((m = new_object->cache) != NULL) {
959 			if ((m->pindex - offidxstart) >= new_object->size) {
960 				/*
961 				 * Return all of the cached pages with
962 				 * offset greater than or equal to the
963 				 * new object's size to the original
964 				 * object's cache.
965 				 */
966 				new_object->cache = m->left;
967 				m->left = orig_object->cache;
968 				orig_object->cache = m;
969 				break;
970 			}
971 			m_next = vm_page_splay(m->pindex, m->right);
972 			/* Update the page's object and offset. */
973 			m->object = new_object;
974 			m->pindex -= offidxstart;
975 			if (m_next == NULL)
976 				break;
977 			m->right = NULL;
978 			m_next->left = m;
979 			new_object->cache = m_next;
980 		}
981 		KASSERT(new_object->cache == NULL ||
982 		    new_object->type == OBJT_SWAP,
983 		    ("vm_page_cache_transfer: object %p's type is incompatible"
984 		    " with cached pages", new_object));
985 	}
986 	mtx_unlock(&vm_page_queue_free_mtx);
987 }
988 
989 /*
990  *	vm_page_alloc:
991  *
992  *	Allocate and return a memory cell associated
993  *	with this VM object/offset pair.
994  *
995  *	page_req classes:
996  *	VM_ALLOC_NORMAL		normal process request
997  *	VM_ALLOC_SYSTEM		system *really* needs a page
998  *	VM_ALLOC_INTERRUPT	interrupt time request
999  *	VM_ALLOC_ZERO		zero page
1000  *
1001  *	This routine may not block.
1002  */
1003 vm_page_t
1004 vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
1005 {
1006 	struct vnode *vp = NULL;
1007 	vm_object_t m_object;
1008 	vm_page_t m;
1009 	int flags, page_req;
1010 
1011 	page_req = req & VM_ALLOC_CLASS_MASK;
1012 	KASSERT(curthread->td_intr_nesting_level == 0 ||
1013 	    page_req == VM_ALLOC_INTERRUPT,
1014 	    ("vm_page_alloc(NORMAL|SYSTEM) in interrupt context"));
1015 
1016 	if ((req & VM_ALLOC_NOOBJ) == 0) {
1017 		KASSERT(object != NULL,
1018 		    ("vm_page_alloc: NULL object."));
1019 		VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1020 	}
1021 
1022 	/*
1023 	 * The pager is allowed to eat deeper into the free page list.
1024 	 */
1025 	if ((curproc == pageproc) && (page_req != VM_ALLOC_INTERRUPT)) {
1026 		page_req = VM_ALLOC_SYSTEM;
1027 	};
1028 
1029 	mtx_lock(&vm_page_queue_free_mtx);
1030 	if (cnt.v_free_count + cnt.v_cache_count > cnt.v_free_reserved ||
1031 	    (page_req == VM_ALLOC_SYSTEM &&
1032 	    cnt.v_free_count + cnt.v_cache_count > cnt.v_interrupt_free_min) ||
1033 	    (page_req == VM_ALLOC_INTERRUPT &&
1034 	    cnt.v_free_count + cnt.v_cache_count > 0)) {
1035 		/*
1036 		 * Allocate from the free queue if the number of free pages
1037 		 * exceeds the minimum for the request class.
1038 		 */
1039 		if (object != NULL &&
1040 		    (m = vm_page_cache_lookup(object, pindex)) != NULL) {
1041 			if ((req & VM_ALLOC_IFNOTCACHED) != 0) {
1042 				mtx_unlock(&vm_page_queue_free_mtx);
1043 				return (NULL);
1044 			}
1045 			if (vm_phys_unfree_page(m))
1046 				vm_phys_set_pool(VM_FREEPOOL_DEFAULT, m, 0);
1047 #if VM_NRESERVLEVEL > 0
1048 			else if (!vm_reserv_reactivate_page(m))
1049 #else
1050 			else
1051 #endif
1052 				panic("vm_page_alloc: cache page %p is missing"
1053 				    " from the free queue", m);
1054 		} else if ((req & VM_ALLOC_IFCACHED) != 0) {
1055 			mtx_unlock(&vm_page_queue_free_mtx);
1056 			return (NULL);
1057 #if VM_NRESERVLEVEL > 0
1058 		} else if (object == NULL ||
1059 		    (object->flags & OBJ_COLORED) == 0 ||
1060 		    (m = vm_reserv_alloc_page(object, pindex)) == NULL) {
1061 #else
1062 		} else {
1063 #endif
1064 			m = vm_phys_alloc_pages(object != NULL ?
1065 			    VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, 0);
1066 #if VM_NRESERVLEVEL > 0
1067 			if (m == NULL && vm_reserv_reclaim()) {
1068 				m = vm_phys_alloc_pages(object != NULL ?
1069 				    VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT,
1070 				    0);
1071 			}
1072 #endif
1073 		}
1074 	} else {
1075 		/*
1076 		 * Not allocatable, give up.
1077 		 */
1078 		mtx_unlock(&vm_page_queue_free_mtx);
1079 		atomic_add_int(&vm_pageout_deficit, 1);
1080 		pagedaemon_wakeup();
1081 		return (NULL);
1082 	}
1083 
1084 	/*
1085 	 *  At this point we had better have found a good page.
1086 	 */
1087 
1088 	KASSERT(
1089 	    m != NULL,
1090 	    ("vm_page_alloc(): missing page on free queue")
1091 	);
1092 	if ((m->flags & PG_CACHED) != 0) {
1093 		KASSERT(m->valid != 0,
1094 		    ("vm_page_alloc: cached page %p is invalid", m));
1095 		if (m->object == object && m->pindex == pindex)
1096 	  		cnt.v_reactivated++;
1097 		else
1098 			m->valid = 0;
1099 		m_object = m->object;
1100 		vm_page_cache_remove(m);
1101 		if (m_object->type == OBJT_VNODE && m_object->cache == NULL)
1102 			vp = m_object->handle;
1103 	} else {
1104 		KASSERT(VM_PAGE_IS_FREE(m),
1105 		    ("vm_page_alloc: page %p is not free", m));
1106 		KASSERT(m->valid == 0,
1107 		    ("vm_page_alloc: free page %p is valid", m));
1108 		cnt.v_free_count--;
1109 	}
1110 
1111 	/*
1112 	 * Initialize structure.  Only the PG_ZERO flag is inherited.
1113 	 */
1114 	flags = 0;
1115 	if (m->flags & PG_ZERO) {
1116 		vm_page_zero_count--;
1117 		if (req & VM_ALLOC_ZERO)
1118 			flags = PG_ZERO;
1119 	}
1120 	if (object == NULL || object->type == OBJT_PHYS)
1121 		flags |= PG_UNMANAGED;
1122 	m->flags = flags;
1123 	if (req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ))
1124 		m->oflags = 0;
1125 	else
1126 		m->oflags = VPO_BUSY;
1127 	if (req & VM_ALLOC_WIRED) {
1128 		atomic_add_int(&cnt.v_wire_count, 1);
1129 		m->wire_count = 1;
1130 	} else
1131 		m->wire_count = 0;
1132 	m->hold_count = 0;
1133 	m->act_count = 0;
1134 	m->busy = 0;
1135 	KASSERT(m->dirty == 0, ("vm_page_alloc: free/cache page %p was dirty", m));
1136 	mtx_unlock(&vm_page_queue_free_mtx);
1137 
1138 	if ((req & VM_ALLOC_NOOBJ) == 0)
1139 		vm_page_insert(m, object, pindex);
1140 	else
1141 		m->pindex = pindex;
1142 
1143 	/*
1144 	 * The following call to vdrop() must come after the above call
1145 	 * to vm_page_insert() in case both affect the same object and
1146 	 * vnode.  Otherwise, the affected vnode's hold count could
1147 	 * temporarily become zero.
1148 	 */
1149 	if (vp != NULL)
1150 		vdrop(vp);
1151 
1152 	/*
1153 	 * Don't wakeup too often - wakeup the pageout daemon when
1154 	 * we would be nearly out of memory.
1155 	 */
1156 	if (vm_paging_needed())
1157 		pagedaemon_wakeup();
1158 
1159 	return (m);
1160 }
1161 
1162 /*
1163  *	vm_wait:	(also see VM_WAIT macro)
1164  *
1165  *	Block until free pages are available for allocation
1166  *	- Called in various places before memory allocations.
1167  */
1168 void
1169 vm_wait(void)
1170 {
1171 
1172 	mtx_lock(&vm_page_queue_free_mtx);
1173 	if (curproc == pageproc) {
1174 		vm_pageout_pages_needed = 1;
1175 		msleep(&vm_pageout_pages_needed, &vm_page_queue_free_mtx,
1176 		    PDROP | PSWP, "VMWait", 0);
1177 	} else {
1178 		if (!vm_pages_needed) {
1179 			vm_pages_needed = 1;
1180 			wakeup(&vm_pages_needed);
1181 		}
1182 		msleep(&cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | PVM,
1183 		    "vmwait", 0);
1184 	}
1185 }
1186 
1187 /*
1188  *	vm_waitpfault:	(also see VM_WAITPFAULT macro)
1189  *
1190  *	Block until free pages are available for allocation
1191  *	- Called only in vm_fault so that processes page faulting
1192  *	  can be easily tracked.
1193  *	- Sleeps at a lower priority than vm_wait() so that vm_wait()ing
1194  *	  processes will be able to grab memory first.  Do not change
1195  *	  this balance without careful testing first.
1196  */
1197 void
1198 vm_waitpfault(void)
1199 {
1200 
1201 	mtx_lock(&vm_page_queue_free_mtx);
1202 	if (!vm_pages_needed) {
1203 		vm_pages_needed = 1;
1204 		wakeup(&vm_pages_needed);
1205 	}
1206 	msleep(&cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | PUSER,
1207 	    "pfault", 0);
1208 }
1209 
1210 /*
1211  *	vm_page_activate:
1212  *
1213  *	Put the specified page on the active list (if appropriate).
1214  *	Ensure that act_count is at least ACT_INIT but do not otherwise
1215  *	mess with it.
1216  *
1217  *	The page queues must be locked.
1218  *	This routine may not block.
1219  */
1220 void
1221 vm_page_activate(vm_page_t m)
1222 {
1223 
1224 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1225 	if (VM_PAGE_GETKNOWNQUEUE2(m) != PQ_ACTIVE) {
1226 		vm_pageq_remove(m);
1227 		if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
1228 			if (m->act_count < ACT_INIT)
1229 				m->act_count = ACT_INIT;
1230 			vm_pageq_enqueue(PQ_ACTIVE, m);
1231 		}
1232 	} else {
1233 		if (m->act_count < ACT_INIT)
1234 			m->act_count = ACT_INIT;
1235 	}
1236 }
1237 
1238 /*
1239  *	vm_page_free_wakeup:
1240  *
1241  *	Helper routine for vm_page_free_toq() and vm_page_cache().  This
1242  *	routine is called when a page has been added to the cache or free
1243  *	queues.
1244  *
1245  *	The page queues must be locked.
1246  *	This routine may not block.
1247  */
1248 static inline void
1249 vm_page_free_wakeup(void)
1250 {
1251 
1252 	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
1253 	/*
1254 	 * if pageout daemon needs pages, then tell it that there are
1255 	 * some free.
1256 	 */
1257 	if (vm_pageout_pages_needed &&
1258 	    cnt.v_cache_count + cnt.v_free_count >= cnt.v_pageout_free_min) {
1259 		wakeup(&vm_pageout_pages_needed);
1260 		vm_pageout_pages_needed = 0;
1261 	}
1262 	/*
1263 	 * wakeup processes that are waiting on memory if we hit a
1264 	 * high water mark. And wakeup scheduler process if we have
1265 	 * lots of memory. this process will swapin processes.
1266 	 */
1267 	if (vm_pages_needed && !vm_page_count_min()) {
1268 		vm_pages_needed = 0;
1269 		wakeup(&cnt.v_free_count);
1270 	}
1271 }
1272 
1273 /*
1274  *	vm_page_free_toq:
1275  *
1276  *	Returns the given page to the free list,
1277  *	disassociating it with any VM object.
1278  *
1279  *	Object and page must be locked prior to entry.
1280  *	This routine may not block.
1281  */
1282 
1283 void
1284 vm_page_free_toq(vm_page_t m)
1285 {
1286 
1287 	if (VM_PAGE_GETQUEUE(m) != PQ_NONE)
1288 		mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1289 	KASSERT(!pmap_page_is_mapped(m),
1290 	    ("vm_page_free_toq: freeing mapped page %p", m));
1291 	PCPU_INC(cnt.v_tfree);
1292 
1293 	if (m->busy || VM_PAGE_IS_FREE(m)) {
1294 		printf(
1295 		"vm_page_free: pindex(%lu), busy(%d), VPO_BUSY(%d), hold(%d)\n",
1296 		    (u_long)m->pindex, m->busy, (m->oflags & VPO_BUSY) ? 1 : 0,
1297 		    m->hold_count);
1298 		if (VM_PAGE_IS_FREE(m))
1299 			panic("vm_page_free: freeing free page");
1300 		else
1301 			panic("vm_page_free: freeing busy page");
1302 	}
1303 
1304 	/*
1305 	 * unqueue, then remove page.  Note that we cannot destroy
1306 	 * the page here because we do not want to call the pager's
1307 	 * callback routine until after we've put the page on the
1308 	 * appropriate free queue.
1309 	 */
1310 	vm_pageq_remove(m);
1311 	vm_page_remove(m);
1312 
1313 	/*
1314 	 * If fictitious remove object association and
1315 	 * return, otherwise delay object association removal.
1316 	 */
1317 	if ((m->flags & PG_FICTITIOUS) != 0) {
1318 		return;
1319 	}
1320 
1321 	m->valid = 0;
1322 	vm_page_undirty(m);
1323 
1324 	if (m->wire_count != 0) {
1325 		if (m->wire_count > 1) {
1326 			panic("vm_page_free: invalid wire count (%d), pindex: 0x%lx",
1327 				m->wire_count, (long)m->pindex);
1328 		}
1329 		panic("vm_page_free: freeing wired page");
1330 	}
1331 	if (m->hold_count != 0) {
1332 		m->flags &= ~PG_ZERO;
1333 		vm_pageq_enqueue(PQ_HOLD, m);
1334 	} else {
1335 		mtx_lock(&vm_page_queue_free_mtx);
1336 		m->flags |= PG_FREE;
1337 		cnt.v_free_count++;
1338 #if VM_NRESERVLEVEL > 0
1339 		if (!vm_reserv_free_page(m))
1340 #else
1341 		if (TRUE)
1342 #endif
1343 			vm_phys_free_pages(m, 0);
1344 		if ((m->flags & PG_ZERO) != 0)
1345 			++vm_page_zero_count;
1346 		else
1347 			vm_page_zero_idle_wakeup();
1348 		vm_page_free_wakeup();
1349 		mtx_unlock(&vm_page_queue_free_mtx);
1350 	}
1351 }
1352 
1353 /*
1354  *	vm_page_wire:
1355  *
1356  *	Mark this page as wired down by yet
1357  *	another map, removing it from paging queues
1358  *	as necessary.
1359  *
1360  *	The page queues must be locked.
1361  *	This routine may not block.
1362  */
1363 void
1364 vm_page_wire(vm_page_t m)
1365 {
1366 
1367 	/*
1368 	 * Only bump the wire statistics if the page is not already wired,
1369 	 * and only unqueue the page if it is on some queue (if it is unmanaged
1370 	 * it is already off the queues).
1371 	 */
1372 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1373 	if (m->flags & PG_FICTITIOUS)
1374 		return;
1375 	if (m->wire_count == 0) {
1376 		if ((m->flags & PG_UNMANAGED) == 0)
1377 			vm_pageq_remove(m);
1378 		atomic_add_int(&cnt.v_wire_count, 1);
1379 	}
1380 	m->wire_count++;
1381 	KASSERT(m->wire_count != 0, ("vm_page_wire: wire_count overflow m=%p", m));
1382 }
1383 
1384 /*
1385  *	vm_page_unwire:
1386  *
1387  *	Release one wiring of this page, potentially
1388  *	enabling it to be paged again.
1389  *
1390  *	Many pages placed on the inactive queue should actually go
1391  *	into the cache, but it is difficult to figure out which.  What
1392  *	we do instead, if the inactive target is well met, is to put
1393  *	clean pages at the head of the inactive queue instead of the tail.
1394  *	This will cause them to be moved to the cache more quickly and
1395  *	if not actively re-referenced, freed more quickly.  If we just
1396  *	stick these pages at the end of the inactive queue, heavy filesystem
1397  *	meta-data accesses can cause an unnecessary paging load on memory bound
1398  *	processes.  This optimization causes one-time-use metadata to be
1399  *	reused more quickly.
1400  *
1401  *	BUT, if we are in a low-memory situation we have no choice but to
1402  *	put clean pages on the cache queue.
1403  *
1404  *	A number of routines use vm_page_unwire() to guarantee that the page
1405  *	will go into either the inactive or active queues, and will NEVER
1406  *	be placed in the cache - for example, just after dirtying a page.
1407  *	dirty pages in the cache are not allowed.
1408  *
1409  *	The page queues must be locked.
1410  *	This routine may not block.
1411  */
1412 void
1413 vm_page_unwire(vm_page_t m, int activate)
1414 {
1415 
1416 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1417 	if (m->flags & PG_FICTITIOUS)
1418 		return;
1419 	if (m->wire_count > 0) {
1420 		m->wire_count--;
1421 		if (m->wire_count == 0) {
1422 			atomic_subtract_int(&cnt.v_wire_count, 1);
1423 			if (m->flags & PG_UNMANAGED) {
1424 				;
1425 			} else if (activate)
1426 				vm_pageq_enqueue(PQ_ACTIVE, m);
1427 			else {
1428 				vm_page_flag_clear(m, PG_WINATCFLS);
1429 				vm_pageq_enqueue(PQ_INACTIVE, m);
1430 			}
1431 		}
1432 	} else {
1433 		panic("vm_page_unwire: invalid wire count: %d", m->wire_count);
1434 	}
1435 }
1436 
1437 
1438 /*
1439  * Move the specified page to the inactive queue.  If the page has
1440  * any associated swap, the swap is deallocated.
1441  *
1442  * Normally athead is 0 resulting in LRU operation.  athead is set
1443  * to 1 if we want this page to be 'as if it were placed in the cache',
1444  * except without unmapping it from the process address space.
1445  *
1446  * This routine may not block.
1447  */
1448 static inline void
1449 _vm_page_deactivate(vm_page_t m, int athead)
1450 {
1451 
1452 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1453 
1454 	/*
1455 	 * Ignore if already inactive.
1456 	 */
1457 	if (VM_PAGE_INQUEUE2(m, PQ_INACTIVE))
1458 		return;
1459 	if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
1460 		vm_page_flag_clear(m, PG_WINATCFLS);
1461 		vm_pageq_remove(m);
1462 		if (athead)
1463 			TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
1464 		else
1465 			TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
1466 		VM_PAGE_SETQUEUE2(m, PQ_INACTIVE);
1467 		cnt.v_inactive_count++;
1468 	}
1469 }
1470 
1471 void
1472 vm_page_deactivate(vm_page_t m)
1473 {
1474     _vm_page_deactivate(m, 0);
1475 }
1476 
1477 /*
1478  * vm_page_try_to_cache:
1479  *
1480  * Returns 0 on failure, 1 on success
1481  */
1482 int
1483 vm_page_try_to_cache(vm_page_t m)
1484 {
1485 
1486 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1487 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1488 	if (m->dirty || m->hold_count || m->busy || m->wire_count ||
1489 	    (m->oflags & VPO_BUSY) || (m->flags & PG_UNMANAGED)) {
1490 		return (0);
1491 	}
1492 	pmap_remove_all(m);
1493 	if (m->dirty)
1494 		return (0);
1495 	vm_page_cache(m);
1496 	return (1);
1497 }
1498 
1499 /*
1500  * vm_page_try_to_free()
1501  *
1502  *	Attempt to free the page.  If we cannot free it, we do nothing.
1503  *	1 is returned on success, 0 on failure.
1504  */
1505 int
1506 vm_page_try_to_free(vm_page_t m)
1507 {
1508 
1509 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1510 	if (m->object != NULL)
1511 		VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1512 	if (m->dirty || m->hold_count || m->busy || m->wire_count ||
1513 	    (m->oflags & VPO_BUSY) || (m->flags & PG_UNMANAGED)) {
1514 		return (0);
1515 	}
1516 	pmap_remove_all(m);
1517 	if (m->dirty)
1518 		return (0);
1519 	vm_page_free(m);
1520 	return (1);
1521 }
1522 
1523 /*
1524  * vm_page_cache
1525  *
1526  * Put the specified page onto the page cache queue (if appropriate).
1527  *
1528  * This routine may not block.
1529  */
1530 void
1531 vm_page_cache(vm_page_t m)
1532 {
1533 	vm_object_t object;
1534 	vm_page_t root;
1535 
1536 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1537 	object = m->object;
1538 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1539 	if ((m->flags & PG_UNMANAGED) || (m->oflags & VPO_BUSY) || m->busy ||
1540 	    m->hold_count || m->wire_count) {
1541 		panic("vm_page_cache: attempting to cache busy page");
1542 	}
1543 	pmap_remove_all(m);
1544 	if (m->dirty != 0)
1545 		panic("vm_page_cache: page %p is dirty", m);
1546 	if (m->valid == 0 || object->type == OBJT_DEFAULT ||
1547 	    (object->type == OBJT_SWAP &&
1548 	    !vm_pager_has_page(object, m->pindex, NULL, NULL))) {
1549 		/*
1550 		 * Hypothesis: A cache-elgible page belonging to a
1551 		 * default object or swap object but without a backing
1552 		 * store must be zero filled.
1553 		 */
1554 		vm_page_free(m);
1555 		return;
1556 	}
1557 	KASSERT((m->flags & PG_CACHED) == 0,
1558 	    ("vm_page_cache: page %p is already cached", m));
1559 	cnt.v_tcached++;
1560 
1561 	/*
1562 	 * Remove the page from the paging queues.
1563 	 */
1564 	vm_pageq_remove(m);
1565 
1566 	/*
1567 	 * Remove the page from the object's collection of resident
1568 	 * pages.
1569 	 */
1570 	if (m != object->root)
1571 		vm_page_splay(m->pindex, object->root);
1572 	if (m->left == NULL)
1573 		root = m->right;
1574 	else {
1575 		root = vm_page_splay(m->pindex, m->left);
1576 		root->right = m->right;
1577 	}
1578 	object->root = root;
1579 	TAILQ_REMOVE(&object->memq, m, listq);
1580 	object->resident_page_count--;
1581 	object->generation++;
1582 
1583 	/*
1584 	 * Insert the page into the object's collection of cached pages
1585 	 * and the physical memory allocator's cache/free page queues.
1586 	 */
1587 	vm_page_flag_clear(m, PG_ZERO);
1588 	mtx_lock(&vm_page_queue_free_mtx);
1589 	m->flags |= PG_CACHED;
1590 	cnt.v_cache_count++;
1591 	root = object->cache;
1592 	if (root == NULL) {
1593 		m->left = NULL;
1594 		m->right = NULL;
1595 	} else {
1596 		root = vm_page_splay(m->pindex, root);
1597 		if (m->pindex < root->pindex) {
1598 			m->left = root->left;
1599 			m->right = root;
1600 			root->left = NULL;
1601 		} else if (__predict_false(m->pindex == root->pindex))
1602 			panic("vm_page_cache: offset already cached");
1603 		else {
1604 			m->right = root->right;
1605 			m->left = root;
1606 			root->right = NULL;
1607 		}
1608 	}
1609 	object->cache = m;
1610 #if VM_NRESERVLEVEL > 0
1611 	if (!vm_reserv_free_page(m)) {
1612 #else
1613 	if (TRUE) {
1614 #endif
1615 		vm_phys_set_pool(VM_FREEPOOL_CACHE, m, 0);
1616 		vm_phys_free_pages(m, 0);
1617 	}
1618 	vm_page_free_wakeup();
1619 	mtx_unlock(&vm_page_queue_free_mtx);
1620 
1621 	/*
1622 	 * Increment the vnode's hold count if this is the object's only
1623 	 * cached page.  Decrement the vnode's hold count if this was
1624 	 * the object's only resident page.
1625 	 */
1626 	if (object->type == OBJT_VNODE) {
1627 		if (root == NULL && object->resident_page_count != 0)
1628 			vhold(object->handle);
1629 		else if (root != NULL && object->resident_page_count == 0)
1630 			vdrop(object->handle);
1631 	}
1632 }
1633 
1634 /*
1635  * vm_page_dontneed
1636  *
1637  *	Cache, deactivate, or do nothing as appropriate.  This routine
1638  *	is typically used by madvise() MADV_DONTNEED.
1639  *
1640  *	Generally speaking we want to move the page into the cache so
1641  *	it gets reused quickly.  However, this can result in a silly syndrome
1642  *	due to the page recycling too quickly.  Small objects will not be
1643  *	fully cached.  On the otherhand, if we move the page to the inactive
1644  *	queue we wind up with a problem whereby very large objects
1645  *	unnecessarily blow away our inactive and cache queues.
1646  *
1647  *	The solution is to move the pages based on a fixed weighting.  We
1648  *	either leave them alone, deactivate them, or move them to the cache,
1649  *	where moving them to the cache has the highest weighting.
1650  *	By forcing some pages into other queues we eventually force the
1651  *	system to balance the queues, potentially recovering other unrelated
1652  *	space from active.  The idea is to not force this to happen too
1653  *	often.
1654  */
1655 void
1656 vm_page_dontneed(vm_page_t m)
1657 {
1658 	static int dnweight;
1659 	int dnw;
1660 	int head;
1661 
1662 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1663 	dnw = ++dnweight;
1664 
1665 	/*
1666 	 * occassionally leave the page alone
1667 	 */
1668 	if ((dnw & 0x01F0) == 0 ||
1669 	    VM_PAGE_INQUEUE2(m, PQ_INACTIVE)) {
1670 		if (m->act_count >= ACT_INIT)
1671 			--m->act_count;
1672 		return;
1673 	}
1674 
1675 	if (m->dirty == 0 && pmap_is_modified(m))
1676 		vm_page_dirty(m);
1677 
1678 	if (m->dirty || (dnw & 0x0070) == 0) {
1679 		/*
1680 		 * Deactivate the page 3 times out of 32.
1681 		 */
1682 		head = 0;
1683 	} else {
1684 		/*
1685 		 * Cache the page 28 times out of every 32.  Note that
1686 		 * the page is deactivated instead of cached, but placed
1687 		 * at the head of the queue instead of the tail.
1688 		 */
1689 		head = 1;
1690 	}
1691 	_vm_page_deactivate(m, head);
1692 }
1693 
1694 /*
1695  * Grab a page, waiting until we are waken up due to the page
1696  * changing state.  We keep on waiting, if the page continues
1697  * to be in the object.  If the page doesn't exist, first allocate it
1698  * and then conditionally zero it.
1699  *
1700  * This routine may block.
1701  */
1702 vm_page_t
1703 vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
1704 {
1705 	vm_page_t m;
1706 
1707 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1708 retrylookup:
1709 	if ((m = vm_page_lookup(object, pindex)) != NULL) {
1710 		if (vm_page_sleep_if_busy(m, TRUE, "pgrbwt")) {
1711 			if ((allocflags & VM_ALLOC_RETRY) == 0)
1712 				return (NULL);
1713 			goto retrylookup;
1714 		} else {
1715 			if ((allocflags & VM_ALLOC_WIRED) != 0) {
1716 				vm_page_lock_queues();
1717 				vm_page_wire(m);
1718 				vm_page_unlock_queues();
1719 			}
1720 			if ((allocflags & VM_ALLOC_NOBUSY) == 0)
1721 				vm_page_busy(m);
1722 			return (m);
1723 		}
1724 	}
1725 	m = vm_page_alloc(object, pindex, allocflags & ~VM_ALLOC_RETRY);
1726 	if (m == NULL) {
1727 		VM_OBJECT_UNLOCK(object);
1728 		VM_WAIT;
1729 		VM_OBJECT_LOCK(object);
1730 		if ((allocflags & VM_ALLOC_RETRY) == 0)
1731 			return (NULL);
1732 		goto retrylookup;
1733 	} else if (m->valid != 0)
1734 		return (m);
1735 	if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0)
1736 		pmap_zero_page(m);
1737 	return (m);
1738 }
1739 
1740 /*
1741  * Mapping function for valid bits or for dirty bits in
1742  * a page.  May not block.
1743  *
1744  * Inputs are required to range within a page.
1745  */
1746 int
1747 vm_page_bits(int base, int size)
1748 {
1749 	int first_bit;
1750 	int last_bit;
1751 
1752 	KASSERT(
1753 	    base + size <= PAGE_SIZE,
1754 	    ("vm_page_bits: illegal base/size %d/%d", base, size)
1755 	);
1756 
1757 	if (size == 0)		/* handle degenerate case */
1758 		return (0);
1759 
1760 	first_bit = base >> DEV_BSHIFT;
1761 	last_bit = (base + size - 1) >> DEV_BSHIFT;
1762 
1763 	return ((2 << last_bit) - (1 << first_bit));
1764 }
1765 
1766 /*
1767  *	vm_page_set_validclean:
1768  *
1769  *	Sets portions of a page valid and clean.  The arguments are expected
1770  *	to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive
1771  *	of any partial chunks touched by the range.  The invalid portion of
1772  *	such chunks will be zero'd.
1773  *
1774  *	This routine may not block.
1775  *
1776  *	(base + size) must be less then or equal to PAGE_SIZE.
1777  */
1778 void
1779 vm_page_set_validclean(vm_page_t m, int base, int size)
1780 {
1781 	int pagebits;
1782 	int frag;
1783 	int endoff;
1784 
1785 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1786 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1787 	if (size == 0)	/* handle degenerate case */
1788 		return;
1789 
1790 	/*
1791 	 * If the base is not DEV_BSIZE aligned and the valid
1792 	 * bit is clear, we have to zero out a portion of the
1793 	 * first block.
1794 	 */
1795 	if ((frag = base & ~(DEV_BSIZE - 1)) != base &&
1796 	    (m->valid & (1 << (base >> DEV_BSHIFT))) == 0)
1797 		pmap_zero_page_area(m, frag, base - frag);
1798 
1799 	/*
1800 	 * If the ending offset is not DEV_BSIZE aligned and the
1801 	 * valid bit is clear, we have to zero out a portion of
1802 	 * the last block.
1803 	 */
1804 	endoff = base + size;
1805 	if ((frag = endoff & ~(DEV_BSIZE - 1)) != endoff &&
1806 	    (m->valid & (1 << (endoff >> DEV_BSHIFT))) == 0)
1807 		pmap_zero_page_area(m, endoff,
1808 		    DEV_BSIZE - (endoff & (DEV_BSIZE - 1)));
1809 
1810 	/*
1811 	 * Set valid, clear dirty bits.  If validating the entire
1812 	 * page we can safely clear the pmap modify bit.  We also
1813 	 * use this opportunity to clear the VPO_NOSYNC flag.  If a process
1814 	 * takes a write fault on a MAP_NOSYNC memory area the flag will
1815 	 * be set again.
1816 	 *
1817 	 * We set valid bits inclusive of any overlap, but we can only
1818 	 * clear dirty bits for DEV_BSIZE chunks that are fully within
1819 	 * the range.
1820 	 */
1821 	pagebits = vm_page_bits(base, size);
1822 	m->valid |= pagebits;
1823 #if 0	/* NOT YET */
1824 	if ((frag = base & (DEV_BSIZE - 1)) != 0) {
1825 		frag = DEV_BSIZE - frag;
1826 		base += frag;
1827 		size -= frag;
1828 		if (size < 0)
1829 			size = 0;
1830 	}
1831 	pagebits = vm_page_bits(base, size & (DEV_BSIZE - 1));
1832 #endif
1833 	m->dirty &= ~pagebits;
1834 	if (base == 0 && size == PAGE_SIZE) {
1835 		pmap_clear_modify(m);
1836 		m->oflags &= ~VPO_NOSYNC;
1837 	}
1838 }
1839 
1840 void
1841 vm_page_clear_dirty(vm_page_t m, int base, int size)
1842 {
1843 
1844 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1845 	m->dirty &= ~vm_page_bits(base, size);
1846 }
1847 
1848 /*
1849  *	vm_page_set_invalid:
1850  *
1851  *	Invalidates DEV_BSIZE'd chunks within a page.  Both the
1852  *	valid and dirty bits for the effected areas are cleared.
1853  *
1854  *	May not block.
1855  */
1856 void
1857 vm_page_set_invalid(vm_page_t m, int base, int size)
1858 {
1859 	int bits;
1860 
1861 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1862 	bits = vm_page_bits(base, size);
1863 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1864 	if (m->valid == VM_PAGE_BITS_ALL && bits != 0)
1865 		pmap_remove_all(m);
1866 	m->valid &= ~bits;
1867 	m->dirty &= ~bits;
1868 	m->object->generation++;
1869 }
1870 
1871 /*
1872  * vm_page_zero_invalid()
1873  *
1874  *	The kernel assumes that the invalid portions of a page contain
1875  *	garbage, but such pages can be mapped into memory by user code.
1876  *	When this occurs, we must zero out the non-valid portions of the
1877  *	page so user code sees what it expects.
1878  *
1879  *	Pages are most often semi-valid when the end of a file is mapped
1880  *	into memory and the file's size is not page aligned.
1881  */
1882 void
1883 vm_page_zero_invalid(vm_page_t m, boolean_t setvalid)
1884 {
1885 	int b;
1886 	int i;
1887 
1888 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1889 	/*
1890 	 * Scan the valid bits looking for invalid sections that
1891 	 * must be zerod.  Invalid sub-DEV_BSIZE'd areas ( where the
1892 	 * valid bit may be set ) have already been zerod by
1893 	 * vm_page_set_validclean().
1894 	 */
1895 	for (b = i = 0; i <= PAGE_SIZE / DEV_BSIZE; ++i) {
1896 		if (i == (PAGE_SIZE / DEV_BSIZE) ||
1897 		    (m->valid & (1 << i))
1898 		) {
1899 			if (i > b) {
1900 				pmap_zero_page_area(m,
1901 				    b << DEV_BSHIFT, (i - b) << DEV_BSHIFT);
1902 			}
1903 			b = i + 1;
1904 		}
1905 	}
1906 
1907 	/*
1908 	 * setvalid is TRUE when we can safely set the zero'd areas
1909 	 * as being valid.  We can do this if there are no cache consistancy
1910 	 * issues.  e.g. it is ok to do with UFS, but not ok to do with NFS.
1911 	 */
1912 	if (setvalid)
1913 		m->valid = VM_PAGE_BITS_ALL;
1914 }
1915 
1916 /*
1917  *	vm_page_is_valid:
1918  *
1919  *	Is (partial) page valid?  Note that the case where size == 0
1920  *	will return FALSE in the degenerate case where the page is
1921  *	entirely invalid, and TRUE otherwise.
1922  *
1923  *	May not block.
1924  */
1925 int
1926 vm_page_is_valid(vm_page_t m, int base, int size)
1927 {
1928 	int bits = vm_page_bits(base, size);
1929 
1930 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1931 	if (m->valid && ((m->valid & bits) == bits))
1932 		return 1;
1933 	else
1934 		return 0;
1935 }
1936 
1937 /*
1938  * update dirty bits from pmap/mmu.  May not block.
1939  */
1940 void
1941 vm_page_test_dirty(vm_page_t m)
1942 {
1943 	if ((m->dirty != VM_PAGE_BITS_ALL) && pmap_is_modified(m)) {
1944 		vm_page_dirty(m);
1945 	}
1946 }
1947 
1948 int so_zerocp_fullpage = 0;
1949 
1950 /*
1951  *	Replace the given page with a copy.  The copied page assumes
1952  *	the portion of the given page's "wire_count" that is not the
1953  *	responsibility of this copy-on-write mechanism.
1954  *
1955  *	The object containing the given page must have a non-zero
1956  *	paging-in-progress count and be locked.
1957  */
1958 void
1959 vm_page_cowfault(vm_page_t m)
1960 {
1961 	vm_page_t mnew;
1962 	vm_object_t object;
1963 	vm_pindex_t pindex;
1964 
1965 	object = m->object;
1966 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1967 	KASSERT(object->paging_in_progress != 0,
1968 	    ("vm_page_cowfault: object %p's paging-in-progress count is zero.",
1969 	    object));
1970 	pindex = m->pindex;
1971 
1972  retry_alloc:
1973 	pmap_remove_all(m);
1974 	vm_page_remove(m);
1975 	mnew = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY);
1976 	if (mnew == NULL) {
1977 		vm_page_insert(m, object, pindex);
1978 		vm_page_unlock_queues();
1979 		VM_OBJECT_UNLOCK(object);
1980 		VM_WAIT;
1981 		VM_OBJECT_LOCK(object);
1982 		if (m == vm_page_lookup(object, pindex)) {
1983 			vm_page_lock_queues();
1984 			goto retry_alloc;
1985 		} else {
1986 			/*
1987 			 * Page disappeared during the wait.
1988 			 */
1989 			vm_page_lock_queues();
1990 			return;
1991 		}
1992 	}
1993 
1994 	if (m->cow == 0) {
1995 		/*
1996 		 * check to see if we raced with an xmit complete when
1997 		 * waiting to allocate a page.  If so, put things back
1998 		 * the way they were
1999 		 */
2000 		vm_page_free(mnew);
2001 		vm_page_insert(m, object, pindex);
2002 	} else { /* clear COW & copy page */
2003 		if (!so_zerocp_fullpage)
2004 			pmap_copy_page(m, mnew);
2005 		mnew->valid = VM_PAGE_BITS_ALL;
2006 		vm_page_dirty(mnew);
2007 		mnew->wire_count = m->wire_count - m->cow;
2008 		m->wire_count = m->cow;
2009 	}
2010 }
2011 
2012 void
2013 vm_page_cowclear(vm_page_t m)
2014 {
2015 
2016 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
2017 	if (m->cow) {
2018 		m->cow--;
2019 		/*
2020 		 * let vm_fault add back write permission  lazily
2021 		 */
2022 	}
2023 	/*
2024 	 *  sf_buf_free() will free the page, so we needn't do it here
2025 	 */
2026 }
2027 
2028 void
2029 vm_page_cowsetup(vm_page_t m)
2030 {
2031 
2032 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
2033 	m->cow++;
2034 	pmap_remove_write(m);
2035 }
2036 
2037 #include "opt_ddb.h"
2038 #ifdef DDB
2039 #include <sys/kernel.h>
2040 
2041 #include <ddb/ddb.h>
2042 
2043 DB_SHOW_COMMAND(page, vm_page_print_page_info)
2044 {
2045 	db_printf("cnt.v_free_count: %d\n", cnt.v_free_count);
2046 	db_printf("cnt.v_cache_count: %d\n", cnt.v_cache_count);
2047 	db_printf("cnt.v_inactive_count: %d\n", cnt.v_inactive_count);
2048 	db_printf("cnt.v_active_count: %d\n", cnt.v_active_count);
2049 	db_printf("cnt.v_wire_count: %d\n", cnt.v_wire_count);
2050 	db_printf("cnt.v_free_reserved: %d\n", cnt.v_free_reserved);
2051 	db_printf("cnt.v_free_min: %d\n", cnt.v_free_min);
2052 	db_printf("cnt.v_free_target: %d\n", cnt.v_free_target);
2053 	db_printf("cnt.v_cache_min: %d\n", cnt.v_cache_min);
2054 	db_printf("cnt.v_inactive_target: %d\n", cnt.v_inactive_target);
2055 }
2056 
2057 DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info)
2058 {
2059 
2060 	db_printf("PQ_FREE:");
2061 	db_printf(" %d", cnt.v_free_count);
2062 	db_printf("\n");
2063 
2064 	db_printf("PQ_CACHE:");
2065 	db_printf(" %d", cnt.v_cache_count);
2066 	db_printf("\n");
2067 
2068 	db_printf("PQ_ACTIVE: %d, PQ_INACTIVE: %d\n",
2069 		*vm_page_queues[PQ_ACTIVE].cnt,
2070 		*vm_page_queues[PQ_INACTIVE].cnt);
2071 }
2072 #endif /* DDB */
2073