xref: /freebsd/sys/vm/vm_fault.c (revision f061a2215f9bf0bea98ac601a34750f89428db67)
1 /*-
2  * Copyright (c) 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  * Copyright (c) 1994 John S. Dyson
5  * All rights reserved.
6  * Copyright (c) 1994 David Greenman
7  * All rights reserved.
8  *
9  *
10  * This code is derived from software contributed to Berkeley by
11  * The Mach Operating System project at Carnegie-Mellon University.
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  * 1. Redistributions of source code must retain the above copyright
17  *    notice, this list of conditions and the following disclaimer.
18  * 2. Redistributions in binary form must reproduce the above copyright
19  *    notice, this list of conditions and the following disclaimer in the
20  *    documentation and/or other materials provided with the distribution.
21  * 3. All advertising materials mentioning features or use of this software
22  *    must display the following acknowledgement:
23  *	This product includes software developed by the University of
24  *	California, Berkeley and its contributors.
25  * 4. Neither the name of the University nor the names of its contributors
26  *    may be used to endorse or promote products derived from this software
27  *    without specific prior written permission.
28  *
29  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
30  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
33  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
35  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
36  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
37  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
38  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39  * SUCH DAMAGE.
40  *
41  *	from: @(#)vm_fault.c	8.4 (Berkeley) 1/12/94
42  *
43  *
44  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
45  * All rights reserved.
46  *
47  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
48  *
49  * Permission to use, copy, modify and distribute this software and
50  * its documentation is hereby granted, provided that both the copyright
51  * notice and this permission notice appear in all copies of the
52  * software, derivative works or modified versions, and any portions
53  * thereof, and that both notices appear in supporting documentation.
54  *
55  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
56  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
57  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
58  *
59  * Carnegie Mellon requests users of this software to return to
60  *
61  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
62  *  School of Computer Science
63  *  Carnegie Mellon University
64  *  Pittsburgh PA 15213-3890
65  *
66  * any improvements or extensions that they make and grant Carnegie the
67  * rights to redistribute these changes.
68  */
69 
70 /*
71  *	Page fault handling module.
72  */
73 
74 #include <sys/cdefs.h>
75 __FBSDID("$FreeBSD$");
76 
77 #include "opt_ktrace.h"
78 #include "opt_vm.h"
79 
80 #include <sys/param.h>
81 #include <sys/systm.h>
82 #include <sys/kernel.h>
83 #include <sys/lock.h>
84 #include <sys/mman.h>
85 #include <sys/proc.h>
86 #include <sys/racct.h>
87 #include <sys/resourcevar.h>
88 #include <sys/rwlock.h>
89 #include <sys/sysctl.h>
90 #include <sys/vmmeter.h>
91 #include <sys/vnode.h>
92 #ifdef KTRACE
93 #include <sys/ktrace.h>
94 #endif
95 
96 #include <vm/vm.h>
97 #include <vm/vm_param.h>
98 #include <vm/pmap.h>
99 #include <vm/vm_map.h>
100 #include <vm/vm_object.h>
101 #include <vm/vm_page.h>
102 #include <vm/vm_pageout.h>
103 #include <vm/vm_kern.h>
104 #include <vm/vm_pager.h>
105 #include <vm/vm_extern.h>
106 #include <vm/vm_reserv.h>
107 
108 #define PFBAK 4
109 #define PFFOR 4
110 
111 #define	VM_FAULT_READ_DEFAULT	(1 + VM_FAULT_READ_AHEAD_INIT)
112 #define	VM_FAULT_READ_MAX	(1 + VM_FAULT_READ_AHEAD_MAX)
113 
114 #define	VM_FAULT_DONTNEED_MIN	1048576
115 
116 struct faultstate {
117 	vm_page_t m;
118 	vm_object_t object;
119 	vm_pindex_t pindex;
120 	vm_page_t first_m;
121 	vm_object_t	first_object;
122 	vm_pindex_t first_pindex;
123 	vm_map_t map;
124 	vm_map_entry_t entry;
125 	int lookup_still_valid;
126 	struct vnode *vp;
127 };
128 
129 static void vm_fault_dontneed(const struct faultstate *fs, vm_offset_t vaddr,
130 	    int ahead);
131 static void vm_fault_prefault(const struct faultstate *fs, vm_offset_t addra,
132 	    int backward, int forward);
133 
134 static inline void
135 release_page(struct faultstate *fs)
136 {
137 
138 	vm_page_xunbusy(fs->m);
139 	vm_page_lock(fs->m);
140 	vm_page_deactivate(fs->m);
141 	vm_page_unlock(fs->m);
142 	fs->m = NULL;
143 }
144 
145 static inline void
146 unlock_map(struct faultstate *fs)
147 {
148 
149 	if (fs->lookup_still_valid) {
150 		vm_map_lookup_done(fs->map, fs->entry);
151 		fs->lookup_still_valid = FALSE;
152 	}
153 }
154 
155 static void
156 unlock_and_deallocate(struct faultstate *fs)
157 {
158 
159 	vm_object_pip_wakeup(fs->object);
160 	VM_OBJECT_WUNLOCK(fs->object);
161 	if (fs->object != fs->first_object) {
162 		VM_OBJECT_WLOCK(fs->first_object);
163 		vm_page_lock(fs->first_m);
164 		vm_page_free(fs->first_m);
165 		vm_page_unlock(fs->first_m);
166 		vm_object_pip_wakeup(fs->first_object);
167 		VM_OBJECT_WUNLOCK(fs->first_object);
168 		fs->first_m = NULL;
169 	}
170 	vm_object_deallocate(fs->first_object);
171 	unlock_map(fs);
172 	if (fs->vp != NULL) {
173 		vput(fs->vp);
174 		fs->vp = NULL;
175 	}
176 }
177 
178 static void
179 vm_fault_dirty(vm_map_entry_t entry, vm_page_t m, vm_prot_t prot,
180     vm_prot_t fault_type, int fault_flags, boolean_t set_wd)
181 {
182 	boolean_t need_dirty;
183 
184 	if (((prot & VM_PROT_WRITE) == 0 &&
185 	    (fault_flags & VM_FAULT_DIRTY) == 0) ||
186 	    (m->oflags & VPO_UNMANAGED) != 0)
187 		return;
188 
189 	VM_OBJECT_ASSERT_LOCKED(m->object);
190 
191 	need_dirty = ((fault_type & VM_PROT_WRITE) != 0 &&
192 	    (fault_flags & VM_FAULT_WIRE) == 0) ||
193 	    (fault_flags & VM_FAULT_DIRTY) != 0;
194 
195 	if (set_wd)
196 		vm_object_set_writeable_dirty(m->object);
197 	else
198 		/*
199 		 * If two callers of vm_fault_dirty() with set_wd ==
200 		 * FALSE, one for the map entry with MAP_ENTRY_NOSYNC
201 		 * flag set, other with flag clear, race, it is
202 		 * possible for the no-NOSYNC thread to see m->dirty
203 		 * != 0 and not clear VPO_NOSYNC.  Take vm_page lock
204 		 * around manipulation of VPO_NOSYNC and
205 		 * vm_page_dirty() call, to avoid the race and keep
206 		 * m->oflags consistent.
207 		 */
208 		vm_page_lock(m);
209 
210 	/*
211 	 * If this is a NOSYNC mmap we do not want to set VPO_NOSYNC
212 	 * if the page is already dirty to prevent data written with
213 	 * the expectation of being synced from not being synced.
214 	 * Likewise if this entry does not request NOSYNC then make
215 	 * sure the page isn't marked NOSYNC.  Applications sharing
216 	 * data should use the same flags to avoid ping ponging.
217 	 */
218 	if ((entry->eflags & MAP_ENTRY_NOSYNC) != 0) {
219 		if (m->dirty == 0) {
220 			m->oflags |= VPO_NOSYNC;
221 		}
222 	} else {
223 		m->oflags &= ~VPO_NOSYNC;
224 	}
225 
226 	/*
227 	 * If the fault is a write, we know that this page is being
228 	 * written NOW so dirty it explicitly to save on
229 	 * pmap_is_modified() calls later.
230 	 *
231 	 * Also tell the backing pager, if any, that it should remove
232 	 * any swap backing since the page is now dirty.
233 	 */
234 	if (need_dirty)
235 		vm_page_dirty(m);
236 	if (!set_wd)
237 		vm_page_unlock(m);
238 	if (need_dirty)
239 		vm_pager_page_unswapped(m);
240 }
241 
242 /*
243  *	vm_fault:
244  *
245  *	Handle a page fault occurring at the given address,
246  *	requiring the given permissions, in the map specified.
247  *	If successful, the page is inserted into the
248  *	associated physical map.
249  *
250  *	NOTE: the given address should be truncated to the
251  *	proper page address.
252  *
253  *	KERN_SUCCESS is returned if the page fault is handled; otherwise,
254  *	a standard error specifying why the fault is fatal is returned.
255  *
256  *	The map in question must be referenced, and remains so.
257  *	Caller may hold no locks.
258  */
259 int
260 vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type,
261     int fault_flags)
262 {
263 	struct thread *td;
264 	int result;
265 
266 	td = curthread;
267 	if ((td->td_pflags & TDP_NOFAULTING) != 0)
268 		return (KERN_PROTECTION_FAILURE);
269 #ifdef KTRACE
270 	if (map != kernel_map && KTRPOINT(td, KTR_FAULT))
271 		ktrfault(vaddr, fault_type);
272 #endif
273 	result = vm_fault_hold(map, trunc_page(vaddr), fault_type, fault_flags,
274 	    NULL);
275 #ifdef KTRACE
276 	if (map != kernel_map && KTRPOINT(td, KTR_FAULTEND))
277 		ktrfaultend(result);
278 #endif
279 	return (result);
280 }
281 
282 int
283 vm_fault_hold(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type,
284     int fault_flags, vm_page_t *m_hold)
285 {
286 	vm_prot_t prot;
287 	int alloc_req, era, faultcount, nera, result;
288 	boolean_t growstack, is_first_object_locked, wired;
289 	int map_generation;
290 	vm_object_t next_object;
291 	int hardfault;
292 	struct faultstate fs;
293 	struct vnode *vp;
294 	vm_page_t m;
295 	int ahead, behind, cluster_offset, error, locked;
296 
297 	hardfault = 0;
298 	growstack = TRUE;
299 	PCPU_INC(cnt.v_vm_faults);
300 	fs.vp = NULL;
301 	faultcount = 0;
302 
303 RetryFault:;
304 
305 	/*
306 	 * Find the backing store object and offset into it to begin the
307 	 * search.
308 	 */
309 	fs.map = map;
310 	result = vm_map_lookup(&fs.map, vaddr, fault_type, &fs.entry,
311 	    &fs.first_object, &fs.first_pindex, &prot, &wired);
312 	if (result != KERN_SUCCESS) {
313 		if (growstack && result == KERN_INVALID_ADDRESS &&
314 		    map != kernel_map) {
315 			result = vm_map_growstack(curproc, vaddr);
316 			if (result != KERN_SUCCESS)
317 				return (KERN_FAILURE);
318 			growstack = FALSE;
319 			goto RetryFault;
320 		}
321 		return (result);
322 	}
323 
324 	map_generation = fs.map->timestamp;
325 
326 	if (fs.entry->eflags & MAP_ENTRY_NOFAULT) {
327 		panic("vm_fault: fault on nofault entry, addr: %lx",
328 		    (u_long)vaddr);
329 	}
330 
331 	if (fs.entry->eflags & MAP_ENTRY_IN_TRANSITION &&
332 	    fs.entry->wiring_thread != curthread) {
333 		vm_map_unlock_read(fs.map);
334 		vm_map_lock(fs.map);
335 		if (vm_map_lookup_entry(fs.map, vaddr, &fs.entry) &&
336 		    (fs.entry->eflags & MAP_ENTRY_IN_TRANSITION)) {
337 			if (fs.vp != NULL) {
338 				vput(fs.vp);
339 				fs.vp = NULL;
340 			}
341 			fs.entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
342 			vm_map_unlock_and_wait(fs.map, 0);
343 		} else
344 			vm_map_unlock(fs.map);
345 		goto RetryFault;
346 	}
347 
348 	if (wired)
349 		fault_type = prot | (fault_type & VM_PROT_COPY);
350 	else
351 		KASSERT((fault_flags & VM_FAULT_WIRE) == 0,
352 		    ("!wired && VM_FAULT_WIRE"));
353 
354 	if (fs.vp == NULL /* avoid locked vnode leak */ &&
355 	    (fault_flags & (VM_FAULT_WIRE | VM_FAULT_DIRTY)) == 0 &&
356 	    /* avoid calling vm_object_set_writeable_dirty() */
357 	    ((prot & VM_PROT_WRITE) == 0 ||
358 	    (fs.first_object->type != OBJT_VNODE &&
359 	    (fs.first_object->flags & OBJ_TMPFS_NODE) == 0) ||
360 	    (fs.first_object->flags & OBJ_MIGHTBEDIRTY) != 0)) {
361 		VM_OBJECT_RLOCK(fs.first_object);
362 		if ((prot & VM_PROT_WRITE) != 0 &&
363 		    (fs.first_object->type == OBJT_VNODE ||
364 		    (fs.first_object->flags & OBJ_TMPFS_NODE) != 0) &&
365 		    (fs.first_object->flags & OBJ_MIGHTBEDIRTY) == 0)
366 			goto fast_failed;
367 		m = vm_page_lookup(fs.first_object, fs.first_pindex);
368 		/* A busy page can be mapped for read|execute access. */
369 		if (m == NULL || ((prot & VM_PROT_WRITE) != 0 &&
370 		    vm_page_busied(m)) || m->valid != VM_PAGE_BITS_ALL)
371 			goto fast_failed;
372 		result = pmap_enter(fs.map->pmap, vaddr, m, prot,
373 		   fault_type | PMAP_ENTER_NOSLEEP | (wired ? PMAP_ENTER_WIRED :
374 		   0), 0);
375 		if (result != KERN_SUCCESS)
376 			goto fast_failed;
377 		if (m_hold != NULL) {
378 			*m_hold = m;
379 			vm_page_lock(m);
380 			vm_page_hold(m);
381 			vm_page_unlock(m);
382 		}
383 		vm_fault_dirty(fs.entry, m, prot, fault_type, fault_flags,
384 		    FALSE);
385 		VM_OBJECT_RUNLOCK(fs.first_object);
386 		if (!wired)
387 			vm_fault_prefault(&fs, vaddr, PFBAK, PFFOR);
388 		vm_map_lookup_done(fs.map, fs.entry);
389 		curthread->td_ru.ru_minflt++;
390 		return (KERN_SUCCESS);
391 fast_failed:
392 		if (!VM_OBJECT_TRYUPGRADE(fs.first_object)) {
393 			VM_OBJECT_RUNLOCK(fs.first_object);
394 			VM_OBJECT_WLOCK(fs.first_object);
395 		}
396 	} else {
397 		VM_OBJECT_WLOCK(fs.first_object);
398 	}
399 
400 	/*
401 	 * Make a reference to this object to prevent its disposal while we
402 	 * are messing with it.  Once we have the reference, the map is free
403 	 * to be diddled.  Since objects reference their shadows (and copies),
404 	 * they will stay around as well.
405 	 *
406 	 * Bump the paging-in-progress count to prevent size changes (e.g.
407 	 * truncation operations) during I/O.  This must be done after
408 	 * obtaining the vnode lock in order to avoid possible deadlocks.
409 	 */
410 	vm_object_reference_locked(fs.first_object);
411 	vm_object_pip_add(fs.first_object, 1);
412 
413 	fs.lookup_still_valid = TRUE;
414 
415 	fs.first_m = NULL;
416 
417 	/*
418 	 * Search for the page at object/offset.
419 	 */
420 	fs.object = fs.first_object;
421 	fs.pindex = fs.first_pindex;
422 	while (TRUE) {
423 		/*
424 		 * If the object is dead, we stop here
425 		 */
426 		if (fs.object->flags & OBJ_DEAD) {
427 			unlock_and_deallocate(&fs);
428 			return (KERN_PROTECTION_FAILURE);
429 		}
430 
431 		/*
432 		 * See if page is resident
433 		 */
434 		fs.m = vm_page_lookup(fs.object, fs.pindex);
435 		if (fs.m != NULL) {
436 			/*
437 			 * Wait/Retry if the page is busy.  We have to do this
438 			 * if the page is either exclusive or shared busy
439 			 * because the vm_pager may be using read busy for
440 			 * pageouts (and even pageins if it is the vnode
441 			 * pager), and we could end up trying to pagein and
442 			 * pageout the same page simultaneously.
443 			 *
444 			 * We can theoretically allow the busy case on a read
445 			 * fault if the page is marked valid, but since such
446 			 * pages are typically already pmap'd, putting that
447 			 * special case in might be more effort then it is
448 			 * worth.  We cannot under any circumstances mess
449 			 * around with a shared busied page except, perhaps,
450 			 * to pmap it.
451 			 */
452 			if (vm_page_busied(fs.m)) {
453 				/*
454 				 * Reference the page before unlocking and
455 				 * sleeping so that the page daemon is less
456 				 * likely to reclaim it.
457 				 */
458 				vm_page_aflag_set(fs.m, PGA_REFERENCED);
459 				if (fs.object != fs.first_object) {
460 					if (!VM_OBJECT_TRYWLOCK(
461 					    fs.first_object)) {
462 						VM_OBJECT_WUNLOCK(fs.object);
463 						VM_OBJECT_WLOCK(fs.first_object);
464 						VM_OBJECT_WLOCK(fs.object);
465 					}
466 					vm_page_lock(fs.first_m);
467 					vm_page_free(fs.first_m);
468 					vm_page_unlock(fs.first_m);
469 					vm_object_pip_wakeup(fs.first_object);
470 					VM_OBJECT_WUNLOCK(fs.first_object);
471 					fs.first_m = NULL;
472 				}
473 				unlock_map(&fs);
474 				if (fs.m == vm_page_lookup(fs.object,
475 				    fs.pindex)) {
476 					vm_page_sleep_if_busy(fs.m, "vmpfw");
477 				}
478 				vm_object_pip_wakeup(fs.object);
479 				VM_OBJECT_WUNLOCK(fs.object);
480 				PCPU_INC(cnt.v_intrans);
481 				vm_object_deallocate(fs.first_object);
482 				goto RetryFault;
483 			}
484 			vm_page_lock(fs.m);
485 			vm_page_remque(fs.m);
486 			vm_page_unlock(fs.m);
487 
488 			/*
489 			 * Mark page busy for other processes, and the
490 			 * pagedaemon.  If it still isn't completely valid
491 			 * (readable), jump to readrest, else break-out ( we
492 			 * found the page ).
493 			 */
494 			vm_page_xbusy(fs.m);
495 			if (fs.m->valid != VM_PAGE_BITS_ALL)
496 				goto readrest;
497 			break;
498 		}
499 		KASSERT(fs.m == NULL, ("fs.m should be NULL, not %p", fs.m));
500 
501 		/*
502 		 * Page is not resident.  If the pager might contain the page
503 		 * or this is the beginning of the search, allocate a new
504 		 * page.  (Default objects are zero-fill, so there is no real
505 		 * pager for them.)
506 		 */
507 		if (fs.object->type != OBJT_DEFAULT ||
508 		    fs.object == fs.first_object) {
509 			if (fs.pindex >= fs.object->size) {
510 				unlock_and_deallocate(&fs);
511 				return (KERN_PROTECTION_FAILURE);
512 			}
513 
514 			/*
515 			 * Allocate a new page for this object/offset pair.
516 			 *
517 			 * Unlocked read of the p_flag is harmless. At
518 			 * worst, the P_KILLED might be not observed
519 			 * there, and allocation can fail, causing
520 			 * restart and new reading of the p_flag.
521 			 */
522 			if (!vm_page_count_severe() || P_KILLED(curproc)) {
523 #if VM_NRESERVLEVEL > 0
524 				vm_object_color(fs.object, atop(vaddr) -
525 				    fs.pindex);
526 #endif
527 				alloc_req = P_KILLED(curproc) ?
528 				    VM_ALLOC_SYSTEM : VM_ALLOC_NORMAL;
529 				if (fs.object->type != OBJT_VNODE &&
530 				    fs.object->backing_object == NULL)
531 					alloc_req |= VM_ALLOC_ZERO;
532 				fs.m = vm_page_alloc(fs.object, fs.pindex,
533 				    alloc_req);
534 			}
535 			if (fs.m == NULL) {
536 				unlock_and_deallocate(&fs);
537 				VM_WAITPFAULT;
538 				goto RetryFault;
539 			} else if (fs.m->valid == VM_PAGE_BITS_ALL)
540 				break;
541 		}
542 
543 readrest:
544 		/*
545 		 * We have either allocated a new page or found an existing
546 		 * page that is only partially valid.
547 		 *
548 		 * Attempt to fault-in the page if there is a chance that the
549 		 * pager has it, and potentially fault in additional pages
550 		 * at the same time.
551 		 */
552 		if (fs.object->type != OBJT_DEFAULT) {
553 			int rv;
554 			u_char behavior = vm_map_entry_behavior(fs.entry);
555 
556 			era = fs.entry->read_ahead;
557 			if (behavior == MAP_ENTRY_BEHAV_RANDOM ||
558 			    P_KILLED(curproc)) {
559 				behind = 0;
560 				nera = 0;
561 				ahead = 0;
562 			} else if (behavior == MAP_ENTRY_BEHAV_SEQUENTIAL) {
563 				behind = 0;
564 				nera = VM_FAULT_READ_AHEAD_MAX;
565 				ahead = nera;
566 				if (fs.pindex == fs.entry->next_read)
567 					vm_fault_dontneed(&fs, vaddr, ahead);
568 			} else if (fs.pindex == fs.entry->next_read) {
569 				/*
570 				 * This is a sequential fault.  Arithmetically
571 				 * increase the requested number of pages in
572 				 * the read-ahead window.  The requested
573 				 * number of pages is "# of sequential faults
574 				 * x (read ahead min + 1) + read ahead min"
575 				 */
576 				behind = 0;
577 				nera = VM_FAULT_READ_AHEAD_MIN;
578 				if (era > 0) {
579 					nera += era + 1;
580 					if (nera > VM_FAULT_READ_AHEAD_MAX)
581 						nera = VM_FAULT_READ_AHEAD_MAX;
582 				}
583 				ahead = nera;
584 				if (era == VM_FAULT_READ_AHEAD_MAX)
585 					vm_fault_dontneed(&fs, vaddr, ahead);
586 			} else {
587 				/*
588 				 * This is a non-sequential fault.  Request a
589 				 * cluster of pages that is aligned to a
590 				 * VM_FAULT_READ_DEFAULT page offset boundary
591 				 * within the object.  Alignment to a page
592 				 * offset boundary is more likely to coincide
593 				 * with the underlying file system block than
594 				 * alignment to a virtual address boundary.
595 				 */
596 				cluster_offset = fs.pindex %
597 				    VM_FAULT_READ_DEFAULT;
598 				behind = ulmin(cluster_offset,
599 				    atop(vaddr - fs.entry->start));
600 				nera = 0;
601 				ahead = VM_FAULT_READ_DEFAULT - 1 -
602 				    cluster_offset;
603 			}
604 			ahead = ulmin(ahead, atop(fs.entry->end - vaddr) - 1);
605 			if (era != nera)
606 				fs.entry->read_ahead = nera;
607 
608 			/*
609 			 * Call the pager to retrieve the data, if any, after
610 			 * releasing the lock on the map.  We hold a ref on
611 			 * fs.object and the pages are exclusive busied.
612 			 */
613 			unlock_map(&fs);
614 
615 			if (fs.object->type == OBJT_VNODE) {
616 				vp = fs.object->handle;
617 				if (vp == fs.vp)
618 					goto vnode_locked;
619 				else if (fs.vp != NULL) {
620 					vput(fs.vp);
621 					fs.vp = NULL;
622 				}
623 				locked = VOP_ISLOCKED(vp);
624 
625 				if (locked != LK_EXCLUSIVE)
626 					locked = LK_SHARED;
627 				/* Do not sleep for vnode lock while fs.m is busy */
628 				error = vget(vp, locked | LK_CANRECURSE |
629 				    LK_NOWAIT, curthread);
630 				if (error != 0) {
631 					vhold(vp);
632 					release_page(&fs);
633 					unlock_and_deallocate(&fs);
634 					error = vget(vp, locked | LK_RETRY |
635 					    LK_CANRECURSE, curthread);
636 					vdrop(vp);
637 					fs.vp = vp;
638 					KASSERT(error == 0,
639 					    ("vm_fault: vget failed"));
640 					goto RetryFault;
641 				}
642 				fs.vp = vp;
643 			}
644 vnode_locked:
645 			KASSERT(fs.vp == NULL || !fs.map->system_map,
646 			    ("vm_fault: vnode-backed object mapped by system map"));
647 
648 			/*
649 			 * Page in the requested page and hint the pager,
650 			 * that it may bring up surrounding pages.
651 			 */
652 			rv = vm_pager_get_pages(fs.object, &fs.m, 1,
653 			    &behind, &ahead);
654 			if (rv == VM_PAGER_OK) {
655 				faultcount = behind + 1 + ahead;
656 				hardfault++;
657 				break; /* break to PAGE HAS BEEN FOUND */
658 			}
659 			if (rv == VM_PAGER_ERROR)
660 				printf("vm_fault: pager read error, pid %d (%s)\n",
661 				    curproc->p_pid, curproc->p_comm);
662 
663 			/*
664 			 * If an I/O error occurred or the requested page was
665 			 * outside the range of the pager, clean up and return
666 			 * an error.
667 			 */
668 			if (rv == VM_PAGER_ERROR || rv == VM_PAGER_BAD) {
669 				vm_page_lock(fs.m);
670 				vm_page_free(fs.m);
671 				vm_page_unlock(fs.m);
672 				fs.m = NULL;
673 				unlock_and_deallocate(&fs);
674 				return (rv == VM_PAGER_ERROR ? KERN_FAILURE :
675 				    KERN_PROTECTION_FAILURE);
676 			}
677 
678 			/*
679 			 * The requested page does not exist at this object/
680 			 * offset.  Remove the invalid page from the object,
681 			 * waking up anyone waiting for it, and continue on to
682 			 * the next object.  However, if this is the top-level
683 			 * object, we must leave the busy page in place to
684 			 * prevent another process from rushing past us, and
685 			 * inserting the page in that object at the same time
686 			 * that we are.
687 			 */
688 			if (fs.object != fs.first_object) {
689 				vm_page_lock(fs.m);
690 				vm_page_free(fs.m);
691 				vm_page_unlock(fs.m);
692 				fs.m = NULL;
693 			}
694 		}
695 
696 		/*
697 		 * We get here if the object has default pager (or unwiring)
698 		 * or the pager doesn't have the page.
699 		 */
700 		if (fs.object == fs.first_object)
701 			fs.first_m = fs.m;
702 
703 		/*
704 		 * Move on to the next object.  Lock the next object before
705 		 * unlocking the current one.
706 		 */
707 		next_object = fs.object->backing_object;
708 		if (next_object == NULL) {
709 			/*
710 			 * If there's no object left, fill the page in the top
711 			 * object with zeros.
712 			 */
713 			if (fs.object != fs.first_object) {
714 				vm_object_pip_wakeup(fs.object);
715 				VM_OBJECT_WUNLOCK(fs.object);
716 
717 				fs.object = fs.first_object;
718 				fs.pindex = fs.first_pindex;
719 				fs.m = fs.first_m;
720 				VM_OBJECT_WLOCK(fs.object);
721 			}
722 			fs.first_m = NULL;
723 
724 			/*
725 			 * Zero the page if necessary and mark it valid.
726 			 */
727 			if ((fs.m->flags & PG_ZERO) == 0) {
728 				pmap_zero_page(fs.m);
729 			} else {
730 				PCPU_INC(cnt.v_ozfod);
731 			}
732 			PCPU_INC(cnt.v_zfod);
733 			fs.m->valid = VM_PAGE_BITS_ALL;
734 			/* Don't try to prefault neighboring pages. */
735 			faultcount = 1;
736 			break;	/* break to PAGE HAS BEEN FOUND */
737 		} else {
738 			KASSERT(fs.object != next_object,
739 			    ("object loop %p", next_object));
740 			VM_OBJECT_WLOCK(next_object);
741 			vm_object_pip_add(next_object, 1);
742 			if (fs.object != fs.first_object)
743 				vm_object_pip_wakeup(fs.object);
744 			fs.pindex +=
745 			    OFF_TO_IDX(fs.object->backing_object_offset);
746 			VM_OBJECT_WUNLOCK(fs.object);
747 			fs.object = next_object;
748 		}
749 	}
750 
751 	vm_page_assert_xbusied(fs.m);
752 
753 	/*
754 	 * PAGE HAS BEEN FOUND. [Loop invariant still holds -- the object lock
755 	 * is held.]
756 	 */
757 
758 	/*
759 	 * If the page is being written, but isn't already owned by the
760 	 * top-level object, we have to copy it into a new page owned by the
761 	 * top-level object.
762 	 */
763 	if (fs.object != fs.first_object) {
764 		/*
765 		 * We only really need to copy if we want to write it.
766 		 */
767 		if ((fault_type & (VM_PROT_COPY | VM_PROT_WRITE)) != 0) {
768 			/*
769 			 * This allows pages to be virtually copied from a
770 			 * backing_object into the first_object, where the
771 			 * backing object has no other refs to it, and cannot
772 			 * gain any more refs.  Instead of a bcopy, we just
773 			 * move the page from the backing object to the
774 			 * first object.  Note that we must mark the page
775 			 * dirty in the first object so that it will go out
776 			 * to swap when needed.
777 			 */
778 			is_first_object_locked = FALSE;
779 			if (
780 				/*
781 				 * Only one shadow object
782 				 */
783 				(fs.object->shadow_count == 1) &&
784 				/*
785 				 * No COW refs, except us
786 				 */
787 				(fs.object->ref_count == 1) &&
788 				/*
789 				 * No one else can look this object up
790 				 */
791 				(fs.object->handle == NULL) &&
792 				/*
793 				 * No other ways to look the object up
794 				 */
795 				((fs.object->type == OBJT_DEFAULT) ||
796 				 (fs.object->type == OBJT_SWAP)) &&
797 			    (is_first_object_locked = VM_OBJECT_TRYWLOCK(fs.first_object)) &&
798 				/*
799 				 * We don't chase down the shadow chain
800 				 */
801 			    fs.object == fs.first_object->backing_object) {
802 				vm_page_lock(fs.m);
803 				vm_page_remove(fs.m);
804 				vm_page_unlock(fs.m);
805 				vm_page_lock(fs.first_m);
806 				vm_page_replace_checked(fs.m, fs.first_object,
807 				    fs.first_pindex, fs.first_m);
808 				vm_page_free(fs.first_m);
809 				vm_page_unlock(fs.first_m);
810 				vm_page_dirty(fs.m);
811 #if VM_NRESERVLEVEL > 0
812 				/*
813 				 * Rename the reservation.
814 				 */
815 				vm_reserv_rename(fs.m, fs.first_object,
816 				    fs.object, OFF_TO_IDX(
817 				    fs.first_object->backing_object_offset));
818 #endif
819 				/*
820 				 * Removing the page from the backing object
821 				 * unbusied it.
822 				 */
823 				vm_page_xbusy(fs.m);
824 				fs.first_m = fs.m;
825 				fs.m = NULL;
826 				PCPU_INC(cnt.v_cow_optim);
827 			} else {
828 				/*
829 				 * Oh, well, lets copy it.
830 				 */
831 				pmap_copy_page(fs.m, fs.first_m);
832 				fs.first_m->valid = VM_PAGE_BITS_ALL;
833 				if (wired && (fault_flags &
834 				    VM_FAULT_WIRE) == 0) {
835 					vm_page_lock(fs.first_m);
836 					vm_page_wire(fs.first_m);
837 					vm_page_unlock(fs.first_m);
838 
839 					vm_page_lock(fs.m);
840 					vm_page_unwire(fs.m, PQ_INACTIVE);
841 					vm_page_unlock(fs.m);
842 				}
843 				/*
844 				 * We no longer need the old page or object.
845 				 */
846 				release_page(&fs);
847 			}
848 			/*
849 			 * fs.object != fs.first_object due to above
850 			 * conditional
851 			 */
852 			vm_object_pip_wakeup(fs.object);
853 			VM_OBJECT_WUNLOCK(fs.object);
854 			/*
855 			 * Only use the new page below...
856 			 */
857 			fs.object = fs.first_object;
858 			fs.pindex = fs.first_pindex;
859 			fs.m = fs.first_m;
860 			if (!is_first_object_locked)
861 				VM_OBJECT_WLOCK(fs.object);
862 			PCPU_INC(cnt.v_cow_faults);
863 			curthread->td_cow++;
864 		} else {
865 			prot &= ~VM_PROT_WRITE;
866 		}
867 	}
868 
869 	/*
870 	 * We must verify that the maps have not changed since our last
871 	 * lookup.
872 	 */
873 	if (!fs.lookup_still_valid) {
874 		vm_object_t retry_object;
875 		vm_pindex_t retry_pindex;
876 		vm_prot_t retry_prot;
877 
878 		if (!vm_map_trylock_read(fs.map)) {
879 			release_page(&fs);
880 			unlock_and_deallocate(&fs);
881 			goto RetryFault;
882 		}
883 		fs.lookup_still_valid = TRUE;
884 		if (fs.map->timestamp != map_generation) {
885 			result = vm_map_lookup_locked(&fs.map, vaddr, fault_type,
886 			    &fs.entry, &retry_object, &retry_pindex, &retry_prot, &wired);
887 
888 			/*
889 			 * If we don't need the page any longer, put it on the inactive
890 			 * list (the easiest thing to do here).  If no one needs it,
891 			 * pageout will grab it eventually.
892 			 */
893 			if (result != KERN_SUCCESS) {
894 				release_page(&fs);
895 				unlock_and_deallocate(&fs);
896 
897 				/*
898 				 * If retry of map lookup would have blocked then
899 				 * retry fault from start.
900 				 */
901 				if (result == KERN_FAILURE)
902 					goto RetryFault;
903 				return (result);
904 			}
905 			if ((retry_object != fs.first_object) ||
906 			    (retry_pindex != fs.first_pindex)) {
907 				release_page(&fs);
908 				unlock_and_deallocate(&fs);
909 				goto RetryFault;
910 			}
911 
912 			/*
913 			 * Check whether the protection has changed or the object has
914 			 * been copied while we left the map unlocked. Changing from
915 			 * read to write permission is OK - we leave the page
916 			 * write-protected, and catch the write fault. Changing from
917 			 * write to read permission means that we can't mark the page
918 			 * write-enabled after all.
919 			 */
920 			prot &= retry_prot;
921 		}
922 	}
923 	/*
924 	 * If the page was filled by a pager, update the map entry's
925 	 * last read offset.
926 	 *
927 	 * XXX The following assignment modifies the map
928 	 * without holding a write lock on it.
929 	 */
930 	if (hardfault)
931 		fs.entry->next_read = fs.pindex + ahead + 1;
932 
933 	vm_fault_dirty(fs.entry, fs.m, prot, fault_type, fault_flags, TRUE);
934 	vm_page_assert_xbusied(fs.m);
935 
936 	/*
937 	 * Page must be completely valid or it is not fit to
938 	 * map into user space.  vm_pager_get_pages() ensures this.
939 	 */
940 	KASSERT(fs.m->valid == VM_PAGE_BITS_ALL,
941 	    ("vm_fault: page %p partially invalid", fs.m));
942 	VM_OBJECT_WUNLOCK(fs.object);
943 
944 	/*
945 	 * Put this page into the physical map.  We had to do the unlock above
946 	 * because pmap_enter() may sleep.  We don't put the page
947 	 * back on the active queue until later so that the pageout daemon
948 	 * won't find it (yet).
949 	 */
950 	pmap_enter(fs.map->pmap, vaddr, fs.m, prot,
951 	    fault_type | (wired ? PMAP_ENTER_WIRED : 0), 0);
952 	if (faultcount != 1 && (fault_flags & VM_FAULT_WIRE) == 0 &&
953 	    wired == 0)
954 		vm_fault_prefault(&fs, vaddr,
955 		    faultcount > 0 ? behind : PFBAK,
956 		    faultcount > 0 ? ahead : PFFOR);
957 	VM_OBJECT_WLOCK(fs.object);
958 	vm_page_lock(fs.m);
959 
960 	/*
961 	 * If the page is not wired down, then put it where the pageout daemon
962 	 * can find it.
963 	 */
964 	if ((fault_flags & VM_FAULT_WIRE) != 0) {
965 		KASSERT(wired, ("VM_FAULT_WIRE && !wired"));
966 		vm_page_wire(fs.m);
967 	} else
968 		vm_page_activate(fs.m);
969 	if (m_hold != NULL) {
970 		*m_hold = fs.m;
971 		vm_page_hold(fs.m);
972 	}
973 	vm_page_unlock(fs.m);
974 	vm_page_xunbusy(fs.m);
975 
976 	/*
977 	 * Unlock everything, and return
978 	 */
979 	unlock_and_deallocate(&fs);
980 	if (hardfault) {
981 		PCPU_INC(cnt.v_io_faults);
982 		curthread->td_ru.ru_majflt++;
983 #ifdef RACCT
984 		if (racct_enable && fs.object->type == OBJT_VNODE) {
985 			PROC_LOCK(curproc);
986 			if ((fault_type & (VM_PROT_COPY | VM_PROT_WRITE)) != 0) {
987 				racct_add_force(curproc, RACCT_WRITEBPS,
988 				    PAGE_SIZE + behind * PAGE_SIZE);
989 				racct_add_force(curproc, RACCT_WRITEIOPS, 1);
990 			} else {
991 				racct_add_force(curproc, RACCT_READBPS,
992 				    PAGE_SIZE + ahead * PAGE_SIZE);
993 				racct_add_force(curproc, RACCT_READIOPS, 1);
994 			}
995 			PROC_UNLOCK(curproc);
996 		}
997 #endif
998 	} else
999 		curthread->td_ru.ru_minflt++;
1000 
1001 	return (KERN_SUCCESS);
1002 }
1003 
1004 /*
1005  * Speed up the reclamation of pages that precede the faulting pindex within
1006  * the first object of the shadow chain.  Essentially, perform the equivalent
1007  * to madvise(..., MADV_DONTNEED) on a large cluster of pages that precedes
1008  * the faulting pindex by the cluster size when the pages read by vm_fault()
1009  * cross a cluster-size boundary.  The cluster size is the greater of the
1010  * smallest superpage size and VM_FAULT_DONTNEED_MIN.
1011  *
1012  * When "fs->first_object" is a shadow object, the pages in the backing object
1013  * that precede the faulting pindex are deactivated by vm_fault().  So, this
1014  * function must only be concerned with pages in the first object.
1015  */
1016 static void
1017 vm_fault_dontneed(const struct faultstate *fs, vm_offset_t vaddr, int ahead)
1018 {
1019 	vm_map_entry_t entry;
1020 	vm_object_t first_object, object;
1021 	vm_offset_t end, start;
1022 	vm_page_t m, m_next;
1023 	vm_pindex_t pend, pstart;
1024 	vm_size_t size;
1025 
1026 	object = fs->object;
1027 	VM_OBJECT_ASSERT_WLOCKED(object);
1028 	first_object = fs->first_object;
1029 	if (first_object != object) {
1030 		if (!VM_OBJECT_TRYWLOCK(first_object)) {
1031 			VM_OBJECT_WUNLOCK(object);
1032 			VM_OBJECT_WLOCK(first_object);
1033 			VM_OBJECT_WLOCK(object);
1034 		}
1035 	}
1036 	/* Neither fictitious nor unmanaged pages can be reclaimed. */
1037 	if ((first_object->flags & (OBJ_FICTITIOUS | OBJ_UNMANAGED)) == 0) {
1038 		size = VM_FAULT_DONTNEED_MIN;
1039 		if (MAXPAGESIZES > 1 && size < pagesizes[1])
1040 			size = pagesizes[1];
1041 		end = rounddown2(vaddr, size);
1042 		if (vaddr - end >= size - PAGE_SIZE - ptoa(ahead) &&
1043 		    (entry = fs->entry)->start < end) {
1044 			if (end - entry->start < size)
1045 				start = entry->start;
1046 			else
1047 				start = end - size;
1048 			pmap_advise(fs->map->pmap, start, end, MADV_DONTNEED);
1049 			pstart = OFF_TO_IDX(entry->offset) + atop(start -
1050 			    entry->start);
1051 			m_next = vm_page_find_least(first_object, pstart);
1052 			pend = OFF_TO_IDX(entry->offset) + atop(end -
1053 			    entry->start);
1054 			while ((m = m_next) != NULL && m->pindex < pend) {
1055 				m_next = TAILQ_NEXT(m, listq);
1056 				if (m->valid != VM_PAGE_BITS_ALL ||
1057 				    vm_page_busied(m))
1058 					continue;
1059 
1060 				/*
1061 				 * Don't clear PGA_REFERENCED, since it would
1062 				 * likely represent a reference by a different
1063 				 * process.
1064 				 *
1065 				 * Typically, at this point, prefetched pages
1066 				 * are still in the inactive queue.  Only
1067 				 * pages that triggered page faults are in the
1068 				 * active queue.
1069 				 */
1070 				vm_page_lock(m);
1071 				vm_page_deactivate(m);
1072 				vm_page_unlock(m);
1073 			}
1074 		}
1075 	}
1076 	if (first_object != object)
1077 		VM_OBJECT_WUNLOCK(first_object);
1078 }
1079 
1080 /*
1081  * vm_fault_prefault provides a quick way of clustering
1082  * pagefaults into a processes address space.  It is a "cousin"
1083  * of vm_map_pmap_enter, except it runs at page fault time instead
1084  * of mmap time.
1085  */
1086 static void
1087 vm_fault_prefault(const struct faultstate *fs, vm_offset_t addra,
1088     int backward, int forward)
1089 {
1090 	pmap_t pmap;
1091 	vm_map_entry_t entry;
1092 	vm_object_t backing_object, lobject;
1093 	vm_offset_t addr, starta;
1094 	vm_pindex_t pindex;
1095 	vm_page_t m;
1096 	int i;
1097 
1098 	pmap = fs->map->pmap;
1099 	if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace))
1100 		return;
1101 
1102 	entry = fs->entry;
1103 
1104 	starta = addra - backward * PAGE_SIZE;
1105 	if (starta < entry->start) {
1106 		starta = entry->start;
1107 	} else if (starta > addra) {
1108 		starta = 0;
1109 	}
1110 
1111 	/*
1112 	 * Generate the sequence of virtual addresses that are candidates for
1113 	 * prefaulting in an outward spiral from the faulting virtual address,
1114 	 * "addra".  Specifically, the sequence is "addra - PAGE_SIZE", "addra
1115 	 * + PAGE_SIZE", "addra - 2 * PAGE_SIZE", "addra + 2 * PAGE_SIZE", ...
1116 	 * If the candidate address doesn't have a backing physical page, then
1117 	 * the loop immediately terminates.
1118 	 */
1119 	for (i = 0; i < 2 * imax(backward, forward); i++) {
1120 		addr = addra + ((i >> 1) + 1) * ((i & 1) == 0 ? -PAGE_SIZE :
1121 		    PAGE_SIZE);
1122 		if (addr > addra + forward * PAGE_SIZE)
1123 			addr = 0;
1124 
1125 		if (addr < starta || addr >= entry->end)
1126 			continue;
1127 
1128 		if (!pmap_is_prefaultable(pmap, addr))
1129 			continue;
1130 
1131 		pindex = ((addr - entry->start) + entry->offset) >> PAGE_SHIFT;
1132 		lobject = entry->object.vm_object;
1133 		VM_OBJECT_RLOCK(lobject);
1134 		while ((m = vm_page_lookup(lobject, pindex)) == NULL &&
1135 		    lobject->type == OBJT_DEFAULT &&
1136 		    (backing_object = lobject->backing_object) != NULL) {
1137 			KASSERT((lobject->backing_object_offset & PAGE_MASK) ==
1138 			    0, ("vm_fault_prefault: unaligned object offset"));
1139 			pindex += lobject->backing_object_offset >> PAGE_SHIFT;
1140 			VM_OBJECT_RLOCK(backing_object);
1141 			VM_OBJECT_RUNLOCK(lobject);
1142 			lobject = backing_object;
1143 		}
1144 		if (m == NULL) {
1145 			VM_OBJECT_RUNLOCK(lobject);
1146 			break;
1147 		}
1148 		if (m->valid == VM_PAGE_BITS_ALL &&
1149 		    (m->flags & PG_FICTITIOUS) == 0)
1150 			pmap_enter_quick(pmap, addr, m, entry->protection);
1151 		VM_OBJECT_RUNLOCK(lobject);
1152 	}
1153 }
1154 
1155 /*
1156  * Hold each of the physical pages that are mapped by the specified range of
1157  * virtual addresses, ["addr", "addr" + "len"), if those mappings are valid
1158  * and allow the specified types of access, "prot".  If all of the implied
1159  * pages are successfully held, then the number of held pages is returned
1160  * together with pointers to those pages in the array "ma".  However, if any
1161  * of the pages cannot be held, -1 is returned.
1162  */
1163 int
1164 vm_fault_quick_hold_pages(vm_map_t map, vm_offset_t addr, vm_size_t len,
1165     vm_prot_t prot, vm_page_t *ma, int max_count)
1166 {
1167 	vm_offset_t end, va;
1168 	vm_page_t *mp;
1169 	int count;
1170 	boolean_t pmap_failed;
1171 
1172 	if (len == 0)
1173 		return (0);
1174 	end = round_page(addr + len);
1175 	addr = trunc_page(addr);
1176 
1177 	/*
1178 	 * Check for illegal addresses.
1179 	 */
1180 	if (addr < vm_map_min(map) || addr > end || end > vm_map_max(map))
1181 		return (-1);
1182 
1183 	if (atop(end - addr) > max_count)
1184 		panic("vm_fault_quick_hold_pages: count > max_count");
1185 	count = atop(end - addr);
1186 
1187 	/*
1188 	 * Most likely, the physical pages are resident in the pmap, so it is
1189 	 * faster to try pmap_extract_and_hold() first.
1190 	 */
1191 	pmap_failed = FALSE;
1192 	for (mp = ma, va = addr; va < end; mp++, va += PAGE_SIZE) {
1193 		*mp = pmap_extract_and_hold(map->pmap, va, prot);
1194 		if (*mp == NULL)
1195 			pmap_failed = TRUE;
1196 		else if ((prot & VM_PROT_WRITE) != 0 &&
1197 		    (*mp)->dirty != VM_PAGE_BITS_ALL) {
1198 			/*
1199 			 * Explicitly dirty the physical page.  Otherwise, the
1200 			 * caller's changes may go unnoticed because they are
1201 			 * performed through an unmanaged mapping or by a DMA
1202 			 * operation.
1203 			 *
1204 			 * The object lock is not held here.
1205 			 * See vm_page_clear_dirty_mask().
1206 			 */
1207 			vm_page_dirty(*mp);
1208 		}
1209 	}
1210 	if (pmap_failed) {
1211 		/*
1212 		 * One or more pages could not be held by the pmap.  Either no
1213 		 * page was mapped at the specified virtual address or that
1214 		 * mapping had insufficient permissions.  Attempt to fault in
1215 		 * and hold these pages.
1216 		 */
1217 		for (mp = ma, va = addr; va < end; mp++, va += PAGE_SIZE)
1218 			if (*mp == NULL && vm_fault_hold(map, va, prot,
1219 			    VM_FAULT_NORMAL, mp) != KERN_SUCCESS)
1220 				goto error;
1221 	}
1222 	return (count);
1223 error:
1224 	for (mp = ma; mp < ma + count; mp++)
1225 		if (*mp != NULL) {
1226 			vm_page_lock(*mp);
1227 			vm_page_unhold(*mp);
1228 			vm_page_unlock(*mp);
1229 		}
1230 	return (-1);
1231 }
1232 
1233 /*
1234  *	Routine:
1235  *		vm_fault_copy_entry
1236  *	Function:
1237  *		Create new shadow object backing dst_entry with private copy of
1238  *		all underlying pages. When src_entry is equal to dst_entry,
1239  *		function implements COW for wired-down map entry. Otherwise,
1240  *		it forks wired entry into dst_map.
1241  *
1242  *	In/out conditions:
1243  *		The source and destination maps must be locked for write.
1244  *		The source map entry must be wired down (or be a sharing map
1245  *		entry corresponding to a main map entry that is wired down).
1246  */
1247 void
1248 vm_fault_copy_entry(vm_map_t dst_map, vm_map_t src_map,
1249     vm_map_entry_t dst_entry, vm_map_entry_t src_entry,
1250     vm_ooffset_t *fork_charge)
1251 {
1252 	vm_object_t backing_object, dst_object, object, src_object;
1253 	vm_pindex_t dst_pindex, pindex, src_pindex;
1254 	vm_prot_t access, prot;
1255 	vm_offset_t vaddr;
1256 	vm_page_t dst_m;
1257 	vm_page_t src_m;
1258 	boolean_t upgrade;
1259 
1260 #ifdef	lint
1261 	src_map++;
1262 #endif	/* lint */
1263 
1264 	upgrade = src_entry == dst_entry;
1265 	access = prot = dst_entry->protection;
1266 
1267 	src_object = src_entry->object.vm_object;
1268 	src_pindex = OFF_TO_IDX(src_entry->offset);
1269 
1270 	if (upgrade && (dst_entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) {
1271 		dst_object = src_object;
1272 		vm_object_reference(dst_object);
1273 	} else {
1274 		/*
1275 		 * Create the top-level object for the destination entry. (Doesn't
1276 		 * actually shadow anything - we copy the pages directly.)
1277 		 */
1278 		dst_object = vm_object_allocate(OBJT_DEFAULT,
1279 		    OFF_TO_IDX(dst_entry->end - dst_entry->start));
1280 #if VM_NRESERVLEVEL > 0
1281 		dst_object->flags |= OBJ_COLORED;
1282 		dst_object->pg_color = atop(dst_entry->start);
1283 #endif
1284 	}
1285 
1286 	VM_OBJECT_WLOCK(dst_object);
1287 	KASSERT(upgrade || dst_entry->object.vm_object == NULL,
1288 	    ("vm_fault_copy_entry: vm_object not NULL"));
1289 	if (src_object != dst_object) {
1290 		dst_entry->object.vm_object = dst_object;
1291 		dst_entry->offset = 0;
1292 		dst_object->charge = dst_entry->end - dst_entry->start;
1293 	}
1294 	if (fork_charge != NULL) {
1295 		KASSERT(dst_entry->cred == NULL,
1296 		    ("vm_fault_copy_entry: leaked swp charge"));
1297 		dst_object->cred = curthread->td_ucred;
1298 		crhold(dst_object->cred);
1299 		*fork_charge += dst_object->charge;
1300 	} else if (dst_object->cred == NULL) {
1301 		KASSERT(dst_entry->cred != NULL, ("no cred for entry %p",
1302 		    dst_entry));
1303 		dst_object->cred = dst_entry->cred;
1304 		dst_entry->cred = NULL;
1305 	}
1306 
1307 	/*
1308 	 * If not an upgrade, then enter the mappings in the pmap as
1309 	 * read and/or execute accesses.  Otherwise, enter them as
1310 	 * write accesses.
1311 	 *
1312 	 * A writeable large page mapping is only created if all of
1313 	 * the constituent small page mappings are modified. Marking
1314 	 * PTEs as modified on inception allows promotion to happen
1315 	 * without taking potentially large number of soft faults.
1316 	 */
1317 	if (!upgrade)
1318 		access &= ~VM_PROT_WRITE;
1319 
1320 	/*
1321 	 * Loop through all of the virtual pages within the entry's
1322 	 * range, copying each page from the source object to the
1323 	 * destination object.  Since the source is wired, those pages
1324 	 * must exist.  In contrast, the destination is pageable.
1325 	 * Since the destination object does share any backing storage
1326 	 * with the source object, all of its pages must be dirtied,
1327 	 * regardless of whether they can be written.
1328 	 */
1329 	for (vaddr = dst_entry->start, dst_pindex = 0;
1330 	    vaddr < dst_entry->end;
1331 	    vaddr += PAGE_SIZE, dst_pindex++) {
1332 again:
1333 		/*
1334 		 * Find the page in the source object, and copy it in.
1335 		 * Because the source is wired down, the page will be
1336 		 * in memory.
1337 		 */
1338 		if (src_object != dst_object)
1339 			VM_OBJECT_RLOCK(src_object);
1340 		object = src_object;
1341 		pindex = src_pindex + dst_pindex;
1342 		while ((src_m = vm_page_lookup(object, pindex)) == NULL &&
1343 		    (backing_object = object->backing_object) != NULL) {
1344 			/*
1345 			 * Unless the source mapping is read-only or
1346 			 * it is presently being upgraded from
1347 			 * read-only, the first object in the shadow
1348 			 * chain should provide all of the pages.  In
1349 			 * other words, this loop body should never be
1350 			 * executed when the source mapping is already
1351 			 * read/write.
1352 			 */
1353 			KASSERT((src_entry->protection & VM_PROT_WRITE) == 0 ||
1354 			    upgrade,
1355 			    ("vm_fault_copy_entry: main object missing page"));
1356 
1357 			VM_OBJECT_RLOCK(backing_object);
1358 			pindex += OFF_TO_IDX(object->backing_object_offset);
1359 			if (object != dst_object)
1360 				VM_OBJECT_RUNLOCK(object);
1361 			object = backing_object;
1362 		}
1363 		KASSERT(src_m != NULL, ("vm_fault_copy_entry: page missing"));
1364 
1365 		if (object != dst_object) {
1366 			/*
1367 			 * Allocate a page in the destination object.
1368 			 */
1369 			dst_m = vm_page_alloc(dst_object, (src_object ==
1370 			    dst_object ? src_pindex : 0) + dst_pindex,
1371 			    VM_ALLOC_NORMAL);
1372 			if (dst_m == NULL) {
1373 				VM_OBJECT_WUNLOCK(dst_object);
1374 				VM_OBJECT_RUNLOCK(object);
1375 				VM_WAIT;
1376 				VM_OBJECT_WLOCK(dst_object);
1377 				goto again;
1378 			}
1379 			pmap_copy_page(src_m, dst_m);
1380 			VM_OBJECT_RUNLOCK(object);
1381 			dst_m->valid = VM_PAGE_BITS_ALL;
1382 			dst_m->dirty = VM_PAGE_BITS_ALL;
1383 		} else {
1384 			dst_m = src_m;
1385 			if (vm_page_sleep_if_busy(dst_m, "fltupg"))
1386 				goto again;
1387 			vm_page_xbusy(dst_m);
1388 			KASSERT(dst_m->valid == VM_PAGE_BITS_ALL,
1389 			    ("invalid dst page %p", dst_m));
1390 		}
1391 		VM_OBJECT_WUNLOCK(dst_object);
1392 
1393 		/*
1394 		 * Enter it in the pmap. If a wired, copy-on-write
1395 		 * mapping is being replaced by a write-enabled
1396 		 * mapping, then wire that new mapping.
1397 		 */
1398 		pmap_enter(dst_map->pmap, vaddr, dst_m, prot,
1399 		    access | (upgrade ? PMAP_ENTER_WIRED : 0), 0);
1400 
1401 		/*
1402 		 * Mark it no longer busy, and put it on the active list.
1403 		 */
1404 		VM_OBJECT_WLOCK(dst_object);
1405 
1406 		if (upgrade) {
1407 			if (src_m != dst_m) {
1408 				vm_page_lock(src_m);
1409 				vm_page_unwire(src_m, PQ_INACTIVE);
1410 				vm_page_unlock(src_m);
1411 				vm_page_lock(dst_m);
1412 				vm_page_wire(dst_m);
1413 				vm_page_unlock(dst_m);
1414 			} else {
1415 				KASSERT(dst_m->wire_count > 0,
1416 				    ("dst_m %p is not wired", dst_m));
1417 			}
1418 		} else {
1419 			vm_page_lock(dst_m);
1420 			vm_page_activate(dst_m);
1421 			vm_page_unlock(dst_m);
1422 		}
1423 		vm_page_xunbusy(dst_m);
1424 	}
1425 	VM_OBJECT_WUNLOCK(dst_object);
1426 	if (upgrade) {
1427 		dst_entry->eflags &= ~(MAP_ENTRY_COW | MAP_ENTRY_NEEDS_COPY);
1428 		vm_object_deallocate(src_object);
1429 	}
1430 }
1431 
1432 /*
1433  * Block entry into the machine-independent layer's page fault handler by
1434  * the calling thread.  Subsequent calls to vm_fault() by that thread will
1435  * return KERN_PROTECTION_FAILURE.  Enable machine-dependent handling of
1436  * spurious page faults.
1437  */
1438 int
1439 vm_fault_disable_pagefaults(void)
1440 {
1441 
1442 	return (curthread_pflags_set(TDP_NOFAULTING | TDP_RESETSPUR));
1443 }
1444 
1445 void
1446 vm_fault_enable_pagefaults(int save)
1447 {
1448 
1449 	curthread_pflags_restore(save);
1450 }
1451