xref: /freebsd/sys/vm/vm_fault.c (revision b9c36cc755002809a7d7c7109e3425fdfca036d2)
1 /*-
2  * Copyright (c) 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  * Copyright (c) 1994 John S. Dyson
5  * All rights reserved.
6  * Copyright (c) 1994 David Greenman
7  * All rights reserved.
8  *
9  *
10  * This code is derived from software contributed to Berkeley by
11  * The Mach Operating System project at Carnegie-Mellon University.
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  * 1. Redistributions of source code must retain the above copyright
17  *    notice, this list of conditions and the following disclaimer.
18  * 2. Redistributions in binary form must reproduce the above copyright
19  *    notice, this list of conditions and the following disclaimer in the
20  *    documentation and/or other materials provided with the distribution.
21  * 3. All advertising materials mentioning features or use of this software
22  *    must display the following acknowledgement:
23  *	This product includes software developed by the University of
24  *	California, Berkeley and its contributors.
25  * 4. Neither the name of the University nor the names of its contributors
26  *    may be used to endorse or promote products derived from this software
27  *    without specific prior written permission.
28  *
29  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
30  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
33  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
35  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
36  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
37  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
38  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39  * SUCH DAMAGE.
40  *
41  *	from: @(#)vm_fault.c	8.4 (Berkeley) 1/12/94
42  *
43  *
44  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
45  * All rights reserved.
46  *
47  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
48  *
49  * Permission to use, copy, modify and distribute this software and
50  * its documentation is hereby granted, provided that both the copyright
51  * notice and this permission notice appear in all copies of the
52  * software, derivative works or modified versions, and any portions
53  * thereof, and that both notices appear in supporting documentation.
54  *
55  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
56  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
57  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
58  *
59  * Carnegie Mellon requests users of this software to return to
60  *
61  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
62  *  School of Computer Science
63  *  Carnegie Mellon University
64  *  Pittsburgh PA 15213-3890
65  *
66  * any improvements or extensions that they make and grant Carnegie the
67  * rights to redistribute these changes.
68  */
69 
70 /*
71  *	Page fault handling module.
72  */
73 
74 #include <sys/cdefs.h>
75 __FBSDID("$FreeBSD$");
76 
77 #include "opt_ktrace.h"
78 #include "opt_vm.h"
79 
80 #include <sys/param.h>
81 #include <sys/systm.h>
82 #include <sys/kernel.h>
83 #include <sys/lock.h>
84 #include <sys/mman.h>
85 #include <sys/proc.h>
86 #include <sys/racct.h>
87 #include <sys/resourcevar.h>
88 #include <sys/rwlock.h>
89 #include <sys/sysctl.h>
90 #include <sys/vmmeter.h>
91 #include <sys/vnode.h>
92 #ifdef KTRACE
93 #include <sys/ktrace.h>
94 #endif
95 
96 #include <vm/vm.h>
97 #include <vm/vm_param.h>
98 #include <vm/pmap.h>
99 #include <vm/vm_map.h>
100 #include <vm/vm_object.h>
101 #include <vm/vm_page.h>
102 #include <vm/vm_pageout.h>
103 #include <vm/vm_kern.h>
104 #include <vm/vm_pager.h>
105 #include <vm/vm_extern.h>
106 #include <vm/vm_reserv.h>
107 
108 #define PFBAK 4
109 #define PFFOR 4
110 
111 #define	VM_FAULT_READ_DEFAULT	(1 + VM_FAULT_READ_AHEAD_INIT)
112 #define	VM_FAULT_READ_MAX	(1 + VM_FAULT_READ_AHEAD_MAX)
113 
114 #define	VM_FAULT_DONTNEED_MIN	1048576
115 
116 struct faultstate {
117 	vm_page_t m;
118 	vm_object_t object;
119 	vm_pindex_t pindex;
120 	vm_page_t first_m;
121 	vm_object_t	first_object;
122 	vm_pindex_t first_pindex;
123 	vm_map_t map;
124 	vm_map_entry_t entry;
125 	int map_generation;
126 	bool lookup_still_valid;
127 	struct vnode *vp;
128 };
129 
130 static void vm_fault_dontneed(const struct faultstate *fs, vm_offset_t vaddr,
131 	    int ahead);
132 static void vm_fault_prefault(const struct faultstate *fs, vm_offset_t addra,
133 	    int backward, int forward);
134 
135 static inline void
136 release_page(struct faultstate *fs)
137 {
138 
139 	vm_page_xunbusy(fs->m);
140 	vm_page_lock(fs->m);
141 	vm_page_deactivate(fs->m);
142 	vm_page_unlock(fs->m);
143 	fs->m = NULL;
144 }
145 
146 static inline void
147 unlock_map(struct faultstate *fs)
148 {
149 
150 	if (fs->lookup_still_valid) {
151 		vm_map_lookup_done(fs->map, fs->entry);
152 		fs->lookup_still_valid = false;
153 	}
154 }
155 
156 static void
157 unlock_vp(struct faultstate *fs)
158 {
159 
160 	if (fs->vp != NULL) {
161 		vput(fs->vp);
162 		fs->vp = NULL;
163 	}
164 }
165 
166 static void
167 unlock_and_deallocate(struct faultstate *fs)
168 {
169 
170 	vm_object_pip_wakeup(fs->object);
171 	VM_OBJECT_WUNLOCK(fs->object);
172 	if (fs->object != fs->first_object) {
173 		VM_OBJECT_WLOCK(fs->first_object);
174 		vm_page_lock(fs->first_m);
175 		vm_page_free(fs->first_m);
176 		vm_page_unlock(fs->first_m);
177 		vm_object_pip_wakeup(fs->first_object);
178 		VM_OBJECT_WUNLOCK(fs->first_object);
179 		fs->first_m = NULL;
180 	}
181 	vm_object_deallocate(fs->first_object);
182 	unlock_map(fs);
183 	unlock_vp(fs);
184 }
185 
186 static void
187 vm_fault_dirty(vm_map_entry_t entry, vm_page_t m, vm_prot_t prot,
188     vm_prot_t fault_type, int fault_flags, bool set_wd)
189 {
190 	bool need_dirty;
191 
192 	if (((prot & VM_PROT_WRITE) == 0 &&
193 	    (fault_flags & VM_FAULT_DIRTY) == 0) ||
194 	    (m->oflags & VPO_UNMANAGED) != 0)
195 		return;
196 
197 	VM_OBJECT_ASSERT_LOCKED(m->object);
198 
199 	need_dirty = ((fault_type & VM_PROT_WRITE) != 0 &&
200 	    (fault_flags & VM_FAULT_WIRE) == 0) ||
201 	    (fault_flags & VM_FAULT_DIRTY) != 0;
202 
203 	if (set_wd)
204 		vm_object_set_writeable_dirty(m->object);
205 	else
206 		/*
207 		 * If two callers of vm_fault_dirty() with set_wd ==
208 		 * FALSE, one for the map entry with MAP_ENTRY_NOSYNC
209 		 * flag set, other with flag clear, race, it is
210 		 * possible for the no-NOSYNC thread to see m->dirty
211 		 * != 0 and not clear VPO_NOSYNC.  Take vm_page lock
212 		 * around manipulation of VPO_NOSYNC and
213 		 * vm_page_dirty() call, to avoid the race and keep
214 		 * m->oflags consistent.
215 		 */
216 		vm_page_lock(m);
217 
218 	/*
219 	 * If this is a NOSYNC mmap we do not want to set VPO_NOSYNC
220 	 * if the page is already dirty to prevent data written with
221 	 * the expectation of being synced from not being synced.
222 	 * Likewise if this entry does not request NOSYNC then make
223 	 * sure the page isn't marked NOSYNC.  Applications sharing
224 	 * data should use the same flags to avoid ping ponging.
225 	 */
226 	if ((entry->eflags & MAP_ENTRY_NOSYNC) != 0) {
227 		if (m->dirty == 0) {
228 			m->oflags |= VPO_NOSYNC;
229 		}
230 	} else {
231 		m->oflags &= ~VPO_NOSYNC;
232 	}
233 
234 	/*
235 	 * If the fault is a write, we know that this page is being
236 	 * written NOW so dirty it explicitly to save on
237 	 * pmap_is_modified() calls later.
238 	 *
239 	 * Also tell the backing pager, if any, that it should remove
240 	 * any swap backing since the page is now dirty.
241 	 */
242 	if (need_dirty)
243 		vm_page_dirty(m);
244 	if (!set_wd)
245 		vm_page_unlock(m);
246 	if (need_dirty)
247 		vm_pager_page_unswapped(m);
248 }
249 
250 static void
251 vm_fault_fill_hold(vm_page_t *m_hold, vm_page_t m)
252 {
253 
254 	if (m_hold != NULL) {
255 		*m_hold = m;
256 		vm_page_lock(m);
257 		vm_page_hold(m);
258 		vm_page_unlock(m);
259 	}
260 }
261 
262 /*
263  * Unlocks fs.first_object and fs.map on success.
264  */
265 static int
266 vm_fault_soft_fast(struct faultstate *fs, vm_offset_t vaddr, vm_prot_t prot,
267     int fault_type, int fault_flags, boolean_t wired, vm_page_t *m_hold)
268 {
269 	vm_page_t m;
270 	int rv;
271 
272 	MPASS(fs->vp == NULL);
273 	m = vm_page_lookup(fs->first_object, fs->first_pindex);
274 	/* A busy page can be mapped for read|execute access. */
275 	if (m == NULL || ((prot & VM_PROT_WRITE) != 0 &&
276 	    vm_page_busied(m)) || m->valid != VM_PAGE_BITS_ALL)
277 		return (KERN_FAILURE);
278 	rv = pmap_enter(fs->map->pmap, vaddr, m, prot, fault_type |
279 	    PMAP_ENTER_NOSLEEP | (wired ? PMAP_ENTER_WIRED : 0), 0);
280 	if (rv != KERN_SUCCESS)
281 		return (rv);
282 	vm_fault_fill_hold(m_hold, m);
283 	vm_fault_dirty(fs->entry, m, prot, fault_type, fault_flags, false);
284 	VM_OBJECT_RUNLOCK(fs->first_object);
285 	if (!wired)
286 		vm_fault_prefault(fs, vaddr, PFBAK, PFFOR);
287 	vm_map_lookup_done(fs->map, fs->entry);
288 	curthread->td_ru.ru_minflt++;
289 	return (KERN_SUCCESS);
290 }
291 
292 static void
293 vm_fault_restore_map_lock(struct faultstate *fs)
294 {
295 
296 	VM_OBJECT_ASSERT_WLOCKED(fs->first_object);
297 	MPASS(fs->first_object->paging_in_progress > 0);
298 
299 	if (!vm_map_trylock_read(fs->map)) {
300 		VM_OBJECT_WUNLOCK(fs->first_object);
301 		vm_map_lock_read(fs->map);
302 		VM_OBJECT_WLOCK(fs->first_object);
303 	}
304 	fs->lookup_still_valid = true;
305 }
306 
307 
308 static int
309 vm_fault_populate(struct faultstate *fs, vm_offset_t vaddr, vm_prot_t prot,
310     int fault_type, int fault_flags, boolean_t wired, vm_page_t *m_hold)
311 {
312 	vm_page_t m;
313 	vm_pindex_t f_first, f_last, pidx;
314 	int rv;
315 
316 	MPASS(fs->object == fs->first_object);
317 	VM_OBJECT_ASSERT_WLOCKED(fs->first_object);
318 	MPASS(fs->first_object->paging_in_progress > 0);
319 	MPASS(fs->first_object->backing_object == NULL);
320 	MPASS(fs->lookup_still_valid);
321 
322 	f_first = OFF_TO_IDX(fs->entry->offset);
323 	f_last = OFF_TO_IDX(fs->entry->offset + fs->entry->end -
324 	    fs->entry->start) - 1;
325 	unlock_map(fs);
326 	unlock_vp(fs);
327 
328 	/*
329 	 * Call the pager (driver) populate() method.
330 	 *
331 	 * There is no guarantee that the method will be called again
332 	 * if the current fault is for read, and a future fault is
333 	 * for write.  Report the entry's maximum allowed protection
334 	 * to the driver.
335 	 */
336 	rv = vm_pager_populate(fs->first_object, fs->first_pindex,
337 	    fault_type, fs->entry->max_protection, &f_first, &f_last);
338 
339 	VM_OBJECT_ASSERT_WLOCKED(fs->first_object);
340 	if (rv == VM_PAGER_BAD) {
341 		/*
342 		 * VM_PAGER_BAD is the backdoor for a pager to request
343 		 * normal fault handling.
344 		 */
345 		vm_fault_restore_map_lock(fs);
346 		if (fs->map->timestamp != fs->map_generation)
347 			return (KERN_RESOURCE_SHORTAGE); /* RetryFault */
348 		return (KERN_NOT_RECEIVER);
349 	}
350 	if (rv != VM_PAGER_OK)
351 		return (KERN_FAILURE); /* AKA SIGSEGV */
352 
353 	/* Ensure that the driver is obeying the interface. */
354 	MPASS(f_first <= f_last);
355 	MPASS(fs->first_pindex <= f_last);
356 	MPASS(fs->first_pindex >= f_first);
357 	MPASS(f_last < fs->first_object->size);
358 
359 	vm_fault_restore_map_lock(fs);
360 	if (fs->map->timestamp != fs->map_generation)
361 		return (KERN_RESOURCE_SHORTAGE); /* RetryFault */
362 
363 	/* Clip pager response to fit into the vm_map_entry. */
364 	f_first = MAX(OFF_TO_IDX(fs->entry->offset), f_first);
365 	f_last = MIN(OFF_TO_IDX(fs->entry->end - fs->entry->start +
366 	    fs->entry->offset), f_last);
367 
368 	pidx = f_first;
369 	for (m = vm_page_lookup(fs->first_object, pidx); pidx <= f_last;
370 	    pidx++, m = vm_page_next(m)) {
371 		/*
372 		 * Check each page to ensure that the driver is
373 		 * obeying the interface: the page must be installed
374 		 * in the object, fully valid, and exclusively busied.
375 		 */
376 		MPASS(m != NULL);
377 		MPASS(vm_page_xbusied(m));
378 		MPASS(m->valid == VM_PAGE_BITS_ALL);
379 		MPASS(m->object == fs->first_object);
380 		MPASS(m->pindex == pidx);
381 
382 		vm_fault_dirty(fs->entry, m, prot, fault_type, fault_flags,
383 		    true);
384 		VM_OBJECT_WUNLOCK(fs->first_object);
385 		pmap_enter(fs->map->pmap, fs->entry->start + IDX_TO_OFF(pidx) -
386 		    fs->entry->offset, m, prot, fault_type | (wired ?
387 		    PMAP_ENTER_WIRED : 0), 0);
388 		VM_OBJECT_WLOCK(fs->first_object);
389 		if (pidx == fs->first_pindex)
390 			vm_fault_fill_hold(m_hold, m);
391 		vm_page_lock(m);
392 		if ((fault_flags & VM_FAULT_WIRE) != 0) {
393 			KASSERT(wired, ("VM_FAULT_WIRE && !wired"));
394 			vm_page_wire(m);
395 		} else {
396 			vm_page_activate(m);
397 		}
398 		vm_page_unlock(m);
399 		vm_page_xunbusy(m);
400 	}
401 	curthread->td_ru.ru_majflt++;
402 	return (KERN_SUCCESS);
403 }
404 
405 /*
406  *	vm_fault:
407  *
408  *	Handle a page fault occurring at the given address,
409  *	requiring the given permissions, in the map specified.
410  *	If successful, the page is inserted into the
411  *	associated physical map.
412  *
413  *	NOTE: the given address should be truncated to the
414  *	proper page address.
415  *
416  *	KERN_SUCCESS is returned if the page fault is handled; otherwise,
417  *	a standard error specifying why the fault is fatal is returned.
418  *
419  *	The map in question must be referenced, and remains so.
420  *	Caller may hold no locks.
421  */
422 int
423 vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type,
424     int fault_flags)
425 {
426 	struct thread *td;
427 	int result;
428 
429 	td = curthread;
430 	if ((td->td_pflags & TDP_NOFAULTING) != 0)
431 		return (KERN_PROTECTION_FAILURE);
432 #ifdef KTRACE
433 	if (map != kernel_map && KTRPOINT(td, KTR_FAULT))
434 		ktrfault(vaddr, fault_type);
435 #endif
436 	result = vm_fault_hold(map, trunc_page(vaddr), fault_type, fault_flags,
437 	    NULL);
438 #ifdef KTRACE
439 	if (map != kernel_map && KTRPOINT(td, KTR_FAULTEND))
440 		ktrfaultend(result);
441 #endif
442 	return (result);
443 }
444 
445 int
446 vm_fault_hold(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type,
447     int fault_flags, vm_page_t *m_hold)
448 {
449 	struct faultstate fs;
450 	struct vnode *vp;
451 	vm_object_t next_object, retry_object;
452 	vm_offset_t e_end, e_start;
453 	vm_pindex_t retry_pindex;
454 	vm_prot_t prot, retry_prot;
455 	int ahead, alloc_req, behind, cluster_offset, error, era, faultcount;
456 	int locked, nera, result, rv;
457 	u_char behavior;
458 	boolean_t wired;	/* Passed by reference. */
459 	bool dead, growstack, hardfault, is_first_object_locked;
460 
461 	PCPU_INC(cnt.v_vm_faults);
462 	fs.vp = NULL;
463 	faultcount = 0;
464 	nera = -1;
465 	growstack = true;
466 	hardfault = false;
467 
468 RetryFault:;
469 
470 	/*
471 	 * Find the backing store object and offset into it to begin the
472 	 * search.
473 	 */
474 	fs.map = map;
475 	result = vm_map_lookup(&fs.map, vaddr, fault_type, &fs.entry,
476 	    &fs.first_object, &fs.first_pindex, &prot, &wired);
477 	if (result != KERN_SUCCESS) {
478 		if (growstack && result == KERN_INVALID_ADDRESS &&
479 		    map != kernel_map) {
480 			result = vm_map_growstack(curproc, vaddr);
481 			if (result != KERN_SUCCESS)
482 				return (KERN_FAILURE);
483 			growstack = false;
484 			goto RetryFault;
485 		}
486 		unlock_vp(&fs);
487 		return (result);
488 	}
489 
490 	fs.map_generation = fs.map->timestamp;
491 
492 	if (fs.entry->eflags & MAP_ENTRY_NOFAULT) {
493 		panic("vm_fault: fault on nofault entry, addr: %lx",
494 		    (u_long)vaddr);
495 	}
496 
497 	if (fs.entry->eflags & MAP_ENTRY_IN_TRANSITION &&
498 	    fs.entry->wiring_thread != curthread) {
499 		vm_map_unlock_read(fs.map);
500 		vm_map_lock(fs.map);
501 		if (vm_map_lookup_entry(fs.map, vaddr, &fs.entry) &&
502 		    (fs.entry->eflags & MAP_ENTRY_IN_TRANSITION)) {
503 			unlock_vp(&fs);
504 			fs.entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
505 			vm_map_unlock_and_wait(fs.map, 0);
506 		} else
507 			vm_map_unlock(fs.map);
508 		goto RetryFault;
509 	}
510 
511 	if (wired)
512 		fault_type = prot | (fault_type & VM_PROT_COPY);
513 	else
514 		KASSERT((fault_flags & VM_FAULT_WIRE) == 0,
515 		    ("!wired && VM_FAULT_WIRE"));
516 
517 	/*
518 	 * Try to avoid lock contention on the top-level object through
519 	 * special-case handling of some types of page faults, specifically,
520 	 * those that are both (1) mapping an existing page from the top-
521 	 * level object and (2) not having to mark that object as containing
522 	 * dirty pages.  Under these conditions, a read lock on the top-level
523 	 * object suffices, allowing multiple page faults of a similar type to
524 	 * run in parallel on the same top-level object.
525 	 */
526 	if (fs.vp == NULL /* avoid locked vnode leak */ &&
527 	    (fault_flags & (VM_FAULT_WIRE | VM_FAULT_DIRTY)) == 0 &&
528 	    /* avoid calling vm_object_set_writeable_dirty() */
529 	    ((prot & VM_PROT_WRITE) == 0 ||
530 	    (fs.first_object->type != OBJT_VNODE &&
531 	    (fs.first_object->flags & OBJ_TMPFS_NODE) == 0) ||
532 	    (fs.first_object->flags & OBJ_MIGHTBEDIRTY) != 0)) {
533 		VM_OBJECT_RLOCK(fs.first_object);
534 		if ((prot & VM_PROT_WRITE) == 0 ||
535 		    (fs.first_object->type != OBJT_VNODE &&
536 		    (fs.first_object->flags & OBJ_TMPFS_NODE) == 0) ||
537 		    (fs.first_object->flags & OBJ_MIGHTBEDIRTY) != 0) {
538 			rv = vm_fault_soft_fast(&fs, vaddr, prot, fault_type,
539 			    fault_flags, wired, m_hold);
540 			if (rv == KERN_SUCCESS)
541 				return (rv);
542 		}
543 		if (!VM_OBJECT_TRYUPGRADE(fs.first_object)) {
544 			VM_OBJECT_RUNLOCK(fs.first_object);
545 			VM_OBJECT_WLOCK(fs.first_object);
546 		}
547 	} else {
548 		VM_OBJECT_WLOCK(fs.first_object);
549 	}
550 
551 	/*
552 	 * Make a reference to this object to prevent its disposal while we
553 	 * are messing with it.  Once we have the reference, the map is free
554 	 * to be diddled.  Since objects reference their shadows (and copies),
555 	 * they will stay around as well.
556 	 *
557 	 * Bump the paging-in-progress count to prevent size changes (e.g.
558 	 * truncation operations) during I/O.
559 	 */
560 	vm_object_reference_locked(fs.first_object);
561 	vm_object_pip_add(fs.first_object, 1);
562 
563 	fs.lookup_still_valid = true;
564 
565 	fs.first_m = NULL;
566 
567 	/*
568 	 * Search for the page at object/offset.
569 	 */
570 	fs.object = fs.first_object;
571 	fs.pindex = fs.first_pindex;
572 	while (TRUE) {
573 		/*
574 		 * If the object is marked for imminent termination,
575 		 * we retry here, since the collapse pass has raced
576 		 * with us.  Otherwise, if we see terminally dead
577 		 * object, return fail.
578 		 */
579 		if ((fs.object->flags & OBJ_DEAD) != 0) {
580 			dead = fs.object->type == OBJT_DEAD;
581 			unlock_and_deallocate(&fs);
582 			if (dead)
583 				return (KERN_PROTECTION_FAILURE);
584 			pause("vmf_de", 1);
585 			goto RetryFault;
586 		}
587 
588 		/*
589 		 * See if page is resident
590 		 */
591 		fs.m = vm_page_lookup(fs.object, fs.pindex);
592 		if (fs.m != NULL) {
593 			/*
594 			 * Wait/Retry if the page is busy.  We have to do this
595 			 * if the page is either exclusive or shared busy
596 			 * because the vm_pager may be using read busy for
597 			 * pageouts (and even pageins if it is the vnode
598 			 * pager), and we could end up trying to pagein and
599 			 * pageout the same page simultaneously.
600 			 *
601 			 * We can theoretically allow the busy case on a read
602 			 * fault if the page is marked valid, but since such
603 			 * pages are typically already pmap'd, putting that
604 			 * special case in might be more effort then it is
605 			 * worth.  We cannot under any circumstances mess
606 			 * around with a shared busied page except, perhaps,
607 			 * to pmap it.
608 			 */
609 			if (vm_page_busied(fs.m)) {
610 				/*
611 				 * Reference the page before unlocking and
612 				 * sleeping so that the page daemon is less
613 				 * likely to reclaim it.
614 				 */
615 				vm_page_aflag_set(fs.m, PGA_REFERENCED);
616 				if (fs.object != fs.first_object) {
617 					if (!VM_OBJECT_TRYWLOCK(
618 					    fs.first_object)) {
619 						VM_OBJECT_WUNLOCK(fs.object);
620 						VM_OBJECT_WLOCK(fs.first_object);
621 						VM_OBJECT_WLOCK(fs.object);
622 					}
623 					vm_page_lock(fs.first_m);
624 					vm_page_free(fs.first_m);
625 					vm_page_unlock(fs.first_m);
626 					vm_object_pip_wakeup(fs.first_object);
627 					VM_OBJECT_WUNLOCK(fs.first_object);
628 					fs.first_m = NULL;
629 				}
630 				unlock_map(&fs);
631 				if (fs.m == vm_page_lookup(fs.object,
632 				    fs.pindex)) {
633 					vm_page_sleep_if_busy(fs.m, "vmpfw");
634 				}
635 				vm_object_pip_wakeup(fs.object);
636 				VM_OBJECT_WUNLOCK(fs.object);
637 				PCPU_INC(cnt.v_intrans);
638 				vm_object_deallocate(fs.first_object);
639 				goto RetryFault;
640 			}
641 			vm_page_lock(fs.m);
642 			vm_page_remque(fs.m);
643 			vm_page_unlock(fs.m);
644 
645 			/*
646 			 * Mark page busy for other processes, and the
647 			 * pagedaemon.  If it still isn't completely valid
648 			 * (readable), jump to readrest, else break-out ( we
649 			 * found the page ).
650 			 */
651 			vm_page_xbusy(fs.m);
652 			if (fs.m->valid != VM_PAGE_BITS_ALL)
653 				goto readrest;
654 			break;
655 		}
656 		KASSERT(fs.m == NULL, ("fs.m should be NULL, not %p", fs.m));
657 
658 		/*
659 		 * Page is not resident.  If the pager might contain the page
660 		 * or this is the beginning of the search, allocate a new
661 		 * page.  (Default objects are zero-fill, so there is no real
662 		 * pager for them.)
663 		 */
664 		if (fs.object->type != OBJT_DEFAULT ||
665 		    fs.object == fs.first_object) {
666 			if (fs.pindex >= fs.object->size) {
667 				unlock_and_deallocate(&fs);
668 				return (KERN_PROTECTION_FAILURE);
669 			}
670 
671 			if (fs.object == fs.first_object &&
672 			    (fs.first_object->flags & OBJ_POPULATE) != 0 &&
673 			    fs.first_object->shadow_count == 0) {
674 				rv = vm_fault_populate(&fs, vaddr, prot,
675 				    fault_type, fault_flags, wired, m_hold);
676 				switch (rv) {
677 				case KERN_SUCCESS:
678 				case KERN_FAILURE:
679 					unlock_and_deallocate(&fs);
680 					return (rv);
681 				case KERN_RESOURCE_SHORTAGE:
682 					unlock_and_deallocate(&fs);
683 					goto RetryFault;
684 				case KERN_NOT_RECEIVER:
685 					/*
686 					 * Pager's populate() method
687 					 * returned VM_PAGER_BAD.
688 					 */
689 					break;
690 				default:
691 					panic("inconsistent return codes");
692 				}
693 			}
694 
695 			/*
696 			 * Allocate a new page for this object/offset pair.
697 			 *
698 			 * Unlocked read of the p_flag is harmless. At
699 			 * worst, the P_KILLED might be not observed
700 			 * there, and allocation can fail, causing
701 			 * restart and new reading of the p_flag.
702 			 */
703 			if (!vm_page_count_severe() || P_KILLED(curproc)) {
704 #if VM_NRESERVLEVEL > 0
705 				vm_object_color(fs.object, atop(vaddr) -
706 				    fs.pindex);
707 #endif
708 				alloc_req = P_KILLED(curproc) ?
709 				    VM_ALLOC_SYSTEM : VM_ALLOC_NORMAL;
710 				if (fs.object->type != OBJT_VNODE &&
711 				    fs.object->backing_object == NULL)
712 					alloc_req |= VM_ALLOC_ZERO;
713 				fs.m = vm_page_alloc(fs.object, fs.pindex,
714 				    alloc_req);
715 			}
716 			if (fs.m == NULL) {
717 				unlock_and_deallocate(&fs);
718 				VM_WAITPFAULT;
719 				goto RetryFault;
720 			}
721 		}
722 
723 readrest:
724 		/*
725 		 * At this point, we have either allocated a new page or found
726 		 * an existing page that is only partially valid.
727 		 *
728 		 * We hold a reference on the current object and the page is
729 		 * exclusive busied.
730 		 */
731 
732 		/*
733 		 * If the pager for the current object might have the page,
734 		 * then determine the number of additional pages to read and
735 		 * potentially reprioritize previously read pages for earlier
736 		 * reclamation.  These operations should only be performed
737 		 * once per page fault.  Even if the current pager doesn't
738 		 * have the page, the number of additional pages to read will
739 		 * apply to subsequent objects in the shadow chain.
740 		 */
741 		if (fs.object->type != OBJT_DEFAULT && nera == -1 &&
742 		    !P_KILLED(curproc)) {
743 			KASSERT(fs.lookup_still_valid, ("map unlocked"));
744 			era = fs.entry->read_ahead;
745 			behavior = vm_map_entry_behavior(fs.entry);
746 			if (behavior == MAP_ENTRY_BEHAV_RANDOM) {
747 				nera = 0;
748 			} else if (behavior == MAP_ENTRY_BEHAV_SEQUENTIAL) {
749 				nera = VM_FAULT_READ_AHEAD_MAX;
750 				if (vaddr == fs.entry->next_read)
751 					vm_fault_dontneed(&fs, vaddr, nera);
752 			} else if (vaddr == fs.entry->next_read) {
753 				/*
754 				 * This is a sequential fault.  Arithmetically
755 				 * increase the requested number of pages in
756 				 * the read-ahead window.  The requested
757 				 * number of pages is "# of sequential faults
758 				 * x (read ahead min + 1) + read ahead min"
759 				 */
760 				nera = VM_FAULT_READ_AHEAD_MIN;
761 				if (era > 0) {
762 					nera += era + 1;
763 					if (nera > VM_FAULT_READ_AHEAD_MAX)
764 						nera = VM_FAULT_READ_AHEAD_MAX;
765 				}
766 				if (era == VM_FAULT_READ_AHEAD_MAX)
767 					vm_fault_dontneed(&fs, vaddr, nera);
768 			} else {
769 				/*
770 				 * This is a non-sequential fault.
771 				 */
772 				nera = 0;
773 			}
774 			if (era != nera) {
775 				/*
776 				 * A read lock on the map suffices to update
777 				 * the read ahead count safely.
778 				 */
779 				fs.entry->read_ahead = nera;
780 			}
781 
782 			/*
783 			 * Prepare for unlocking the map.  Save the map
784 			 * entry's start and end addresses, which are used to
785 			 * optimize the size of the pager operation below.
786 			 * Even if the map entry's addresses change after
787 			 * unlocking the map, using the saved addresses is
788 			 * safe.
789 			 */
790 			e_start = fs.entry->start;
791 			e_end = fs.entry->end;
792 		}
793 
794 		/*
795 		 * Call the pager to retrieve the page if there is a chance
796 		 * that the pager has it, and potentially retrieve additional
797 		 * pages at the same time.
798 		 */
799 		if (fs.object->type != OBJT_DEFAULT) {
800 			/*
801 			 * Release the map lock before locking the vnode or
802 			 * sleeping in the pager.  (If the current object has
803 			 * a shadow, then an earlier iteration of this loop
804 			 * may have already unlocked the map.)
805 			 */
806 			unlock_map(&fs);
807 
808 			if (fs.object->type == OBJT_VNODE &&
809 			    (vp = fs.object->handle) != fs.vp) {
810 				/*
811 				 * Perform an unlock in case the desired vnode
812 				 * changed while the map was unlocked during a
813 				 * retry.
814 				 */
815 				unlock_vp(&fs);
816 
817 				locked = VOP_ISLOCKED(vp);
818 				if (locked != LK_EXCLUSIVE)
819 					locked = LK_SHARED;
820 
821 				/*
822 				 * We must not sleep acquiring the vnode lock
823 				 * while we have the page exclusive busied or
824 				 * the object's paging-in-progress count
825 				 * incremented.  Otherwise, we could deadlock.
826 				 */
827 				error = vget(vp, locked | LK_CANRECURSE |
828 				    LK_NOWAIT, curthread);
829 				if (error != 0) {
830 					vhold(vp);
831 					release_page(&fs);
832 					unlock_and_deallocate(&fs);
833 					error = vget(vp, locked | LK_RETRY |
834 					    LK_CANRECURSE, curthread);
835 					vdrop(vp);
836 					fs.vp = vp;
837 					KASSERT(error == 0,
838 					    ("vm_fault: vget failed"));
839 					goto RetryFault;
840 				}
841 				fs.vp = vp;
842 			}
843 			KASSERT(fs.vp == NULL || !fs.map->system_map,
844 			    ("vm_fault: vnode-backed object mapped by system map"));
845 
846 			/*
847 			 * Page in the requested page and hint the pager,
848 			 * that it may bring up surrounding pages.
849 			 */
850 			if (nera == -1 || behavior == MAP_ENTRY_BEHAV_RANDOM ||
851 			    P_KILLED(curproc)) {
852 				behind = 0;
853 				ahead = 0;
854 			} else {
855 				/* Is this a sequential fault? */
856 				if (nera > 0) {
857 					behind = 0;
858 					ahead = nera;
859 				} else {
860 					/*
861 					 * Request a cluster of pages that is
862 					 * aligned to a VM_FAULT_READ_DEFAULT
863 					 * page offset boundary within the
864 					 * object.  Alignment to a page offset
865 					 * boundary is more likely to coincide
866 					 * with the underlying file system
867 					 * block than alignment to a virtual
868 					 * address boundary.
869 					 */
870 					cluster_offset = fs.pindex %
871 					    VM_FAULT_READ_DEFAULT;
872 					behind = ulmin(cluster_offset,
873 					    atop(vaddr - e_start));
874 					ahead = VM_FAULT_READ_DEFAULT - 1 -
875 					    cluster_offset;
876 				}
877 				ahead = ulmin(ahead, atop(e_end - vaddr) - 1);
878 			}
879 			rv = vm_pager_get_pages(fs.object, &fs.m, 1,
880 			    &behind, &ahead);
881 			if (rv == VM_PAGER_OK) {
882 				faultcount = behind + 1 + ahead;
883 				hardfault = true;
884 				break; /* break to PAGE HAS BEEN FOUND */
885 			}
886 			if (rv == VM_PAGER_ERROR)
887 				printf("vm_fault: pager read error, pid %d (%s)\n",
888 				    curproc->p_pid, curproc->p_comm);
889 
890 			/*
891 			 * If an I/O error occurred or the requested page was
892 			 * outside the range of the pager, clean up and return
893 			 * an error.
894 			 */
895 			if (rv == VM_PAGER_ERROR || rv == VM_PAGER_BAD) {
896 				vm_page_lock(fs.m);
897 				if (fs.m->wire_count == 0)
898 					vm_page_free(fs.m);
899 				else
900 					vm_page_xunbusy_maybelocked(fs.m);
901 				vm_page_unlock(fs.m);
902 				fs.m = NULL;
903 				unlock_and_deallocate(&fs);
904 				return (rv == VM_PAGER_ERROR ? KERN_FAILURE :
905 				    KERN_PROTECTION_FAILURE);
906 			}
907 
908 			/*
909 			 * The requested page does not exist at this object/
910 			 * offset.  Remove the invalid page from the object,
911 			 * waking up anyone waiting for it, and continue on to
912 			 * the next object.  However, if this is the top-level
913 			 * object, we must leave the busy page in place to
914 			 * prevent another process from rushing past us, and
915 			 * inserting the page in that object at the same time
916 			 * that we are.
917 			 */
918 			if (fs.object != fs.first_object) {
919 				vm_page_lock(fs.m);
920 				if (fs.m->wire_count == 0)
921 					vm_page_free(fs.m);
922 				else
923 					vm_page_xunbusy_maybelocked(fs.m);
924 				vm_page_unlock(fs.m);
925 				fs.m = NULL;
926 			}
927 		}
928 
929 		/*
930 		 * We get here if the object has default pager (or unwiring)
931 		 * or the pager doesn't have the page.
932 		 */
933 		if (fs.object == fs.first_object)
934 			fs.first_m = fs.m;
935 
936 		/*
937 		 * Move on to the next object.  Lock the next object before
938 		 * unlocking the current one.
939 		 */
940 		next_object = fs.object->backing_object;
941 		if (next_object == NULL) {
942 			/*
943 			 * If there's no object left, fill the page in the top
944 			 * object with zeros.
945 			 */
946 			if (fs.object != fs.first_object) {
947 				vm_object_pip_wakeup(fs.object);
948 				VM_OBJECT_WUNLOCK(fs.object);
949 
950 				fs.object = fs.first_object;
951 				fs.pindex = fs.first_pindex;
952 				fs.m = fs.first_m;
953 				VM_OBJECT_WLOCK(fs.object);
954 			}
955 			fs.first_m = NULL;
956 
957 			/*
958 			 * Zero the page if necessary and mark it valid.
959 			 */
960 			if ((fs.m->flags & PG_ZERO) == 0) {
961 				pmap_zero_page(fs.m);
962 			} else {
963 				PCPU_INC(cnt.v_ozfod);
964 			}
965 			PCPU_INC(cnt.v_zfod);
966 			fs.m->valid = VM_PAGE_BITS_ALL;
967 			/* Don't try to prefault neighboring pages. */
968 			faultcount = 1;
969 			break;	/* break to PAGE HAS BEEN FOUND */
970 		} else {
971 			KASSERT(fs.object != next_object,
972 			    ("object loop %p", next_object));
973 			VM_OBJECT_WLOCK(next_object);
974 			vm_object_pip_add(next_object, 1);
975 			if (fs.object != fs.first_object)
976 				vm_object_pip_wakeup(fs.object);
977 			fs.pindex +=
978 			    OFF_TO_IDX(fs.object->backing_object_offset);
979 			VM_OBJECT_WUNLOCK(fs.object);
980 			fs.object = next_object;
981 		}
982 	}
983 
984 	vm_page_assert_xbusied(fs.m);
985 
986 	/*
987 	 * PAGE HAS BEEN FOUND. [Loop invariant still holds -- the object lock
988 	 * is held.]
989 	 */
990 
991 	/*
992 	 * If the page is being written, but isn't already owned by the
993 	 * top-level object, we have to copy it into a new page owned by the
994 	 * top-level object.
995 	 */
996 	if (fs.object != fs.first_object) {
997 		/*
998 		 * We only really need to copy if we want to write it.
999 		 */
1000 		if ((fault_type & (VM_PROT_COPY | VM_PROT_WRITE)) != 0) {
1001 			/*
1002 			 * This allows pages to be virtually copied from a
1003 			 * backing_object into the first_object, where the
1004 			 * backing object has no other refs to it, and cannot
1005 			 * gain any more refs.  Instead of a bcopy, we just
1006 			 * move the page from the backing object to the
1007 			 * first object.  Note that we must mark the page
1008 			 * dirty in the first object so that it will go out
1009 			 * to swap when needed.
1010 			 */
1011 			is_first_object_locked = false;
1012 			if (
1013 				/*
1014 				 * Only one shadow object
1015 				 */
1016 				(fs.object->shadow_count == 1) &&
1017 				/*
1018 				 * No COW refs, except us
1019 				 */
1020 				(fs.object->ref_count == 1) &&
1021 				/*
1022 				 * No one else can look this object up
1023 				 */
1024 				(fs.object->handle == NULL) &&
1025 				/*
1026 				 * No other ways to look the object up
1027 				 */
1028 				((fs.object->type == OBJT_DEFAULT) ||
1029 				 (fs.object->type == OBJT_SWAP)) &&
1030 			    (is_first_object_locked = VM_OBJECT_TRYWLOCK(fs.first_object)) &&
1031 				/*
1032 				 * We don't chase down the shadow chain
1033 				 */
1034 			    fs.object == fs.first_object->backing_object) {
1035 				vm_page_lock(fs.m);
1036 				vm_page_remove(fs.m);
1037 				vm_page_unlock(fs.m);
1038 				vm_page_lock(fs.first_m);
1039 				vm_page_replace_checked(fs.m, fs.first_object,
1040 				    fs.first_pindex, fs.first_m);
1041 				vm_page_free(fs.first_m);
1042 				vm_page_unlock(fs.first_m);
1043 				vm_page_dirty(fs.m);
1044 #if VM_NRESERVLEVEL > 0
1045 				/*
1046 				 * Rename the reservation.
1047 				 */
1048 				vm_reserv_rename(fs.m, fs.first_object,
1049 				    fs.object, OFF_TO_IDX(
1050 				    fs.first_object->backing_object_offset));
1051 #endif
1052 				/*
1053 				 * Removing the page from the backing object
1054 				 * unbusied it.
1055 				 */
1056 				vm_page_xbusy(fs.m);
1057 				fs.first_m = fs.m;
1058 				fs.m = NULL;
1059 				PCPU_INC(cnt.v_cow_optim);
1060 			} else {
1061 				/*
1062 				 * Oh, well, lets copy it.
1063 				 */
1064 				pmap_copy_page(fs.m, fs.first_m);
1065 				fs.first_m->valid = VM_PAGE_BITS_ALL;
1066 				if (wired && (fault_flags &
1067 				    VM_FAULT_WIRE) == 0) {
1068 					vm_page_lock(fs.first_m);
1069 					vm_page_wire(fs.first_m);
1070 					vm_page_unlock(fs.first_m);
1071 
1072 					vm_page_lock(fs.m);
1073 					vm_page_unwire(fs.m, PQ_INACTIVE);
1074 					vm_page_unlock(fs.m);
1075 				}
1076 				/*
1077 				 * We no longer need the old page or object.
1078 				 */
1079 				release_page(&fs);
1080 			}
1081 			/*
1082 			 * fs.object != fs.first_object due to above
1083 			 * conditional
1084 			 */
1085 			vm_object_pip_wakeup(fs.object);
1086 			VM_OBJECT_WUNLOCK(fs.object);
1087 			/*
1088 			 * Only use the new page below...
1089 			 */
1090 			fs.object = fs.first_object;
1091 			fs.pindex = fs.first_pindex;
1092 			fs.m = fs.first_m;
1093 			if (!is_first_object_locked)
1094 				VM_OBJECT_WLOCK(fs.object);
1095 			PCPU_INC(cnt.v_cow_faults);
1096 			curthread->td_cow++;
1097 		} else {
1098 			prot &= ~VM_PROT_WRITE;
1099 		}
1100 	}
1101 
1102 	/*
1103 	 * We must verify that the maps have not changed since our last
1104 	 * lookup.
1105 	 */
1106 	if (!fs.lookup_still_valid) {
1107 		if (!vm_map_trylock_read(fs.map)) {
1108 			release_page(&fs);
1109 			unlock_and_deallocate(&fs);
1110 			goto RetryFault;
1111 		}
1112 		fs.lookup_still_valid = true;
1113 		if (fs.map->timestamp != fs.map_generation) {
1114 			result = vm_map_lookup_locked(&fs.map, vaddr, fault_type,
1115 			    &fs.entry, &retry_object, &retry_pindex, &retry_prot, &wired);
1116 
1117 			/*
1118 			 * If we don't need the page any longer, put it on the inactive
1119 			 * list (the easiest thing to do here).  If no one needs it,
1120 			 * pageout will grab it eventually.
1121 			 */
1122 			if (result != KERN_SUCCESS) {
1123 				release_page(&fs);
1124 				unlock_and_deallocate(&fs);
1125 
1126 				/*
1127 				 * If retry of map lookup would have blocked then
1128 				 * retry fault from start.
1129 				 */
1130 				if (result == KERN_FAILURE)
1131 					goto RetryFault;
1132 				return (result);
1133 			}
1134 			if ((retry_object != fs.first_object) ||
1135 			    (retry_pindex != fs.first_pindex)) {
1136 				release_page(&fs);
1137 				unlock_and_deallocate(&fs);
1138 				goto RetryFault;
1139 			}
1140 
1141 			/*
1142 			 * Check whether the protection has changed or the object has
1143 			 * been copied while we left the map unlocked. Changing from
1144 			 * read to write permission is OK - we leave the page
1145 			 * write-protected, and catch the write fault. Changing from
1146 			 * write to read permission means that we can't mark the page
1147 			 * write-enabled after all.
1148 			 */
1149 			prot &= retry_prot;
1150 		}
1151 	}
1152 
1153 	/*
1154 	 * If the page was filled by a pager, save the virtual address that
1155 	 * should be faulted on next under a sequential access pattern to the
1156 	 * map entry.  A read lock on the map suffices to update this address
1157 	 * safely.
1158 	 */
1159 	if (hardfault)
1160 		fs.entry->next_read = vaddr + ptoa(ahead) + PAGE_SIZE;
1161 
1162 	vm_fault_dirty(fs.entry, fs.m, prot, fault_type, fault_flags, true);
1163 	vm_page_assert_xbusied(fs.m);
1164 
1165 	/*
1166 	 * Page must be completely valid or it is not fit to
1167 	 * map into user space.  vm_pager_get_pages() ensures this.
1168 	 */
1169 	KASSERT(fs.m->valid == VM_PAGE_BITS_ALL,
1170 	    ("vm_fault: page %p partially invalid", fs.m));
1171 	VM_OBJECT_WUNLOCK(fs.object);
1172 
1173 	/*
1174 	 * Put this page into the physical map.  We had to do the unlock above
1175 	 * because pmap_enter() may sleep.  We don't put the page
1176 	 * back on the active queue until later so that the pageout daemon
1177 	 * won't find it (yet).
1178 	 */
1179 	pmap_enter(fs.map->pmap, vaddr, fs.m, prot,
1180 	    fault_type | (wired ? PMAP_ENTER_WIRED : 0), 0);
1181 	if (faultcount != 1 && (fault_flags & VM_FAULT_WIRE) == 0 &&
1182 	    wired == 0)
1183 		vm_fault_prefault(&fs, vaddr,
1184 		    faultcount > 0 ? behind : PFBAK,
1185 		    faultcount > 0 ? ahead : PFFOR);
1186 	VM_OBJECT_WLOCK(fs.object);
1187 	vm_page_lock(fs.m);
1188 
1189 	/*
1190 	 * If the page is not wired down, then put it where the pageout daemon
1191 	 * can find it.
1192 	 */
1193 	if ((fault_flags & VM_FAULT_WIRE) != 0) {
1194 		KASSERT(wired, ("VM_FAULT_WIRE && !wired"));
1195 		vm_page_wire(fs.m);
1196 	} else
1197 		vm_page_activate(fs.m);
1198 	if (m_hold != NULL) {
1199 		*m_hold = fs.m;
1200 		vm_page_hold(fs.m);
1201 	}
1202 	vm_page_unlock(fs.m);
1203 	vm_page_xunbusy(fs.m);
1204 
1205 	/*
1206 	 * Unlock everything, and return
1207 	 */
1208 	unlock_and_deallocate(&fs);
1209 	if (hardfault) {
1210 		PCPU_INC(cnt.v_io_faults);
1211 		curthread->td_ru.ru_majflt++;
1212 #ifdef RACCT
1213 		if (racct_enable && fs.object->type == OBJT_VNODE) {
1214 			PROC_LOCK(curproc);
1215 			if ((fault_type & (VM_PROT_COPY | VM_PROT_WRITE)) != 0) {
1216 				racct_add_force(curproc, RACCT_WRITEBPS,
1217 				    PAGE_SIZE + behind * PAGE_SIZE);
1218 				racct_add_force(curproc, RACCT_WRITEIOPS, 1);
1219 			} else {
1220 				racct_add_force(curproc, RACCT_READBPS,
1221 				    PAGE_SIZE + ahead * PAGE_SIZE);
1222 				racct_add_force(curproc, RACCT_READIOPS, 1);
1223 			}
1224 			PROC_UNLOCK(curproc);
1225 		}
1226 #endif
1227 	} else
1228 		curthread->td_ru.ru_minflt++;
1229 
1230 	return (KERN_SUCCESS);
1231 }
1232 
1233 /*
1234  * Speed up the reclamation of pages that precede the faulting pindex within
1235  * the first object of the shadow chain.  Essentially, perform the equivalent
1236  * to madvise(..., MADV_DONTNEED) on a large cluster of pages that precedes
1237  * the faulting pindex by the cluster size when the pages read by vm_fault()
1238  * cross a cluster-size boundary.  The cluster size is the greater of the
1239  * smallest superpage size and VM_FAULT_DONTNEED_MIN.
1240  *
1241  * When "fs->first_object" is a shadow object, the pages in the backing object
1242  * that precede the faulting pindex are deactivated by vm_fault().  So, this
1243  * function must only be concerned with pages in the first object.
1244  */
1245 static void
1246 vm_fault_dontneed(const struct faultstate *fs, vm_offset_t vaddr, int ahead)
1247 {
1248 	vm_map_entry_t entry;
1249 	vm_object_t first_object, object;
1250 	vm_offset_t end, start;
1251 	vm_page_t m, m_next;
1252 	vm_pindex_t pend, pstart;
1253 	vm_size_t size;
1254 
1255 	object = fs->object;
1256 	VM_OBJECT_ASSERT_WLOCKED(object);
1257 	first_object = fs->first_object;
1258 	if (first_object != object) {
1259 		if (!VM_OBJECT_TRYWLOCK(first_object)) {
1260 			VM_OBJECT_WUNLOCK(object);
1261 			VM_OBJECT_WLOCK(first_object);
1262 			VM_OBJECT_WLOCK(object);
1263 		}
1264 	}
1265 	/* Neither fictitious nor unmanaged pages can be reclaimed. */
1266 	if ((first_object->flags & (OBJ_FICTITIOUS | OBJ_UNMANAGED)) == 0) {
1267 		size = VM_FAULT_DONTNEED_MIN;
1268 		if (MAXPAGESIZES > 1 && size < pagesizes[1])
1269 			size = pagesizes[1];
1270 		end = rounddown2(vaddr, size);
1271 		if (vaddr - end >= size - PAGE_SIZE - ptoa(ahead) &&
1272 		    (entry = fs->entry)->start < end) {
1273 			if (end - entry->start < size)
1274 				start = entry->start;
1275 			else
1276 				start = end - size;
1277 			pmap_advise(fs->map->pmap, start, end, MADV_DONTNEED);
1278 			pstart = OFF_TO_IDX(entry->offset) + atop(start -
1279 			    entry->start);
1280 			m_next = vm_page_find_least(first_object, pstart);
1281 			pend = OFF_TO_IDX(entry->offset) + atop(end -
1282 			    entry->start);
1283 			while ((m = m_next) != NULL && m->pindex < pend) {
1284 				m_next = TAILQ_NEXT(m, listq);
1285 				if (m->valid != VM_PAGE_BITS_ALL ||
1286 				    vm_page_busied(m))
1287 					continue;
1288 
1289 				/*
1290 				 * Don't clear PGA_REFERENCED, since it would
1291 				 * likely represent a reference by a different
1292 				 * process.
1293 				 *
1294 				 * Typically, at this point, prefetched pages
1295 				 * are still in the inactive queue.  Only
1296 				 * pages that triggered page faults are in the
1297 				 * active queue.
1298 				 */
1299 				vm_page_lock(m);
1300 				vm_page_deactivate(m);
1301 				vm_page_unlock(m);
1302 			}
1303 		}
1304 	}
1305 	if (first_object != object)
1306 		VM_OBJECT_WUNLOCK(first_object);
1307 }
1308 
1309 /*
1310  * vm_fault_prefault provides a quick way of clustering
1311  * pagefaults into a processes address space.  It is a "cousin"
1312  * of vm_map_pmap_enter, except it runs at page fault time instead
1313  * of mmap time.
1314  */
1315 static void
1316 vm_fault_prefault(const struct faultstate *fs, vm_offset_t addra,
1317     int backward, int forward)
1318 {
1319 	pmap_t pmap;
1320 	vm_map_entry_t entry;
1321 	vm_object_t backing_object, lobject;
1322 	vm_offset_t addr, starta;
1323 	vm_pindex_t pindex;
1324 	vm_page_t m;
1325 	int i;
1326 
1327 	pmap = fs->map->pmap;
1328 	if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace))
1329 		return;
1330 
1331 	entry = fs->entry;
1332 
1333 	starta = addra - backward * PAGE_SIZE;
1334 	if (starta < entry->start) {
1335 		starta = entry->start;
1336 	} else if (starta > addra) {
1337 		starta = 0;
1338 	}
1339 
1340 	/*
1341 	 * Generate the sequence of virtual addresses that are candidates for
1342 	 * prefaulting in an outward spiral from the faulting virtual address,
1343 	 * "addra".  Specifically, the sequence is "addra - PAGE_SIZE", "addra
1344 	 * + PAGE_SIZE", "addra - 2 * PAGE_SIZE", "addra + 2 * PAGE_SIZE", ...
1345 	 * If the candidate address doesn't have a backing physical page, then
1346 	 * the loop immediately terminates.
1347 	 */
1348 	for (i = 0; i < 2 * imax(backward, forward); i++) {
1349 		addr = addra + ((i >> 1) + 1) * ((i & 1) == 0 ? -PAGE_SIZE :
1350 		    PAGE_SIZE);
1351 		if (addr > addra + forward * PAGE_SIZE)
1352 			addr = 0;
1353 
1354 		if (addr < starta || addr >= entry->end)
1355 			continue;
1356 
1357 		if (!pmap_is_prefaultable(pmap, addr))
1358 			continue;
1359 
1360 		pindex = ((addr - entry->start) + entry->offset) >> PAGE_SHIFT;
1361 		lobject = entry->object.vm_object;
1362 		VM_OBJECT_RLOCK(lobject);
1363 		while ((m = vm_page_lookup(lobject, pindex)) == NULL &&
1364 		    lobject->type == OBJT_DEFAULT &&
1365 		    (backing_object = lobject->backing_object) != NULL) {
1366 			KASSERT((lobject->backing_object_offset & PAGE_MASK) ==
1367 			    0, ("vm_fault_prefault: unaligned object offset"));
1368 			pindex += lobject->backing_object_offset >> PAGE_SHIFT;
1369 			VM_OBJECT_RLOCK(backing_object);
1370 			VM_OBJECT_RUNLOCK(lobject);
1371 			lobject = backing_object;
1372 		}
1373 		if (m == NULL) {
1374 			VM_OBJECT_RUNLOCK(lobject);
1375 			break;
1376 		}
1377 		if (m->valid == VM_PAGE_BITS_ALL &&
1378 		    (m->flags & PG_FICTITIOUS) == 0)
1379 			pmap_enter_quick(pmap, addr, m, entry->protection);
1380 		VM_OBJECT_RUNLOCK(lobject);
1381 	}
1382 }
1383 
1384 /*
1385  * Hold each of the physical pages that are mapped by the specified range of
1386  * virtual addresses, ["addr", "addr" + "len"), if those mappings are valid
1387  * and allow the specified types of access, "prot".  If all of the implied
1388  * pages are successfully held, then the number of held pages is returned
1389  * together with pointers to those pages in the array "ma".  However, if any
1390  * of the pages cannot be held, -1 is returned.
1391  */
1392 int
1393 vm_fault_quick_hold_pages(vm_map_t map, vm_offset_t addr, vm_size_t len,
1394     vm_prot_t prot, vm_page_t *ma, int max_count)
1395 {
1396 	vm_offset_t end, va;
1397 	vm_page_t *mp;
1398 	int count;
1399 	boolean_t pmap_failed;
1400 
1401 	if (len == 0)
1402 		return (0);
1403 	end = round_page(addr + len);
1404 	addr = trunc_page(addr);
1405 
1406 	/*
1407 	 * Check for illegal addresses.
1408 	 */
1409 	if (addr < vm_map_min(map) || addr > end || end > vm_map_max(map))
1410 		return (-1);
1411 
1412 	if (atop(end - addr) > max_count)
1413 		panic("vm_fault_quick_hold_pages: count > max_count");
1414 	count = atop(end - addr);
1415 
1416 	/*
1417 	 * Most likely, the physical pages are resident in the pmap, so it is
1418 	 * faster to try pmap_extract_and_hold() first.
1419 	 */
1420 	pmap_failed = FALSE;
1421 	for (mp = ma, va = addr; va < end; mp++, va += PAGE_SIZE) {
1422 		*mp = pmap_extract_and_hold(map->pmap, va, prot);
1423 		if (*mp == NULL)
1424 			pmap_failed = TRUE;
1425 		else if ((prot & VM_PROT_WRITE) != 0 &&
1426 		    (*mp)->dirty != VM_PAGE_BITS_ALL) {
1427 			/*
1428 			 * Explicitly dirty the physical page.  Otherwise, the
1429 			 * caller's changes may go unnoticed because they are
1430 			 * performed through an unmanaged mapping or by a DMA
1431 			 * operation.
1432 			 *
1433 			 * The object lock is not held here.
1434 			 * See vm_page_clear_dirty_mask().
1435 			 */
1436 			vm_page_dirty(*mp);
1437 		}
1438 	}
1439 	if (pmap_failed) {
1440 		/*
1441 		 * One or more pages could not be held by the pmap.  Either no
1442 		 * page was mapped at the specified virtual address or that
1443 		 * mapping had insufficient permissions.  Attempt to fault in
1444 		 * and hold these pages.
1445 		 */
1446 		for (mp = ma, va = addr; va < end; mp++, va += PAGE_SIZE)
1447 			if (*mp == NULL && vm_fault_hold(map, va, prot,
1448 			    VM_FAULT_NORMAL, mp) != KERN_SUCCESS)
1449 				goto error;
1450 	}
1451 	return (count);
1452 error:
1453 	for (mp = ma; mp < ma + count; mp++)
1454 		if (*mp != NULL) {
1455 			vm_page_lock(*mp);
1456 			vm_page_unhold(*mp);
1457 			vm_page_unlock(*mp);
1458 		}
1459 	return (-1);
1460 }
1461 
1462 /*
1463  *	Routine:
1464  *		vm_fault_copy_entry
1465  *	Function:
1466  *		Create new shadow object backing dst_entry with private copy of
1467  *		all underlying pages. When src_entry is equal to dst_entry,
1468  *		function implements COW for wired-down map entry. Otherwise,
1469  *		it forks wired entry into dst_map.
1470  *
1471  *	In/out conditions:
1472  *		The source and destination maps must be locked for write.
1473  *		The source map entry must be wired down (or be a sharing map
1474  *		entry corresponding to a main map entry that is wired down).
1475  */
1476 void
1477 vm_fault_copy_entry(vm_map_t dst_map, vm_map_t src_map,
1478     vm_map_entry_t dst_entry, vm_map_entry_t src_entry,
1479     vm_ooffset_t *fork_charge)
1480 {
1481 	vm_object_t backing_object, dst_object, object, src_object;
1482 	vm_pindex_t dst_pindex, pindex, src_pindex;
1483 	vm_prot_t access, prot;
1484 	vm_offset_t vaddr;
1485 	vm_page_t dst_m;
1486 	vm_page_t src_m;
1487 	boolean_t upgrade;
1488 
1489 #ifdef	lint
1490 	src_map++;
1491 #endif	/* lint */
1492 
1493 	upgrade = src_entry == dst_entry;
1494 	access = prot = dst_entry->protection;
1495 
1496 	src_object = src_entry->object.vm_object;
1497 	src_pindex = OFF_TO_IDX(src_entry->offset);
1498 
1499 	if (upgrade && (dst_entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) {
1500 		dst_object = src_object;
1501 		vm_object_reference(dst_object);
1502 	} else {
1503 		/*
1504 		 * Create the top-level object for the destination entry. (Doesn't
1505 		 * actually shadow anything - we copy the pages directly.)
1506 		 */
1507 		dst_object = vm_object_allocate(OBJT_DEFAULT,
1508 		    OFF_TO_IDX(dst_entry->end - dst_entry->start));
1509 #if VM_NRESERVLEVEL > 0
1510 		dst_object->flags |= OBJ_COLORED;
1511 		dst_object->pg_color = atop(dst_entry->start);
1512 #endif
1513 	}
1514 
1515 	VM_OBJECT_WLOCK(dst_object);
1516 	KASSERT(upgrade || dst_entry->object.vm_object == NULL,
1517 	    ("vm_fault_copy_entry: vm_object not NULL"));
1518 	if (src_object != dst_object) {
1519 		dst_entry->object.vm_object = dst_object;
1520 		dst_entry->offset = 0;
1521 		dst_object->charge = dst_entry->end - dst_entry->start;
1522 	}
1523 	if (fork_charge != NULL) {
1524 		KASSERT(dst_entry->cred == NULL,
1525 		    ("vm_fault_copy_entry: leaked swp charge"));
1526 		dst_object->cred = curthread->td_ucred;
1527 		crhold(dst_object->cred);
1528 		*fork_charge += dst_object->charge;
1529 	} else if (dst_object->cred == NULL) {
1530 		KASSERT(dst_entry->cred != NULL, ("no cred for entry %p",
1531 		    dst_entry));
1532 		dst_object->cred = dst_entry->cred;
1533 		dst_entry->cred = NULL;
1534 	}
1535 
1536 	/*
1537 	 * If not an upgrade, then enter the mappings in the pmap as
1538 	 * read and/or execute accesses.  Otherwise, enter them as
1539 	 * write accesses.
1540 	 *
1541 	 * A writeable large page mapping is only created if all of
1542 	 * the constituent small page mappings are modified. Marking
1543 	 * PTEs as modified on inception allows promotion to happen
1544 	 * without taking potentially large number of soft faults.
1545 	 */
1546 	if (!upgrade)
1547 		access &= ~VM_PROT_WRITE;
1548 
1549 	/*
1550 	 * Loop through all of the virtual pages within the entry's
1551 	 * range, copying each page from the source object to the
1552 	 * destination object.  Since the source is wired, those pages
1553 	 * must exist.  In contrast, the destination is pageable.
1554 	 * Since the destination object does share any backing storage
1555 	 * with the source object, all of its pages must be dirtied,
1556 	 * regardless of whether they can be written.
1557 	 */
1558 	for (vaddr = dst_entry->start, dst_pindex = 0;
1559 	    vaddr < dst_entry->end;
1560 	    vaddr += PAGE_SIZE, dst_pindex++) {
1561 again:
1562 		/*
1563 		 * Find the page in the source object, and copy it in.
1564 		 * Because the source is wired down, the page will be
1565 		 * in memory.
1566 		 */
1567 		if (src_object != dst_object)
1568 			VM_OBJECT_RLOCK(src_object);
1569 		object = src_object;
1570 		pindex = src_pindex + dst_pindex;
1571 		while ((src_m = vm_page_lookup(object, pindex)) == NULL &&
1572 		    (backing_object = object->backing_object) != NULL) {
1573 			/*
1574 			 * Unless the source mapping is read-only or
1575 			 * it is presently being upgraded from
1576 			 * read-only, the first object in the shadow
1577 			 * chain should provide all of the pages.  In
1578 			 * other words, this loop body should never be
1579 			 * executed when the source mapping is already
1580 			 * read/write.
1581 			 */
1582 			KASSERT((src_entry->protection & VM_PROT_WRITE) == 0 ||
1583 			    upgrade,
1584 			    ("vm_fault_copy_entry: main object missing page"));
1585 
1586 			VM_OBJECT_RLOCK(backing_object);
1587 			pindex += OFF_TO_IDX(object->backing_object_offset);
1588 			if (object != dst_object)
1589 				VM_OBJECT_RUNLOCK(object);
1590 			object = backing_object;
1591 		}
1592 		KASSERT(src_m != NULL, ("vm_fault_copy_entry: page missing"));
1593 
1594 		if (object != dst_object) {
1595 			/*
1596 			 * Allocate a page in the destination object.
1597 			 */
1598 			dst_m = vm_page_alloc(dst_object, (src_object ==
1599 			    dst_object ? src_pindex : 0) + dst_pindex,
1600 			    VM_ALLOC_NORMAL);
1601 			if (dst_m == NULL) {
1602 				VM_OBJECT_WUNLOCK(dst_object);
1603 				VM_OBJECT_RUNLOCK(object);
1604 				VM_WAIT;
1605 				VM_OBJECT_WLOCK(dst_object);
1606 				goto again;
1607 			}
1608 			pmap_copy_page(src_m, dst_m);
1609 			VM_OBJECT_RUNLOCK(object);
1610 			dst_m->valid = VM_PAGE_BITS_ALL;
1611 			dst_m->dirty = VM_PAGE_BITS_ALL;
1612 		} else {
1613 			dst_m = src_m;
1614 			if (vm_page_sleep_if_busy(dst_m, "fltupg"))
1615 				goto again;
1616 			vm_page_xbusy(dst_m);
1617 			KASSERT(dst_m->valid == VM_PAGE_BITS_ALL,
1618 			    ("invalid dst page %p", dst_m));
1619 		}
1620 		VM_OBJECT_WUNLOCK(dst_object);
1621 
1622 		/*
1623 		 * Enter it in the pmap. If a wired, copy-on-write
1624 		 * mapping is being replaced by a write-enabled
1625 		 * mapping, then wire that new mapping.
1626 		 */
1627 		pmap_enter(dst_map->pmap, vaddr, dst_m, prot,
1628 		    access | (upgrade ? PMAP_ENTER_WIRED : 0), 0);
1629 
1630 		/*
1631 		 * Mark it no longer busy, and put it on the active list.
1632 		 */
1633 		VM_OBJECT_WLOCK(dst_object);
1634 
1635 		if (upgrade) {
1636 			if (src_m != dst_m) {
1637 				vm_page_lock(src_m);
1638 				vm_page_unwire(src_m, PQ_INACTIVE);
1639 				vm_page_unlock(src_m);
1640 				vm_page_lock(dst_m);
1641 				vm_page_wire(dst_m);
1642 				vm_page_unlock(dst_m);
1643 			} else {
1644 				KASSERT(dst_m->wire_count > 0,
1645 				    ("dst_m %p is not wired", dst_m));
1646 			}
1647 		} else {
1648 			vm_page_lock(dst_m);
1649 			vm_page_activate(dst_m);
1650 			vm_page_unlock(dst_m);
1651 		}
1652 		vm_page_xunbusy(dst_m);
1653 	}
1654 	VM_OBJECT_WUNLOCK(dst_object);
1655 	if (upgrade) {
1656 		dst_entry->eflags &= ~(MAP_ENTRY_COW | MAP_ENTRY_NEEDS_COPY);
1657 		vm_object_deallocate(src_object);
1658 	}
1659 }
1660 
1661 /*
1662  * Block entry into the machine-independent layer's page fault handler by
1663  * the calling thread.  Subsequent calls to vm_fault() by that thread will
1664  * return KERN_PROTECTION_FAILURE.  Enable machine-dependent handling of
1665  * spurious page faults.
1666  */
1667 int
1668 vm_fault_disable_pagefaults(void)
1669 {
1670 
1671 	return (curthread_pflags_set(TDP_NOFAULTING | TDP_RESETSPUR));
1672 }
1673 
1674 void
1675 vm_fault_enable_pagefaults(int save)
1676 {
1677 
1678 	curthread_pflags_restore(save);
1679 }
1680