xref: /freebsd/sys/vm/vm_fault.c (revision 380a989b3223d455375b4fae70fd0b9bdd43bafb)
1 /*
2  * Copyright (c) 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  * Copyright (c) 1994 John S. Dyson
5  * All rights reserved.
6  * Copyright (c) 1994 David Greenman
7  * All rights reserved.
8  *
9  *
10  * This code is derived from software contributed to Berkeley by
11  * The Mach Operating System project at Carnegie-Mellon University.
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  * 1. Redistributions of source code must retain the above copyright
17  *    notice, this list of conditions and the following disclaimer.
18  * 2. Redistributions in binary form must reproduce the above copyright
19  *    notice, this list of conditions and the following disclaimer in the
20  *    documentation and/or other materials provided with the distribution.
21  * 3. All advertising materials mentioning features or use of this software
22  *    must display the following acknowledgement:
23  *	This product includes software developed by the University of
24  *	California, Berkeley and its contributors.
25  * 4. Neither the name of the University nor the names of its contributors
26  *    may be used to endorse or promote products derived from this software
27  *    without specific prior written permission.
28  *
29  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
30  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
33  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
35  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
36  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
37  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
38  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39  * SUCH DAMAGE.
40  *
41  *	from: @(#)vm_fault.c	8.4 (Berkeley) 1/12/94
42  *
43  *
44  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
45  * All rights reserved.
46  *
47  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
48  *
49  * Permission to use, copy, modify and distribute this software and
50  * its documentation is hereby granted, provided that both the copyright
51  * notice and this permission notice appear in all copies of the
52  * software, derivative works or modified versions, and any portions
53  * thereof, and that both notices appear in supporting documentation.
54  *
55  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
56  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
57  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
58  *
59  * Carnegie Mellon requests users of this software to return to
60  *
61  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
62  *  School of Computer Science
63  *  Carnegie Mellon University
64  *  Pittsburgh PA 15213-3890
65  *
66  * any improvements or extensions that they make and grant Carnegie the
67  * rights to redistribute these changes.
68  *
69  * $Id: vm_fault.c,v 1.91 1998/11/25 07:40:49 dg Exp $
70  */
71 
72 /*
73  *	Page fault handling module.
74  */
75 
76 #include <sys/param.h>
77 #include <sys/systm.h>
78 #include <sys/proc.h>
79 #include <sys/vnode.h>
80 #include <sys/resourcevar.h>
81 #include <sys/vmmeter.h>
82 
83 #include <vm/vm.h>
84 #include <vm/vm_param.h>
85 #include <vm/vm_prot.h>
86 #include <sys/lock.h>
87 #include <vm/pmap.h>
88 #include <vm/vm_map.h>
89 #include <vm/vm_object.h>
90 #include <vm/vm_page.h>
91 #include <vm/vm_pageout.h>
92 #include <vm/vm_kern.h>
93 #include <vm/vm_pager.h>
94 #include <vm/vnode_pager.h>
95 #include <vm/vm_extern.h>
96 
97 static int vm_fault_additional_pages __P((vm_page_t, int,
98 					  int, vm_page_t *, int *));
99 
100 #define VM_FAULT_READ_AHEAD 8
101 #define VM_FAULT_READ_BEHIND 7
102 #define VM_FAULT_READ (VM_FAULT_READ_AHEAD+VM_FAULT_READ_BEHIND+1)
103 
104 struct faultstate {
105 	vm_page_t m;
106 	vm_object_t object;
107 	vm_pindex_t pindex;
108 	vm_page_t first_m;
109 	vm_object_t	first_object;
110 	vm_pindex_t first_pindex;
111 	vm_map_t map;
112 	vm_map_entry_t entry;
113 	int lookup_still_valid;
114 	struct vnode *vp;
115 };
116 
117 static void
118 release_page(struct faultstate *fs)
119 {
120 	vm_page_wakeup(fs->m);
121 	vm_page_deactivate(fs->m);
122 	fs->m = NULL;
123 }
124 
125 static void
126 unlock_map(struct faultstate *fs)
127 {
128 	if (fs->lookup_still_valid) {
129 		vm_map_lookup_done(fs->map, fs->entry);
130 		fs->lookup_still_valid = FALSE;
131 	}
132 }
133 
134 static void
135 _unlock_things(struct faultstate *fs, int dealloc)
136 {
137 	vm_object_pip_wakeup(fs->object);
138 	if (fs->object != fs->first_object) {
139 		vm_page_free(fs->first_m);
140 		vm_object_pip_wakeup(fs->first_object);
141 		fs->first_m = NULL;
142 	}
143 	if (dealloc) {
144 		vm_object_deallocate(fs->first_object);
145 	}
146 	unlock_map(fs);
147 	if (fs->vp != NULL) {
148 		vput(fs->vp);
149 		fs->vp = NULL;
150 	}
151 }
152 
153 #define unlock_things(fs) _unlock_things(fs, 0)
154 #define unlock_and_deallocate(fs) _unlock_things(fs, 1)
155 
156 /*
157  *	vm_fault:
158  *
159  *	Handle a page fault occuring at the given address,
160  *	requiring the given permissions, in the map specified.
161  *	If successful, the page is inserted into the
162  *	associated physical map.
163  *
164  *	NOTE: the given address should be truncated to the
165  *	proper page address.
166  *
167  *	KERN_SUCCESS is returned if the page fault is handled; otherwise,
168  *	a standard error specifying why the fault is fatal is returned.
169  *
170  *
171  *	The map in question must be referenced, and remains so.
172  *	Caller may hold no locks.
173  */
174 int
175 vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, int fault_flags)
176 {
177 	vm_prot_t prot;
178 	int result;
179 	boolean_t wired;
180 	int map_generation;
181 	vm_page_t old_m;
182 	vm_object_t next_object;
183 	vm_page_t marray[VM_FAULT_READ];
184 	int hardfault;
185 	int faultcount;
186 	struct faultstate fs;
187 
188 	cnt.v_vm_faults++;	/* needs lock XXX */
189 	hardfault = 0;
190 
191 RetryFault:;
192 	fs.map = map;
193 
194 	/*
195 	 * Find the backing store object and offset into it to begin the
196 	 * search.
197 	 */
198 	if ((result = vm_map_lookup(&fs.map, vaddr,
199 		fault_type, &fs.entry, &fs.first_object,
200 		&fs.first_pindex, &prot, &wired)) != KERN_SUCCESS) {
201 		if ((result != KERN_PROTECTION_FAILURE) ||
202 			((fault_flags & VM_FAULT_WIRE_MASK) != VM_FAULT_USER_WIRE)) {
203 			return result;
204 		}
205 
206 		/*
207    		 * If we are user-wiring a r/w segment, and it is COW, then
208    		 * we need to do the COW operation.  Note that we don't COW
209    		 * currently RO sections now, because it is NOT desirable
210    		 * to COW .text.  We simply keep .text from ever being COW'ed
211    		 * and take the heat that one cannot debug wired .text sections.
212    		 */
213 		result = vm_map_lookup(&fs.map, vaddr,
214 			VM_PROT_READ|VM_PROT_WRITE|VM_PROT_OVERRIDE_WRITE,
215 			&fs.entry, &fs.first_object, &fs.first_pindex, &prot, &wired);
216 		if (result != KERN_SUCCESS) {
217 			return result;
218 		}
219 
220 		/*
221 		 * If we don't COW now, on a user wire, the user will never
222 		 * be able to write to the mapping.  If we don't make this
223 		 * restriction, the bookkeeping would be nearly impossible.
224 		 */
225 		if ((fs.entry->protection & VM_PROT_WRITE) == 0)
226 			fs.entry->max_protection &= ~VM_PROT_WRITE;
227 	}
228 
229 	map_generation = fs.map->timestamp;
230 
231 	if (fs.entry->eflags & MAP_ENTRY_NOFAULT) {
232 		panic("vm_fault: fault on nofault entry, addr: %lx",
233 		    (u_long)vaddr);
234 	}
235 
236 	/*
237 	 * Make a reference to this object to prevent its disposal while we
238 	 * are messing with it.  Once we have the reference, the map is free
239 	 * to be diddled.  Since objects reference their shadows (and copies),
240 	 * they will stay around as well.
241 	 */
242 	vm_object_reference(fs.first_object);
243 	vm_object_pip_add(fs.first_object, 1);
244 
245 	fs.vp = vnode_pager_lock(fs.first_object);
246 	if ((fault_type & VM_PROT_WRITE) &&
247 		(fs.first_object->type == OBJT_VNODE)) {
248 		vm_freeze_copyopts(fs.first_object,
249 			fs.first_pindex, fs.first_pindex + 1);
250 	}
251 
252 	fs.lookup_still_valid = TRUE;
253 
254 	if (wired)
255 		fault_type = prot;
256 
257 	fs.first_m = NULL;
258 
259 	/*
260 	 * Search for the page at object/offset.
261 	 */
262 
263 	fs.object = fs.first_object;
264 	fs.pindex = fs.first_pindex;
265 
266 	/*
267 	 * See whether this page is resident
268 	 */
269 	while (TRUE) {
270 
271 		if (fs.object->flags & OBJ_DEAD) {
272 			unlock_and_deallocate(&fs);
273 			return (KERN_PROTECTION_FAILURE);
274 		}
275 
276 		fs.m = vm_page_lookup(fs.object, fs.pindex);
277 		if (fs.m != NULL) {
278 			int queue, s;
279 			/*
280 			 * If the page is being brought in, wait for it and
281 			 * then retry.
282 			 */
283 			if ((fs.m->flags & PG_BUSY) ||
284 				(fs.m->busy &&
285 				 (fs.m->valid & VM_PAGE_BITS_ALL) != VM_PAGE_BITS_ALL)) {
286 				unlock_things(&fs);
287 				s = splvm();
288 				if ((fs.m->flags & PG_BUSY) ||
289 					(fs.m->busy &&
290 					 (fs.m->valid & VM_PAGE_BITS_ALL) != VM_PAGE_BITS_ALL)) {
291 					vm_page_flag_set(fs.m, PG_WANTED | PG_REFERENCED);
292 					cnt.v_intrans++;
293 					tsleep(fs.m, PSWP, "vmpfw", 0);
294 				}
295 				splx(s);
296 				vm_object_deallocate(fs.first_object);
297 				goto RetryFault;
298 			}
299 
300 			queue = fs.m->queue;
301 			s = splvm();
302 			vm_page_unqueue_nowakeup(fs.m);
303 			splx(s);
304 
305 			/*
306 			 * Mark page busy for other processes, and the pagedaemon.
307 			 */
308 			if (((queue - fs.m->pc) == PQ_CACHE) &&
309 			    (cnt.v_free_count + cnt.v_cache_count) < cnt.v_free_min) {
310 				vm_page_activate(fs.m);
311 				unlock_and_deallocate(&fs);
312 				VM_WAIT;
313 				goto RetryFault;
314 			}
315 
316 			vm_page_busy(fs.m);
317 			if (((fs.m->valid & VM_PAGE_BITS_ALL) != VM_PAGE_BITS_ALL) &&
318 				fs.m->object != kernel_object && fs.m->object != kmem_object) {
319 				goto readrest;
320 			}
321 
322 			break;
323 		}
324 		if (((fs.object->type != OBJT_DEFAULT) &&
325 				(((fault_flags & VM_FAULT_WIRE_MASK) == 0) || wired))
326 		    || (fs.object == fs.first_object)) {
327 
328 			if (fs.pindex >= fs.object->size) {
329 				unlock_and_deallocate(&fs);
330 				return (KERN_PROTECTION_FAILURE);
331 			}
332 
333 			/*
334 			 * Allocate a new page for this object/offset pair.
335 			 */
336 			fs.m = vm_page_alloc(fs.object, fs.pindex,
337 				(fs.vp || fs.object->backing_object)? VM_ALLOC_NORMAL: VM_ALLOC_ZERO);
338 
339 			if (fs.m == NULL) {
340 				unlock_and_deallocate(&fs);
341 				VM_WAIT;
342 				goto RetryFault;
343 			}
344 		}
345 
346 readrest:
347 		if (fs.object->type != OBJT_DEFAULT &&
348 			(((fault_flags & VM_FAULT_WIRE_MASK) == 0) || wired)) {
349 			int rv;
350 			int reqpage;
351 			int ahead, behind;
352 
353 			if (fs.first_object->behavior == OBJ_RANDOM) {
354 				ahead = 0;
355 				behind = 0;
356 			} else {
357 				behind = (vaddr - fs.entry->start) >> PAGE_SHIFT;
358 				if (behind > VM_FAULT_READ_BEHIND)
359 					behind = VM_FAULT_READ_BEHIND;
360 
361 				ahead = ((fs.entry->end - vaddr) >> PAGE_SHIFT) - 1;
362 				if (ahead > VM_FAULT_READ_AHEAD)
363 					ahead = VM_FAULT_READ_AHEAD;
364 			}
365 
366 			if ((fs.first_object->type != OBJT_DEVICE) &&
367 				(fs.first_object->behavior == OBJ_SEQUENTIAL)) {
368 				vm_pindex_t firstpindex, tmppindex;
369 				if (fs.first_pindex <
370 					2*(VM_FAULT_READ_BEHIND + VM_FAULT_READ_AHEAD + 1))
371 					firstpindex = 0;
372 				else
373 					firstpindex = fs.first_pindex -
374 						2*(VM_FAULT_READ_BEHIND + VM_FAULT_READ_AHEAD + 1);
375 
376 				for(tmppindex = fs.first_pindex - 1;
377 					tmppindex >= firstpindex;
378 					--tmppindex) {
379 					vm_page_t mt;
380 					mt = vm_page_lookup( fs.first_object, tmppindex);
381 					if (mt == NULL || (mt->valid != VM_PAGE_BITS_ALL))
382 						break;
383 					if (mt->busy ||
384 						(mt->flags & (PG_BUSY | PG_FICTITIOUS)) ||
385 						mt->hold_count ||
386 						mt->wire_count)
387 						continue;
388 					if (mt->dirty == 0)
389 						vm_page_test_dirty(mt);
390 					if (mt->dirty) {
391 						vm_page_protect(mt, VM_PROT_NONE);
392 						vm_page_deactivate(mt);
393 					} else {
394 						vm_page_cache(mt);
395 					}
396 				}
397 
398 				ahead += behind;
399 				behind = 0;
400 			}
401 
402 			/*
403 			 * now we find out if any other pages should be paged
404 			 * in at this time this routine checks to see if the
405 			 * pages surrounding this fault reside in the same
406 			 * object as the page for this fault.  If they do,
407 			 * then they are faulted in also into the object.  The
408 			 * array "marray" returned contains an array of
409 			 * vm_page_t structs where one of them is the
410 			 * vm_page_t passed to the routine.  The reqpage
411 			 * return value is the index into the marray for the
412 			 * vm_page_t passed to the routine.
413 			 */
414 			faultcount = vm_fault_additional_pages(
415 			    fs.m, behind, ahead, marray, &reqpage);
416 
417 			/*
418 			 * Call the pager to retrieve the data, if any, after
419 			 * releasing the lock on the map.
420 			 */
421 			unlock_map(&fs);
422 
423 			rv = faultcount ?
424 			    vm_pager_get_pages(fs.object, marray, faultcount,
425 				reqpage) : VM_PAGER_FAIL;
426 
427 			if (rv == VM_PAGER_OK) {
428 				/*
429 				 * Found the page. Leave it busy while we play
430 				 * with it.
431 				 */
432 
433 				/*
434 				 * Relookup in case pager changed page. Pager
435 				 * is responsible for disposition of old page
436 				 * if moved.
437 				 */
438 				fs.m = vm_page_lookup(fs.object, fs.pindex);
439 				if(!fs.m) {
440 					unlock_and_deallocate(&fs);
441 					goto RetryFault;
442 				}
443 
444 				hardfault++;
445 				break;
446 			}
447 			/*
448 			 * Remove the bogus page (which does not exist at this
449 			 * object/offset); before doing so, we must get back
450 			 * our object lock to preserve our invariant.
451 			 *
452 			 * Also wake up any other process that may want to bring
453 			 * in this page.
454 			 *
455 			 * If this is the top-level object, we must leave the
456 			 * busy page to prevent another process from rushing
457 			 * past us, and inserting the page in that object at
458 			 * the same time that we are.
459 			 */
460 
461 			if (rv == VM_PAGER_ERROR)
462 				printf("vm_fault: pager read error, pid %d (%s)\n",
463 				    curproc->p_pid, curproc->p_comm);
464 			/*
465 			 * Data outside the range of the pager or an I/O error
466 			 */
467 			/*
468 			 * XXX - the check for kernel_map is a kludge to work
469 			 * around having the machine panic on a kernel space
470 			 * fault w/ I/O error.
471 			 */
472 			if (((fs.map != kernel_map) && (rv == VM_PAGER_ERROR)) ||
473 				(rv == VM_PAGER_BAD)) {
474 				vm_page_free(fs.m);
475 				fs.m = NULL;
476 				unlock_and_deallocate(&fs);
477 				return ((rv == VM_PAGER_ERROR) ? KERN_FAILURE : KERN_PROTECTION_FAILURE);
478 			}
479 			if (fs.object != fs.first_object) {
480 				vm_page_free(fs.m);
481 				fs.m = NULL;
482 				/*
483 				 * XXX - we cannot just fall out at this
484 				 * point, m has been freed and is invalid!
485 				 */
486 			}
487 		}
488 		/*
489 		 * We get here if the object has default pager (or unwiring) or the
490 		 * pager doesn't have the page.
491 		 */
492 		if (fs.object == fs.first_object)
493 			fs.first_m = fs.m;
494 
495 		/*
496 		 * Move on to the next object.  Lock the next object before
497 		 * unlocking the current one.
498 		 */
499 
500 		fs.pindex += OFF_TO_IDX(fs.object->backing_object_offset);
501 		next_object = fs.object->backing_object;
502 		if (next_object == NULL) {
503 			/*
504 			 * If there's no object left, fill the page in the top
505 			 * object with zeros.
506 			 */
507 			if (fs.object != fs.first_object) {
508 				vm_object_pip_wakeup(fs.object);
509 
510 				fs.object = fs.first_object;
511 				fs.pindex = fs.first_pindex;
512 				fs.m = fs.first_m;
513 			}
514 			fs.first_m = NULL;
515 
516 			if ((fs.m->flags & PG_ZERO) == 0) {
517 				vm_page_zero_fill(fs.m);
518 				cnt.v_ozfod++;
519 			}
520 			cnt.v_zfod++;
521 			break;
522 		} else {
523 			if (fs.object != fs.first_object) {
524 				vm_object_pip_wakeup(fs.object);
525 			}
526 			fs.object = next_object;
527 			vm_object_pip_add(fs.object, 1);
528 		}
529 	}
530 
531 	KASSERT((fs.m->flags & PG_BUSY) != 0,
532 	    ("vm_fault: not busy after main loop"));
533 
534 	/*
535 	 * PAGE HAS BEEN FOUND. [Loop invariant still holds -- the object lock
536 	 * is held.]
537 	 */
538 
539 	old_m = fs.m;	/* save page that would be copied */
540 
541 	/*
542 	 * If the page is being written, but isn't already owned by the
543 	 * top-level object, we have to copy it into a new page owned by the
544 	 * top-level object.
545 	 */
546 
547 	if (fs.object != fs.first_object) {
548 		/*
549 		 * We only really need to copy if we want to write it.
550 		 */
551 
552 		if (fault_type & VM_PROT_WRITE) {
553 
554 			/*
555 			 * This allows pages to be virtually copied from a backing_object
556 			 * into the first_object, where the backing object has no other
557 			 * refs to it, and cannot gain any more refs.  Instead of a
558 			 * bcopy, we just move the page from the backing object to the
559 			 * first object.  Note that we must mark the page dirty in the
560 			 * first object so that it will go out to swap when needed.
561 			 */
562 			if (map_generation == fs.map->timestamp &&
563 				/*
564 				 * Only one shadow object
565 				 */
566 				(fs.object->shadow_count == 1) &&
567 				/*
568 				 * No COW refs, except us
569 				 */
570 				(fs.object->ref_count == 1) &&
571 				/*
572 				 * Noone else can look this object up
573 				 */
574 				(fs.object->handle == NULL) &&
575 				/*
576 				 * No other ways to look the object up
577 				 */
578 				((fs.object->type == OBJT_DEFAULT) ||
579 				 (fs.object->type == OBJT_SWAP)) &&
580 				/*
581 				 * We don't chase down the shadow chain
582 				 */
583 				(fs.object == fs.first_object->backing_object) &&
584 
585 				/*
586 				 * grab the lock if we need to
587 				 */
588 				(fs.lookup_still_valid ||
589 						(((fs.entry->eflags & MAP_ENTRY_IS_A_MAP) == 0) &&
590 						 lockmgr(&fs.map->lock,
591 							LK_EXCLUSIVE|LK_NOWAIT, (void *)0, curproc) == 0))) {
592 
593 				fs.lookup_still_valid = 1;
594 				/*
595 				 * get rid of the unnecessary page
596 				 */
597 				vm_page_protect(fs.first_m, VM_PROT_NONE);
598 				vm_page_free(fs.first_m);
599 				fs.first_m = NULL;
600 
601 				/*
602 				 * grab the page and put it into the process'es object
603 				 */
604 				vm_page_rename(fs.m, fs.first_object, fs.first_pindex);
605 				fs.first_m = fs.m;
606 				fs.first_m->dirty = VM_PAGE_BITS_ALL;
607 				vm_page_busy(fs.first_m);
608 				fs.m = NULL;
609 				cnt.v_cow_optim++;
610 			} else {
611 				/*
612 				 * Oh, well, lets copy it.
613 				 */
614 				vm_page_copy(fs.m, fs.first_m);
615 			}
616 
617 			if (fs.m) {
618 				/*
619 				 * We no longer need the old page or object.
620 				 */
621 				release_page(&fs);
622 			}
623 
624 			vm_object_pip_wakeup(fs.object);
625 			/*
626 			 * Only use the new page below...
627 			 */
628 
629 			cnt.v_cow_faults++;
630 			fs.m = fs.first_m;
631 			fs.object = fs.first_object;
632 			fs.pindex = fs.first_pindex;
633 
634 		} else {
635 			prot &= ~VM_PROT_WRITE;
636 		}
637 	}
638 
639 	/*
640 	 * We must verify that the maps have not changed since our last
641 	 * lookup.
642 	 */
643 
644 	if (!fs.lookup_still_valid &&
645 		(fs.map->timestamp != map_generation)) {
646 		vm_object_t retry_object;
647 		vm_pindex_t retry_pindex;
648 		vm_prot_t retry_prot;
649 
650 		/*
651 		 * Since map entries may be pageable, make sure we can take a
652 		 * page fault on them.
653 		 */
654 
655 		/*
656 		 * To avoid trying to write_lock the map while another process
657 		 * has it read_locked (in vm_map_pageable), we do not try for
658 		 * write permission.  If the page is still writable, we will
659 		 * get write permission.  If it is not, or has been marked
660 		 * needs_copy, we enter the mapping without write permission,
661 		 * and will merely take another fault.
662 		 */
663 		result = vm_map_lookup(&fs.map, vaddr, fault_type & ~VM_PROT_WRITE,
664 		    &fs.entry, &retry_object, &retry_pindex, &retry_prot, &wired);
665 		map_generation = fs.map->timestamp;
666 
667 		/*
668 		 * If we don't need the page any longer, put it on the active
669 		 * list (the easiest thing to do here).  If no one needs it,
670 		 * pageout will grab it eventually.
671 		 */
672 
673 		if (result != KERN_SUCCESS) {
674 			release_page(&fs);
675 			unlock_and_deallocate(&fs);
676 			return (result);
677 		}
678 		fs.lookup_still_valid = TRUE;
679 
680 		if ((retry_object != fs.first_object) ||
681 		    (retry_pindex != fs.first_pindex)) {
682 			release_page(&fs);
683 			unlock_and_deallocate(&fs);
684 			goto RetryFault;
685 		}
686 		/*
687 		 * Check whether the protection has changed or the object has
688 		 * been copied while we left the map unlocked. Changing from
689 		 * read to write permission is OK - we leave the page
690 		 * write-protected, and catch the write fault. Changing from
691 		 * write to read permission means that we can't mark the page
692 		 * write-enabled after all.
693 		 */
694 		prot &= retry_prot;
695 	}
696 
697 	/*
698 	 * Put this page into the physical map. We had to do the unlock above
699 	 * because pmap_enter may cause other faults.   We don't put the page
700 	 * back on the active queue until later so that the page-out daemon
701 	 * won't find us (yet).
702 	 */
703 
704 	if (prot & VM_PROT_WRITE) {
705 		vm_page_flag_set(fs.m, PG_WRITEABLE);
706 		vm_object_set_flag(fs.m->object,
707 				   OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY);
708 		/*
709 		 * If the fault is a write, we know that this page is being
710 		 * written NOW. This will save on the pmap_is_modified() calls
711 		 * later.
712 		 */
713 		if (fault_flags & VM_FAULT_DIRTY) {
714 			fs.m->dirty = VM_PAGE_BITS_ALL;
715 		}
716 	}
717 
718 	unlock_things(&fs);
719 	fs.m->valid = VM_PAGE_BITS_ALL;
720 	vm_page_flag_clear(fs.m, PG_ZERO);
721 
722 	pmap_enter(fs.map->pmap, vaddr, VM_PAGE_TO_PHYS(fs.m), prot, wired);
723 	if (((fault_flags & VM_FAULT_WIRE_MASK) == 0) && (wired == 0)) {
724 		pmap_prefault(fs.map->pmap, vaddr, fs.entry);
725 	}
726 
727 	vm_page_flag_set(fs.m, PG_MAPPED|PG_REFERENCED);
728 	if (fault_flags & VM_FAULT_HOLD)
729 		vm_page_hold(fs.m);
730 
731 	/*
732 	 * If the page is not wired down, then put it where the pageout daemon
733 	 * can find it.
734 	 */
735 	if (fault_flags & VM_FAULT_WIRE_MASK) {
736 		if (wired)
737 			vm_page_wire(fs.m);
738 		else
739 			vm_page_unwire(fs.m, 1);
740 	} else {
741 		vm_page_activate(fs.m);
742 	}
743 
744 	if (curproc && (curproc->p_flag & P_INMEM) && curproc->p_stats) {
745 		if (hardfault) {
746 			curproc->p_stats->p_ru.ru_majflt++;
747 		} else {
748 			curproc->p_stats->p_ru.ru_minflt++;
749 		}
750 	}
751 
752 	/*
753 	 * Unlock everything, and return
754 	 */
755 
756 	vm_page_wakeup(fs.m);
757 	vm_object_deallocate(fs.first_object);
758 
759 	return (KERN_SUCCESS);
760 
761 }
762 
763 /*
764  *	vm_fault_wire:
765  *
766  *	Wire down a range of virtual addresses in a map.
767  */
768 int
769 vm_fault_wire(map, start, end)
770 	vm_map_t map;
771 	vm_offset_t start, end;
772 {
773 
774 	register vm_offset_t va;
775 	register pmap_t pmap;
776 	int rv;
777 
778 	pmap = vm_map_pmap(map);
779 
780 	/*
781 	 * Inform the physical mapping system that the range of addresses may
782 	 * not fault, so that page tables and such can be locked down as well.
783 	 */
784 
785 	pmap_pageable(pmap, start, end, FALSE);
786 
787 	/*
788 	 * We simulate a fault to get the page and enter it in the physical
789 	 * map.
790 	 */
791 
792 	for (va = start; va < end; va += PAGE_SIZE) {
793 		rv = vm_fault(map, va, VM_PROT_READ|VM_PROT_WRITE,
794 			VM_FAULT_CHANGE_WIRING);
795 		if (rv) {
796 			if (va != start)
797 				vm_fault_unwire(map, start, va);
798 			return (rv);
799 		}
800 	}
801 	return (KERN_SUCCESS);
802 }
803 
804 /*
805  *	vm_fault_user_wire:
806  *
807  *	Wire down a range of virtual addresses in a map.  This
808  *	is for user mode though, so we only ask for read access
809  *	on currently read only sections.
810  */
811 int
812 vm_fault_user_wire(map, start, end)
813 	vm_map_t map;
814 	vm_offset_t start, end;
815 {
816 
817 	register vm_offset_t va;
818 	register pmap_t pmap;
819 	int rv;
820 
821 	pmap = vm_map_pmap(map);
822 
823 	/*
824 	 * Inform the physical mapping system that the range of addresses may
825 	 * not fault, so that page tables and such can be locked down as well.
826 	 */
827 
828 	pmap_pageable(pmap, start, end, FALSE);
829 
830 	/*
831 	 * We simulate a fault to get the page and enter it in the physical
832 	 * map.
833 	 */
834 	for (va = start; va < end; va += PAGE_SIZE) {
835 		rv = vm_fault(map, va, VM_PROT_READ, VM_FAULT_USER_WIRE);
836 		if (rv) {
837 			if (va != start)
838 				vm_fault_unwire(map, start, va);
839 			return (rv);
840 		}
841 	}
842 	return (KERN_SUCCESS);
843 }
844 
845 
846 /*
847  *	vm_fault_unwire:
848  *
849  *	Unwire a range of virtual addresses in a map.
850  */
851 void
852 vm_fault_unwire(map, start, end)
853 	vm_map_t map;
854 	vm_offset_t start, end;
855 {
856 
857 	register vm_offset_t va, pa;
858 	register pmap_t pmap;
859 
860 	pmap = vm_map_pmap(map);
861 
862 	/*
863 	 * Since the pages are wired down, we must be able to get their
864 	 * mappings from the physical map system.
865 	 */
866 
867 	for (va = start; va < end; va += PAGE_SIZE) {
868 		pa = pmap_extract(pmap, va);
869 		if (pa != (vm_offset_t) 0) {
870 			pmap_change_wiring(pmap, va, FALSE);
871 			vm_page_unwire(PHYS_TO_VM_PAGE(pa), 1);
872 		}
873 	}
874 
875 	/*
876 	 * Inform the physical mapping system that the range of addresses may
877 	 * fault, so that page tables and such may be unwired themselves.
878 	 */
879 
880 	pmap_pageable(pmap, start, end, TRUE);
881 
882 }
883 
884 /*
885  *	Routine:
886  *		vm_fault_copy_entry
887  *	Function:
888  *		Copy all of the pages from a wired-down map entry to another.
889  *
890  *	In/out conditions:
891  *		The source and destination maps must be locked for write.
892  *		The source map entry must be wired down (or be a sharing map
893  *		entry corresponding to a main map entry that is wired down).
894  */
895 
896 void
897 vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry)
898 	vm_map_t dst_map;
899 	vm_map_t src_map;
900 	vm_map_entry_t dst_entry;
901 	vm_map_entry_t src_entry;
902 {
903 	vm_object_t dst_object;
904 	vm_object_t src_object;
905 	vm_ooffset_t dst_offset;
906 	vm_ooffset_t src_offset;
907 	vm_prot_t prot;
908 	vm_offset_t vaddr;
909 	vm_page_t dst_m;
910 	vm_page_t src_m;
911 
912 #ifdef	lint
913 	src_map++;
914 #endif	/* lint */
915 
916 	src_object = src_entry->object.vm_object;
917 	src_offset = src_entry->offset;
918 
919 	/*
920 	 * Create the top-level object for the destination entry. (Doesn't
921 	 * actually shadow anything - we copy the pages directly.)
922 	 */
923 	dst_object = vm_object_allocate(OBJT_DEFAULT,
924 	    (vm_size_t) OFF_TO_IDX(dst_entry->end - dst_entry->start));
925 
926 	dst_entry->object.vm_object = dst_object;
927 	dst_entry->offset = 0;
928 
929 	prot = dst_entry->max_protection;
930 
931 	/*
932 	 * Loop through all of the pages in the entry's range, copying each
933 	 * one from the source object (it should be there) to the destination
934 	 * object.
935 	 */
936 	for (vaddr = dst_entry->start, dst_offset = 0;
937 	    vaddr < dst_entry->end;
938 	    vaddr += PAGE_SIZE, dst_offset += PAGE_SIZE) {
939 
940 		/*
941 		 * Allocate a page in the destination object
942 		 */
943 		do {
944 			dst_m = vm_page_alloc(dst_object,
945 				OFF_TO_IDX(dst_offset), VM_ALLOC_NORMAL);
946 			if (dst_m == NULL) {
947 				VM_WAIT;
948 			}
949 		} while (dst_m == NULL);
950 
951 		/*
952 		 * Find the page in the source object, and copy it in.
953 		 * (Because the source is wired down, the page will be in
954 		 * memory.)
955 		 */
956 		src_m = vm_page_lookup(src_object,
957 			OFF_TO_IDX(dst_offset + src_offset));
958 		if (src_m == NULL)
959 			panic("vm_fault_copy_wired: page missing");
960 
961 		vm_page_copy(src_m, dst_m);
962 
963 		/*
964 		 * Enter it in the pmap...
965 		 */
966 
967 		vm_page_flag_clear(dst_m, PG_ZERO);
968 		pmap_enter(dst_map->pmap, vaddr, VM_PAGE_TO_PHYS(dst_m),
969 		    prot, FALSE);
970 		vm_page_flag_set(dst_m, PG_WRITEABLE|PG_MAPPED);
971 
972 		/*
973 		 * Mark it no longer busy, and put it on the active list.
974 		 */
975 		vm_page_activate(dst_m);
976 		vm_page_wakeup(dst_m);
977 	}
978 }
979 
980 
981 /*
982  * This routine checks around the requested page for other pages that
983  * might be able to be faulted in.  This routine brackets the viable
984  * pages for the pages to be paged in.
985  *
986  * Inputs:
987  *	m, rbehind, rahead
988  *
989  * Outputs:
990  *  marray (array of vm_page_t), reqpage (index of requested page)
991  *
992  * Return value:
993  *  number of pages in marray
994  */
995 static int
996 vm_fault_additional_pages(m, rbehind, rahead, marray, reqpage)
997 	vm_page_t m;
998 	int rbehind;
999 	int rahead;
1000 	vm_page_t *marray;
1001 	int *reqpage;
1002 {
1003 	int i,j;
1004 	vm_object_t object;
1005 	vm_pindex_t pindex, startpindex, endpindex, tpindex;
1006 	vm_page_t rtm;
1007 	int cbehind, cahead;
1008 
1009 	object = m->object;
1010 	pindex = m->pindex;
1011 
1012 	/*
1013 	 * we don't fault-ahead for device pager
1014 	 */
1015 	if (object->type == OBJT_DEVICE) {
1016 		*reqpage = 0;
1017 		marray[0] = m;
1018 		return 1;
1019 	}
1020 
1021 	/*
1022 	 * if the requested page is not available, then give up now
1023 	 */
1024 
1025 	if (!vm_pager_has_page(object,
1026 		OFF_TO_IDX(object->paging_offset) + pindex, &cbehind, &cahead)) {
1027 		return 0;
1028 	}
1029 
1030 	if ((cbehind == 0) && (cahead == 0)) {
1031 		*reqpage = 0;
1032 		marray[0] = m;
1033 		return 1;
1034 	}
1035 
1036 	if (rahead > cahead) {
1037 		rahead = cahead;
1038 	}
1039 
1040 	if (rbehind > cbehind) {
1041 		rbehind = cbehind;
1042 	}
1043 
1044 	/*
1045 	 * try to do any readahead that we might have free pages for.
1046 	 */
1047 	if ((rahead + rbehind) >
1048 		((cnt.v_free_count + cnt.v_cache_count) - cnt.v_free_reserved)) {
1049 		pagedaemon_wakeup();
1050 		marray[0] = m;
1051 		*reqpage = 0;
1052 		return 1;
1053 	}
1054 
1055 	/*
1056 	 * scan backward for the read behind pages -- in memory
1057 	 */
1058 	if (pindex > 0) {
1059 		if (rbehind > pindex) {
1060 			rbehind = pindex;
1061 			startpindex = 0;
1062 		} else {
1063 			startpindex = pindex - rbehind;
1064 		}
1065 
1066 		for ( tpindex = pindex - 1; tpindex >= startpindex; tpindex -= 1) {
1067 			if (vm_page_lookup( object, tpindex)) {
1068 				startpindex = tpindex + 1;
1069 				break;
1070 			}
1071 			if (tpindex == 0)
1072 				break;
1073 		}
1074 
1075 		for(i = 0, tpindex = startpindex; tpindex < pindex; i++, tpindex++) {
1076 
1077 			rtm = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);
1078 			if (rtm == NULL) {
1079 				for (j = 0; j < i; j++) {
1080 					vm_page_free(marray[j]);
1081 				}
1082 				marray[0] = m;
1083 				*reqpage = 0;
1084 				return 1;
1085 			}
1086 
1087 			marray[i] = rtm;
1088 		}
1089 	} else {
1090 		startpindex = 0;
1091 		i = 0;
1092 	}
1093 
1094 	marray[i] = m;
1095 	/* page offset of the required page */
1096 	*reqpage = i;
1097 
1098 	tpindex = pindex + 1;
1099 	i++;
1100 
1101 	/*
1102 	 * scan forward for the read ahead pages
1103 	 */
1104 	endpindex = tpindex + rahead;
1105 	if (endpindex > object->size)
1106 		endpindex = object->size;
1107 
1108 	for( ; tpindex < endpindex; i++, tpindex++) {
1109 
1110 		if (vm_page_lookup(object, tpindex)) {
1111 			break;
1112 		}
1113 
1114 		rtm = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);
1115 		if (rtm == NULL) {
1116 			break;
1117 		}
1118 
1119 		marray[i] = rtm;
1120 	}
1121 
1122 	/* return number of bytes of pages */
1123 	return i;
1124 }
1125