xref: /freebsd/sys/vm/vm_map.c (revision 195ebc7e9e4b129de810833791a19dfb4349d6a9)
1 /*-
2  * Copyright (c) 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * This code is derived from software contributed to Berkeley by
6  * The Mach Operating System project at Carnegie-Mellon University.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  * 4. Neither the name of the University nor the names of its contributors
17  *    may be used to endorse or promote products derived from this software
18  *    without specific prior written permission.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30  * SUCH DAMAGE.
31  *
32  *	from: @(#)vm_map.c	8.3 (Berkeley) 1/12/94
33  *
34  *
35  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
36  * All rights reserved.
37  *
38  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
39  *
40  * Permission to use, copy, modify and distribute this software and
41  * its documentation is hereby granted, provided that both the copyright
42  * notice and this permission notice appear in all copies of the
43  * software, derivative works or modified versions, and any portions
44  * thereof, and that both notices appear in supporting documentation.
45  *
46  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
47  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
48  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
49  *
50  * Carnegie Mellon requests users of this software to return to
51  *
52  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
53  *  School of Computer Science
54  *  Carnegie Mellon University
55  *  Pittsburgh PA 15213-3890
56  *
57  * any improvements or extensions that they make and grant Carnegie the
58  * rights to redistribute these changes.
59  */
60 
61 /*
62  *	Virtual memory mapping module.
63  */
64 
65 #include <sys/cdefs.h>
66 __FBSDID("$FreeBSD$");
67 
68 #include <sys/param.h>
69 #include <sys/systm.h>
70 #include <sys/ktr.h>
71 #include <sys/lock.h>
72 #include <sys/mutex.h>
73 #include <sys/proc.h>
74 #include <sys/vmmeter.h>
75 #include <sys/mman.h>
76 #include <sys/vnode.h>
77 #include <sys/resourcevar.h>
78 #include <sys/file.h>
79 #include <sys/sysent.h>
80 #include <sys/shm.h>
81 
82 #include <vm/vm.h>
83 #include <vm/vm_param.h>
84 #include <vm/pmap.h>
85 #include <vm/vm_map.h>
86 #include <vm/vm_page.h>
87 #include <vm/vm_object.h>
88 #include <vm/vm_pager.h>
89 #include <vm/vm_kern.h>
90 #include <vm/vm_extern.h>
91 #include <vm/swap_pager.h>
92 #include <vm/uma.h>
93 
94 /*
95  *	Virtual memory maps provide for the mapping, protection,
96  *	and sharing of virtual memory objects.  In addition,
97  *	this module provides for an efficient virtual copy of
98  *	memory from one map to another.
99  *
100  *	Synchronization is required prior to most operations.
101  *
102  *	Maps consist of an ordered doubly-linked list of simple
103  *	entries; a self-adjusting binary search tree of these
104  *	entries is used to speed up lookups.
105  *
106  *	Since portions of maps are specified by start/end addresses,
107  *	which may not align with existing map entries, all
108  *	routines merely "clip" entries to these start/end values.
109  *	[That is, an entry is split into two, bordering at a
110  *	start or end value.]  Note that these clippings may not
111  *	always be necessary (as the two resulting entries are then
112  *	not changed); however, the clipping is done for convenience.
113  *
114  *	As mentioned above, virtual copy operations are performed
115  *	by copying VM object references from one map to
116  *	another, and then marking both regions as copy-on-write.
117  */
118 
119 /*
120  *	vm_map_startup:
121  *
122  *	Initialize the vm_map module.  Must be called before
123  *	any other vm_map routines.
124  *
125  *	Map and entry structures are allocated from the general
126  *	purpose memory pool with some exceptions:
127  *
128  *	- The kernel map and kmem submap are allocated statically.
129  *	- Kernel map entries are allocated out of a static pool.
130  *
131  *	These restrictions are necessary since malloc() uses the
132  *	maps and requires map entries.
133  */
134 
135 static struct mtx map_sleep_mtx;
136 static uma_zone_t mapentzone;
137 static uma_zone_t kmapentzone;
138 static uma_zone_t mapzone;
139 static uma_zone_t vmspace_zone;
140 static struct vm_object kmapentobj;
141 static int vmspace_zinit(void *mem, int size, int flags);
142 static void vmspace_zfini(void *mem, int size);
143 static int vm_map_zinit(void *mem, int ize, int flags);
144 static void vm_map_zfini(void *mem, int size);
145 static void _vm_map_init(vm_map_t map, vm_offset_t min, vm_offset_t max);
146 static void vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry);
147 #ifdef INVARIANTS
148 static void vm_map_zdtor(void *mem, int size, void *arg);
149 static void vmspace_zdtor(void *mem, int size, void *arg);
150 #endif
151 
152 /*
153  * PROC_VMSPACE_{UN,}LOCK() can be a noop as long as vmspaces are type
154  * stable.
155  */
156 #define PROC_VMSPACE_LOCK(p) do { } while (0)
157 #define PROC_VMSPACE_UNLOCK(p) do { } while (0)
158 
159 /*
160  *	VM_MAP_RANGE_CHECK:	[ internal use only ]
161  *
162  *	Asserts that the starting and ending region
163  *	addresses fall within the valid range of the map.
164  */
165 #define	VM_MAP_RANGE_CHECK(map, start, end)		\
166 		{					\
167 		if (start < vm_map_min(map))		\
168 			start = vm_map_min(map);	\
169 		if (end > vm_map_max(map))		\
170 			end = vm_map_max(map);		\
171 		if (start > end)			\
172 			start = end;			\
173 		}
174 
175 void
176 vm_map_startup(void)
177 {
178 	mtx_init(&map_sleep_mtx, "vm map sleep mutex", NULL, MTX_DEF);
179 	mapzone = uma_zcreate("MAP", sizeof(struct vm_map), NULL,
180 #ifdef INVARIANTS
181 	    vm_map_zdtor,
182 #else
183 	    NULL,
184 #endif
185 	    vm_map_zinit, vm_map_zfini, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
186 	uma_prealloc(mapzone, MAX_KMAP);
187 	kmapentzone = uma_zcreate("KMAP ENTRY", sizeof(struct vm_map_entry),
188 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
189 	    UMA_ZONE_MTXCLASS | UMA_ZONE_VM);
190 	uma_prealloc(kmapentzone, MAX_KMAPENT);
191 	mapentzone = uma_zcreate("MAP ENTRY", sizeof(struct vm_map_entry),
192 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
193 }
194 
195 static void
196 vmspace_zfini(void *mem, int size)
197 {
198 	struct vmspace *vm;
199 
200 	vm = (struct vmspace *)mem;
201 	vm_map_zfini(&vm->vm_map, sizeof(vm->vm_map));
202 }
203 
204 static int
205 vmspace_zinit(void *mem, int size, int flags)
206 {
207 	struct vmspace *vm;
208 
209 	vm = (struct vmspace *)mem;
210 
211 	vm->vm_map.pmap = NULL;
212 	(void)vm_map_zinit(&vm->vm_map, sizeof(vm->vm_map), flags);
213 	return (0);
214 }
215 
216 static void
217 vm_map_zfini(void *mem, int size)
218 {
219 	vm_map_t map;
220 
221 	map = (vm_map_t)mem;
222 	mtx_destroy(&map->system_mtx);
223 	sx_destroy(&map->lock);
224 }
225 
226 static int
227 vm_map_zinit(void *mem, int size, int flags)
228 {
229 	vm_map_t map;
230 
231 	map = (vm_map_t)mem;
232 	map->nentries = 0;
233 	map->size = 0;
234 	mtx_init(&map->system_mtx, "system map", NULL, MTX_DEF | MTX_DUPOK);
235 	sx_init(&map->lock, "user map");
236 	return (0);
237 }
238 
239 #ifdef INVARIANTS
240 static void
241 vmspace_zdtor(void *mem, int size, void *arg)
242 {
243 	struct vmspace *vm;
244 
245 	vm = (struct vmspace *)mem;
246 
247 	vm_map_zdtor(&vm->vm_map, sizeof(vm->vm_map), arg);
248 }
249 static void
250 vm_map_zdtor(void *mem, int size, void *arg)
251 {
252 	vm_map_t map;
253 
254 	map = (vm_map_t)mem;
255 	KASSERT(map->nentries == 0,
256 	    ("map %p nentries == %d on free.",
257 	    map, map->nentries));
258 	KASSERT(map->size == 0,
259 	    ("map %p size == %lu on free.",
260 	    map, (unsigned long)map->size));
261 }
262 #endif	/* INVARIANTS */
263 
264 /*
265  * Allocate a vmspace structure, including a vm_map and pmap,
266  * and initialize those structures.  The refcnt is set to 1.
267  */
268 struct vmspace *
269 vmspace_alloc(min, max)
270 	vm_offset_t min, max;
271 {
272 	struct vmspace *vm;
273 
274 	vm = uma_zalloc(vmspace_zone, M_WAITOK);
275 	if (vm->vm_map.pmap == NULL && !pmap_pinit(vmspace_pmap(vm))) {
276 		uma_zfree(vmspace_zone, vm);
277 		return (NULL);
278 	}
279 	CTR1(KTR_VM, "vmspace_alloc: %p", vm);
280 	_vm_map_init(&vm->vm_map, min, max);
281 	vm->vm_map.pmap = vmspace_pmap(vm);		/* XXX */
282 	vm->vm_refcnt = 1;
283 	vm->vm_shm = NULL;
284 	vm->vm_swrss = 0;
285 	vm->vm_tsize = 0;
286 	vm->vm_dsize = 0;
287 	vm->vm_ssize = 0;
288 	vm->vm_taddr = 0;
289 	vm->vm_daddr = 0;
290 	vm->vm_maxsaddr = 0;
291 	return (vm);
292 }
293 
294 void
295 vm_init2(void)
296 {
297 	uma_zone_set_obj(kmapentzone, &kmapentobj, lmin(cnt.v_page_count,
298 	    (VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / PAGE_SIZE) / 8 +
299 	     maxproc * 2 + maxfiles);
300 	vmspace_zone = uma_zcreate("VMSPACE", sizeof(struct vmspace), NULL,
301 #ifdef INVARIANTS
302 	    vmspace_zdtor,
303 #else
304 	    NULL,
305 #endif
306 	    vmspace_zinit, vmspace_zfini, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
307 }
308 
309 static inline void
310 vmspace_dofree(struct vmspace *vm)
311 {
312 	CTR1(KTR_VM, "vmspace_free: %p", vm);
313 
314 	/*
315 	 * Make sure any SysV shm is freed, it might not have been in
316 	 * exit1().
317 	 */
318 	shmexit(vm);
319 
320 	/*
321 	 * Lock the map, to wait out all other references to it.
322 	 * Delete all of the mappings and pages they hold, then call
323 	 * the pmap module to reclaim anything left.
324 	 */
325 	(void)vm_map_remove(&vm->vm_map, vm->vm_map.min_offset,
326 	    vm->vm_map.max_offset);
327 
328 	/*
329 	 * XXX Comment out the pmap_release call for now. The
330 	 * vmspace_zone is marked as UMA_ZONE_NOFREE, and bugs cause
331 	 * pmap.resident_count to be != 0 on exit sometimes.
332 	 */
333 /* 	pmap_release(vmspace_pmap(vm)); */
334 	uma_zfree(vmspace_zone, vm);
335 }
336 
337 void
338 vmspace_free(struct vmspace *vm)
339 {
340 	int refcnt;
341 
342 	if (vm->vm_refcnt == 0)
343 		panic("vmspace_free: attempt to free already freed vmspace");
344 
345 	do
346 		refcnt = vm->vm_refcnt;
347 	while (!atomic_cmpset_int(&vm->vm_refcnt, refcnt, refcnt - 1));
348 	if (refcnt == 1)
349 		vmspace_dofree(vm);
350 }
351 
352 void
353 vmspace_exitfree(struct proc *p)
354 {
355 	struct vmspace *vm;
356 
357 	PROC_VMSPACE_LOCK(p);
358 	vm = p->p_vmspace;
359 	p->p_vmspace = NULL;
360 	PROC_VMSPACE_UNLOCK(p);
361 	KASSERT(vm == &vmspace0, ("vmspace_exitfree: wrong vmspace"));
362 	vmspace_free(vm);
363 }
364 
365 void
366 vmspace_exit(struct thread *td)
367 {
368 	int refcnt;
369 	struct vmspace *vm;
370 	struct proc *p;
371 
372 	/*
373 	 * Release user portion of address space.
374 	 * This releases references to vnodes,
375 	 * which could cause I/O if the file has been unlinked.
376 	 * Need to do this early enough that we can still sleep.
377 	 *
378 	 * The last exiting process to reach this point releases as
379 	 * much of the environment as it can. vmspace_dofree() is the
380 	 * slower fallback in case another process had a temporary
381 	 * reference to the vmspace.
382 	 */
383 
384 	p = td->td_proc;
385 	vm = p->p_vmspace;
386 	atomic_add_int(&vmspace0.vm_refcnt, 1);
387 	do {
388 		refcnt = vm->vm_refcnt;
389 		if (refcnt > 1 && p->p_vmspace != &vmspace0) {
390 			/* Switch now since other proc might free vmspace */
391 			PROC_VMSPACE_LOCK(p);
392 			p->p_vmspace = &vmspace0;
393 			PROC_VMSPACE_UNLOCK(p);
394 			pmap_activate(td);
395 		}
396 	} while (!atomic_cmpset_int(&vm->vm_refcnt, refcnt, refcnt - 1));
397 	if (refcnt == 1) {
398 		if (p->p_vmspace != vm) {
399 			/* vmspace not yet freed, switch back */
400 			PROC_VMSPACE_LOCK(p);
401 			p->p_vmspace = vm;
402 			PROC_VMSPACE_UNLOCK(p);
403 			pmap_activate(td);
404 		}
405 		pmap_remove_pages(vmspace_pmap(vm));
406 		/* Switch now since this proc will free vmspace */
407 		PROC_VMSPACE_LOCK(p);
408 		p->p_vmspace = &vmspace0;
409 		PROC_VMSPACE_UNLOCK(p);
410 		pmap_activate(td);
411 		vmspace_dofree(vm);
412 	}
413 }
414 
415 /* Acquire reference to vmspace owned by another process. */
416 
417 struct vmspace *
418 vmspace_acquire_ref(struct proc *p)
419 {
420 	struct vmspace *vm;
421 	int refcnt;
422 
423 	PROC_VMSPACE_LOCK(p);
424 	vm = p->p_vmspace;
425 	if (vm == NULL) {
426 		PROC_VMSPACE_UNLOCK(p);
427 		return (NULL);
428 	}
429 	do {
430 		refcnt = vm->vm_refcnt;
431 		if (refcnt <= 0) { 	/* Avoid 0->1 transition */
432 			PROC_VMSPACE_UNLOCK(p);
433 			return (NULL);
434 		}
435 	} while (!atomic_cmpset_int(&vm->vm_refcnt, refcnt, refcnt + 1));
436 	if (vm != p->p_vmspace) {
437 		PROC_VMSPACE_UNLOCK(p);
438 		vmspace_free(vm);
439 		return (NULL);
440 	}
441 	PROC_VMSPACE_UNLOCK(p);
442 	return (vm);
443 }
444 
445 void
446 _vm_map_lock(vm_map_t map, const char *file, int line)
447 {
448 
449 	if (map->system_map)
450 		_mtx_lock_flags(&map->system_mtx, 0, file, line);
451 	else
452 		(void)_sx_xlock(&map->lock, 0, file, line);
453 	map->timestamp++;
454 }
455 
456 void
457 _vm_map_unlock(vm_map_t map, const char *file, int line)
458 {
459 	vm_map_entry_t free_entry, entry;
460 	vm_object_t object;
461 
462 	free_entry = map->deferred_freelist;
463 	map->deferred_freelist = NULL;
464 
465 	if (map->system_map)
466 		_mtx_unlock_flags(&map->system_mtx, 0, file, line);
467 	else
468 		_sx_xunlock(&map->lock, file, line);
469 
470 	while (free_entry != NULL) {
471 		entry = free_entry;
472 		free_entry = free_entry->next;
473 
474 		if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
475 			object = entry->object.vm_object;
476 			vm_object_deallocate(object);
477 		}
478 
479 		vm_map_entry_dispose(map, entry);
480 	}
481 }
482 
483 void
484 _vm_map_lock_read(vm_map_t map, const char *file, int line)
485 {
486 
487 	if (map->system_map)
488 		_mtx_lock_flags(&map->system_mtx, 0, file, line);
489 	else
490 		(void)_sx_slock(&map->lock, 0, file, line);
491 }
492 
493 void
494 _vm_map_unlock_read(vm_map_t map, const char *file, int line)
495 {
496 
497 	if (map->system_map)
498 		_mtx_unlock_flags(&map->system_mtx, 0, file, line);
499 	else
500 		_sx_sunlock(&map->lock, file, line);
501 }
502 
503 int
504 _vm_map_trylock(vm_map_t map, const char *file, int line)
505 {
506 	int error;
507 
508 	error = map->system_map ?
509 	    !_mtx_trylock(&map->system_mtx, 0, file, line) :
510 	    !_sx_try_xlock(&map->lock, file, line);
511 	if (error == 0)
512 		map->timestamp++;
513 	return (error == 0);
514 }
515 
516 int
517 _vm_map_trylock_read(vm_map_t map, const char *file, int line)
518 {
519 	int error;
520 
521 	error = map->system_map ?
522 	    !_mtx_trylock(&map->system_mtx, 0, file, line) :
523 	    !_sx_try_slock(&map->lock, file, line);
524 	return (error == 0);
525 }
526 
527 /*
528  *	_vm_map_lock_upgrade:	[ internal use only ]
529  *
530  *	Tries to upgrade a read (shared) lock on the specified map to a write
531  *	(exclusive) lock.  Returns the value "0" if the upgrade succeeds and a
532  *	non-zero value if the upgrade fails.  If the upgrade fails, the map is
533  *	returned without a read or write lock held.
534  *
535  *	Requires that the map be read locked.
536  */
537 int
538 _vm_map_lock_upgrade(vm_map_t map, const char *file, int line)
539 {
540 	unsigned int last_timestamp;
541 
542 	if (map->system_map) {
543 #ifdef INVARIANTS
544 		_mtx_assert(&map->system_mtx, MA_OWNED, file, line);
545 #endif
546 	} else {
547 		if (!_sx_try_upgrade(&map->lock, file, line)) {
548 			last_timestamp = map->timestamp;
549 			_sx_sunlock(&map->lock, file, line);
550 			/*
551 			 * If the map's timestamp does not change while the
552 			 * map is unlocked, then the upgrade succeeds.
553 			 */
554 			(void)_sx_xlock(&map->lock, 0, file, line);
555 			if (last_timestamp != map->timestamp) {
556 				_sx_xunlock(&map->lock, file, line);
557 				return (1);
558 			}
559 		}
560 	}
561 	map->timestamp++;
562 	return (0);
563 }
564 
565 void
566 _vm_map_lock_downgrade(vm_map_t map, const char *file, int line)
567 {
568 
569 	if (map->system_map) {
570 #ifdef INVARIANTS
571 		_mtx_assert(&map->system_mtx, MA_OWNED, file, line);
572 #endif
573 	} else
574 		_sx_downgrade(&map->lock, file, line);
575 }
576 
577 /*
578  *	vm_map_locked:
579  *
580  *	Returns a non-zero value if the caller holds a write (exclusive) lock
581  *	on the specified map and the value "0" otherwise.
582  */
583 int
584 vm_map_locked(vm_map_t map)
585 {
586 
587 	if (map->system_map)
588 		return (mtx_owned(&map->system_mtx));
589 	else
590 		return (sx_xlocked(&map->lock));
591 }
592 
593 #ifdef INVARIANTS
594 static void
595 _vm_map_assert_locked(vm_map_t map, const char *file, int line)
596 {
597 
598 	if (map->system_map)
599 		_mtx_assert(&map->system_mtx, MA_OWNED, file, line);
600 	else
601 		_sx_assert(&map->lock, SA_XLOCKED, file, line);
602 }
603 
604 #if 0
605 static void
606 _vm_map_assert_locked_read(vm_map_t map, const char *file, int line)
607 {
608 
609 	if (map->system_map)
610 		_mtx_assert(&map->system_mtx, MA_OWNED, file, line);
611 	else
612 		_sx_assert(&map->lock, SA_SLOCKED, file, line);
613 }
614 #endif
615 
616 #define	VM_MAP_ASSERT_LOCKED(map) \
617     _vm_map_assert_locked(map, LOCK_FILE, LOCK_LINE)
618 #define	VM_MAP_ASSERT_LOCKED_READ(map) \
619     _vm_map_assert_locked_read(map, LOCK_FILE, LOCK_LINE)
620 #else
621 #define	VM_MAP_ASSERT_LOCKED(map)
622 #define	VM_MAP_ASSERT_LOCKED_READ(map)
623 #endif
624 
625 /*
626  *	vm_map_unlock_and_wait:
627  */
628 int
629 vm_map_unlock_and_wait(vm_map_t map, int timo)
630 {
631 
632 	mtx_lock(&map_sleep_mtx);
633 	vm_map_unlock(map);
634 	return (msleep(&map->root, &map_sleep_mtx, PDROP | PVM, "vmmaps", timo));
635 }
636 
637 /*
638  *	vm_map_wakeup:
639  */
640 void
641 vm_map_wakeup(vm_map_t map)
642 {
643 
644 	/*
645 	 * Acquire and release map_sleep_mtx to prevent a wakeup()
646 	 * from being performed (and lost) between the vm_map_unlock()
647 	 * and the msleep() in vm_map_unlock_and_wait().
648 	 */
649 	mtx_lock(&map_sleep_mtx);
650 	mtx_unlock(&map_sleep_mtx);
651 	wakeup(&map->root);
652 }
653 
654 long
655 vmspace_resident_count(struct vmspace *vmspace)
656 {
657 	return pmap_resident_count(vmspace_pmap(vmspace));
658 }
659 
660 long
661 vmspace_wired_count(struct vmspace *vmspace)
662 {
663 	return pmap_wired_count(vmspace_pmap(vmspace));
664 }
665 
666 /*
667  *	vm_map_create:
668  *
669  *	Creates and returns a new empty VM map with
670  *	the given physical map structure, and having
671  *	the given lower and upper address bounds.
672  */
673 vm_map_t
674 vm_map_create(pmap_t pmap, vm_offset_t min, vm_offset_t max)
675 {
676 	vm_map_t result;
677 
678 	result = uma_zalloc(mapzone, M_WAITOK);
679 	CTR1(KTR_VM, "vm_map_create: %p", result);
680 	_vm_map_init(result, min, max);
681 	result->pmap = pmap;
682 	return (result);
683 }
684 
685 /*
686  * Initialize an existing vm_map structure
687  * such as that in the vmspace structure.
688  * The pmap is set elsewhere.
689  */
690 static void
691 _vm_map_init(vm_map_t map, vm_offset_t min, vm_offset_t max)
692 {
693 
694 	map->header.next = map->header.prev = &map->header;
695 	map->needs_wakeup = FALSE;
696 	map->system_map = 0;
697 	map->min_offset = min;
698 	map->max_offset = max;
699 	map->flags = 0;
700 	map->root = NULL;
701 	map->timestamp = 0;
702 	map->deferred_freelist = NULL;
703 }
704 
705 void
706 vm_map_init(vm_map_t map, vm_offset_t min, vm_offset_t max)
707 {
708 	_vm_map_init(map, min, max);
709 	mtx_init(&map->system_mtx, "system map", NULL, MTX_DEF | MTX_DUPOK);
710 	sx_init(&map->lock, "user map");
711 }
712 
713 /*
714  *	vm_map_entry_dispose:	[ internal use only ]
715  *
716  *	Inverse of vm_map_entry_create.
717  */
718 static void
719 vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry)
720 {
721 	uma_zfree(map->system_map ? kmapentzone : mapentzone, entry);
722 }
723 
724 /*
725  *	vm_map_entry_create:	[ internal use only ]
726  *
727  *	Allocates a VM map entry for insertion.
728  *	No entry fields are filled in.
729  */
730 static vm_map_entry_t
731 vm_map_entry_create(vm_map_t map)
732 {
733 	vm_map_entry_t new_entry;
734 
735 	if (map->system_map)
736 		new_entry = uma_zalloc(kmapentzone, M_NOWAIT);
737 	else
738 		new_entry = uma_zalloc(mapentzone, M_WAITOK);
739 	if (new_entry == NULL)
740 		panic("vm_map_entry_create: kernel resources exhausted");
741 	return (new_entry);
742 }
743 
744 /*
745  *	vm_map_entry_set_behavior:
746  *
747  *	Set the expected access behavior, either normal, random, or
748  *	sequential.
749  */
750 static inline void
751 vm_map_entry_set_behavior(vm_map_entry_t entry, u_char behavior)
752 {
753 	entry->eflags = (entry->eflags & ~MAP_ENTRY_BEHAV_MASK) |
754 	    (behavior & MAP_ENTRY_BEHAV_MASK);
755 }
756 
757 /*
758  *	vm_map_entry_set_max_free:
759  *
760  *	Set the max_free field in a vm_map_entry.
761  */
762 static inline void
763 vm_map_entry_set_max_free(vm_map_entry_t entry)
764 {
765 
766 	entry->max_free = entry->adj_free;
767 	if (entry->left != NULL && entry->left->max_free > entry->max_free)
768 		entry->max_free = entry->left->max_free;
769 	if (entry->right != NULL && entry->right->max_free > entry->max_free)
770 		entry->max_free = entry->right->max_free;
771 }
772 
773 /*
774  *	vm_map_entry_splay:
775  *
776  *	The Sleator and Tarjan top-down splay algorithm with the
777  *	following variation.  Max_free must be computed bottom-up, so
778  *	on the downward pass, maintain the left and right spines in
779  *	reverse order.  Then, make a second pass up each side to fix
780  *	the pointers and compute max_free.  The time bound is O(log n)
781  *	amortized.
782  *
783  *	The new root is the vm_map_entry containing "addr", or else an
784  *	adjacent entry (lower or higher) if addr is not in the tree.
785  *
786  *	The map must be locked, and leaves it so.
787  *
788  *	Returns: the new root.
789  */
790 static vm_map_entry_t
791 vm_map_entry_splay(vm_offset_t addr, vm_map_entry_t root)
792 {
793 	vm_map_entry_t llist, rlist;
794 	vm_map_entry_t ltree, rtree;
795 	vm_map_entry_t y;
796 
797 	/* Special case of empty tree. */
798 	if (root == NULL)
799 		return (root);
800 
801 	/*
802 	 * Pass One: Splay down the tree until we find addr or a NULL
803 	 * pointer where addr would go.  llist and rlist are the two
804 	 * sides in reverse order (bottom-up), with llist linked by
805 	 * the right pointer and rlist linked by the left pointer in
806 	 * the vm_map_entry.  Wait until Pass Two to set max_free on
807 	 * the two spines.
808 	 */
809 	llist = NULL;
810 	rlist = NULL;
811 	for (;;) {
812 		/* root is never NULL in here. */
813 		if (addr < root->start) {
814 			y = root->left;
815 			if (y == NULL)
816 				break;
817 			if (addr < y->start && y->left != NULL) {
818 				/* Rotate right and put y on rlist. */
819 				root->left = y->right;
820 				y->right = root;
821 				vm_map_entry_set_max_free(root);
822 				root = y->left;
823 				y->left = rlist;
824 				rlist = y;
825 			} else {
826 				/* Put root on rlist. */
827 				root->left = rlist;
828 				rlist = root;
829 				root = y;
830 			}
831 		} else if (addr >= root->end) {
832 			y = root->right;
833 			if (y == NULL)
834 				break;
835 			if (addr >= y->end && y->right != NULL) {
836 				/* Rotate left and put y on llist. */
837 				root->right = y->left;
838 				y->left = root;
839 				vm_map_entry_set_max_free(root);
840 				root = y->right;
841 				y->right = llist;
842 				llist = y;
843 			} else {
844 				/* Put root on llist. */
845 				root->right = llist;
846 				llist = root;
847 				root = y;
848 			}
849 		} else
850 			break;
851 	}
852 
853 	/*
854 	 * Pass Two: Walk back up the two spines, flip the pointers
855 	 * and set max_free.  The subtrees of the root go at the
856 	 * bottom of llist and rlist.
857 	 */
858 	ltree = root->left;
859 	while (llist != NULL) {
860 		y = llist->right;
861 		llist->right = ltree;
862 		vm_map_entry_set_max_free(llist);
863 		ltree = llist;
864 		llist = y;
865 	}
866 	rtree = root->right;
867 	while (rlist != NULL) {
868 		y = rlist->left;
869 		rlist->left = rtree;
870 		vm_map_entry_set_max_free(rlist);
871 		rtree = rlist;
872 		rlist = y;
873 	}
874 
875 	/*
876 	 * Final assembly: add ltree and rtree as subtrees of root.
877 	 */
878 	root->left = ltree;
879 	root->right = rtree;
880 	vm_map_entry_set_max_free(root);
881 
882 	return (root);
883 }
884 
885 /*
886  *	vm_map_entry_{un,}link:
887  *
888  *	Insert/remove entries from maps.
889  */
890 static void
891 vm_map_entry_link(vm_map_t map,
892 		  vm_map_entry_t after_where,
893 		  vm_map_entry_t entry)
894 {
895 
896 	CTR4(KTR_VM,
897 	    "vm_map_entry_link: map %p, nentries %d, entry %p, after %p", map,
898 	    map->nentries, entry, after_where);
899 	VM_MAP_ASSERT_LOCKED(map);
900 	map->nentries++;
901 	entry->prev = after_where;
902 	entry->next = after_where->next;
903 	entry->next->prev = entry;
904 	after_where->next = entry;
905 
906 	if (after_where != &map->header) {
907 		if (after_where != map->root)
908 			vm_map_entry_splay(after_where->start, map->root);
909 		entry->right = after_where->right;
910 		entry->left = after_where;
911 		after_where->right = NULL;
912 		after_where->adj_free = entry->start - after_where->end;
913 		vm_map_entry_set_max_free(after_where);
914 	} else {
915 		entry->right = map->root;
916 		entry->left = NULL;
917 	}
918 	entry->adj_free = (entry->next == &map->header ? map->max_offset :
919 	    entry->next->start) - entry->end;
920 	vm_map_entry_set_max_free(entry);
921 	map->root = entry;
922 }
923 
924 static void
925 vm_map_entry_unlink(vm_map_t map,
926 		    vm_map_entry_t entry)
927 {
928 	vm_map_entry_t next, prev, root;
929 
930 	VM_MAP_ASSERT_LOCKED(map);
931 	if (entry != map->root)
932 		vm_map_entry_splay(entry->start, map->root);
933 	if (entry->left == NULL)
934 		root = entry->right;
935 	else {
936 		root = vm_map_entry_splay(entry->start, entry->left);
937 		root->right = entry->right;
938 		root->adj_free = (entry->next == &map->header ? map->max_offset :
939 		    entry->next->start) - root->end;
940 		vm_map_entry_set_max_free(root);
941 	}
942 	map->root = root;
943 
944 	prev = entry->prev;
945 	next = entry->next;
946 	next->prev = prev;
947 	prev->next = next;
948 	map->nentries--;
949 	CTR3(KTR_VM, "vm_map_entry_unlink: map %p, nentries %d, entry %p", map,
950 	    map->nentries, entry);
951 }
952 
953 /*
954  *	vm_map_entry_resize_free:
955  *
956  *	Recompute the amount of free space following a vm_map_entry
957  *	and propagate that value up the tree.  Call this function after
958  *	resizing a map entry in-place, that is, without a call to
959  *	vm_map_entry_link() or _unlink().
960  *
961  *	The map must be locked, and leaves it so.
962  */
963 static void
964 vm_map_entry_resize_free(vm_map_t map, vm_map_entry_t entry)
965 {
966 
967 	/*
968 	 * Using splay trees without parent pointers, propagating
969 	 * max_free up the tree is done by moving the entry to the
970 	 * root and making the change there.
971 	 */
972 	if (entry != map->root)
973 		map->root = vm_map_entry_splay(entry->start, map->root);
974 
975 	entry->adj_free = (entry->next == &map->header ? map->max_offset :
976 	    entry->next->start) - entry->end;
977 	vm_map_entry_set_max_free(entry);
978 }
979 
980 /*
981  *	vm_map_lookup_entry:	[ internal use only ]
982  *
983  *	Finds the map entry containing (or
984  *	immediately preceding) the specified address
985  *	in the given map; the entry is returned
986  *	in the "entry" parameter.  The boolean
987  *	result indicates whether the address is
988  *	actually contained in the map.
989  */
990 boolean_t
991 vm_map_lookup_entry(
992 	vm_map_t map,
993 	vm_offset_t address,
994 	vm_map_entry_t *entry)	/* OUT */
995 {
996 	vm_map_entry_t cur;
997 	boolean_t locked;
998 
999 	/*
1000 	 * If the map is empty, then the map entry immediately preceding
1001 	 * "address" is the map's header.
1002 	 */
1003 	cur = map->root;
1004 	if (cur == NULL)
1005 		*entry = &map->header;
1006 	else if (address >= cur->start && cur->end > address) {
1007 		*entry = cur;
1008 		return (TRUE);
1009 	} else if ((locked = vm_map_locked(map)) ||
1010 	    sx_try_upgrade(&map->lock)) {
1011 		/*
1012 		 * Splay requires a write lock on the map.  However, it only
1013 		 * restructures the binary search tree; it does not otherwise
1014 		 * change the map.  Thus, the map's timestamp need not change
1015 		 * on a temporary upgrade.
1016 		 */
1017 		map->root = cur = vm_map_entry_splay(address, cur);
1018 		if (!locked)
1019 			sx_downgrade(&map->lock);
1020 
1021 		/*
1022 		 * If "address" is contained within a map entry, the new root
1023 		 * is that map entry.  Otherwise, the new root is a map entry
1024 		 * immediately before or after "address".
1025 		 */
1026 		if (address >= cur->start) {
1027 			*entry = cur;
1028 			if (cur->end > address)
1029 				return (TRUE);
1030 		} else
1031 			*entry = cur->prev;
1032 	} else
1033 		/*
1034 		 * Since the map is only locked for read access, perform a
1035 		 * standard binary search tree lookup for "address".
1036 		 */
1037 		for (;;) {
1038 			if (address < cur->start) {
1039 				if (cur->left == NULL) {
1040 					*entry = cur->prev;
1041 					break;
1042 				}
1043 				cur = cur->left;
1044 			} else if (cur->end > address) {
1045 				*entry = cur;
1046 				return (TRUE);
1047 			} else {
1048 				if (cur->right == NULL) {
1049 					*entry = cur;
1050 					break;
1051 				}
1052 				cur = cur->right;
1053 			}
1054 		}
1055 	return (FALSE);
1056 }
1057 
1058 /*
1059  *	vm_map_insert:
1060  *
1061  *	Inserts the given whole VM object into the target
1062  *	map at the specified address range.  The object's
1063  *	size should match that of the address range.
1064  *
1065  *	Requires that the map be locked, and leaves it so.
1066  *
1067  *	If object is non-NULL, ref count must be bumped by caller
1068  *	prior to making call to account for the new entry.
1069  */
1070 int
1071 vm_map_insert(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
1072 	      vm_offset_t start, vm_offset_t end, vm_prot_t prot, vm_prot_t max,
1073 	      int cow)
1074 {
1075 	vm_map_entry_t new_entry;
1076 	vm_map_entry_t prev_entry;
1077 	vm_map_entry_t temp_entry;
1078 	vm_eflags_t protoeflags;
1079 
1080 	VM_MAP_ASSERT_LOCKED(map);
1081 
1082 	/*
1083 	 * Check that the start and end points are not bogus.
1084 	 */
1085 	if ((start < map->min_offset) || (end > map->max_offset) ||
1086 	    (start >= end))
1087 		return (KERN_INVALID_ADDRESS);
1088 
1089 	/*
1090 	 * Find the entry prior to the proposed starting address; if it's part
1091 	 * of an existing entry, this range is bogus.
1092 	 */
1093 	if (vm_map_lookup_entry(map, start, &temp_entry))
1094 		return (KERN_NO_SPACE);
1095 
1096 	prev_entry = temp_entry;
1097 
1098 	/*
1099 	 * Assert that the next entry doesn't overlap the end point.
1100 	 */
1101 	if ((prev_entry->next != &map->header) &&
1102 	    (prev_entry->next->start < end))
1103 		return (KERN_NO_SPACE);
1104 
1105 	protoeflags = 0;
1106 
1107 	if (cow & MAP_COPY_ON_WRITE)
1108 		protoeflags |= MAP_ENTRY_COW|MAP_ENTRY_NEEDS_COPY;
1109 
1110 	if (cow & MAP_NOFAULT) {
1111 		protoeflags |= MAP_ENTRY_NOFAULT;
1112 
1113 		KASSERT(object == NULL,
1114 			("vm_map_insert: paradoxical MAP_NOFAULT request"));
1115 	}
1116 	if (cow & MAP_DISABLE_SYNCER)
1117 		protoeflags |= MAP_ENTRY_NOSYNC;
1118 	if (cow & MAP_DISABLE_COREDUMP)
1119 		protoeflags |= MAP_ENTRY_NOCOREDUMP;
1120 
1121 	if (object != NULL) {
1122 		/*
1123 		 * OBJ_ONEMAPPING must be cleared unless this mapping
1124 		 * is trivially proven to be the only mapping for any
1125 		 * of the object's pages.  (Object granularity
1126 		 * reference counting is insufficient to recognize
1127 		 * aliases with precision.)
1128 		 */
1129 		VM_OBJECT_LOCK(object);
1130 		if (object->ref_count > 1 || object->shadow_count != 0)
1131 			vm_object_clear_flag(object, OBJ_ONEMAPPING);
1132 		VM_OBJECT_UNLOCK(object);
1133 	}
1134 	else if ((prev_entry != &map->header) &&
1135 		 (prev_entry->eflags == protoeflags) &&
1136 		 (prev_entry->end == start) &&
1137 		 (prev_entry->wired_count == 0) &&
1138 		 ((prev_entry->object.vm_object == NULL) ||
1139 		  vm_object_coalesce(prev_entry->object.vm_object,
1140 				     prev_entry->offset,
1141 				     (vm_size_t)(prev_entry->end - prev_entry->start),
1142 				     (vm_size_t)(end - prev_entry->end)))) {
1143 		/*
1144 		 * We were able to extend the object.  Determine if we
1145 		 * can extend the previous map entry to include the
1146 		 * new range as well.
1147 		 */
1148 		if ((prev_entry->inheritance == VM_INHERIT_DEFAULT) &&
1149 		    (prev_entry->protection == prot) &&
1150 		    (prev_entry->max_protection == max)) {
1151 			map->size += (end - prev_entry->end);
1152 			prev_entry->end = end;
1153 			vm_map_entry_resize_free(map, prev_entry);
1154 			vm_map_simplify_entry(map, prev_entry);
1155 			return (KERN_SUCCESS);
1156 		}
1157 
1158 		/*
1159 		 * If we can extend the object but cannot extend the
1160 		 * map entry, we have to create a new map entry.  We
1161 		 * must bump the ref count on the extended object to
1162 		 * account for it.  object may be NULL.
1163 		 */
1164 		object = prev_entry->object.vm_object;
1165 		offset = prev_entry->offset +
1166 			(prev_entry->end - prev_entry->start);
1167 		vm_object_reference(object);
1168 	}
1169 
1170 	/*
1171 	 * NOTE: if conditionals fail, object can be NULL here.  This occurs
1172 	 * in things like the buffer map where we manage kva but do not manage
1173 	 * backing objects.
1174 	 */
1175 
1176 	/*
1177 	 * Create a new entry
1178 	 */
1179 	new_entry = vm_map_entry_create(map);
1180 	new_entry->start = start;
1181 	new_entry->end = end;
1182 
1183 	new_entry->eflags = protoeflags;
1184 	new_entry->object.vm_object = object;
1185 	new_entry->offset = offset;
1186 	new_entry->avail_ssize = 0;
1187 
1188 	new_entry->inheritance = VM_INHERIT_DEFAULT;
1189 	new_entry->protection = prot;
1190 	new_entry->max_protection = max;
1191 	new_entry->wired_count = 0;
1192 
1193 	/*
1194 	 * Insert the new entry into the list
1195 	 */
1196 	vm_map_entry_link(map, prev_entry, new_entry);
1197 	map->size += new_entry->end - new_entry->start;
1198 
1199 #if 0
1200 	/*
1201 	 * Temporarily removed to avoid MAP_STACK panic, due to
1202 	 * MAP_STACK being a huge hack.  Will be added back in
1203 	 * when MAP_STACK (and the user stack mapping) is fixed.
1204 	 */
1205 	/*
1206 	 * It may be possible to simplify the entry
1207 	 */
1208 	vm_map_simplify_entry(map, new_entry);
1209 #endif
1210 
1211 	if (cow & (MAP_PREFAULT|MAP_PREFAULT_PARTIAL)) {
1212 		vm_map_pmap_enter(map, start, prot,
1213 				    object, OFF_TO_IDX(offset), end - start,
1214 				    cow & MAP_PREFAULT_PARTIAL);
1215 	}
1216 
1217 	return (KERN_SUCCESS);
1218 }
1219 
1220 /*
1221  *	vm_map_findspace:
1222  *
1223  *	Find the first fit (lowest VM address) for "length" free bytes
1224  *	beginning at address >= start in the given map.
1225  *
1226  *	In a vm_map_entry, "adj_free" is the amount of free space
1227  *	adjacent (higher address) to this entry, and "max_free" is the
1228  *	maximum amount of contiguous free space in its subtree.  This
1229  *	allows finding a free region in one path down the tree, so
1230  *	O(log n) amortized with splay trees.
1231  *
1232  *	The map must be locked, and leaves it so.
1233  *
1234  *	Returns: 0 on success, and starting address in *addr,
1235  *		 1 if insufficient space.
1236  */
1237 int
1238 vm_map_findspace(vm_map_t map, vm_offset_t start, vm_size_t length,
1239     vm_offset_t *addr)	/* OUT */
1240 {
1241 	vm_map_entry_t entry;
1242 	vm_offset_t end, st;
1243 
1244 	/*
1245 	 * Request must fit within min/max VM address and must avoid
1246 	 * address wrap.
1247 	 */
1248 	if (start < map->min_offset)
1249 		start = map->min_offset;
1250 	if (start + length > map->max_offset || start + length < start)
1251 		return (1);
1252 
1253 	/* Empty tree means wide open address space. */
1254 	if (map->root == NULL) {
1255 		*addr = start;
1256 		goto found;
1257 	}
1258 
1259 	/*
1260 	 * After splay, if start comes before root node, then there
1261 	 * must be a gap from start to the root.
1262 	 */
1263 	map->root = vm_map_entry_splay(start, map->root);
1264 	if (start + length <= map->root->start) {
1265 		*addr = start;
1266 		goto found;
1267 	}
1268 
1269 	/*
1270 	 * Root is the last node that might begin its gap before
1271 	 * start, and this is the last comparison where address
1272 	 * wrap might be a problem.
1273 	 */
1274 	st = (start > map->root->end) ? start : map->root->end;
1275 	if (length <= map->root->end + map->root->adj_free - st) {
1276 		*addr = st;
1277 		goto found;
1278 	}
1279 
1280 	/* With max_free, can immediately tell if no solution. */
1281 	entry = map->root->right;
1282 	if (entry == NULL || length > entry->max_free)
1283 		return (1);
1284 
1285 	/*
1286 	 * Search the right subtree in the order: left subtree, root,
1287 	 * right subtree (first fit).  The previous splay implies that
1288 	 * all regions in the right subtree have addresses > start.
1289 	 */
1290 	while (entry != NULL) {
1291 		if (entry->left != NULL && entry->left->max_free >= length)
1292 			entry = entry->left;
1293 		else if (entry->adj_free >= length) {
1294 			*addr = entry->end;
1295 			goto found;
1296 		} else
1297 			entry = entry->right;
1298 	}
1299 
1300 	/* Can't get here, so panic if we do. */
1301 	panic("vm_map_findspace: max_free corrupt");
1302 
1303 found:
1304 	/* Expand the kernel pmap, if necessary. */
1305 	if (map == kernel_map) {
1306 		end = round_page(*addr + length);
1307 		if (end > kernel_vm_end)
1308 			pmap_growkernel(end);
1309 	}
1310 	return (0);
1311 }
1312 
1313 int
1314 vm_map_fixed(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
1315     vm_offset_t start, vm_size_t length, vm_prot_t prot,
1316     vm_prot_t max, int cow)
1317 {
1318 	vm_offset_t end;
1319 	int result;
1320 
1321 	end = start + length;
1322 	vm_map_lock(map);
1323 	VM_MAP_RANGE_CHECK(map, start, end);
1324 	(void) vm_map_delete(map, start, end);
1325 	result = vm_map_insert(map, object, offset, start, end, prot,
1326 	    max, cow);
1327 	vm_map_unlock(map);
1328 	return (result);
1329 }
1330 
1331 /*
1332  *	vm_map_find finds an unallocated region in the target address
1333  *	map with the given length.  The search is defined to be
1334  *	first-fit from the specified address; the region found is
1335  *	returned in the same parameter.
1336  *
1337  *	If object is non-NULL, ref count must be bumped by caller
1338  *	prior to making call to account for the new entry.
1339  */
1340 int
1341 vm_map_find(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
1342 	    vm_offset_t *addr,	/* IN/OUT */
1343 	    vm_size_t length, int find_space, vm_prot_t prot,
1344 	    vm_prot_t max, int cow)
1345 {
1346 	vm_offset_t start;
1347 	int result;
1348 
1349 	start = *addr;
1350 	vm_map_lock(map);
1351 	do {
1352 		if (find_space != VMFS_NO_SPACE) {
1353 			if (vm_map_findspace(map, start, length, addr)) {
1354 				vm_map_unlock(map);
1355 				return (KERN_NO_SPACE);
1356 			}
1357 			if (find_space == VMFS_ALIGNED_SPACE)
1358 				pmap_align_superpage(object, offset, addr,
1359 				    length);
1360 			start = *addr;
1361 		}
1362 		result = vm_map_insert(map, object, offset, start, start +
1363 		    length, prot, max, cow);
1364 	} while (result == KERN_NO_SPACE && find_space == VMFS_ALIGNED_SPACE);
1365 	vm_map_unlock(map);
1366 	return (result);
1367 }
1368 
1369 /*
1370  *	vm_map_simplify_entry:
1371  *
1372  *	Simplify the given map entry by merging with either neighbor.  This
1373  *	routine also has the ability to merge with both neighbors.
1374  *
1375  *	The map must be locked.
1376  *
1377  *	This routine guarentees that the passed entry remains valid (though
1378  *	possibly extended).  When merging, this routine may delete one or
1379  *	both neighbors.
1380  */
1381 void
1382 vm_map_simplify_entry(vm_map_t map, vm_map_entry_t entry)
1383 {
1384 	vm_map_entry_t next, prev;
1385 	vm_size_t prevsize, esize;
1386 
1387 	if (entry->eflags & (MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_IS_SUB_MAP))
1388 		return;
1389 
1390 	prev = entry->prev;
1391 	if (prev != &map->header) {
1392 		prevsize = prev->end - prev->start;
1393 		if ( (prev->end == entry->start) &&
1394 		     (prev->object.vm_object == entry->object.vm_object) &&
1395 		     (!prev->object.vm_object ||
1396 			(prev->offset + prevsize == entry->offset)) &&
1397 		     (prev->eflags == entry->eflags) &&
1398 		     (prev->protection == entry->protection) &&
1399 		     (prev->max_protection == entry->max_protection) &&
1400 		     (prev->inheritance == entry->inheritance) &&
1401 		     (prev->wired_count == entry->wired_count)) {
1402 			vm_map_entry_unlink(map, prev);
1403 			entry->start = prev->start;
1404 			entry->offset = prev->offset;
1405 			if (entry->prev != &map->header)
1406 				vm_map_entry_resize_free(map, entry->prev);
1407 
1408 			/*
1409 			 * If the backing object is a vnode object,
1410 			 * vm_object_deallocate() calls vrele().
1411 			 * However, vrele() does not lock the vnode
1412 			 * because the vnode has additional
1413 			 * references.  Thus, the map lock can be kept
1414 			 * without causing a lock-order reversal with
1415 			 * the vnode lock.
1416 			 */
1417 			if (prev->object.vm_object)
1418 				vm_object_deallocate(prev->object.vm_object);
1419 			vm_map_entry_dispose(map, prev);
1420 		}
1421 	}
1422 
1423 	next = entry->next;
1424 	if (next != &map->header) {
1425 		esize = entry->end - entry->start;
1426 		if ((entry->end == next->start) &&
1427 		    (next->object.vm_object == entry->object.vm_object) &&
1428 		     (!entry->object.vm_object ||
1429 			(entry->offset + esize == next->offset)) &&
1430 		    (next->eflags == entry->eflags) &&
1431 		    (next->protection == entry->protection) &&
1432 		    (next->max_protection == entry->max_protection) &&
1433 		    (next->inheritance == entry->inheritance) &&
1434 		    (next->wired_count == entry->wired_count)) {
1435 			vm_map_entry_unlink(map, next);
1436 			entry->end = next->end;
1437 			vm_map_entry_resize_free(map, entry);
1438 
1439 			/*
1440 			 * See comment above.
1441 			 */
1442 			if (next->object.vm_object)
1443 				vm_object_deallocate(next->object.vm_object);
1444 			vm_map_entry_dispose(map, next);
1445 		}
1446 	}
1447 }
1448 /*
1449  *	vm_map_clip_start:	[ internal use only ]
1450  *
1451  *	Asserts that the given entry begins at or after
1452  *	the specified address; if necessary,
1453  *	it splits the entry into two.
1454  */
1455 #define vm_map_clip_start(map, entry, startaddr) \
1456 { \
1457 	if (startaddr > entry->start) \
1458 		_vm_map_clip_start(map, entry, startaddr); \
1459 }
1460 
1461 /*
1462  *	This routine is called only when it is known that
1463  *	the entry must be split.
1464  */
1465 static void
1466 _vm_map_clip_start(vm_map_t map, vm_map_entry_t entry, vm_offset_t start)
1467 {
1468 	vm_map_entry_t new_entry;
1469 
1470 	VM_MAP_ASSERT_LOCKED(map);
1471 
1472 	/*
1473 	 * Split off the front portion -- note that we must insert the new
1474 	 * entry BEFORE this one, so that this entry has the specified
1475 	 * starting address.
1476 	 */
1477 	vm_map_simplify_entry(map, entry);
1478 
1479 	/*
1480 	 * If there is no object backing this entry, we might as well create
1481 	 * one now.  If we defer it, an object can get created after the map
1482 	 * is clipped, and individual objects will be created for the split-up
1483 	 * map.  This is a bit of a hack, but is also about the best place to
1484 	 * put this improvement.
1485 	 */
1486 	if (entry->object.vm_object == NULL && !map->system_map) {
1487 		vm_object_t object;
1488 		object = vm_object_allocate(OBJT_DEFAULT,
1489 				atop(entry->end - entry->start));
1490 		entry->object.vm_object = object;
1491 		entry->offset = 0;
1492 	}
1493 
1494 	new_entry = vm_map_entry_create(map);
1495 	*new_entry = *entry;
1496 
1497 	new_entry->end = start;
1498 	entry->offset += (start - entry->start);
1499 	entry->start = start;
1500 
1501 	vm_map_entry_link(map, entry->prev, new_entry);
1502 
1503 	if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
1504 		vm_object_reference(new_entry->object.vm_object);
1505 	}
1506 }
1507 
1508 /*
1509  *	vm_map_clip_end:	[ internal use only ]
1510  *
1511  *	Asserts that the given entry ends at or before
1512  *	the specified address; if necessary,
1513  *	it splits the entry into two.
1514  */
1515 #define vm_map_clip_end(map, entry, endaddr) \
1516 { \
1517 	if ((endaddr) < (entry->end)) \
1518 		_vm_map_clip_end((map), (entry), (endaddr)); \
1519 }
1520 
1521 /*
1522  *	This routine is called only when it is known that
1523  *	the entry must be split.
1524  */
1525 static void
1526 _vm_map_clip_end(vm_map_t map, vm_map_entry_t entry, vm_offset_t end)
1527 {
1528 	vm_map_entry_t new_entry;
1529 
1530 	VM_MAP_ASSERT_LOCKED(map);
1531 
1532 	/*
1533 	 * If there is no object backing this entry, we might as well create
1534 	 * one now.  If we defer it, an object can get created after the map
1535 	 * is clipped, and individual objects will be created for the split-up
1536 	 * map.  This is a bit of a hack, but is also about the best place to
1537 	 * put this improvement.
1538 	 */
1539 	if (entry->object.vm_object == NULL && !map->system_map) {
1540 		vm_object_t object;
1541 		object = vm_object_allocate(OBJT_DEFAULT,
1542 				atop(entry->end - entry->start));
1543 		entry->object.vm_object = object;
1544 		entry->offset = 0;
1545 	}
1546 
1547 	/*
1548 	 * Create a new entry and insert it AFTER the specified entry
1549 	 */
1550 	new_entry = vm_map_entry_create(map);
1551 	*new_entry = *entry;
1552 
1553 	new_entry->start = entry->end = end;
1554 	new_entry->offset += (end - entry->start);
1555 
1556 	vm_map_entry_link(map, entry, new_entry);
1557 
1558 	if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
1559 		vm_object_reference(new_entry->object.vm_object);
1560 	}
1561 }
1562 
1563 /*
1564  *	vm_map_submap:		[ kernel use only ]
1565  *
1566  *	Mark the given range as handled by a subordinate map.
1567  *
1568  *	This range must have been created with vm_map_find,
1569  *	and no other operations may have been performed on this
1570  *	range prior to calling vm_map_submap.
1571  *
1572  *	Only a limited number of operations can be performed
1573  *	within this rage after calling vm_map_submap:
1574  *		vm_fault
1575  *	[Don't try vm_map_copy!]
1576  *
1577  *	To remove a submapping, one must first remove the
1578  *	range from the superior map, and then destroy the
1579  *	submap (if desired).  [Better yet, don't try it.]
1580  */
1581 int
1582 vm_map_submap(
1583 	vm_map_t map,
1584 	vm_offset_t start,
1585 	vm_offset_t end,
1586 	vm_map_t submap)
1587 {
1588 	vm_map_entry_t entry;
1589 	int result = KERN_INVALID_ARGUMENT;
1590 
1591 	vm_map_lock(map);
1592 
1593 	VM_MAP_RANGE_CHECK(map, start, end);
1594 
1595 	if (vm_map_lookup_entry(map, start, &entry)) {
1596 		vm_map_clip_start(map, entry, start);
1597 	} else
1598 		entry = entry->next;
1599 
1600 	vm_map_clip_end(map, entry, end);
1601 
1602 	if ((entry->start == start) && (entry->end == end) &&
1603 	    ((entry->eflags & MAP_ENTRY_COW) == 0) &&
1604 	    (entry->object.vm_object == NULL)) {
1605 		entry->object.sub_map = submap;
1606 		entry->eflags |= MAP_ENTRY_IS_SUB_MAP;
1607 		result = KERN_SUCCESS;
1608 	}
1609 	vm_map_unlock(map);
1610 
1611 	return (result);
1612 }
1613 
1614 /*
1615  * The maximum number of pages to map
1616  */
1617 #define	MAX_INIT_PT	96
1618 
1619 /*
1620  *	vm_map_pmap_enter:
1621  *
1622  *	Preload read-only mappings for the given object's resident pages into
1623  *	the given map.  This eliminates the soft faults on process startup and
1624  *	immediately after an mmap(2).  Because these are speculative mappings,
1625  *	cached pages are not reactivated and mapped.
1626  */
1627 void
1628 vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot,
1629     vm_object_t object, vm_pindex_t pindex, vm_size_t size, int flags)
1630 {
1631 	vm_offset_t start;
1632 	vm_page_t p, p_start;
1633 	vm_pindex_t psize, tmpidx;
1634 	boolean_t are_queues_locked;
1635 
1636 	if ((prot & (VM_PROT_READ | VM_PROT_EXECUTE)) == 0 || object == NULL)
1637 		return;
1638 	VM_OBJECT_LOCK(object);
1639 	if (object->type == OBJT_DEVICE) {
1640 		pmap_object_init_pt(map->pmap, addr, object, pindex, size);
1641 		goto unlock_return;
1642 	}
1643 
1644 	psize = atop(size);
1645 
1646 	if (object->type != OBJT_VNODE ||
1647 	    ((flags & MAP_PREFAULT_PARTIAL) && (psize > MAX_INIT_PT) &&
1648 	     (object->resident_page_count > MAX_INIT_PT))) {
1649 		goto unlock_return;
1650 	}
1651 
1652 	if (psize + pindex > object->size) {
1653 		if (object->size < pindex)
1654 			goto unlock_return;
1655 		psize = object->size - pindex;
1656 	}
1657 
1658 	are_queues_locked = FALSE;
1659 	start = 0;
1660 	p_start = NULL;
1661 
1662 	if ((p = TAILQ_FIRST(&object->memq)) != NULL) {
1663 		if (p->pindex < pindex) {
1664 			p = vm_page_splay(pindex, object->root);
1665 			if ((object->root = p)->pindex < pindex)
1666 				p = TAILQ_NEXT(p, listq);
1667 		}
1668 	}
1669 	/*
1670 	 * Assert: the variable p is either (1) the page with the
1671 	 * least pindex greater than or equal to the parameter pindex
1672 	 * or (2) NULL.
1673 	 */
1674 	for (;
1675 	     p != NULL && (tmpidx = p->pindex - pindex) < psize;
1676 	     p = TAILQ_NEXT(p, listq)) {
1677 		/*
1678 		 * don't allow an madvise to blow away our really
1679 		 * free pages allocating pv entries.
1680 		 */
1681 		if ((flags & MAP_PREFAULT_MADVISE) &&
1682 		    cnt.v_free_count < cnt.v_free_reserved) {
1683 			psize = tmpidx;
1684 			break;
1685 		}
1686 		if ((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) {
1687 			if (p_start == NULL) {
1688 				start = addr + ptoa(tmpidx);
1689 				p_start = p;
1690 			}
1691 		} else if (p_start != NULL) {
1692 			if (!are_queues_locked) {
1693 				are_queues_locked = TRUE;
1694 				vm_page_lock_queues();
1695 			}
1696 			pmap_enter_object(map->pmap, start, addr +
1697 			    ptoa(tmpidx), p_start, prot);
1698 			p_start = NULL;
1699 		}
1700 	}
1701 	if (p_start != NULL) {
1702 		if (!are_queues_locked) {
1703 			are_queues_locked = TRUE;
1704 			vm_page_lock_queues();
1705 		}
1706 		pmap_enter_object(map->pmap, start, addr + ptoa(psize),
1707 		    p_start, prot);
1708 	}
1709 	if (are_queues_locked)
1710 		vm_page_unlock_queues();
1711 unlock_return:
1712 	VM_OBJECT_UNLOCK(object);
1713 }
1714 
1715 /*
1716  *	vm_map_protect:
1717  *
1718  *	Sets the protection of the specified address
1719  *	region in the target map.  If "set_max" is
1720  *	specified, the maximum protection is to be set;
1721  *	otherwise, only the current protection is affected.
1722  */
1723 int
1724 vm_map_protect(vm_map_t map, vm_offset_t start, vm_offset_t end,
1725 	       vm_prot_t new_prot, boolean_t set_max)
1726 {
1727 	vm_map_entry_t current;
1728 	vm_map_entry_t entry;
1729 
1730 	vm_map_lock(map);
1731 
1732 	VM_MAP_RANGE_CHECK(map, start, end);
1733 
1734 	if (vm_map_lookup_entry(map, start, &entry)) {
1735 		vm_map_clip_start(map, entry, start);
1736 	} else {
1737 		entry = entry->next;
1738 	}
1739 
1740 	/*
1741 	 * Make a first pass to check for protection violations.
1742 	 */
1743 	current = entry;
1744 	while ((current != &map->header) && (current->start < end)) {
1745 		if (current->eflags & MAP_ENTRY_IS_SUB_MAP) {
1746 			vm_map_unlock(map);
1747 			return (KERN_INVALID_ARGUMENT);
1748 		}
1749 		if ((new_prot & current->max_protection) != new_prot) {
1750 			vm_map_unlock(map);
1751 			return (KERN_PROTECTION_FAILURE);
1752 		}
1753 		current = current->next;
1754 	}
1755 
1756 	/*
1757 	 * Go back and fix up protections. [Note that clipping is not
1758 	 * necessary the second time.]
1759 	 */
1760 	current = entry;
1761 	while ((current != &map->header) && (current->start < end)) {
1762 		vm_prot_t old_prot;
1763 
1764 		vm_map_clip_end(map, current, end);
1765 
1766 		old_prot = current->protection;
1767 		if (set_max)
1768 			current->protection =
1769 			    (current->max_protection = new_prot) &
1770 			    old_prot;
1771 		else
1772 			current->protection = new_prot;
1773 
1774 		/*
1775 		 * Update physical map if necessary. Worry about copy-on-write
1776 		 * here.
1777 		 */
1778 		if (current->protection != old_prot) {
1779 #define MASK(entry)	(((entry)->eflags & MAP_ENTRY_COW) ? ~VM_PROT_WRITE : \
1780 							VM_PROT_ALL)
1781 			pmap_protect(map->pmap, current->start,
1782 			    current->end,
1783 			    current->protection & MASK(current));
1784 #undef	MASK
1785 		}
1786 		vm_map_simplify_entry(map, current);
1787 		current = current->next;
1788 	}
1789 	vm_map_unlock(map);
1790 	return (KERN_SUCCESS);
1791 }
1792 
1793 /*
1794  *	vm_map_madvise:
1795  *
1796  *	This routine traverses a processes map handling the madvise
1797  *	system call.  Advisories are classified as either those effecting
1798  *	the vm_map_entry structure, or those effecting the underlying
1799  *	objects.
1800  */
1801 int
1802 vm_map_madvise(
1803 	vm_map_t map,
1804 	vm_offset_t start,
1805 	vm_offset_t end,
1806 	int behav)
1807 {
1808 	vm_map_entry_t current, entry;
1809 	int modify_map = 0;
1810 
1811 	/*
1812 	 * Some madvise calls directly modify the vm_map_entry, in which case
1813 	 * we need to use an exclusive lock on the map and we need to perform
1814 	 * various clipping operations.  Otherwise we only need a read-lock
1815 	 * on the map.
1816 	 */
1817 	switch(behav) {
1818 	case MADV_NORMAL:
1819 	case MADV_SEQUENTIAL:
1820 	case MADV_RANDOM:
1821 	case MADV_NOSYNC:
1822 	case MADV_AUTOSYNC:
1823 	case MADV_NOCORE:
1824 	case MADV_CORE:
1825 		modify_map = 1;
1826 		vm_map_lock(map);
1827 		break;
1828 	case MADV_WILLNEED:
1829 	case MADV_DONTNEED:
1830 	case MADV_FREE:
1831 		vm_map_lock_read(map);
1832 		break;
1833 	default:
1834 		return (KERN_INVALID_ARGUMENT);
1835 	}
1836 
1837 	/*
1838 	 * Locate starting entry and clip if necessary.
1839 	 */
1840 	VM_MAP_RANGE_CHECK(map, start, end);
1841 
1842 	if (vm_map_lookup_entry(map, start, &entry)) {
1843 		if (modify_map)
1844 			vm_map_clip_start(map, entry, start);
1845 	} else {
1846 		entry = entry->next;
1847 	}
1848 
1849 	if (modify_map) {
1850 		/*
1851 		 * madvise behaviors that are implemented in the vm_map_entry.
1852 		 *
1853 		 * We clip the vm_map_entry so that behavioral changes are
1854 		 * limited to the specified address range.
1855 		 */
1856 		for (current = entry;
1857 		     (current != &map->header) && (current->start < end);
1858 		     current = current->next
1859 		) {
1860 			if (current->eflags & MAP_ENTRY_IS_SUB_MAP)
1861 				continue;
1862 
1863 			vm_map_clip_end(map, current, end);
1864 
1865 			switch (behav) {
1866 			case MADV_NORMAL:
1867 				vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_NORMAL);
1868 				break;
1869 			case MADV_SEQUENTIAL:
1870 				vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_SEQUENTIAL);
1871 				break;
1872 			case MADV_RANDOM:
1873 				vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_RANDOM);
1874 				break;
1875 			case MADV_NOSYNC:
1876 				current->eflags |= MAP_ENTRY_NOSYNC;
1877 				break;
1878 			case MADV_AUTOSYNC:
1879 				current->eflags &= ~MAP_ENTRY_NOSYNC;
1880 				break;
1881 			case MADV_NOCORE:
1882 				current->eflags |= MAP_ENTRY_NOCOREDUMP;
1883 				break;
1884 			case MADV_CORE:
1885 				current->eflags &= ~MAP_ENTRY_NOCOREDUMP;
1886 				break;
1887 			default:
1888 				break;
1889 			}
1890 			vm_map_simplify_entry(map, current);
1891 		}
1892 		vm_map_unlock(map);
1893 	} else {
1894 		vm_pindex_t pindex;
1895 		int count;
1896 
1897 		/*
1898 		 * madvise behaviors that are implemented in the underlying
1899 		 * vm_object.
1900 		 *
1901 		 * Since we don't clip the vm_map_entry, we have to clip
1902 		 * the vm_object pindex and count.
1903 		 */
1904 		for (current = entry;
1905 		     (current != &map->header) && (current->start < end);
1906 		     current = current->next
1907 		) {
1908 			vm_offset_t useStart;
1909 
1910 			if (current->eflags & MAP_ENTRY_IS_SUB_MAP)
1911 				continue;
1912 
1913 			pindex = OFF_TO_IDX(current->offset);
1914 			count = atop(current->end - current->start);
1915 			useStart = current->start;
1916 
1917 			if (current->start < start) {
1918 				pindex += atop(start - current->start);
1919 				count -= atop(start - current->start);
1920 				useStart = start;
1921 			}
1922 			if (current->end > end)
1923 				count -= atop(current->end - end);
1924 
1925 			if (count <= 0)
1926 				continue;
1927 
1928 			vm_object_madvise(current->object.vm_object,
1929 					  pindex, count, behav);
1930 			if (behav == MADV_WILLNEED) {
1931 				vm_map_pmap_enter(map,
1932 				    useStart,
1933 				    current->protection,
1934 				    current->object.vm_object,
1935 				    pindex,
1936 				    (count << PAGE_SHIFT),
1937 				    MAP_PREFAULT_MADVISE
1938 				);
1939 			}
1940 		}
1941 		vm_map_unlock_read(map);
1942 	}
1943 	return (0);
1944 }
1945 
1946 
1947 /*
1948  *	vm_map_inherit:
1949  *
1950  *	Sets the inheritance of the specified address
1951  *	range in the target map.  Inheritance
1952  *	affects how the map will be shared with
1953  *	child maps at the time of vmspace_fork.
1954  */
1955 int
1956 vm_map_inherit(vm_map_t map, vm_offset_t start, vm_offset_t end,
1957 	       vm_inherit_t new_inheritance)
1958 {
1959 	vm_map_entry_t entry;
1960 	vm_map_entry_t temp_entry;
1961 
1962 	switch (new_inheritance) {
1963 	case VM_INHERIT_NONE:
1964 	case VM_INHERIT_COPY:
1965 	case VM_INHERIT_SHARE:
1966 		break;
1967 	default:
1968 		return (KERN_INVALID_ARGUMENT);
1969 	}
1970 	vm_map_lock(map);
1971 	VM_MAP_RANGE_CHECK(map, start, end);
1972 	if (vm_map_lookup_entry(map, start, &temp_entry)) {
1973 		entry = temp_entry;
1974 		vm_map_clip_start(map, entry, start);
1975 	} else
1976 		entry = temp_entry->next;
1977 	while ((entry != &map->header) && (entry->start < end)) {
1978 		vm_map_clip_end(map, entry, end);
1979 		entry->inheritance = new_inheritance;
1980 		vm_map_simplify_entry(map, entry);
1981 		entry = entry->next;
1982 	}
1983 	vm_map_unlock(map);
1984 	return (KERN_SUCCESS);
1985 }
1986 
1987 /*
1988  *	vm_map_unwire:
1989  *
1990  *	Implements both kernel and user unwiring.
1991  */
1992 int
1993 vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end,
1994     int flags)
1995 {
1996 	vm_map_entry_t entry, first_entry, tmp_entry;
1997 	vm_offset_t saved_start;
1998 	unsigned int last_timestamp;
1999 	int rv;
2000 	boolean_t need_wakeup, result, user_unwire;
2001 
2002 	user_unwire = (flags & VM_MAP_WIRE_USER) ? TRUE : FALSE;
2003 	vm_map_lock(map);
2004 	VM_MAP_RANGE_CHECK(map, start, end);
2005 	if (!vm_map_lookup_entry(map, start, &first_entry)) {
2006 		if (flags & VM_MAP_WIRE_HOLESOK)
2007 			first_entry = first_entry->next;
2008 		else {
2009 			vm_map_unlock(map);
2010 			return (KERN_INVALID_ADDRESS);
2011 		}
2012 	}
2013 	last_timestamp = map->timestamp;
2014 	entry = first_entry;
2015 	while (entry != &map->header && entry->start < end) {
2016 		if (entry->eflags & MAP_ENTRY_IN_TRANSITION) {
2017 			/*
2018 			 * We have not yet clipped the entry.
2019 			 */
2020 			saved_start = (start >= entry->start) ? start :
2021 			    entry->start;
2022 			entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
2023 			if (vm_map_unlock_and_wait(map, 0)) {
2024 				/*
2025 				 * Allow interruption of user unwiring?
2026 				 */
2027 			}
2028 			vm_map_lock(map);
2029 			if (last_timestamp+1 != map->timestamp) {
2030 				/*
2031 				 * Look again for the entry because the map was
2032 				 * modified while it was unlocked.
2033 				 * Specifically, the entry may have been
2034 				 * clipped, merged, or deleted.
2035 				 */
2036 				if (!vm_map_lookup_entry(map, saved_start,
2037 				    &tmp_entry)) {
2038 					if (flags & VM_MAP_WIRE_HOLESOK)
2039 						tmp_entry = tmp_entry->next;
2040 					else {
2041 						if (saved_start == start) {
2042 							/*
2043 							 * First_entry has been deleted.
2044 							 */
2045 							vm_map_unlock(map);
2046 							return (KERN_INVALID_ADDRESS);
2047 						}
2048 						end = saved_start;
2049 						rv = KERN_INVALID_ADDRESS;
2050 						goto done;
2051 					}
2052 				}
2053 				if (entry == first_entry)
2054 					first_entry = tmp_entry;
2055 				else
2056 					first_entry = NULL;
2057 				entry = tmp_entry;
2058 			}
2059 			last_timestamp = map->timestamp;
2060 			continue;
2061 		}
2062 		vm_map_clip_start(map, entry, start);
2063 		vm_map_clip_end(map, entry, end);
2064 		/*
2065 		 * Mark the entry in case the map lock is released.  (See
2066 		 * above.)
2067 		 */
2068 		entry->eflags |= MAP_ENTRY_IN_TRANSITION;
2069 		/*
2070 		 * Check the map for holes in the specified region.
2071 		 * If VM_MAP_WIRE_HOLESOK was specified, skip this check.
2072 		 */
2073 		if (((flags & VM_MAP_WIRE_HOLESOK) == 0) &&
2074 		    (entry->end < end && (entry->next == &map->header ||
2075 		    entry->next->start > entry->end))) {
2076 			end = entry->end;
2077 			rv = KERN_INVALID_ADDRESS;
2078 			goto done;
2079 		}
2080 		/*
2081 		 * If system unwiring, require that the entry is system wired.
2082 		 */
2083 		if (!user_unwire &&
2084 		    vm_map_entry_system_wired_count(entry) == 0) {
2085 			end = entry->end;
2086 			rv = KERN_INVALID_ARGUMENT;
2087 			goto done;
2088 		}
2089 		entry = entry->next;
2090 	}
2091 	rv = KERN_SUCCESS;
2092 done:
2093 	need_wakeup = FALSE;
2094 	if (first_entry == NULL) {
2095 		result = vm_map_lookup_entry(map, start, &first_entry);
2096 		if (!result && (flags & VM_MAP_WIRE_HOLESOK))
2097 			first_entry = first_entry->next;
2098 		else
2099 			KASSERT(result, ("vm_map_unwire: lookup failed"));
2100 	}
2101 	entry = first_entry;
2102 	while (entry != &map->header && entry->start < end) {
2103 		if (rv == KERN_SUCCESS && (!user_unwire ||
2104 		    (entry->eflags & MAP_ENTRY_USER_WIRED))) {
2105 			if (user_unwire)
2106 				entry->eflags &= ~MAP_ENTRY_USER_WIRED;
2107 			entry->wired_count--;
2108 			if (entry->wired_count == 0) {
2109 				/*
2110 				 * Retain the map lock.
2111 				 */
2112 				vm_fault_unwire(map, entry->start, entry->end,
2113 				    entry->object.vm_object != NULL &&
2114 				    entry->object.vm_object->type == OBJT_DEVICE);
2115 			}
2116 		}
2117 		KASSERT(entry->eflags & MAP_ENTRY_IN_TRANSITION,
2118 			("vm_map_unwire: in-transition flag missing"));
2119 		entry->eflags &= ~MAP_ENTRY_IN_TRANSITION;
2120 		if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) {
2121 			entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP;
2122 			need_wakeup = TRUE;
2123 		}
2124 		vm_map_simplify_entry(map, entry);
2125 		entry = entry->next;
2126 	}
2127 	vm_map_unlock(map);
2128 	if (need_wakeup)
2129 		vm_map_wakeup(map);
2130 	return (rv);
2131 }
2132 
2133 /*
2134  *	vm_map_wire:
2135  *
2136  *	Implements both kernel and user wiring.
2137  */
2138 int
2139 vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end,
2140     int flags)
2141 {
2142 	vm_map_entry_t entry, first_entry, tmp_entry;
2143 	vm_offset_t saved_end, saved_start;
2144 	unsigned int last_timestamp;
2145 	int rv;
2146 	boolean_t fictitious, need_wakeup, result, user_wire;
2147 
2148 	user_wire = (flags & VM_MAP_WIRE_USER) ? TRUE : FALSE;
2149 	vm_map_lock(map);
2150 	VM_MAP_RANGE_CHECK(map, start, end);
2151 	if (!vm_map_lookup_entry(map, start, &first_entry)) {
2152 		if (flags & VM_MAP_WIRE_HOLESOK)
2153 			first_entry = first_entry->next;
2154 		else {
2155 			vm_map_unlock(map);
2156 			return (KERN_INVALID_ADDRESS);
2157 		}
2158 	}
2159 	last_timestamp = map->timestamp;
2160 	entry = first_entry;
2161 	while (entry != &map->header && entry->start < end) {
2162 		if (entry->eflags & MAP_ENTRY_IN_TRANSITION) {
2163 			/*
2164 			 * We have not yet clipped the entry.
2165 			 */
2166 			saved_start = (start >= entry->start) ? start :
2167 			    entry->start;
2168 			entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
2169 			if (vm_map_unlock_and_wait(map, 0)) {
2170 				/*
2171 				 * Allow interruption of user wiring?
2172 				 */
2173 			}
2174 			vm_map_lock(map);
2175 			if (last_timestamp + 1 != map->timestamp) {
2176 				/*
2177 				 * Look again for the entry because the map was
2178 				 * modified while it was unlocked.
2179 				 * Specifically, the entry may have been
2180 				 * clipped, merged, or deleted.
2181 				 */
2182 				if (!vm_map_lookup_entry(map, saved_start,
2183 				    &tmp_entry)) {
2184 					if (flags & VM_MAP_WIRE_HOLESOK)
2185 						tmp_entry = tmp_entry->next;
2186 					else {
2187 						if (saved_start == start) {
2188 							/*
2189 							 * first_entry has been deleted.
2190 							 */
2191 							vm_map_unlock(map);
2192 							return (KERN_INVALID_ADDRESS);
2193 						}
2194 						end = saved_start;
2195 						rv = KERN_INVALID_ADDRESS;
2196 						goto done;
2197 					}
2198 				}
2199 				if (entry == first_entry)
2200 					first_entry = tmp_entry;
2201 				else
2202 					first_entry = NULL;
2203 				entry = tmp_entry;
2204 			}
2205 			last_timestamp = map->timestamp;
2206 			continue;
2207 		}
2208 		vm_map_clip_start(map, entry, start);
2209 		vm_map_clip_end(map, entry, end);
2210 		/*
2211 		 * Mark the entry in case the map lock is released.  (See
2212 		 * above.)
2213 		 */
2214 		entry->eflags |= MAP_ENTRY_IN_TRANSITION;
2215 		/*
2216 		 *
2217 		 */
2218 		if (entry->wired_count == 0) {
2219 			if ((entry->protection & (VM_PROT_READ|VM_PROT_EXECUTE))
2220 			    == 0) {
2221 				if ((flags & VM_MAP_WIRE_HOLESOK) == 0) {
2222 					end = entry->end;
2223 					rv = KERN_INVALID_ADDRESS;
2224 					goto done;
2225 				}
2226 				entry->eflags |= MAP_ENTRY_WIRE_SKIPPED;
2227 				goto next_entry;
2228 			}
2229 			entry->wired_count++;
2230 			saved_start = entry->start;
2231 			saved_end = entry->end;
2232 			fictitious = entry->object.vm_object != NULL &&
2233 			    entry->object.vm_object->type == OBJT_DEVICE;
2234 			/*
2235 			 * Release the map lock, relying on the in-transition
2236 			 * mark.
2237 			 */
2238 			vm_map_unlock(map);
2239 			rv = vm_fault_wire(map, saved_start, saved_end,
2240 			    user_wire, fictitious);
2241 			vm_map_lock(map);
2242 			if (last_timestamp + 1 != map->timestamp) {
2243 				/*
2244 				 * Look again for the entry because the map was
2245 				 * modified while it was unlocked.  The entry
2246 				 * may have been clipped, but NOT merged or
2247 				 * deleted.
2248 				 */
2249 				result = vm_map_lookup_entry(map, saved_start,
2250 				    &tmp_entry);
2251 				KASSERT(result, ("vm_map_wire: lookup failed"));
2252 				if (entry == first_entry)
2253 					first_entry = tmp_entry;
2254 				else
2255 					first_entry = NULL;
2256 				entry = tmp_entry;
2257 				while (entry->end < saved_end) {
2258 					if (rv != KERN_SUCCESS) {
2259 						KASSERT(entry->wired_count == 1,
2260 						    ("vm_map_wire: bad count"));
2261 						entry->wired_count = -1;
2262 					}
2263 					entry = entry->next;
2264 				}
2265 			}
2266 			last_timestamp = map->timestamp;
2267 			if (rv != KERN_SUCCESS) {
2268 				KASSERT(entry->wired_count == 1,
2269 				    ("vm_map_wire: bad count"));
2270 				/*
2271 				 * Assign an out-of-range value to represent
2272 				 * the failure to wire this entry.
2273 				 */
2274 				entry->wired_count = -1;
2275 				end = entry->end;
2276 				goto done;
2277 			}
2278 		} else if (!user_wire ||
2279 			   (entry->eflags & MAP_ENTRY_USER_WIRED) == 0) {
2280 			entry->wired_count++;
2281 		}
2282 		/*
2283 		 * Check the map for holes in the specified region.
2284 		 * If VM_MAP_WIRE_HOLESOK was specified, skip this check.
2285 		 */
2286 	next_entry:
2287 		if (((flags & VM_MAP_WIRE_HOLESOK) == 0) &&
2288 		    (entry->end < end && (entry->next == &map->header ||
2289 		    entry->next->start > entry->end))) {
2290 			end = entry->end;
2291 			rv = KERN_INVALID_ADDRESS;
2292 			goto done;
2293 		}
2294 		entry = entry->next;
2295 	}
2296 	rv = KERN_SUCCESS;
2297 done:
2298 	need_wakeup = FALSE;
2299 	if (first_entry == NULL) {
2300 		result = vm_map_lookup_entry(map, start, &first_entry);
2301 		if (!result && (flags & VM_MAP_WIRE_HOLESOK))
2302 			first_entry = first_entry->next;
2303 		else
2304 			KASSERT(result, ("vm_map_wire: lookup failed"));
2305 	}
2306 	entry = first_entry;
2307 	while (entry != &map->header && entry->start < end) {
2308 		if ((entry->eflags & MAP_ENTRY_WIRE_SKIPPED) != 0)
2309 			goto next_entry_done;
2310 		if (rv == KERN_SUCCESS) {
2311 			if (user_wire)
2312 				entry->eflags |= MAP_ENTRY_USER_WIRED;
2313 		} else if (entry->wired_count == -1) {
2314 			/*
2315 			 * Wiring failed on this entry.  Thus, unwiring is
2316 			 * unnecessary.
2317 			 */
2318 			entry->wired_count = 0;
2319 		} else {
2320 			if (!user_wire ||
2321 			    (entry->eflags & MAP_ENTRY_USER_WIRED) == 0)
2322 				entry->wired_count--;
2323 			if (entry->wired_count == 0) {
2324 				/*
2325 				 * Retain the map lock.
2326 				 */
2327 				vm_fault_unwire(map, entry->start, entry->end,
2328 				    entry->object.vm_object != NULL &&
2329 				    entry->object.vm_object->type == OBJT_DEVICE);
2330 			}
2331 		}
2332 	next_entry_done:
2333 		KASSERT(entry->eflags & MAP_ENTRY_IN_TRANSITION,
2334 			("vm_map_wire: in-transition flag missing"));
2335 		entry->eflags &= ~(MAP_ENTRY_IN_TRANSITION|MAP_ENTRY_WIRE_SKIPPED);
2336 		if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) {
2337 			entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP;
2338 			need_wakeup = TRUE;
2339 		}
2340 		vm_map_simplify_entry(map, entry);
2341 		entry = entry->next;
2342 	}
2343 	vm_map_unlock(map);
2344 	if (need_wakeup)
2345 		vm_map_wakeup(map);
2346 	return (rv);
2347 }
2348 
2349 /*
2350  * vm_map_sync
2351  *
2352  * Push any dirty cached pages in the address range to their pager.
2353  * If syncio is TRUE, dirty pages are written synchronously.
2354  * If invalidate is TRUE, any cached pages are freed as well.
2355  *
2356  * If the size of the region from start to end is zero, we are
2357  * supposed to flush all modified pages within the region containing
2358  * start.  Unfortunately, a region can be split or coalesced with
2359  * neighboring regions, making it difficult to determine what the
2360  * original region was.  Therefore, we approximate this requirement by
2361  * flushing the current region containing start.
2362  *
2363  * Returns an error if any part of the specified range is not mapped.
2364  */
2365 int
2366 vm_map_sync(
2367 	vm_map_t map,
2368 	vm_offset_t start,
2369 	vm_offset_t end,
2370 	boolean_t syncio,
2371 	boolean_t invalidate)
2372 {
2373 	vm_map_entry_t current;
2374 	vm_map_entry_t entry;
2375 	vm_size_t size;
2376 	vm_object_t object;
2377 	vm_ooffset_t offset;
2378 	unsigned int last_timestamp;
2379 
2380 	vm_map_lock_read(map);
2381 	VM_MAP_RANGE_CHECK(map, start, end);
2382 	if (!vm_map_lookup_entry(map, start, &entry)) {
2383 		vm_map_unlock_read(map);
2384 		return (KERN_INVALID_ADDRESS);
2385 	} else if (start == end) {
2386 		start = entry->start;
2387 		end = entry->end;
2388 	}
2389 	/*
2390 	 * Make a first pass to check for user-wired memory and holes.
2391 	 */
2392 	for (current = entry; current != &map->header && current->start < end;
2393 	    current = current->next) {
2394 		if (invalidate && (current->eflags & MAP_ENTRY_USER_WIRED)) {
2395 			vm_map_unlock_read(map);
2396 			return (KERN_INVALID_ARGUMENT);
2397 		}
2398 		if (end > current->end &&
2399 		    (current->next == &map->header ||
2400 			current->end != current->next->start)) {
2401 			vm_map_unlock_read(map);
2402 			return (KERN_INVALID_ADDRESS);
2403 		}
2404 	}
2405 
2406 	if (invalidate)
2407 		pmap_remove(map->pmap, start, end);
2408 
2409 	/*
2410 	 * Make a second pass, cleaning/uncaching pages from the indicated
2411 	 * objects as we go.
2412 	 */
2413 	for (current = entry; current != &map->header && current->start < end;) {
2414 		offset = current->offset + (start - current->start);
2415 		size = (end <= current->end ? end : current->end) - start;
2416 		if (current->eflags & MAP_ENTRY_IS_SUB_MAP) {
2417 			vm_map_t smap;
2418 			vm_map_entry_t tentry;
2419 			vm_size_t tsize;
2420 
2421 			smap = current->object.sub_map;
2422 			vm_map_lock_read(smap);
2423 			(void) vm_map_lookup_entry(smap, offset, &tentry);
2424 			tsize = tentry->end - offset;
2425 			if (tsize < size)
2426 				size = tsize;
2427 			object = tentry->object.vm_object;
2428 			offset = tentry->offset + (offset - tentry->start);
2429 			vm_map_unlock_read(smap);
2430 		} else {
2431 			object = current->object.vm_object;
2432 		}
2433 		vm_object_reference(object);
2434 		last_timestamp = map->timestamp;
2435 		vm_map_unlock_read(map);
2436 		vm_object_sync(object, offset, size, syncio, invalidate);
2437 		start += size;
2438 		vm_object_deallocate(object);
2439 		vm_map_lock_read(map);
2440 		if (last_timestamp == map->timestamp ||
2441 		    !vm_map_lookup_entry(map, start, &current))
2442 			current = current->next;
2443 	}
2444 
2445 	vm_map_unlock_read(map);
2446 	return (KERN_SUCCESS);
2447 }
2448 
2449 /*
2450  *	vm_map_entry_unwire:	[ internal use only ]
2451  *
2452  *	Make the region specified by this entry pageable.
2453  *
2454  *	The map in question should be locked.
2455  *	[This is the reason for this routine's existence.]
2456  */
2457 static void
2458 vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry)
2459 {
2460 	vm_fault_unwire(map, entry->start, entry->end,
2461 	    entry->object.vm_object != NULL &&
2462 	    entry->object.vm_object->type == OBJT_DEVICE);
2463 	entry->wired_count = 0;
2464 }
2465 
2466 /*
2467  *	vm_map_entry_delete:	[ internal use only ]
2468  *
2469  *	Deallocate the given entry from the target map.
2470  */
2471 static void
2472 vm_map_entry_delete(vm_map_t map, vm_map_entry_t entry)
2473 {
2474 	vm_object_t object;
2475 	vm_pindex_t offidxstart, offidxend, count;
2476 
2477 	vm_map_entry_unlink(map, entry);
2478 	map->size -= entry->end - entry->start;
2479 
2480 	if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
2481 	    (object = entry->object.vm_object) != NULL) {
2482 		count = OFF_TO_IDX(entry->end - entry->start);
2483 		offidxstart = OFF_TO_IDX(entry->offset);
2484 		offidxend = offidxstart + count;
2485 		VM_OBJECT_LOCK(object);
2486 		if (object->ref_count != 1 &&
2487 		    ((object->flags & (OBJ_NOSPLIT|OBJ_ONEMAPPING)) == OBJ_ONEMAPPING ||
2488 		    object == kernel_object || object == kmem_object)) {
2489 			vm_object_collapse(object);
2490 			vm_object_page_remove(object, offidxstart, offidxend, FALSE);
2491 			if (object->type == OBJT_SWAP)
2492 				swap_pager_freespace(object, offidxstart, count);
2493 			if (offidxend >= object->size &&
2494 			    offidxstart < object->size)
2495 				object->size = offidxstart;
2496 		}
2497 		VM_OBJECT_UNLOCK(object);
2498 	} else
2499 		entry->object.vm_object = NULL;
2500 }
2501 
2502 /*
2503  *	vm_map_delete:	[ internal use only ]
2504  *
2505  *	Deallocates the given address range from the target
2506  *	map.
2507  */
2508 int
2509 vm_map_delete(vm_map_t map, vm_offset_t start, vm_offset_t end)
2510 {
2511 	vm_map_entry_t entry;
2512 	vm_map_entry_t first_entry;
2513 
2514 	VM_MAP_ASSERT_LOCKED(map);
2515 
2516 	/*
2517 	 * Find the start of the region, and clip it
2518 	 */
2519 	if (!vm_map_lookup_entry(map, start, &first_entry))
2520 		entry = first_entry->next;
2521 	else {
2522 		entry = first_entry;
2523 		vm_map_clip_start(map, entry, start);
2524 	}
2525 
2526 	/*
2527 	 * Step through all entries in this region
2528 	 */
2529 	while ((entry != &map->header) && (entry->start < end)) {
2530 		vm_map_entry_t next;
2531 
2532 		/*
2533 		 * Wait for wiring or unwiring of an entry to complete.
2534 		 * Also wait for any system wirings to disappear on
2535 		 * user maps.
2536 		 */
2537 		if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0 ||
2538 		    (vm_map_pmap(map) != kernel_pmap &&
2539 		    vm_map_entry_system_wired_count(entry) != 0)) {
2540 			unsigned int last_timestamp;
2541 			vm_offset_t saved_start;
2542 			vm_map_entry_t tmp_entry;
2543 
2544 			saved_start = entry->start;
2545 			entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
2546 			last_timestamp = map->timestamp;
2547 			(void) vm_map_unlock_and_wait(map, 0);
2548 			vm_map_lock(map);
2549 			if (last_timestamp + 1 != map->timestamp) {
2550 				/*
2551 				 * Look again for the entry because the map was
2552 				 * modified while it was unlocked.
2553 				 * Specifically, the entry may have been
2554 				 * clipped, merged, or deleted.
2555 				 */
2556 				if (!vm_map_lookup_entry(map, saved_start,
2557 							 &tmp_entry))
2558 					entry = tmp_entry->next;
2559 				else {
2560 					entry = tmp_entry;
2561 					vm_map_clip_start(map, entry,
2562 							  saved_start);
2563 				}
2564 			}
2565 			continue;
2566 		}
2567 		vm_map_clip_end(map, entry, end);
2568 
2569 		next = entry->next;
2570 
2571 		/*
2572 		 * Unwire before removing addresses from the pmap; otherwise,
2573 		 * unwiring will put the entries back in the pmap.
2574 		 */
2575 		if (entry->wired_count != 0) {
2576 			vm_map_entry_unwire(map, entry);
2577 		}
2578 
2579 		pmap_remove(map->pmap, entry->start, entry->end);
2580 
2581 		/*
2582 		 * Delete the entry only after removing all pmap
2583 		 * entries pointing to its pages.  (Otherwise, its
2584 		 * page frames may be reallocated, and any modify bits
2585 		 * will be set in the wrong object!)
2586 		 */
2587 		vm_map_entry_delete(map, entry);
2588 		entry->next = map->deferred_freelist;
2589 		map->deferred_freelist = entry;
2590 		entry = next;
2591 	}
2592 	return (KERN_SUCCESS);
2593 }
2594 
2595 /*
2596  *	vm_map_remove:
2597  *
2598  *	Remove the given address range from the target map.
2599  *	This is the exported form of vm_map_delete.
2600  */
2601 int
2602 vm_map_remove(vm_map_t map, vm_offset_t start, vm_offset_t end)
2603 {
2604 	int result;
2605 
2606 	vm_map_lock(map);
2607 	VM_MAP_RANGE_CHECK(map, start, end);
2608 	result = vm_map_delete(map, start, end);
2609 	vm_map_unlock(map);
2610 	return (result);
2611 }
2612 
2613 /*
2614  *	vm_map_check_protection:
2615  *
2616  *	Assert that the target map allows the specified privilege on the
2617  *	entire address region given.  The entire region must be allocated.
2618  *
2619  *	WARNING!  This code does not and should not check whether the
2620  *	contents of the region is accessible.  For example a smaller file
2621  *	might be mapped into a larger address space.
2622  *
2623  *	NOTE!  This code is also called by munmap().
2624  *
2625  *	The map must be locked.  A read lock is sufficient.
2626  */
2627 boolean_t
2628 vm_map_check_protection(vm_map_t map, vm_offset_t start, vm_offset_t end,
2629 			vm_prot_t protection)
2630 {
2631 	vm_map_entry_t entry;
2632 	vm_map_entry_t tmp_entry;
2633 
2634 	if (!vm_map_lookup_entry(map, start, &tmp_entry))
2635 		return (FALSE);
2636 	entry = tmp_entry;
2637 
2638 	while (start < end) {
2639 		if (entry == &map->header)
2640 			return (FALSE);
2641 		/*
2642 		 * No holes allowed!
2643 		 */
2644 		if (start < entry->start)
2645 			return (FALSE);
2646 		/*
2647 		 * Check protection associated with entry.
2648 		 */
2649 		if ((entry->protection & protection) != protection)
2650 			return (FALSE);
2651 		/* go to next entry */
2652 		start = entry->end;
2653 		entry = entry->next;
2654 	}
2655 	return (TRUE);
2656 }
2657 
2658 /*
2659  *	vm_map_copy_entry:
2660  *
2661  *	Copies the contents of the source entry to the destination
2662  *	entry.  The entries *must* be aligned properly.
2663  */
2664 static void
2665 vm_map_copy_entry(
2666 	vm_map_t src_map,
2667 	vm_map_t dst_map,
2668 	vm_map_entry_t src_entry,
2669 	vm_map_entry_t dst_entry)
2670 {
2671 	vm_object_t src_object;
2672 
2673 	VM_MAP_ASSERT_LOCKED(dst_map);
2674 
2675 	if ((dst_entry->eflags|src_entry->eflags) & MAP_ENTRY_IS_SUB_MAP)
2676 		return;
2677 
2678 	if (src_entry->wired_count == 0) {
2679 
2680 		/*
2681 		 * If the source entry is marked needs_copy, it is already
2682 		 * write-protected.
2683 		 */
2684 		if ((src_entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) {
2685 			pmap_protect(src_map->pmap,
2686 			    src_entry->start,
2687 			    src_entry->end,
2688 			    src_entry->protection & ~VM_PROT_WRITE);
2689 		}
2690 
2691 		/*
2692 		 * Make a copy of the object.
2693 		 */
2694 		if ((src_object = src_entry->object.vm_object) != NULL) {
2695 			VM_OBJECT_LOCK(src_object);
2696 			if ((src_object->handle == NULL) &&
2697 				(src_object->type == OBJT_DEFAULT ||
2698 				 src_object->type == OBJT_SWAP)) {
2699 				vm_object_collapse(src_object);
2700 				if ((src_object->flags & (OBJ_NOSPLIT|OBJ_ONEMAPPING)) == OBJ_ONEMAPPING) {
2701 					vm_object_split(src_entry);
2702 					src_object = src_entry->object.vm_object;
2703 				}
2704 			}
2705 			vm_object_reference_locked(src_object);
2706 			vm_object_clear_flag(src_object, OBJ_ONEMAPPING);
2707 			VM_OBJECT_UNLOCK(src_object);
2708 			dst_entry->object.vm_object = src_object;
2709 			src_entry->eflags |= (MAP_ENTRY_COW|MAP_ENTRY_NEEDS_COPY);
2710 			dst_entry->eflags |= (MAP_ENTRY_COW|MAP_ENTRY_NEEDS_COPY);
2711 			dst_entry->offset = src_entry->offset;
2712 		} else {
2713 			dst_entry->object.vm_object = NULL;
2714 			dst_entry->offset = 0;
2715 		}
2716 
2717 		pmap_copy(dst_map->pmap, src_map->pmap, dst_entry->start,
2718 		    dst_entry->end - dst_entry->start, src_entry->start);
2719 	} else {
2720 		/*
2721 		 * Of course, wired down pages can't be set copy-on-write.
2722 		 * Cause wired pages to be copied into the new map by
2723 		 * simulating faults (the new pages are pageable)
2724 		 */
2725 		vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry);
2726 	}
2727 }
2728 
2729 /*
2730  * vmspace_map_entry_forked:
2731  * Update the newly-forked vmspace each time a map entry is inherited
2732  * or copied.  The values for vm_dsize and vm_tsize are approximate
2733  * (and mostly-obsolete ideas in the face of mmap(2) et al.)
2734  */
2735 static void
2736 vmspace_map_entry_forked(const struct vmspace *vm1, struct vmspace *vm2,
2737     vm_map_entry_t entry)
2738 {
2739 	vm_size_t entrysize;
2740 	vm_offset_t newend;
2741 
2742 	entrysize = entry->end - entry->start;
2743 	vm2->vm_map.size += entrysize;
2744 	if (entry->eflags & (MAP_ENTRY_GROWS_DOWN | MAP_ENTRY_GROWS_UP)) {
2745 		vm2->vm_ssize += btoc(entrysize);
2746 	} else if (entry->start >= (vm_offset_t)vm1->vm_daddr &&
2747 	    entry->start < (vm_offset_t)vm1->vm_daddr + ctob(vm1->vm_dsize)) {
2748 		newend = MIN(entry->end,
2749 		    (vm_offset_t)vm1->vm_daddr + ctob(vm1->vm_dsize));
2750 		vm2->vm_dsize += btoc(newend - entry->start);
2751 	} else if (entry->start >= (vm_offset_t)vm1->vm_taddr &&
2752 	    entry->start < (vm_offset_t)vm1->vm_taddr + ctob(vm1->vm_tsize)) {
2753 		newend = MIN(entry->end,
2754 		    (vm_offset_t)vm1->vm_taddr + ctob(vm1->vm_tsize));
2755 		vm2->vm_tsize += btoc(newend - entry->start);
2756 	}
2757 }
2758 
2759 /*
2760  * vmspace_fork:
2761  * Create a new process vmspace structure and vm_map
2762  * based on those of an existing process.  The new map
2763  * is based on the old map, according to the inheritance
2764  * values on the regions in that map.
2765  *
2766  * XXX It might be worth coalescing the entries added to the new vmspace.
2767  *
2768  * The source map must not be locked.
2769  */
2770 struct vmspace *
2771 vmspace_fork(struct vmspace *vm1)
2772 {
2773 	struct vmspace *vm2;
2774 	vm_map_t old_map = &vm1->vm_map;
2775 	vm_map_t new_map;
2776 	vm_map_entry_t old_entry;
2777 	vm_map_entry_t new_entry;
2778 	vm_object_t object;
2779 	int locked;
2780 
2781 	vm_map_lock(old_map);
2782 
2783 	vm2 = vmspace_alloc(old_map->min_offset, old_map->max_offset);
2784 	if (vm2 == NULL)
2785 		goto unlock_and_return;
2786 	vm2->vm_taddr = vm1->vm_taddr;
2787 	vm2->vm_daddr = vm1->vm_daddr;
2788 	vm2->vm_maxsaddr = vm1->vm_maxsaddr;
2789 	new_map = &vm2->vm_map;	/* XXX */
2790 	locked = vm_map_trylock(new_map); /* trylock to silence WITNESS */
2791 	KASSERT(locked, ("vmspace_fork: lock failed"));
2792 	new_map->timestamp = 1;
2793 
2794 	old_entry = old_map->header.next;
2795 
2796 	while (old_entry != &old_map->header) {
2797 		if (old_entry->eflags & MAP_ENTRY_IS_SUB_MAP)
2798 			panic("vm_map_fork: encountered a submap");
2799 
2800 		switch (old_entry->inheritance) {
2801 		case VM_INHERIT_NONE:
2802 			break;
2803 
2804 		case VM_INHERIT_SHARE:
2805 			/*
2806 			 * Clone the entry, creating the shared object if necessary.
2807 			 */
2808 			object = old_entry->object.vm_object;
2809 			if (object == NULL) {
2810 				object = vm_object_allocate(OBJT_DEFAULT,
2811 					atop(old_entry->end - old_entry->start));
2812 				old_entry->object.vm_object = object;
2813 				old_entry->offset = 0;
2814 			}
2815 
2816 			/*
2817 			 * Add the reference before calling vm_object_shadow
2818 			 * to insure that a shadow object is created.
2819 			 */
2820 			vm_object_reference(object);
2821 			if (old_entry->eflags & MAP_ENTRY_NEEDS_COPY) {
2822 				vm_object_shadow(&old_entry->object.vm_object,
2823 					&old_entry->offset,
2824 					atop(old_entry->end - old_entry->start));
2825 				old_entry->eflags &= ~MAP_ENTRY_NEEDS_COPY;
2826 				/* Transfer the second reference too. */
2827 				vm_object_reference(
2828 				    old_entry->object.vm_object);
2829 
2830 				/*
2831 				 * As in vm_map_simplify_entry(), the
2832 				 * vnode lock will not be acquired in
2833 				 * this call to vm_object_deallocate().
2834 				 */
2835 				vm_object_deallocate(object);
2836 				object = old_entry->object.vm_object;
2837 			}
2838 			VM_OBJECT_LOCK(object);
2839 			vm_object_clear_flag(object, OBJ_ONEMAPPING);
2840 			VM_OBJECT_UNLOCK(object);
2841 
2842 			/*
2843 			 * Clone the entry, referencing the shared object.
2844 			 */
2845 			new_entry = vm_map_entry_create(new_map);
2846 			*new_entry = *old_entry;
2847 			new_entry->eflags &= ~(MAP_ENTRY_USER_WIRED |
2848 			    MAP_ENTRY_IN_TRANSITION);
2849 			new_entry->wired_count = 0;
2850 
2851 			/*
2852 			 * Insert the entry into the new map -- we know we're
2853 			 * inserting at the end of the new map.
2854 			 */
2855 			vm_map_entry_link(new_map, new_map->header.prev,
2856 			    new_entry);
2857 			vmspace_map_entry_forked(vm1, vm2, new_entry);
2858 
2859 			/*
2860 			 * Update the physical map
2861 			 */
2862 			pmap_copy(new_map->pmap, old_map->pmap,
2863 			    new_entry->start,
2864 			    (old_entry->end - old_entry->start),
2865 			    old_entry->start);
2866 			break;
2867 
2868 		case VM_INHERIT_COPY:
2869 			/*
2870 			 * Clone the entry and link into the map.
2871 			 */
2872 			new_entry = vm_map_entry_create(new_map);
2873 			*new_entry = *old_entry;
2874 			new_entry->eflags &= ~(MAP_ENTRY_USER_WIRED |
2875 			    MAP_ENTRY_IN_TRANSITION);
2876 			new_entry->wired_count = 0;
2877 			new_entry->object.vm_object = NULL;
2878 			vm_map_entry_link(new_map, new_map->header.prev,
2879 			    new_entry);
2880 			vmspace_map_entry_forked(vm1, vm2, new_entry);
2881 			vm_map_copy_entry(old_map, new_map, old_entry,
2882 			    new_entry);
2883 			break;
2884 		}
2885 		old_entry = old_entry->next;
2886 	}
2887 unlock_and_return:
2888 	vm_map_unlock(old_map);
2889 	if (vm2 != NULL)
2890 		vm_map_unlock(new_map);
2891 
2892 	return (vm2);
2893 }
2894 
2895 int
2896 vm_map_stack(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize,
2897     vm_prot_t prot, vm_prot_t max, int cow)
2898 {
2899 	vm_map_entry_t new_entry, prev_entry;
2900 	vm_offset_t bot, top;
2901 	vm_size_t init_ssize;
2902 	int orient, rv;
2903 	rlim_t vmemlim;
2904 
2905 	/*
2906 	 * The stack orientation is piggybacked with the cow argument.
2907 	 * Extract it into orient and mask the cow argument so that we
2908 	 * don't pass it around further.
2909 	 * NOTE: We explicitly allow bi-directional stacks.
2910 	 */
2911 	orient = cow & (MAP_STACK_GROWS_DOWN|MAP_STACK_GROWS_UP);
2912 	cow &= ~orient;
2913 	KASSERT(orient != 0, ("No stack grow direction"));
2914 
2915 	if (addrbos < vm_map_min(map) ||
2916 	    addrbos > vm_map_max(map) ||
2917 	    addrbos + max_ssize < addrbos)
2918 		return (KERN_NO_SPACE);
2919 
2920 	init_ssize = (max_ssize < sgrowsiz) ? max_ssize : sgrowsiz;
2921 
2922 	PROC_LOCK(curthread->td_proc);
2923 	vmemlim = lim_cur(curthread->td_proc, RLIMIT_VMEM);
2924 	PROC_UNLOCK(curthread->td_proc);
2925 
2926 	vm_map_lock(map);
2927 
2928 	/* If addr is already mapped, no go */
2929 	if (vm_map_lookup_entry(map, addrbos, &prev_entry)) {
2930 		vm_map_unlock(map);
2931 		return (KERN_NO_SPACE);
2932 	}
2933 
2934 	/* If we would blow our VMEM resource limit, no go */
2935 	if (map->size + init_ssize > vmemlim) {
2936 		vm_map_unlock(map);
2937 		return (KERN_NO_SPACE);
2938 	}
2939 
2940 	/*
2941 	 * If we can't accomodate max_ssize in the current mapping, no go.
2942 	 * However, we need to be aware that subsequent user mappings might
2943 	 * map into the space we have reserved for stack, and currently this
2944 	 * space is not protected.
2945 	 *
2946 	 * Hopefully we will at least detect this condition when we try to
2947 	 * grow the stack.
2948 	 */
2949 	if ((prev_entry->next != &map->header) &&
2950 	    (prev_entry->next->start < addrbos + max_ssize)) {
2951 		vm_map_unlock(map);
2952 		return (KERN_NO_SPACE);
2953 	}
2954 
2955 	/*
2956 	 * We initially map a stack of only init_ssize.  We will grow as
2957 	 * needed later.  Depending on the orientation of the stack (i.e.
2958 	 * the grow direction) we either map at the top of the range, the
2959 	 * bottom of the range or in the middle.
2960 	 *
2961 	 * Note: we would normally expect prot and max to be VM_PROT_ALL,
2962 	 * and cow to be 0.  Possibly we should eliminate these as input
2963 	 * parameters, and just pass these values here in the insert call.
2964 	 */
2965 	if (orient == MAP_STACK_GROWS_DOWN)
2966 		bot = addrbos + max_ssize - init_ssize;
2967 	else if (orient == MAP_STACK_GROWS_UP)
2968 		bot = addrbos;
2969 	else
2970 		bot = round_page(addrbos + max_ssize/2 - init_ssize/2);
2971 	top = bot + init_ssize;
2972 	rv = vm_map_insert(map, NULL, 0, bot, top, prot, max, cow);
2973 
2974 	/* Now set the avail_ssize amount. */
2975 	if (rv == KERN_SUCCESS) {
2976 		if (prev_entry != &map->header)
2977 			vm_map_clip_end(map, prev_entry, bot);
2978 		new_entry = prev_entry->next;
2979 		if (new_entry->end != top || new_entry->start != bot)
2980 			panic("Bad entry start/end for new stack entry");
2981 
2982 		new_entry->avail_ssize = max_ssize - init_ssize;
2983 		if (orient & MAP_STACK_GROWS_DOWN)
2984 			new_entry->eflags |= MAP_ENTRY_GROWS_DOWN;
2985 		if (orient & MAP_STACK_GROWS_UP)
2986 			new_entry->eflags |= MAP_ENTRY_GROWS_UP;
2987 	}
2988 
2989 	vm_map_unlock(map);
2990 	return (rv);
2991 }
2992 
2993 /* Attempts to grow a vm stack entry.  Returns KERN_SUCCESS if the
2994  * desired address is already mapped, or if we successfully grow
2995  * the stack.  Also returns KERN_SUCCESS if addr is outside the
2996  * stack range (this is strange, but preserves compatibility with
2997  * the grow function in vm_machdep.c).
2998  */
2999 int
3000 vm_map_growstack(struct proc *p, vm_offset_t addr)
3001 {
3002 	vm_map_entry_t next_entry, prev_entry;
3003 	vm_map_entry_t new_entry, stack_entry;
3004 	struct vmspace *vm = p->p_vmspace;
3005 	vm_map_t map = &vm->vm_map;
3006 	vm_offset_t end;
3007 	size_t grow_amount, max_grow;
3008 	rlim_t stacklim, vmemlim;
3009 	int is_procstack, rv;
3010 
3011 Retry:
3012 	PROC_LOCK(p);
3013 	stacklim = lim_cur(p, RLIMIT_STACK);
3014 	vmemlim = lim_cur(p, RLIMIT_VMEM);
3015 	PROC_UNLOCK(p);
3016 
3017 	vm_map_lock_read(map);
3018 
3019 	/* If addr is already in the entry range, no need to grow.*/
3020 	if (vm_map_lookup_entry(map, addr, &prev_entry)) {
3021 		vm_map_unlock_read(map);
3022 		return (KERN_SUCCESS);
3023 	}
3024 
3025 	next_entry = prev_entry->next;
3026 	if (!(prev_entry->eflags & MAP_ENTRY_GROWS_UP)) {
3027 		/*
3028 		 * This entry does not grow upwards. Since the address lies
3029 		 * beyond this entry, the next entry (if one exists) has to
3030 		 * be a downward growable entry. The entry list header is
3031 		 * never a growable entry, so it suffices to check the flags.
3032 		 */
3033 		if (!(next_entry->eflags & MAP_ENTRY_GROWS_DOWN)) {
3034 			vm_map_unlock_read(map);
3035 			return (KERN_SUCCESS);
3036 		}
3037 		stack_entry = next_entry;
3038 	} else {
3039 		/*
3040 		 * This entry grows upward. If the next entry does not at
3041 		 * least grow downwards, this is the entry we need to grow.
3042 		 * otherwise we have two possible choices and we have to
3043 		 * select one.
3044 		 */
3045 		if (next_entry->eflags & MAP_ENTRY_GROWS_DOWN) {
3046 			/*
3047 			 * We have two choices; grow the entry closest to
3048 			 * the address to minimize the amount of growth.
3049 			 */
3050 			if (addr - prev_entry->end <= next_entry->start - addr)
3051 				stack_entry = prev_entry;
3052 			else
3053 				stack_entry = next_entry;
3054 		} else
3055 			stack_entry = prev_entry;
3056 	}
3057 
3058 	if (stack_entry == next_entry) {
3059 		KASSERT(stack_entry->eflags & MAP_ENTRY_GROWS_DOWN, ("foo"));
3060 		KASSERT(addr < stack_entry->start, ("foo"));
3061 		end = (prev_entry != &map->header) ? prev_entry->end :
3062 		    stack_entry->start - stack_entry->avail_ssize;
3063 		grow_amount = roundup(stack_entry->start - addr, PAGE_SIZE);
3064 		max_grow = stack_entry->start - end;
3065 	} else {
3066 		KASSERT(stack_entry->eflags & MAP_ENTRY_GROWS_UP, ("foo"));
3067 		KASSERT(addr >= stack_entry->end, ("foo"));
3068 		end = (next_entry != &map->header) ? next_entry->start :
3069 		    stack_entry->end + stack_entry->avail_ssize;
3070 		grow_amount = roundup(addr + 1 - stack_entry->end, PAGE_SIZE);
3071 		max_grow = end - stack_entry->end;
3072 	}
3073 
3074 	if (grow_amount > stack_entry->avail_ssize) {
3075 		vm_map_unlock_read(map);
3076 		return (KERN_NO_SPACE);
3077 	}
3078 
3079 	/*
3080 	 * If there is no longer enough space between the entries nogo, and
3081 	 * adjust the available space.  Note: this  should only happen if the
3082 	 * user has mapped into the stack area after the stack was created,
3083 	 * and is probably an error.
3084 	 *
3085 	 * This also effectively destroys any guard page the user might have
3086 	 * intended by limiting the stack size.
3087 	 */
3088 	if (grow_amount > max_grow) {
3089 		if (vm_map_lock_upgrade(map))
3090 			goto Retry;
3091 
3092 		stack_entry->avail_ssize = max_grow;
3093 
3094 		vm_map_unlock(map);
3095 		return (KERN_NO_SPACE);
3096 	}
3097 
3098 	is_procstack = (addr >= (vm_offset_t)vm->vm_maxsaddr) ? 1 : 0;
3099 
3100 	/*
3101 	 * If this is the main process stack, see if we're over the stack
3102 	 * limit.
3103 	 */
3104 	if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > stacklim)) {
3105 		vm_map_unlock_read(map);
3106 		return (KERN_NO_SPACE);
3107 	}
3108 
3109 	/* Round up the grow amount modulo SGROWSIZ */
3110 	grow_amount = roundup (grow_amount, sgrowsiz);
3111 	if (grow_amount > stack_entry->avail_ssize)
3112 		grow_amount = stack_entry->avail_ssize;
3113 	if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > stacklim)) {
3114 		grow_amount = stacklim - ctob(vm->vm_ssize);
3115 	}
3116 
3117 	/* If we would blow our VMEM resource limit, no go */
3118 	if (map->size + grow_amount > vmemlim) {
3119 		vm_map_unlock_read(map);
3120 		return (KERN_NO_SPACE);
3121 	}
3122 
3123 	if (vm_map_lock_upgrade(map))
3124 		goto Retry;
3125 
3126 	if (stack_entry == next_entry) {
3127 		/*
3128 		 * Growing downward.
3129 		 */
3130 		/* Get the preliminary new entry start value */
3131 		addr = stack_entry->start - grow_amount;
3132 
3133 		/*
3134 		 * If this puts us into the previous entry, cut back our
3135 		 * growth to the available space. Also, see the note above.
3136 		 */
3137 		if (addr < end) {
3138 			stack_entry->avail_ssize = max_grow;
3139 			addr = end;
3140 		}
3141 
3142 		rv = vm_map_insert(map, NULL, 0, addr, stack_entry->start,
3143 		    p->p_sysent->sv_stackprot, VM_PROT_ALL, 0);
3144 
3145 		/* Adjust the available stack space by the amount we grew. */
3146 		if (rv == KERN_SUCCESS) {
3147 			if (prev_entry != &map->header)
3148 				vm_map_clip_end(map, prev_entry, addr);
3149 			new_entry = prev_entry->next;
3150 			KASSERT(new_entry == stack_entry->prev, ("foo"));
3151 			KASSERT(new_entry->end == stack_entry->start, ("foo"));
3152 			KASSERT(new_entry->start == addr, ("foo"));
3153 			grow_amount = new_entry->end - new_entry->start;
3154 			new_entry->avail_ssize = stack_entry->avail_ssize -
3155 			    grow_amount;
3156 			stack_entry->eflags &= ~MAP_ENTRY_GROWS_DOWN;
3157 			new_entry->eflags |= MAP_ENTRY_GROWS_DOWN;
3158 		}
3159 	} else {
3160 		/*
3161 		 * Growing upward.
3162 		 */
3163 		addr = stack_entry->end + grow_amount;
3164 
3165 		/*
3166 		 * If this puts us into the next entry, cut back our growth
3167 		 * to the available space. Also, see the note above.
3168 		 */
3169 		if (addr > end) {
3170 			stack_entry->avail_ssize = end - stack_entry->end;
3171 			addr = end;
3172 		}
3173 
3174 		grow_amount = addr - stack_entry->end;
3175 
3176 		/* Grow the underlying object if applicable. */
3177 		if (stack_entry->object.vm_object == NULL ||
3178 		    vm_object_coalesce(stack_entry->object.vm_object,
3179 		    stack_entry->offset,
3180 		    (vm_size_t)(stack_entry->end - stack_entry->start),
3181 		    (vm_size_t)grow_amount)) {
3182 			map->size += (addr - stack_entry->end);
3183 			/* Update the current entry. */
3184 			stack_entry->end = addr;
3185 			stack_entry->avail_ssize -= grow_amount;
3186 			vm_map_entry_resize_free(map, stack_entry);
3187 			rv = KERN_SUCCESS;
3188 
3189 			if (next_entry != &map->header)
3190 				vm_map_clip_start(map, next_entry, addr);
3191 		} else
3192 			rv = KERN_FAILURE;
3193 	}
3194 
3195 	if (rv == KERN_SUCCESS && is_procstack)
3196 		vm->vm_ssize += btoc(grow_amount);
3197 
3198 	vm_map_unlock(map);
3199 
3200 	/*
3201 	 * Heed the MAP_WIREFUTURE flag if it was set for this process.
3202 	 */
3203 	if (rv == KERN_SUCCESS && (map->flags & MAP_WIREFUTURE)) {
3204 		vm_map_wire(map,
3205 		    (stack_entry == next_entry) ? addr : addr - grow_amount,
3206 		    (stack_entry == next_entry) ? stack_entry->start : addr,
3207 		    (p->p_flag & P_SYSTEM)
3208 		    ? VM_MAP_WIRE_SYSTEM|VM_MAP_WIRE_NOHOLES
3209 		    : VM_MAP_WIRE_USER|VM_MAP_WIRE_NOHOLES);
3210 	}
3211 
3212 	return (rv);
3213 }
3214 
3215 /*
3216  * Unshare the specified VM space for exec.  If other processes are
3217  * mapped to it, then create a new one.  The new vmspace is null.
3218  */
3219 int
3220 vmspace_exec(struct proc *p, vm_offset_t minuser, vm_offset_t maxuser)
3221 {
3222 	struct vmspace *oldvmspace = p->p_vmspace;
3223 	struct vmspace *newvmspace;
3224 
3225 	newvmspace = vmspace_alloc(minuser, maxuser);
3226 	if (newvmspace == NULL)
3227 		return (ENOMEM);
3228 	newvmspace->vm_swrss = oldvmspace->vm_swrss;
3229 	/*
3230 	 * This code is written like this for prototype purposes.  The
3231 	 * goal is to avoid running down the vmspace here, but let the
3232 	 * other process's that are still using the vmspace to finally
3233 	 * run it down.  Even though there is little or no chance of blocking
3234 	 * here, it is a good idea to keep this form for future mods.
3235 	 */
3236 	PROC_VMSPACE_LOCK(p);
3237 	p->p_vmspace = newvmspace;
3238 	PROC_VMSPACE_UNLOCK(p);
3239 	if (p == curthread->td_proc)
3240 		pmap_activate(curthread);
3241 	vmspace_free(oldvmspace);
3242 	return (0);
3243 }
3244 
3245 /*
3246  * Unshare the specified VM space for forcing COW.  This
3247  * is called by rfork, for the (RFMEM|RFPROC) == 0 case.
3248  */
3249 int
3250 vmspace_unshare(struct proc *p)
3251 {
3252 	struct vmspace *oldvmspace = p->p_vmspace;
3253 	struct vmspace *newvmspace;
3254 
3255 	if (oldvmspace->vm_refcnt == 1)
3256 		return (0);
3257 	newvmspace = vmspace_fork(oldvmspace);
3258 	if (newvmspace == NULL)
3259 		return (ENOMEM);
3260 	PROC_VMSPACE_LOCK(p);
3261 	p->p_vmspace = newvmspace;
3262 	PROC_VMSPACE_UNLOCK(p);
3263 	if (p == curthread->td_proc)
3264 		pmap_activate(curthread);
3265 	vmspace_free(oldvmspace);
3266 	return (0);
3267 }
3268 
3269 /*
3270  *	vm_map_lookup:
3271  *
3272  *	Finds the VM object, offset, and
3273  *	protection for a given virtual address in the
3274  *	specified map, assuming a page fault of the
3275  *	type specified.
3276  *
3277  *	Leaves the map in question locked for read; return
3278  *	values are guaranteed until a vm_map_lookup_done
3279  *	call is performed.  Note that the map argument
3280  *	is in/out; the returned map must be used in
3281  *	the call to vm_map_lookup_done.
3282  *
3283  *	A handle (out_entry) is returned for use in
3284  *	vm_map_lookup_done, to make that fast.
3285  *
3286  *	If a lookup is requested with "write protection"
3287  *	specified, the map may be changed to perform virtual
3288  *	copying operations, although the data referenced will
3289  *	remain the same.
3290  */
3291 int
3292 vm_map_lookup(vm_map_t *var_map,		/* IN/OUT */
3293 	      vm_offset_t vaddr,
3294 	      vm_prot_t fault_typea,
3295 	      vm_map_entry_t *out_entry,	/* OUT */
3296 	      vm_object_t *object,		/* OUT */
3297 	      vm_pindex_t *pindex,		/* OUT */
3298 	      vm_prot_t *out_prot,		/* OUT */
3299 	      boolean_t *wired)			/* OUT */
3300 {
3301 	vm_map_entry_t entry;
3302 	vm_map_t map = *var_map;
3303 	vm_prot_t prot;
3304 	vm_prot_t fault_type = fault_typea;
3305 
3306 RetryLookup:;
3307 
3308 	vm_map_lock_read(map);
3309 
3310 	/*
3311 	 * Lookup the faulting address.
3312 	 */
3313 	if (!vm_map_lookup_entry(map, vaddr, out_entry)) {
3314 		vm_map_unlock_read(map);
3315 		return (KERN_INVALID_ADDRESS);
3316 	}
3317 
3318 	entry = *out_entry;
3319 
3320 	/*
3321 	 * Handle submaps.
3322 	 */
3323 	if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
3324 		vm_map_t old_map = map;
3325 
3326 		*var_map = map = entry->object.sub_map;
3327 		vm_map_unlock_read(old_map);
3328 		goto RetryLookup;
3329 	}
3330 
3331 	/*
3332 	 * Check whether this task is allowed to have this page.
3333 	 * Note the special case for MAP_ENTRY_COW
3334 	 * pages with an override.  This is to implement a forced
3335 	 * COW for debuggers.
3336 	 */
3337 	if (fault_type & VM_PROT_OVERRIDE_WRITE)
3338 		prot = entry->max_protection;
3339 	else
3340 		prot = entry->protection;
3341 	fault_type &= (VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE);
3342 	if ((fault_type & prot) != fault_type) {
3343 		vm_map_unlock_read(map);
3344 		return (KERN_PROTECTION_FAILURE);
3345 	}
3346 	if ((entry->eflags & MAP_ENTRY_USER_WIRED) &&
3347 	    (entry->eflags & MAP_ENTRY_COW) &&
3348 	    (fault_type & VM_PROT_WRITE) &&
3349 	    (fault_typea & VM_PROT_OVERRIDE_WRITE) == 0) {
3350 		vm_map_unlock_read(map);
3351 		return (KERN_PROTECTION_FAILURE);
3352 	}
3353 
3354 	/*
3355 	 * If this page is not pageable, we have to get it for all possible
3356 	 * accesses.
3357 	 */
3358 	*wired = (entry->wired_count != 0);
3359 	if (*wired)
3360 		prot = fault_type = entry->protection;
3361 
3362 	/*
3363 	 * If the entry was copy-on-write, we either ...
3364 	 */
3365 	if (entry->eflags & MAP_ENTRY_NEEDS_COPY) {
3366 		/*
3367 		 * If we want to write the page, we may as well handle that
3368 		 * now since we've got the map locked.
3369 		 *
3370 		 * If we don't need to write the page, we just demote the
3371 		 * permissions allowed.
3372 		 */
3373 		if (fault_type & VM_PROT_WRITE) {
3374 			/*
3375 			 * Make a new object, and place it in the object
3376 			 * chain.  Note that no new references have appeared
3377 			 * -- one just moved from the map to the new
3378 			 * object.
3379 			 */
3380 			if (vm_map_lock_upgrade(map))
3381 				goto RetryLookup;
3382 
3383 			vm_object_shadow(
3384 			    &entry->object.vm_object,
3385 			    &entry->offset,
3386 			    atop(entry->end - entry->start));
3387 			entry->eflags &= ~MAP_ENTRY_NEEDS_COPY;
3388 
3389 			vm_map_lock_downgrade(map);
3390 		} else {
3391 			/*
3392 			 * We're attempting to read a copy-on-write page --
3393 			 * don't allow writes.
3394 			 */
3395 			prot &= ~VM_PROT_WRITE;
3396 		}
3397 	}
3398 
3399 	/*
3400 	 * Create an object if necessary.
3401 	 */
3402 	if (entry->object.vm_object == NULL &&
3403 	    !map->system_map) {
3404 		if (vm_map_lock_upgrade(map))
3405 			goto RetryLookup;
3406 		entry->object.vm_object = vm_object_allocate(OBJT_DEFAULT,
3407 		    atop(entry->end - entry->start));
3408 		entry->offset = 0;
3409 		vm_map_lock_downgrade(map);
3410 	}
3411 
3412 	/*
3413 	 * Return the object/offset from this entry.  If the entry was
3414 	 * copy-on-write or empty, it has been fixed up.
3415 	 */
3416 	*pindex = OFF_TO_IDX((vaddr - entry->start) + entry->offset);
3417 	*object = entry->object.vm_object;
3418 
3419 	*out_prot = prot;
3420 	return (KERN_SUCCESS);
3421 }
3422 
3423 /*
3424  *	vm_map_lookup_locked:
3425  *
3426  *	Lookup the faulting address.  A version of vm_map_lookup that returns
3427  *      KERN_FAILURE instead of blocking on map lock or memory allocation.
3428  */
3429 int
3430 vm_map_lookup_locked(vm_map_t *var_map,		/* IN/OUT */
3431 		     vm_offset_t vaddr,
3432 		     vm_prot_t fault_typea,
3433 		     vm_map_entry_t *out_entry,	/* OUT */
3434 		     vm_object_t *object,	/* OUT */
3435 		     vm_pindex_t *pindex,	/* OUT */
3436 		     vm_prot_t *out_prot,	/* OUT */
3437 		     boolean_t *wired)		/* OUT */
3438 {
3439 	vm_map_entry_t entry;
3440 	vm_map_t map = *var_map;
3441 	vm_prot_t prot;
3442 	vm_prot_t fault_type = fault_typea;
3443 
3444 	/*
3445 	 * Lookup the faulting address.
3446 	 */
3447 	if (!vm_map_lookup_entry(map, vaddr, out_entry))
3448 		return (KERN_INVALID_ADDRESS);
3449 
3450 	entry = *out_entry;
3451 
3452 	/*
3453 	 * Fail if the entry refers to a submap.
3454 	 */
3455 	if (entry->eflags & MAP_ENTRY_IS_SUB_MAP)
3456 		return (KERN_FAILURE);
3457 
3458 	/*
3459 	 * Check whether this task is allowed to have this page.
3460 	 * Note the special case for MAP_ENTRY_COW
3461 	 * pages with an override.  This is to implement a forced
3462 	 * COW for debuggers.
3463 	 */
3464 	if (fault_type & VM_PROT_OVERRIDE_WRITE)
3465 		prot = entry->max_protection;
3466 	else
3467 		prot = entry->protection;
3468 	fault_type &= VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE;
3469 	if ((fault_type & prot) != fault_type)
3470 		return (KERN_PROTECTION_FAILURE);
3471 	if ((entry->eflags & MAP_ENTRY_USER_WIRED) &&
3472 	    (entry->eflags & MAP_ENTRY_COW) &&
3473 	    (fault_type & VM_PROT_WRITE) &&
3474 	    (fault_typea & VM_PROT_OVERRIDE_WRITE) == 0)
3475 		return (KERN_PROTECTION_FAILURE);
3476 
3477 	/*
3478 	 * If this page is not pageable, we have to get it for all possible
3479 	 * accesses.
3480 	 */
3481 	*wired = (entry->wired_count != 0);
3482 	if (*wired)
3483 		prot = fault_type = entry->protection;
3484 
3485 	if (entry->eflags & MAP_ENTRY_NEEDS_COPY) {
3486 		/*
3487 		 * Fail if the entry was copy-on-write for a write fault.
3488 		 */
3489 		if (fault_type & VM_PROT_WRITE)
3490 			return (KERN_FAILURE);
3491 		/*
3492 		 * We're attempting to read a copy-on-write page --
3493 		 * don't allow writes.
3494 		 */
3495 		prot &= ~VM_PROT_WRITE;
3496 	}
3497 
3498 	/*
3499 	 * Fail if an object should be created.
3500 	 */
3501 	if (entry->object.vm_object == NULL && !map->system_map)
3502 		return (KERN_FAILURE);
3503 
3504 	/*
3505 	 * Return the object/offset from this entry.  If the entry was
3506 	 * copy-on-write or empty, it has been fixed up.
3507 	 */
3508 	*pindex = OFF_TO_IDX((vaddr - entry->start) + entry->offset);
3509 	*object = entry->object.vm_object;
3510 
3511 	*out_prot = prot;
3512 	return (KERN_SUCCESS);
3513 }
3514 
3515 /*
3516  *	vm_map_lookup_done:
3517  *
3518  *	Releases locks acquired by a vm_map_lookup
3519  *	(according to the handle returned by that lookup).
3520  */
3521 void
3522 vm_map_lookup_done(vm_map_t map, vm_map_entry_t entry)
3523 {
3524 	/*
3525 	 * Unlock the main-level map
3526 	 */
3527 	vm_map_unlock_read(map);
3528 }
3529 
3530 #include "opt_ddb.h"
3531 #ifdef DDB
3532 #include <sys/kernel.h>
3533 
3534 #include <ddb/ddb.h>
3535 
3536 /*
3537  *	vm_map_print:	[ debug ]
3538  */
3539 DB_SHOW_COMMAND(map, vm_map_print)
3540 {
3541 	static int nlines;
3542 	/* XXX convert args. */
3543 	vm_map_t map = (vm_map_t)addr;
3544 	boolean_t full = have_addr;
3545 
3546 	vm_map_entry_t entry;
3547 
3548 	db_iprintf("Task map %p: pmap=%p, nentries=%d, version=%u\n",
3549 	    (void *)map,
3550 	    (void *)map->pmap, map->nentries, map->timestamp);
3551 	nlines++;
3552 
3553 	if (!full && db_indent)
3554 		return;
3555 
3556 	db_indent += 2;
3557 	for (entry = map->header.next; entry != &map->header;
3558 	    entry = entry->next) {
3559 		db_iprintf("map entry %p: start=%p, end=%p\n",
3560 		    (void *)entry, (void *)entry->start, (void *)entry->end);
3561 		nlines++;
3562 		{
3563 			static char *inheritance_name[4] =
3564 			{"share", "copy", "none", "donate_copy"};
3565 
3566 			db_iprintf(" prot=%x/%x/%s",
3567 			    entry->protection,
3568 			    entry->max_protection,
3569 			    inheritance_name[(int)(unsigned char)entry->inheritance]);
3570 			if (entry->wired_count != 0)
3571 				db_printf(", wired");
3572 		}
3573 		if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
3574 			db_printf(", share=%p, offset=0x%jx\n",
3575 			    (void *)entry->object.sub_map,
3576 			    (uintmax_t)entry->offset);
3577 			nlines++;
3578 			if ((entry->prev == &map->header) ||
3579 			    (entry->prev->object.sub_map !=
3580 				entry->object.sub_map)) {
3581 				db_indent += 2;
3582 				vm_map_print((db_expr_t)(intptr_t)
3583 					     entry->object.sub_map,
3584 					     full, 0, (char *)0);
3585 				db_indent -= 2;
3586 			}
3587 		} else {
3588 			db_printf(", object=%p, offset=0x%jx",
3589 			    (void *)entry->object.vm_object,
3590 			    (uintmax_t)entry->offset);
3591 			if (entry->eflags & MAP_ENTRY_COW)
3592 				db_printf(", copy (%s)",
3593 				    (entry->eflags & MAP_ENTRY_NEEDS_COPY) ? "needed" : "done");
3594 			db_printf("\n");
3595 			nlines++;
3596 
3597 			if ((entry->prev == &map->header) ||
3598 			    (entry->prev->object.vm_object !=
3599 				entry->object.vm_object)) {
3600 				db_indent += 2;
3601 				vm_object_print((db_expr_t)(intptr_t)
3602 						entry->object.vm_object,
3603 						full, 0, (char *)0);
3604 				nlines += 4;
3605 				db_indent -= 2;
3606 			}
3607 		}
3608 	}
3609 	db_indent -= 2;
3610 	if (db_indent == 0)
3611 		nlines = 0;
3612 }
3613 
3614 
3615 DB_SHOW_COMMAND(procvm, procvm)
3616 {
3617 	struct proc *p;
3618 
3619 	if (have_addr) {
3620 		p = (struct proc *) addr;
3621 	} else {
3622 		p = curproc;
3623 	}
3624 
3625 	db_printf("p = %p, vmspace = %p, map = %p, pmap = %p\n",
3626 	    (void *)p, (void *)p->p_vmspace, (void *)&p->p_vmspace->vm_map,
3627 	    (void *)vmspace_pmap(p->p_vmspace));
3628 
3629 	vm_map_print((db_expr_t)(intptr_t)&p->p_vmspace->vm_map, 1, 0, NULL);
3630 }
3631 
3632 #endif /* DDB */
3633