xref: /freebsd/sys/vm/vm_map.c (revision b4e38a41f584ad4391c04b8cfec81f46176b18b0)
1 /*-
2  * SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU)
3  *
4  * Copyright (c) 1991, 1993
5  *	The Regents of the University of California.  All rights reserved.
6  *
7  * This code is derived from software contributed to Berkeley by
8  * The Mach Operating System project at Carnegie-Mellon University.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 3. Neither the name of the University nor the names of its contributors
19  *    may be used to endorse or promote products derived from this software
20  *    without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  *	from: @(#)vm_map.c	8.3 (Berkeley) 1/12/94
35  *
36  *
37  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
38  * All rights reserved.
39  *
40  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
41  *
42  * Permission to use, copy, modify and distribute this software and
43  * its documentation is hereby granted, provided that both the copyright
44  * notice and this permission notice appear in all copies of the
45  * software, derivative works or modified versions, and any portions
46  * thereof, and that both notices appear in supporting documentation.
47  *
48  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
49  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
50  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
51  *
52  * Carnegie Mellon requests users of this software to return to
53  *
54  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
55  *  School of Computer Science
56  *  Carnegie Mellon University
57  *  Pittsburgh PA 15213-3890
58  *
59  * any improvements or extensions that they make and grant Carnegie the
60  * rights to redistribute these changes.
61  */
62 
63 /*
64  *	Virtual memory mapping module.
65  */
66 
67 #include <sys/cdefs.h>
68 __FBSDID("$FreeBSD$");
69 
70 #include <sys/param.h>
71 #include <sys/systm.h>
72 #include <sys/elf.h>
73 #include <sys/kernel.h>
74 #include <sys/ktr.h>
75 #include <sys/lock.h>
76 #include <sys/mutex.h>
77 #include <sys/proc.h>
78 #include <sys/vmmeter.h>
79 #include <sys/mman.h>
80 #include <sys/vnode.h>
81 #include <sys/racct.h>
82 #include <sys/resourcevar.h>
83 #include <sys/rwlock.h>
84 #include <sys/file.h>
85 #include <sys/sysctl.h>
86 #include <sys/sysent.h>
87 #include <sys/shm.h>
88 
89 #include <vm/vm.h>
90 #include <vm/vm_param.h>
91 #include <vm/pmap.h>
92 #include <vm/vm_map.h>
93 #include <vm/vm_page.h>
94 #include <vm/vm_pageout.h>
95 #include <vm/vm_object.h>
96 #include <vm/vm_pager.h>
97 #include <vm/vm_kern.h>
98 #include <vm/vm_extern.h>
99 #include <vm/vnode_pager.h>
100 #include <vm/swap_pager.h>
101 #include <vm/uma.h>
102 
103 /*
104  *	Virtual memory maps provide for the mapping, protection,
105  *	and sharing of virtual memory objects.  In addition,
106  *	this module provides for an efficient virtual copy of
107  *	memory from one map to another.
108  *
109  *	Synchronization is required prior to most operations.
110  *
111  *	Maps consist of an ordered doubly-linked list of simple
112  *	entries; a self-adjusting binary search tree of these
113  *	entries is used to speed up lookups.
114  *
115  *	Since portions of maps are specified by start/end addresses,
116  *	which may not align with existing map entries, all
117  *	routines merely "clip" entries to these start/end values.
118  *	[That is, an entry is split into two, bordering at a
119  *	start or end value.]  Note that these clippings may not
120  *	always be necessary (as the two resulting entries are then
121  *	not changed); however, the clipping is done for convenience.
122  *
123  *	As mentioned above, virtual copy operations are performed
124  *	by copying VM object references from one map to
125  *	another, and then marking both regions as copy-on-write.
126  */
127 
128 static struct mtx map_sleep_mtx;
129 static uma_zone_t mapentzone;
130 static uma_zone_t kmapentzone;
131 static uma_zone_t mapzone;
132 static uma_zone_t vmspace_zone;
133 static int vmspace_zinit(void *mem, int size, int flags);
134 static int vm_map_zinit(void *mem, int ize, int flags);
135 static void _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min,
136     vm_offset_t max);
137 static void vm_map_entry_deallocate(vm_map_entry_t entry, boolean_t system_map);
138 static void vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry);
139 static void vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry);
140 static int vm_map_growstack(vm_map_t map, vm_offset_t addr,
141     vm_map_entry_t gap_entry);
142 static void vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot,
143     vm_object_t object, vm_pindex_t pindex, vm_size_t size, int flags);
144 #ifdef INVARIANTS
145 static void vm_map_zdtor(void *mem, int size, void *arg);
146 static void vmspace_zdtor(void *mem, int size, void *arg);
147 #endif
148 static int vm_map_stack_locked(vm_map_t map, vm_offset_t addrbos,
149     vm_size_t max_ssize, vm_size_t growsize, vm_prot_t prot, vm_prot_t max,
150     int cow);
151 static void vm_map_wire_entry_failure(vm_map_t map, vm_map_entry_t entry,
152     vm_offset_t failed_addr);
153 
154 #define	ENTRY_CHARGED(e) ((e)->cred != NULL || \
155     ((e)->object.vm_object != NULL && (e)->object.vm_object->cred != NULL && \
156      !((e)->eflags & MAP_ENTRY_NEEDS_COPY)))
157 
158 /*
159  * PROC_VMSPACE_{UN,}LOCK() can be a noop as long as vmspaces are type
160  * stable.
161  */
162 #define PROC_VMSPACE_LOCK(p) do { } while (0)
163 #define PROC_VMSPACE_UNLOCK(p) do { } while (0)
164 
165 /*
166  *	VM_MAP_RANGE_CHECK:	[ internal use only ]
167  *
168  *	Asserts that the starting and ending region
169  *	addresses fall within the valid range of the map.
170  */
171 #define	VM_MAP_RANGE_CHECK(map, start, end)		\
172 		{					\
173 		if (start < vm_map_min(map))		\
174 			start = vm_map_min(map);	\
175 		if (end > vm_map_max(map))		\
176 			end = vm_map_max(map);		\
177 		if (start > end)			\
178 			start = end;			\
179 		}
180 
181 /*
182  *	vm_map_startup:
183  *
184  *	Initialize the vm_map module.  Must be called before
185  *	any other vm_map routines.
186  *
187  *	Map and entry structures are allocated from the general
188  *	purpose memory pool with some exceptions:
189  *
190  *	- The kernel map and kmem submap are allocated statically.
191  *	- Kernel map entries are allocated out of a static pool.
192  *
193  *	These restrictions are necessary since malloc() uses the
194  *	maps and requires map entries.
195  */
196 
197 void
198 vm_map_startup(void)
199 {
200 	mtx_init(&map_sleep_mtx, "vm map sleep mutex", NULL, MTX_DEF);
201 	mapzone = uma_zcreate("MAP", sizeof(struct vm_map), NULL,
202 #ifdef INVARIANTS
203 	    vm_map_zdtor,
204 #else
205 	    NULL,
206 #endif
207 	    vm_map_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
208 	uma_prealloc(mapzone, MAX_KMAP);
209 	kmapentzone = uma_zcreate("KMAP ENTRY", sizeof(struct vm_map_entry),
210 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
211 	    UMA_ZONE_MTXCLASS | UMA_ZONE_VM);
212 	mapentzone = uma_zcreate("MAP ENTRY", sizeof(struct vm_map_entry),
213 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
214 	vmspace_zone = uma_zcreate("VMSPACE", sizeof(struct vmspace), NULL,
215 #ifdef INVARIANTS
216 	    vmspace_zdtor,
217 #else
218 	    NULL,
219 #endif
220 	    vmspace_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
221 }
222 
223 static int
224 vmspace_zinit(void *mem, int size, int flags)
225 {
226 	struct vmspace *vm;
227 
228 	vm = (struct vmspace *)mem;
229 
230 	vm->vm_map.pmap = NULL;
231 	(void)vm_map_zinit(&vm->vm_map, sizeof(vm->vm_map), flags);
232 	PMAP_LOCK_INIT(vmspace_pmap(vm));
233 	return (0);
234 }
235 
236 static int
237 vm_map_zinit(void *mem, int size, int flags)
238 {
239 	vm_map_t map;
240 
241 	map = (vm_map_t)mem;
242 	memset(map, 0, sizeof(*map));
243 	mtx_init(&map->system_mtx, "vm map (system)", NULL, MTX_DEF | MTX_DUPOK);
244 	sx_init(&map->lock, "vm map (user)");
245 	return (0);
246 }
247 
248 #ifdef INVARIANTS
249 static void
250 vmspace_zdtor(void *mem, int size, void *arg)
251 {
252 	struct vmspace *vm;
253 
254 	vm = (struct vmspace *)mem;
255 
256 	vm_map_zdtor(&vm->vm_map, sizeof(vm->vm_map), arg);
257 }
258 static void
259 vm_map_zdtor(void *mem, int size, void *arg)
260 {
261 	vm_map_t map;
262 
263 	map = (vm_map_t)mem;
264 	KASSERT(map->nentries == 0,
265 	    ("map %p nentries == %d on free.",
266 	    map, map->nentries));
267 	KASSERT(map->size == 0,
268 	    ("map %p size == %lu on free.",
269 	    map, (unsigned long)map->size));
270 }
271 #endif	/* INVARIANTS */
272 
273 /*
274  * Allocate a vmspace structure, including a vm_map and pmap,
275  * and initialize those structures.  The refcnt is set to 1.
276  *
277  * If 'pinit' is NULL then the embedded pmap is initialized via pmap_pinit().
278  */
279 struct vmspace *
280 vmspace_alloc(vm_offset_t min, vm_offset_t max, pmap_pinit_t pinit)
281 {
282 	struct vmspace *vm;
283 
284 	vm = uma_zalloc(vmspace_zone, M_WAITOK);
285 	KASSERT(vm->vm_map.pmap == NULL, ("vm_map.pmap must be NULL"));
286 	if (!pinit(vmspace_pmap(vm))) {
287 		uma_zfree(vmspace_zone, vm);
288 		return (NULL);
289 	}
290 	CTR1(KTR_VM, "vmspace_alloc: %p", vm);
291 	_vm_map_init(&vm->vm_map, vmspace_pmap(vm), min, max);
292 	vm->vm_refcnt = 1;
293 	vm->vm_shm = NULL;
294 	vm->vm_swrss = 0;
295 	vm->vm_tsize = 0;
296 	vm->vm_dsize = 0;
297 	vm->vm_ssize = 0;
298 	vm->vm_taddr = 0;
299 	vm->vm_daddr = 0;
300 	vm->vm_maxsaddr = 0;
301 	return (vm);
302 }
303 
304 #ifdef RACCT
305 static void
306 vmspace_container_reset(struct proc *p)
307 {
308 
309 	PROC_LOCK(p);
310 	racct_set(p, RACCT_DATA, 0);
311 	racct_set(p, RACCT_STACK, 0);
312 	racct_set(p, RACCT_RSS, 0);
313 	racct_set(p, RACCT_MEMLOCK, 0);
314 	racct_set(p, RACCT_VMEM, 0);
315 	PROC_UNLOCK(p);
316 }
317 #endif
318 
319 static inline void
320 vmspace_dofree(struct vmspace *vm)
321 {
322 
323 	CTR1(KTR_VM, "vmspace_free: %p", vm);
324 
325 	/*
326 	 * Make sure any SysV shm is freed, it might not have been in
327 	 * exit1().
328 	 */
329 	shmexit(vm);
330 
331 	/*
332 	 * Lock the map, to wait out all other references to it.
333 	 * Delete all of the mappings and pages they hold, then call
334 	 * the pmap module to reclaim anything left.
335 	 */
336 	(void)vm_map_remove(&vm->vm_map, vm_map_min(&vm->vm_map),
337 	    vm_map_max(&vm->vm_map));
338 
339 	pmap_release(vmspace_pmap(vm));
340 	vm->vm_map.pmap = NULL;
341 	uma_zfree(vmspace_zone, vm);
342 }
343 
344 void
345 vmspace_free(struct vmspace *vm)
346 {
347 
348 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
349 	    "vmspace_free() called");
350 
351 	if (vm->vm_refcnt == 0)
352 		panic("vmspace_free: attempt to free already freed vmspace");
353 
354 	if (atomic_fetchadd_int(&vm->vm_refcnt, -1) == 1)
355 		vmspace_dofree(vm);
356 }
357 
358 void
359 vmspace_exitfree(struct proc *p)
360 {
361 	struct vmspace *vm;
362 
363 	PROC_VMSPACE_LOCK(p);
364 	vm = p->p_vmspace;
365 	p->p_vmspace = NULL;
366 	PROC_VMSPACE_UNLOCK(p);
367 	KASSERT(vm == &vmspace0, ("vmspace_exitfree: wrong vmspace"));
368 	vmspace_free(vm);
369 }
370 
371 void
372 vmspace_exit(struct thread *td)
373 {
374 	int refcnt;
375 	struct vmspace *vm;
376 	struct proc *p;
377 
378 	/*
379 	 * Release user portion of address space.
380 	 * This releases references to vnodes,
381 	 * which could cause I/O if the file has been unlinked.
382 	 * Need to do this early enough that we can still sleep.
383 	 *
384 	 * The last exiting process to reach this point releases as
385 	 * much of the environment as it can. vmspace_dofree() is the
386 	 * slower fallback in case another process had a temporary
387 	 * reference to the vmspace.
388 	 */
389 
390 	p = td->td_proc;
391 	vm = p->p_vmspace;
392 	atomic_add_int(&vmspace0.vm_refcnt, 1);
393 	refcnt = vm->vm_refcnt;
394 	do {
395 		if (refcnt > 1 && p->p_vmspace != &vmspace0) {
396 			/* Switch now since other proc might free vmspace */
397 			PROC_VMSPACE_LOCK(p);
398 			p->p_vmspace = &vmspace0;
399 			PROC_VMSPACE_UNLOCK(p);
400 			pmap_activate(td);
401 		}
402 	} while (!atomic_fcmpset_int(&vm->vm_refcnt, &refcnt, refcnt - 1));
403 	if (refcnt == 1) {
404 		if (p->p_vmspace != vm) {
405 			/* vmspace not yet freed, switch back */
406 			PROC_VMSPACE_LOCK(p);
407 			p->p_vmspace = vm;
408 			PROC_VMSPACE_UNLOCK(p);
409 			pmap_activate(td);
410 		}
411 		pmap_remove_pages(vmspace_pmap(vm));
412 		/* Switch now since this proc will free vmspace */
413 		PROC_VMSPACE_LOCK(p);
414 		p->p_vmspace = &vmspace0;
415 		PROC_VMSPACE_UNLOCK(p);
416 		pmap_activate(td);
417 		vmspace_dofree(vm);
418 	}
419 #ifdef RACCT
420 	if (racct_enable)
421 		vmspace_container_reset(p);
422 #endif
423 }
424 
425 /* Acquire reference to vmspace owned by another process. */
426 
427 struct vmspace *
428 vmspace_acquire_ref(struct proc *p)
429 {
430 	struct vmspace *vm;
431 	int refcnt;
432 
433 	PROC_VMSPACE_LOCK(p);
434 	vm = p->p_vmspace;
435 	if (vm == NULL) {
436 		PROC_VMSPACE_UNLOCK(p);
437 		return (NULL);
438 	}
439 	refcnt = vm->vm_refcnt;
440 	do {
441 		if (refcnt <= 0) { 	/* Avoid 0->1 transition */
442 			PROC_VMSPACE_UNLOCK(p);
443 			return (NULL);
444 		}
445 	} while (!atomic_fcmpset_int(&vm->vm_refcnt, &refcnt, refcnt + 1));
446 	if (vm != p->p_vmspace) {
447 		PROC_VMSPACE_UNLOCK(p);
448 		vmspace_free(vm);
449 		return (NULL);
450 	}
451 	PROC_VMSPACE_UNLOCK(p);
452 	return (vm);
453 }
454 
455 /*
456  * Switch between vmspaces in an AIO kernel process.
457  *
458  * The new vmspace is either the vmspace of a user process obtained
459  * from an active AIO request or the initial vmspace of the AIO kernel
460  * process (when it is idling).  Because user processes will block to
461  * drain any active AIO requests before proceeding in exit() or
462  * execve(), the reference count for vmspaces from AIO requests can
463  * never be 0.  Similarly, AIO kernel processes hold an extra
464  * reference on their initial vmspace for the life of the process.  As
465  * a result, the 'newvm' vmspace always has a non-zero reference
466  * count.  This permits an additional reference on 'newvm' to be
467  * acquired via a simple atomic increment rather than the loop in
468  * vmspace_acquire_ref() above.
469  */
470 void
471 vmspace_switch_aio(struct vmspace *newvm)
472 {
473 	struct vmspace *oldvm;
474 
475 	/* XXX: Need some way to assert that this is an aio daemon. */
476 
477 	KASSERT(newvm->vm_refcnt > 0,
478 	    ("vmspace_switch_aio: newvm unreferenced"));
479 
480 	oldvm = curproc->p_vmspace;
481 	if (oldvm == newvm)
482 		return;
483 
484 	/*
485 	 * Point to the new address space and refer to it.
486 	 */
487 	curproc->p_vmspace = newvm;
488 	atomic_add_int(&newvm->vm_refcnt, 1);
489 
490 	/* Activate the new mapping. */
491 	pmap_activate(curthread);
492 
493 	vmspace_free(oldvm);
494 }
495 
496 void
497 _vm_map_lock(vm_map_t map, const char *file, int line)
498 {
499 
500 	if (map->system_map)
501 		mtx_lock_flags_(&map->system_mtx, 0, file, line);
502 	else
503 		sx_xlock_(&map->lock, file, line);
504 	map->timestamp++;
505 }
506 
507 void
508 vm_map_entry_set_vnode_text(vm_map_entry_t entry, bool add)
509 {
510 	vm_object_t object;
511 	struct vnode *vp;
512 	bool vp_held;
513 
514 	if ((entry->eflags & MAP_ENTRY_VN_EXEC) == 0)
515 		return;
516 	KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0,
517 	    ("Submap with execs"));
518 	object = entry->object.vm_object;
519 	KASSERT(object != NULL, ("No object for text, entry %p", entry));
520 	if ((object->flags & OBJ_ANON) != 0)
521 		object = object->handle;
522 	else
523 		KASSERT(object->backing_object == NULL,
524 		    ("non-anon object %p shadows", object));
525 	KASSERT(object != NULL, ("No content object for text, entry %p obj %p",
526 	    entry, entry->object.vm_object));
527 
528 	/*
529 	 * Mostly, we do not lock the backing object.  It is
530 	 * referenced by the entry we are processing, so it cannot go
531 	 * away.
532 	 */
533 	vp = NULL;
534 	vp_held = false;
535 	if (object->type == OBJT_DEAD) {
536 		/*
537 		 * For OBJT_DEAD objects, v_writecount was handled in
538 		 * vnode_pager_dealloc().
539 		 */
540 	} else if (object->type == OBJT_VNODE) {
541 		vp = object->handle;
542 	} else if (object->type == OBJT_SWAP) {
543 		KASSERT((object->flags & OBJ_TMPFS_NODE) != 0,
544 		    ("vm_map_entry_set_vnode_text: swap and !TMPFS "
545 		    "entry %p, object %p, add %d", entry, object, add));
546 		/*
547 		 * Tmpfs VREG node, which was reclaimed, has
548 		 * OBJ_TMPFS_NODE flag set, but not OBJ_TMPFS.  In
549 		 * this case there is no v_writecount to adjust.
550 		 */
551 		VM_OBJECT_RLOCK(object);
552 		if ((object->flags & OBJ_TMPFS) != 0) {
553 			vp = object->un_pager.swp.swp_tmpfs;
554 			if (vp != NULL) {
555 				vhold(vp);
556 				vp_held = true;
557 			}
558 		}
559 		VM_OBJECT_RUNLOCK(object);
560 	} else {
561 		KASSERT(0,
562 		    ("vm_map_entry_set_vnode_text: wrong object type, "
563 		    "entry %p, object %p, add %d", entry, object, add));
564 	}
565 	if (vp != NULL) {
566 		if (add) {
567 			VOP_SET_TEXT_CHECKED(vp);
568 		} else {
569 			vn_lock(vp, LK_SHARED | LK_RETRY);
570 			VOP_UNSET_TEXT_CHECKED(vp);
571 			VOP_UNLOCK(vp);
572 		}
573 		if (vp_held)
574 			vdrop(vp);
575 	}
576 }
577 
578 /*
579  * Use a different name for this vm_map_entry field when it's use
580  * is not consistent with its use as part of an ordered search tree.
581  */
582 #define defer_next right
583 
584 static void
585 vm_map_process_deferred(void)
586 {
587 	struct thread *td;
588 	vm_map_entry_t entry, next;
589 	vm_object_t object;
590 
591 	td = curthread;
592 	entry = td->td_map_def_user;
593 	td->td_map_def_user = NULL;
594 	while (entry != NULL) {
595 		next = entry->defer_next;
596 		MPASS((entry->eflags & (MAP_ENTRY_WRITECNT |
597 		    MAP_ENTRY_VN_EXEC)) != (MAP_ENTRY_WRITECNT |
598 		    MAP_ENTRY_VN_EXEC));
599 		if ((entry->eflags & MAP_ENTRY_WRITECNT) != 0) {
600 			/*
601 			 * Decrement the object's writemappings and
602 			 * possibly the vnode's v_writecount.
603 			 */
604 			KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0,
605 			    ("Submap with writecount"));
606 			object = entry->object.vm_object;
607 			KASSERT(object != NULL, ("No object for writecount"));
608 			vm_pager_release_writecount(object, entry->start,
609 			    entry->end);
610 		}
611 		vm_map_entry_set_vnode_text(entry, false);
612 		vm_map_entry_deallocate(entry, FALSE);
613 		entry = next;
614 	}
615 }
616 
617 #ifdef INVARIANTS
618 static void
619 _vm_map_assert_locked(vm_map_t map, const char *file, int line)
620 {
621 
622 	if (map->system_map)
623 		mtx_assert_(&map->system_mtx, MA_OWNED, file, line);
624 	else
625 		sx_assert_(&map->lock, SA_XLOCKED, file, line);
626 }
627 
628 #define	VM_MAP_ASSERT_LOCKED(map) \
629     _vm_map_assert_locked(map, LOCK_FILE, LOCK_LINE)
630 
631 enum { VMMAP_CHECK_NONE, VMMAP_CHECK_UNLOCK, VMMAP_CHECK_ALL };
632 #ifdef DIAGNOSTIC
633 static int enable_vmmap_check = VMMAP_CHECK_UNLOCK;
634 #else
635 static int enable_vmmap_check = VMMAP_CHECK_NONE;
636 #endif
637 SYSCTL_INT(_debug, OID_AUTO, vmmap_check, CTLFLAG_RWTUN,
638     &enable_vmmap_check, 0, "Enable vm map consistency checking");
639 
640 static void _vm_map_assert_consistent(vm_map_t map, int check);
641 
642 #define VM_MAP_ASSERT_CONSISTENT(map) \
643     _vm_map_assert_consistent(map, VMMAP_CHECK_ALL)
644 #ifdef DIAGNOSTIC
645 #define VM_MAP_UNLOCK_CONSISTENT(map) do {				\
646 	if (map->nupdates > map->nentries) {				\
647 		_vm_map_assert_consistent(map, VMMAP_CHECK_UNLOCK);	\
648 		map->nupdates = 0;					\
649 	}								\
650 } while (0)
651 #else
652 #define VM_MAP_UNLOCK_CONSISTENT(map)
653 #endif
654 #else
655 #define	VM_MAP_ASSERT_LOCKED(map)
656 #define VM_MAP_ASSERT_CONSISTENT(map)
657 #define VM_MAP_UNLOCK_CONSISTENT(map)
658 #endif /* INVARIANTS */
659 
660 void
661 _vm_map_unlock(vm_map_t map, const char *file, int line)
662 {
663 
664 	VM_MAP_UNLOCK_CONSISTENT(map);
665 	if (map->system_map)
666 		mtx_unlock_flags_(&map->system_mtx, 0, file, line);
667 	else {
668 		sx_xunlock_(&map->lock, file, line);
669 		vm_map_process_deferred();
670 	}
671 }
672 
673 void
674 _vm_map_lock_read(vm_map_t map, const char *file, int line)
675 {
676 
677 	if (map->system_map)
678 		mtx_lock_flags_(&map->system_mtx, 0, file, line);
679 	else
680 		sx_slock_(&map->lock, file, line);
681 }
682 
683 void
684 _vm_map_unlock_read(vm_map_t map, const char *file, int line)
685 {
686 
687 	if (map->system_map)
688 		mtx_unlock_flags_(&map->system_mtx, 0, file, line);
689 	else {
690 		sx_sunlock_(&map->lock, file, line);
691 		vm_map_process_deferred();
692 	}
693 }
694 
695 int
696 _vm_map_trylock(vm_map_t map, const char *file, int line)
697 {
698 	int error;
699 
700 	error = map->system_map ?
701 	    !mtx_trylock_flags_(&map->system_mtx, 0, file, line) :
702 	    !sx_try_xlock_(&map->lock, file, line);
703 	if (error == 0)
704 		map->timestamp++;
705 	return (error == 0);
706 }
707 
708 int
709 _vm_map_trylock_read(vm_map_t map, const char *file, int line)
710 {
711 	int error;
712 
713 	error = map->system_map ?
714 	    !mtx_trylock_flags_(&map->system_mtx, 0, file, line) :
715 	    !sx_try_slock_(&map->lock, file, line);
716 	return (error == 0);
717 }
718 
719 /*
720  *	_vm_map_lock_upgrade:	[ internal use only ]
721  *
722  *	Tries to upgrade a read (shared) lock on the specified map to a write
723  *	(exclusive) lock.  Returns the value "0" if the upgrade succeeds and a
724  *	non-zero value if the upgrade fails.  If the upgrade fails, the map is
725  *	returned without a read or write lock held.
726  *
727  *	Requires that the map be read locked.
728  */
729 int
730 _vm_map_lock_upgrade(vm_map_t map, const char *file, int line)
731 {
732 	unsigned int last_timestamp;
733 
734 	if (map->system_map) {
735 		mtx_assert_(&map->system_mtx, MA_OWNED, file, line);
736 	} else {
737 		if (!sx_try_upgrade_(&map->lock, file, line)) {
738 			last_timestamp = map->timestamp;
739 			sx_sunlock_(&map->lock, file, line);
740 			vm_map_process_deferred();
741 			/*
742 			 * If the map's timestamp does not change while the
743 			 * map is unlocked, then the upgrade succeeds.
744 			 */
745 			sx_xlock_(&map->lock, file, line);
746 			if (last_timestamp != map->timestamp) {
747 				sx_xunlock_(&map->lock, file, line);
748 				return (1);
749 			}
750 		}
751 	}
752 	map->timestamp++;
753 	return (0);
754 }
755 
756 void
757 _vm_map_lock_downgrade(vm_map_t map, const char *file, int line)
758 {
759 
760 	if (map->system_map) {
761 		mtx_assert_(&map->system_mtx, MA_OWNED, file, line);
762 	} else {
763 		VM_MAP_UNLOCK_CONSISTENT(map);
764 		sx_downgrade_(&map->lock, file, line);
765 	}
766 }
767 
768 /*
769  *	vm_map_locked:
770  *
771  *	Returns a non-zero value if the caller holds a write (exclusive) lock
772  *	on the specified map and the value "0" otherwise.
773  */
774 int
775 vm_map_locked(vm_map_t map)
776 {
777 
778 	if (map->system_map)
779 		return (mtx_owned(&map->system_mtx));
780 	else
781 		return (sx_xlocked(&map->lock));
782 }
783 
784 /*
785  *	_vm_map_unlock_and_wait:
786  *
787  *	Atomically releases the lock on the specified map and puts the calling
788  *	thread to sleep.  The calling thread will remain asleep until either
789  *	vm_map_wakeup() is performed on the map or the specified timeout is
790  *	exceeded.
791  *
792  *	WARNING!  This function does not perform deferred deallocations of
793  *	objects and map	entries.  Therefore, the calling thread is expected to
794  *	reacquire the map lock after reawakening and later perform an ordinary
795  *	unlock operation, such as vm_map_unlock(), before completing its
796  *	operation on the map.
797  */
798 int
799 _vm_map_unlock_and_wait(vm_map_t map, int timo, const char *file, int line)
800 {
801 
802 	VM_MAP_UNLOCK_CONSISTENT(map);
803 	mtx_lock(&map_sleep_mtx);
804 	if (map->system_map)
805 		mtx_unlock_flags_(&map->system_mtx, 0, file, line);
806 	else
807 		sx_xunlock_(&map->lock, file, line);
808 	return (msleep(&map->root, &map_sleep_mtx, PDROP | PVM, "vmmaps",
809 	    timo));
810 }
811 
812 /*
813  *	vm_map_wakeup:
814  *
815  *	Awaken any threads that have slept on the map using
816  *	vm_map_unlock_and_wait().
817  */
818 void
819 vm_map_wakeup(vm_map_t map)
820 {
821 
822 	/*
823 	 * Acquire and release map_sleep_mtx to prevent a wakeup()
824 	 * from being performed (and lost) between the map unlock
825 	 * and the msleep() in _vm_map_unlock_and_wait().
826 	 */
827 	mtx_lock(&map_sleep_mtx);
828 	mtx_unlock(&map_sleep_mtx);
829 	wakeup(&map->root);
830 }
831 
832 void
833 vm_map_busy(vm_map_t map)
834 {
835 
836 	VM_MAP_ASSERT_LOCKED(map);
837 	map->busy++;
838 }
839 
840 void
841 vm_map_unbusy(vm_map_t map)
842 {
843 
844 	VM_MAP_ASSERT_LOCKED(map);
845 	KASSERT(map->busy, ("vm_map_unbusy: not busy"));
846 	if (--map->busy == 0 && (map->flags & MAP_BUSY_WAKEUP)) {
847 		vm_map_modflags(map, 0, MAP_BUSY_WAKEUP);
848 		wakeup(&map->busy);
849 	}
850 }
851 
852 void
853 vm_map_wait_busy(vm_map_t map)
854 {
855 
856 	VM_MAP_ASSERT_LOCKED(map);
857 	while (map->busy) {
858 		vm_map_modflags(map, MAP_BUSY_WAKEUP, 0);
859 		if (map->system_map)
860 			msleep(&map->busy, &map->system_mtx, 0, "mbusy", 0);
861 		else
862 			sx_sleep(&map->busy, &map->lock, 0, "mbusy", 0);
863 	}
864 	map->timestamp++;
865 }
866 
867 long
868 vmspace_resident_count(struct vmspace *vmspace)
869 {
870 	return pmap_resident_count(vmspace_pmap(vmspace));
871 }
872 
873 /*
874  *	vm_map_create:
875  *
876  *	Creates and returns a new empty VM map with
877  *	the given physical map structure, and having
878  *	the given lower and upper address bounds.
879  */
880 vm_map_t
881 vm_map_create(pmap_t pmap, vm_offset_t min, vm_offset_t max)
882 {
883 	vm_map_t result;
884 
885 	result = uma_zalloc(mapzone, M_WAITOK);
886 	CTR1(KTR_VM, "vm_map_create: %p", result);
887 	_vm_map_init(result, pmap, min, max);
888 	return (result);
889 }
890 
891 /*
892  * Initialize an existing vm_map structure
893  * such as that in the vmspace structure.
894  */
895 static void
896 _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max)
897 {
898 
899 	map->header.eflags = MAP_ENTRY_HEADER;
900 	map->needs_wakeup = FALSE;
901 	map->system_map = 0;
902 	map->pmap = pmap;
903 	map->header.end = min;
904 	map->header.start = max;
905 	map->flags = 0;
906 	map->header.left = map->header.right = &map->header;
907 	map->root = NULL;
908 	map->timestamp = 0;
909 	map->busy = 0;
910 	map->anon_loc = 0;
911 #ifdef DIAGNOSTIC
912 	map->nupdates = 0;
913 #endif
914 }
915 
916 void
917 vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max)
918 {
919 
920 	_vm_map_init(map, pmap, min, max);
921 	mtx_init(&map->system_mtx, "system map", NULL, MTX_DEF | MTX_DUPOK);
922 	sx_init(&map->lock, "user map");
923 }
924 
925 /*
926  *	vm_map_entry_dispose:	[ internal use only ]
927  *
928  *	Inverse of vm_map_entry_create.
929  */
930 static void
931 vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry)
932 {
933 	uma_zfree(map->system_map ? kmapentzone : mapentzone, entry);
934 }
935 
936 /*
937  *	vm_map_entry_create:	[ internal use only ]
938  *
939  *	Allocates a VM map entry for insertion.
940  *	No entry fields are filled in.
941  */
942 static vm_map_entry_t
943 vm_map_entry_create(vm_map_t map)
944 {
945 	vm_map_entry_t new_entry;
946 
947 	if (map->system_map)
948 		new_entry = uma_zalloc(kmapentzone, M_NOWAIT);
949 	else
950 		new_entry = uma_zalloc(mapentzone, M_WAITOK);
951 	if (new_entry == NULL)
952 		panic("vm_map_entry_create: kernel resources exhausted");
953 	return (new_entry);
954 }
955 
956 /*
957  *	vm_map_entry_set_behavior:
958  *
959  *	Set the expected access behavior, either normal, random, or
960  *	sequential.
961  */
962 static inline void
963 vm_map_entry_set_behavior(vm_map_entry_t entry, u_char behavior)
964 {
965 	entry->eflags = (entry->eflags & ~MAP_ENTRY_BEHAV_MASK) |
966 	    (behavior & MAP_ENTRY_BEHAV_MASK);
967 }
968 
969 /*
970  *	vm_map_entry_max_free_{left,right}:
971  *
972  *	Compute the size of the largest free gap between two entries,
973  *	one the root of a tree and the other the ancestor of that root
974  *	that is the least or greatest ancestor found on the search path.
975  */
976 static inline vm_size_t
977 vm_map_entry_max_free_left(vm_map_entry_t root, vm_map_entry_t left_ancestor)
978 {
979 
980 	return (root->left != left_ancestor ?
981 	    root->left->max_free : root->start - left_ancestor->end);
982 }
983 
984 static inline vm_size_t
985 vm_map_entry_max_free_right(vm_map_entry_t root, vm_map_entry_t right_ancestor)
986 {
987 
988 	return (root->right != right_ancestor ?
989 	    root->right->max_free : right_ancestor->start - root->end);
990 }
991 
992 /*
993  *	vm_map_entry_{pred,succ}:
994  *
995  *	Find the {predecessor, successor} of the entry by taking one step
996  *	in the appropriate direction and backtracking as much as necessary.
997  *	vm_map_entry_succ is defined in vm_map.h.
998  */
999 static inline vm_map_entry_t
1000 vm_map_entry_pred(vm_map_entry_t entry)
1001 {
1002 	vm_map_entry_t prior;
1003 
1004 	prior = entry->left;
1005 	if (prior->right->start < entry->start) {
1006 		do
1007 			prior = prior->right;
1008 		while (prior->right != entry);
1009 	}
1010 	return (prior);
1011 }
1012 
1013 static inline vm_size_t
1014 vm_size_max(vm_size_t a, vm_size_t b)
1015 {
1016 
1017 	return (a > b ? a : b);
1018 }
1019 
1020 #define SPLAY_LEFT_STEP(root, y, llist, rlist, test) do {		\
1021 	vm_map_entry_t z;						\
1022 	vm_size_t max_free;						\
1023 									\
1024 	/*								\
1025 	 * Infer root->right->max_free == root->max_free when		\
1026 	 * y->max_free < root->max_free || root->max_free == 0.		\
1027 	 * Otherwise, look right to find it.				\
1028 	 */								\
1029 	y = root->left;							\
1030 	max_free = root->max_free;					\
1031 	KASSERT(max_free == vm_size_max(				\
1032 	    vm_map_entry_max_free_left(root, llist),			\
1033 	    vm_map_entry_max_free_right(root, rlist)),			\
1034 	    ("%s: max_free invariant fails", __func__));		\
1035 	if (max_free - 1 < vm_map_entry_max_free_left(root, llist))	\
1036 		max_free = vm_map_entry_max_free_right(root, rlist);	\
1037 	if (y != llist && (test)) {					\
1038 		/* Rotate right and make y root. */			\
1039 		z = y->right;						\
1040 		if (z != root) {					\
1041 			root->left = z;					\
1042 			y->right = root;				\
1043 			if (max_free < y->max_free)			\
1044 			    root->max_free = max_free =			\
1045 			    vm_size_max(max_free, z->max_free);		\
1046 		} else if (max_free < y->max_free)			\
1047 			root->max_free = max_free =			\
1048 			    vm_size_max(max_free, root->start - y->end);\
1049 		root = y;						\
1050 		y = root->left;						\
1051 	}								\
1052 	/* Copy right->max_free.  Put root on rlist. */			\
1053 	root->max_free = max_free;					\
1054 	KASSERT(max_free == vm_map_entry_max_free_right(root, rlist),	\
1055 	    ("%s: max_free not copied from right", __func__));		\
1056 	root->left = rlist;						\
1057 	rlist = root;							\
1058 	root = y != llist ? y : NULL;					\
1059 } while (0)
1060 
1061 #define SPLAY_RIGHT_STEP(root, y, llist, rlist, test) do {		\
1062 	vm_map_entry_t z;						\
1063 	vm_size_t max_free;						\
1064 									\
1065 	/*								\
1066 	 * Infer root->left->max_free == root->max_free when		\
1067 	 * y->max_free < root->max_free || root->max_free == 0.		\
1068 	 * Otherwise, look left to find it.				\
1069 	 */								\
1070 	y = root->right;						\
1071 	max_free = root->max_free;					\
1072 	KASSERT(max_free == vm_size_max(				\
1073 	    vm_map_entry_max_free_left(root, llist),			\
1074 	    vm_map_entry_max_free_right(root, rlist)),			\
1075 	    ("%s: max_free invariant fails", __func__));		\
1076 	if (max_free - 1 < vm_map_entry_max_free_right(root, rlist))	\
1077 		max_free = vm_map_entry_max_free_left(root, llist);	\
1078 	if (y != rlist && (test)) {					\
1079 		/* Rotate left and make y root. */			\
1080 		z = y->left;						\
1081 		if (z != root) {					\
1082 			root->right = z;				\
1083 			y->left = root;					\
1084 			if (max_free < y->max_free)			\
1085 			    root->max_free = max_free =			\
1086 			    vm_size_max(max_free, z->max_free);		\
1087 		} else if (max_free < y->max_free)			\
1088 			root->max_free = max_free =			\
1089 			    vm_size_max(max_free, y->start - root->end);\
1090 		root = y;						\
1091 		y = root->right;					\
1092 	}								\
1093 	/* Copy left->max_free.  Put root on llist. */			\
1094 	root->max_free = max_free;					\
1095 	KASSERT(max_free == vm_map_entry_max_free_left(root, llist),	\
1096 	    ("%s: max_free not copied from left", __func__));		\
1097 	root->right = llist;						\
1098 	llist = root;							\
1099 	root = y != rlist ? y : NULL;					\
1100 } while (0)
1101 
1102 /*
1103  * Walk down the tree until we find addr or a gap where addr would go, breaking
1104  * off left and right subtrees of nodes less than, or greater than addr.  Treat
1105  * subtrees with root->max_free < length as empty trees.  llist and rlist are
1106  * the two sides in reverse order (bottom-up), with llist linked by the right
1107  * pointer and rlist linked by the left pointer in the vm_map_entry, and both
1108  * lists terminated by &map->header.  This function, and the subsequent call to
1109  * vm_map_splay_merge_{left,right,pred,succ}, rely on the start and end address
1110  * values in &map->header.
1111  */
1112 static __always_inline vm_map_entry_t
1113 vm_map_splay_split(vm_map_t map, vm_offset_t addr, vm_size_t length,
1114     vm_map_entry_t *llist, vm_map_entry_t *rlist)
1115 {
1116 	vm_map_entry_t left, right, root, y;
1117 
1118 	left = right = &map->header;
1119 	root = map->root;
1120 	while (root != NULL && root->max_free >= length) {
1121 		KASSERT(left->end <= root->start &&
1122 		    root->end <= right->start,
1123 		    ("%s: root not within tree bounds", __func__));
1124 		if (addr < root->start) {
1125 			SPLAY_LEFT_STEP(root, y, left, right,
1126 			    y->max_free >= length && addr < y->start);
1127 		} else if (addr >= root->end) {
1128 			SPLAY_RIGHT_STEP(root, y, left, right,
1129 			    y->max_free >= length && addr >= y->end);
1130 		} else
1131 			break;
1132 	}
1133 	*llist = left;
1134 	*rlist = right;
1135 	return (root);
1136 }
1137 
1138 static __always_inline void
1139 vm_map_splay_findnext(vm_map_entry_t root, vm_map_entry_t *rlist)
1140 {
1141 	vm_map_entry_t hi, right, y;
1142 
1143 	right = *rlist;
1144 	hi = root->right == right ? NULL : root->right;
1145 	if (hi == NULL)
1146 		return;
1147 	do
1148 		SPLAY_LEFT_STEP(hi, y, root, right, true);
1149 	while (hi != NULL);
1150 	*rlist = right;
1151 }
1152 
1153 static __always_inline void
1154 vm_map_splay_findprev(vm_map_entry_t root, vm_map_entry_t *llist)
1155 {
1156 	vm_map_entry_t left, lo, y;
1157 
1158 	left = *llist;
1159 	lo = root->left == left ? NULL : root->left;
1160 	if (lo == NULL)
1161 		return;
1162 	do
1163 		SPLAY_RIGHT_STEP(lo, y, left, root, true);
1164 	while (lo != NULL);
1165 	*llist = left;
1166 }
1167 
1168 static inline void
1169 vm_map_entry_swap(vm_map_entry_t *a, vm_map_entry_t *b)
1170 {
1171 	vm_map_entry_t tmp;
1172 
1173 	tmp = *b;
1174 	*b = *a;
1175 	*a = tmp;
1176 }
1177 
1178 /*
1179  * Walk back up the two spines, flip the pointers and set max_free.  The
1180  * subtrees of the root go at the bottom of llist and rlist.
1181  */
1182 static vm_size_t
1183 vm_map_splay_merge_left_walk(vm_map_entry_t header, vm_map_entry_t root,
1184     vm_map_entry_t tail, vm_size_t max_free, vm_map_entry_t llist)
1185 {
1186 	do {
1187 		/*
1188 		 * The max_free values of the children of llist are in
1189 		 * llist->max_free and max_free.  Update with the
1190 		 * max value.
1191 		 */
1192 		llist->max_free = max_free =
1193 		    vm_size_max(llist->max_free, max_free);
1194 		vm_map_entry_swap(&llist->right, &tail);
1195 		vm_map_entry_swap(&tail, &llist);
1196 	} while (llist != header);
1197 	root->left = tail;
1198 	return (max_free);
1199 }
1200 
1201 /*
1202  * When llist is known to be the predecessor of root.
1203  */
1204 static inline vm_size_t
1205 vm_map_splay_merge_pred(vm_map_entry_t header, vm_map_entry_t root,
1206     vm_map_entry_t llist)
1207 {
1208 	vm_size_t max_free;
1209 
1210 	max_free = root->start - llist->end;
1211 	if (llist != header) {
1212 		max_free = vm_map_splay_merge_left_walk(header, root,
1213 		    root, max_free, llist);
1214 	} else {
1215 		root->left = header;
1216 		header->right = root;
1217 	}
1218 	return (max_free);
1219 }
1220 
1221 /*
1222  * When llist may or may not be the predecessor of root.
1223  */
1224 static inline vm_size_t
1225 vm_map_splay_merge_left(vm_map_entry_t header, vm_map_entry_t root,
1226     vm_map_entry_t llist)
1227 {
1228 	vm_size_t max_free;
1229 
1230 	max_free = vm_map_entry_max_free_left(root, llist);
1231 	if (llist != header) {
1232 		max_free = vm_map_splay_merge_left_walk(header, root,
1233 		    root->left == llist ? root : root->left,
1234 		    max_free, llist);
1235 	}
1236 	return (max_free);
1237 }
1238 
1239 static vm_size_t
1240 vm_map_splay_merge_right_walk(vm_map_entry_t header, vm_map_entry_t root,
1241     vm_map_entry_t tail, vm_size_t max_free, vm_map_entry_t rlist)
1242 {
1243 	do {
1244 		/*
1245 		 * The max_free values of the children of rlist are in
1246 		 * rlist->max_free and max_free.  Update with the
1247 		 * max value.
1248 		 */
1249 		rlist->max_free = max_free =
1250 		    vm_size_max(rlist->max_free, max_free);
1251 		vm_map_entry_swap(&rlist->left, &tail);
1252 		vm_map_entry_swap(&tail, &rlist);
1253 	} while (rlist != header);
1254 	root->right = tail;
1255 	return (max_free);
1256 }
1257 
1258 /*
1259  * When rlist is known to be the succecessor of root.
1260  */
1261 static inline vm_size_t
1262 vm_map_splay_merge_succ(vm_map_entry_t header, vm_map_entry_t root,
1263     vm_map_entry_t rlist)
1264 {
1265 	vm_size_t max_free;
1266 
1267 	max_free = rlist->start - root->end;
1268 	if (rlist != header) {
1269 		max_free = vm_map_splay_merge_right_walk(header, root,
1270 		    root, max_free, rlist);
1271 	} else {
1272 		root->right = header;
1273 		header->left = root;
1274 	}
1275 	return (max_free);
1276 }
1277 
1278 /*
1279  * When rlist may or may not be the succecessor of root.
1280  */
1281 static inline vm_size_t
1282 vm_map_splay_merge_right(vm_map_entry_t header, vm_map_entry_t root,
1283     vm_map_entry_t rlist)
1284 {
1285 	vm_size_t max_free;
1286 
1287 	max_free = vm_map_entry_max_free_right(root, rlist);
1288 	if (rlist != header) {
1289 		max_free = vm_map_splay_merge_right_walk(header, root,
1290 		    root->right == rlist ? root : root->right,
1291 		    max_free, rlist);
1292 	}
1293 	return (max_free);
1294 }
1295 
1296 /*
1297  *	vm_map_splay:
1298  *
1299  *	The Sleator and Tarjan top-down splay algorithm with the
1300  *	following variation.  Max_free must be computed bottom-up, so
1301  *	on the downward pass, maintain the left and right spines in
1302  *	reverse order.  Then, make a second pass up each side to fix
1303  *	the pointers and compute max_free.  The time bound is O(log n)
1304  *	amortized.
1305  *
1306  *	The tree is threaded, which means that there are no null pointers.
1307  *	When a node has no left child, its left pointer points to its
1308  *	predecessor, which the last ancestor on the search path from the root
1309  *	where the search branched right.  Likewise, when a node has no right
1310  *	child, its right pointer points to its successor.  The map header node
1311  *	is the predecessor of the first map entry, and the successor of the
1312  *	last.
1313  *
1314  *	The new root is the vm_map_entry containing "addr", or else an
1315  *	adjacent entry (lower if possible) if addr is not in the tree.
1316  *
1317  *	The map must be locked, and leaves it so.
1318  *
1319  *	Returns: the new root.
1320  */
1321 static vm_map_entry_t
1322 vm_map_splay(vm_map_t map, vm_offset_t addr)
1323 {
1324 	vm_map_entry_t header, llist, rlist, root;
1325 	vm_size_t max_free_left, max_free_right;
1326 
1327 	header = &map->header;
1328 	root = vm_map_splay_split(map, addr, 0, &llist, &rlist);
1329 	if (root != NULL) {
1330 		max_free_left = vm_map_splay_merge_left(header, root, llist);
1331 		max_free_right = vm_map_splay_merge_right(header, root, rlist);
1332 	} else if (llist != header) {
1333 		/*
1334 		 * Recover the greatest node in the left
1335 		 * subtree and make it the root.
1336 		 */
1337 		root = llist;
1338 		llist = root->right;
1339 		max_free_left = vm_map_splay_merge_left(header, root, llist);
1340 		max_free_right = vm_map_splay_merge_succ(header, root, rlist);
1341 	} else if (rlist != header) {
1342 		/*
1343 		 * Recover the least node in the right
1344 		 * subtree and make it the root.
1345 		 */
1346 		root = rlist;
1347 		rlist = root->left;
1348 		max_free_left = vm_map_splay_merge_pred(header, root, llist);
1349 		max_free_right = vm_map_splay_merge_right(header, root, rlist);
1350 	} else {
1351 		/* There is no root. */
1352 		return (NULL);
1353 	}
1354 	root->max_free = vm_size_max(max_free_left, max_free_right);
1355 	map->root = root;
1356 	VM_MAP_ASSERT_CONSISTENT(map);
1357 	return (root);
1358 }
1359 
1360 /*
1361  *	vm_map_entry_{un,}link:
1362  *
1363  *	Insert/remove entries from maps.  On linking, if new entry clips
1364  *	existing entry, trim existing entry to avoid overlap, and manage
1365  *	offsets.  On unlinking, merge disappearing entry with neighbor, if
1366  *	called for, and manage offsets.  Callers should not modify fields in
1367  *	entries already mapped.
1368  */
1369 static void
1370 vm_map_entry_link(vm_map_t map, vm_map_entry_t entry)
1371 {
1372 	vm_map_entry_t header, llist, rlist, root;
1373 	vm_size_t max_free_left, max_free_right;
1374 
1375 	CTR3(KTR_VM,
1376 	    "vm_map_entry_link: map %p, nentries %d, entry %p", map,
1377 	    map->nentries, entry);
1378 	VM_MAP_ASSERT_LOCKED(map);
1379 	map->nentries++;
1380 	header = &map->header;
1381 	root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist);
1382 	if (root == NULL) {
1383 		/*
1384 		 * The new entry does not overlap any existing entry in the
1385 		 * map, so it becomes the new root of the map tree.
1386 		 */
1387 		max_free_left = vm_map_splay_merge_pred(header, entry, llist);
1388 		max_free_right = vm_map_splay_merge_succ(header, entry, rlist);
1389 	} else if (entry->start == root->start) {
1390 		/*
1391 		 * The new entry is a clone of root, with only the end field
1392 		 * changed.  The root entry will be shrunk to abut the new
1393 		 * entry, and will be the right child of the new root entry in
1394 		 * the modified map.
1395 		 */
1396 		KASSERT(entry->end < root->end,
1397 		    ("%s: clip_start not within entry", __func__));
1398 		vm_map_splay_findprev(root, &llist);
1399 		root->offset += entry->end - root->start;
1400 		root->start = entry->end;
1401 		max_free_left = vm_map_splay_merge_pred(header, entry, llist);
1402 		max_free_right = root->max_free = vm_size_max(
1403 		    vm_map_splay_merge_pred(entry, root, entry),
1404 		    vm_map_splay_merge_right(header, root, rlist));
1405 	} else {
1406 		/*
1407 		 * The new entry is a clone of root, with only the start field
1408 		 * changed.  The root entry will be shrunk to abut the new
1409 		 * entry, and will be the left child of the new root entry in
1410 		 * the modified map.
1411 		 */
1412 		KASSERT(entry->end == root->end,
1413 		    ("%s: clip_start not within entry", __func__));
1414 		vm_map_splay_findnext(root, &rlist);
1415 		entry->offset += entry->start - root->start;
1416 		root->end = entry->start;
1417 		max_free_left = root->max_free = vm_size_max(
1418 		    vm_map_splay_merge_left(header, root, llist),
1419 		    vm_map_splay_merge_succ(entry, root, entry));
1420 		max_free_right = vm_map_splay_merge_succ(header, entry, rlist);
1421 	}
1422 	entry->max_free = vm_size_max(max_free_left, max_free_right);
1423 	map->root = entry;
1424 	VM_MAP_ASSERT_CONSISTENT(map);
1425 }
1426 
1427 enum unlink_merge_type {
1428 	UNLINK_MERGE_NONE,
1429 	UNLINK_MERGE_NEXT
1430 };
1431 
1432 static void
1433 vm_map_entry_unlink(vm_map_t map, vm_map_entry_t entry,
1434     enum unlink_merge_type op)
1435 {
1436 	vm_map_entry_t header, llist, rlist, root;
1437 	vm_size_t max_free_left, max_free_right;
1438 
1439 	VM_MAP_ASSERT_LOCKED(map);
1440 	header = &map->header;
1441 	root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist);
1442 	KASSERT(root != NULL,
1443 	    ("vm_map_entry_unlink: unlink object not mapped"));
1444 
1445 	vm_map_splay_findprev(root, &llist);
1446 	vm_map_splay_findnext(root, &rlist);
1447 	if (op == UNLINK_MERGE_NEXT) {
1448 		rlist->start = root->start;
1449 		rlist->offset = root->offset;
1450 	}
1451 	if (llist != header) {
1452 		root = llist;
1453 		llist = root->right;
1454 		max_free_left = vm_map_splay_merge_left(header, root, llist);
1455 		max_free_right = vm_map_splay_merge_succ(header, root, rlist);
1456 	} else if (rlist != header) {
1457 		root = rlist;
1458 		rlist = root->left;
1459 		max_free_left = vm_map_splay_merge_pred(header, root, llist);
1460 		max_free_right = vm_map_splay_merge_right(header, root, rlist);
1461 	} else {
1462 		header->left = header->right = header;
1463 		root = NULL;
1464 	}
1465 	if (root != NULL)
1466 		root->max_free = vm_size_max(max_free_left, max_free_right);
1467 	map->root = root;
1468 	VM_MAP_ASSERT_CONSISTENT(map);
1469 	map->nentries--;
1470 	CTR3(KTR_VM, "vm_map_entry_unlink: map %p, nentries %d, entry %p", map,
1471 	    map->nentries, entry);
1472 }
1473 
1474 /*
1475  *	vm_map_entry_resize:
1476  *
1477  *	Resize a vm_map_entry, recompute the amount of free space that
1478  *	follows it and propagate that value up the tree.
1479  *
1480  *	The map must be locked, and leaves it so.
1481  */
1482 static void
1483 vm_map_entry_resize(vm_map_t map, vm_map_entry_t entry, vm_size_t grow_amount)
1484 {
1485 	vm_map_entry_t header, llist, rlist, root;
1486 
1487 	VM_MAP_ASSERT_LOCKED(map);
1488 	header = &map->header;
1489 	root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist);
1490 	KASSERT(root != NULL, ("%s: resize object not mapped", __func__));
1491 	vm_map_splay_findnext(root, &rlist);
1492 	entry->end += grow_amount;
1493 	root->max_free = vm_size_max(
1494 	    vm_map_splay_merge_left(header, root, llist),
1495 	    vm_map_splay_merge_succ(header, root, rlist));
1496 	map->root = root;
1497 	VM_MAP_ASSERT_CONSISTENT(map);
1498 	CTR4(KTR_VM, "%s: map %p, nentries %d, entry %p",
1499 	    __func__, map, map->nentries, entry);
1500 }
1501 
1502 /*
1503  *	vm_map_lookup_entry:	[ internal use only ]
1504  *
1505  *	Finds the map entry containing (or
1506  *	immediately preceding) the specified address
1507  *	in the given map; the entry is returned
1508  *	in the "entry" parameter.  The boolean
1509  *	result indicates whether the address is
1510  *	actually contained in the map.
1511  */
1512 boolean_t
1513 vm_map_lookup_entry(
1514 	vm_map_t map,
1515 	vm_offset_t address,
1516 	vm_map_entry_t *entry)	/* OUT */
1517 {
1518 	vm_map_entry_t cur, header, lbound, ubound;
1519 	boolean_t locked;
1520 
1521 	/*
1522 	 * If the map is empty, then the map entry immediately preceding
1523 	 * "address" is the map's header.
1524 	 */
1525 	header = &map->header;
1526 	cur = map->root;
1527 	if (cur == NULL) {
1528 		*entry = header;
1529 		return (FALSE);
1530 	}
1531 	if (address >= cur->start && cur->end > address) {
1532 		*entry = cur;
1533 		return (TRUE);
1534 	}
1535 	if ((locked = vm_map_locked(map)) ||
1536 	    sx_try_upgrade(&map->lock)) {
1537 		/*
1538 		 * Splay requires a write lock on the map.  However, it only
1539 		 * restructures the binary search tree; it does not otherwise
1540 		 * change the map.  Thus, the map's timestamp need not change
1541 		 * on a temporary upgrade.
1542 		 */
1543 		cur = vm_map_splay(map, address);
1544 		if (!locked) {
1545 			VM_MAP_UNLOCK_CONSISTENT(map);
1546 			sx_downgrade(&map->lock);
1547 		}
1548 
1549 		/*
1550 		 * If "address" is contained within a map entry, the new root
1551 		 * is that map entry.  Otherwise, the new root is a map entry
1552 		 * immediately before or after "address".
1553 		 */
1554 		if (address < cur->start) {
1555 			*entry = header;
1556 			return (FALSE);
1557 		}
1558 		*entry = cur;
1559 		return (address < cur->end);
1560 	}
1561 	/*
1562 	 * Since the map is only locked for read access, perform a
1563 	 * standard binary search tree lookup for "address".
1564 	 */
1565 	lbound = ubound = header;
1566 	for (;;) {
1567 		if (address < cur->start) {
1568 			ubound = cur;
1569 			cur = cur->left;
1570 			if (cur == lbound)
1571 				break;
1572 		} else if (cur->end <= address) {
1573 			lbound = cur;
1574 			cur = cur->right;
1575 			if (cur == ubound)
1576 				break;
1577 		} else {
1578 			*entry = cur;
1579 			return (TRUE);
1580 		}
1581 	}
1582 	*entry = lbound;
1583 	return (FALSE);
1584 }
1585 
1586 /*
1587  *	vm_map_insert:
1588  *
1589  *	Inserts the given whole VM object into the target
1590  *	map at the specified address range.  The object's
1591  *	size should match that of the address range.
1592  *
1593  *	Requires that the map be locked, and leaves it so.
1594  *
1595  *	If object is non-NULL, ref count must be bumped by caller
1596  *	prior to making call to account for the new entry.
1597  */
1598 int
1599 vm_map_insert(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
1600     vm_offset_t start, vm_offset_t end, vm_prot_t prot, vm_prot_t max, int cow)
1601 {
1602 	vm_map_entry_t new_entry, next_entry, prev_entry;
1603 	struct ucred *cred;
1604 	vm_eflags_t protoeflags;
1605 	vm_inherit_t inheritance;
1606 
1607 	VM_MAP_ASSERT_LOCKED(map);
1608 	KASSERT(object != kernel_object ||
1609 	    (cow & MAP_COPY_ON_WRITE) == 0,
1610 	    ("vm_map_insert: kernel object and COW"));
1611 	KASSERT(object == NULL || (cow & MAP_NOFAULT) == 0,
1612 	    ("vm_map_insert: paradoxical MAP_NOFAULT request"));
1613 	KASSERT((prot & ~max) == 0,
1614 	    ("prot %#x is not subset of max_prot %#x", prot, max));
1615 
1616 	/*
1617 	 * Check that the start and end points are not bogus.
1618 	 */
1619 	if (start == end || !vm_map_range_valid(map, start, end))
1620 		return (KERN_INVALID_ADDRESS);
1621 
1622 	/*
1623 	 * Find the entry prior to the proposed starting address; if it's part
1624 	 * of an existing entry, this range is bogus.
1625 	 */
1626 	if (vm_map_lookup_entry(map, start, &prev_entry))
1627 		return (KERN_NO_SPACE);
1628 
1629 	/*
1630 	 * Assert that the next entry doesn't overlap the end point.
1631 	 */
1632 	next_entry = vm_map_entry_succ(prev_entry);
1633 	if (next_entry->start < end)
1634 		return (KERN_NO_SPACE);
1635 
1636 	if ((cow & MAP_CREATE_GUARD) != 0 && (object != NULL ||
1637 	    max != VM_PROT_NONE))
1638 		return (KERN_INVALID_ARGUMENT);
1639 
1640 	protoeflags = 0;
1641 	if (cow & MAP_COPY_ON_WRITE)
1642 		protoeflags |= MAP_ENTRY_COW | MAP_ENTRY_NEEDS_COPY;
1643 	if (cow & MAP_NOFAULT)
1644 		protoeflags |= MAP_ENTRY_NOFAULT;
1645 	if (cow & MAP_DISABLE_SYNCER)
1646 		protoeflags |= MAP_ENTRY_NOSYNC;
1647 	if (cow & MAP_DISABLE_COREDUMP)
1648 		protoeflags |= MAP_ENTRY_NOCOREDUMP;
1649 	if (cow & MAP_STACK_GROWS_DOWN)
1650 		protoeflags |= MAP_ENTRY_GROWS_DOWN;
1651 	if (cow & MAP_STACK_GROWS_UP)
1652 		protoeflags |= MAP_ENTRY_GROWS_UP;
1653 	if (cow & MAP_WRITECOUNT)
1654 		protoeflags |= MAP_ENTRY_WRITECNT;
1655 	if (cow & MAP_VN_EXEC)
1656 		protoeflags |= MAP_ENTRY_VN_EXEC;
1657 	if ((cow & MAP_CREATE_GUARD) != 0)
1658 		protoeflags |= MAP_ENTRY_GUARD;
1659 	if ((cow & MAP_CREATE_STACK_GAP_DN) != 0)
1660 		protoeflags |= MAP_ENTRY_STACK_GAP_DN;
1661 	if ((cow & MAP_CREATE_STACK_GAP_UP) != 0)
1662 		protoeflags |= MAP_ENTRY_STACK_GAP_UP;
1663 	if (cow & MAP_INHERIT_SHARE)
1664 		inheritance = VM_INHERIT_SHARE;
1665 	else
1666 		inheritance = VM_INHERIT_DEFAULT;
1667 
1668 	cred = NULL;
1669 	if ((cow & (MAP_ACC_NO_CHARGE | MAP_NOFAULT | MAP_CREATE_GUARD)) != 0)
1670 		goto charged;
1671 	if ((cow & MAP_ACC_CHARGED) || ((prot & VM_PROT_WRITE) &&
1672 	    ((protoeflags & MAP_ENTRY_NEEDS_COPY) || object == NULL))) {
1673 		if (!(cow & MAP_ACC_CHARGED) && !swap_reserve(end - start))
1674 			return (KERN_RESOURCE_SHORTAGE);
1675 		KASSERT(object == NULL ||
1676 		    (protoeflags & MAP_ENTRY_NEEDS_COPY) != 0 ||
1677 		    object->cred == NULL,
1678 		    ("overcommit: vm_map_insert o %p", object));
1679 		cred = curthread->td_ucred;
1680 	}
1681 
1682 charged:
1683 	/* Expand the kernel pmap, if necessary. */
1684 	if (map == kernel_map && end > kernel_vm_end)
1685 		pmap_growkernel(end);
1686 	if (object != NULL) {
1687 		/*
1688 		 * OBJ_ONEMAPPING must be cleared unless this mapping
1689 		 * is trivially proven to be the only mapping for any
1690 		 * of the object's pages.  (Object granularity
1691 		 * reference counting is insufficient to recognize
1692 		 * aliases with precision.)
1693 		 */
1694 		if ((object->flags & OBJ_ANON) != 0) {
1695 			VM_OBJECT_WLOCK(object);
1696 			if (object->ref_count > 1 || object->shadow_count != 0)
1697 				vm_object_clear_flag(object, OBJ_ONEMAPPING);
1698 			VM_OBJECT_WUNLOCK(object);
1699 		}
1700 	} else if ((prev_entry->eflags & ~MAP_ENTRY_USER_WIRED) ==
1701 	    protoeflags &&
1702 	    (cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP |
1703 	    MAP_VN_EXEC)) == 0 &&
1704 	    prev_entry->end == start && (prev_entry->cred == cred ||
1705 	    (prev_entry->object.vm_object != NULL &&
1706 	    prev_entry->object.vm_object->cred == cred)) &&
1707 	    vm_object_coalesce(prev_entry->object.vm_object,
1708 	    prev_entry->offset,
1709 	    (vm_size_t)(prev_entry->end - prev_entry->start),
1710 	    (vm_size_t)(end - prev_entry->end), cred != NULL &&
1711 	    (protoeflags & MAP_ENTRY_NEEDS_COPY) == 0)) {
1712 		/*
1713 		 * We were able to extend the object.  Determine if we
1714 		 * can extend the previous map entry to include the
1715 		 * new range as well.
1716 		 */
1717 		if (prev_entry->inheritance == inheritance &&
1718 		    prev_entry->protection == prot &&
1719 		    prev_entry->max_protection == max &&
1720 		    prev_entry->wired_count == 0) {
1721 			KASSERT((prev_entry->eflags & MAP_ENTRY_USER_WIRED) ==
1722 			    0, ("prev_entry %p has incoherent wiring",
1723 			    prev_entry));
1724 			if ((prev_entry->eflags & MAP_ENTRY_GUARD) == 0)
1725 				map->size += end - prev_entry->end;
1726 			vm_map_entry_resize(map, prev_entry,
1727 			    end - prev_entry->end);
1728 			vm_map_try_merge_entries(map, prev_entry, next_entry);
1729 			return (KERN_SUCCESS);
1730 		}
1731 
1732 		/*
1733 		 * If we can extend the object but cannot extend the
1734 		 * map entry, we have to create a new map entry.  We
1735 		 * must bump the ref count on the extended object to
1736 		 * account for it.  object may be NULL.
1737 		 */
1738 		object = prev_entry->object.vm_object;
1739 		offset = prev_entry->offset +
1740 		    (prev_entry->end - prev_entry->start);
1741 		vm_object_reference(object);
1742 		if (cred != NULL && object != NULL && object->cred != NULL &&
1743 		    !(prev_entry->eflags & MAP_ENTRY_NEEDS_COPY)) {
1744 			/* Object already accounts for this uid. */
1745 			cred = NULL;
1746 		}
1747 	}
1748 	if (cred != NULL)
1749 		crhold(cred);
1750 
1751 	/*
1752 	 * Create a new entry
1753 	 */
1754 	new_entry = vm_map_entry_create(map);
1755 	new_entry->start = start;
1756 	new_entry->end = end;
1757 	new_entry->cred = NULL;
1758 
1759 	new_entry->eflags = protoeflags;
1760 	new_entry->object.vm_object = object;
1761 	new_entry->offset = offset;
1762 
1763 	new_entry->inheritance = inheritance;
1764 	new_entry->protection = prot;
1765 	new_entry->max_protection = max;
1766 	new_entry->wired_count = 0;
1767 	new_entry->wiring_thread = NULL;
1768 	new_entry->read_ahead = VM_FAULT_READ_AHEAD_INIT;
1769 	new_entry->next_read = start;
1770 
1771 	KASSERT(cred == NULL || !ENTRY_CHARGED(new_entry),
1772 	    ("overcommit: vm_map_insert leaks vm_map %p", new_entry));
1773 	new_entry->cred = cred;
1774 
1775 	/*
1776 	 * Insert the new entry into the list
1777 	 */
1778 	vm_map_entry_link(map, new_entry);
1779 	if ((new_entry->eflags & MAP_ENTRY_GUARD) == 0)
1780 		map->size += new_entry->end - new_entry->start;
1781 
1782 	/*
1783 	 * Try to coalesce the new entry with both the previous and next
1784 	 * entries in the list.  Previously, we only attempted to coalesce
1785 	 * with the previous entry when object is NULL.  Here, we handle the
1786 	 * other cases, which are less common.
1787 	 */
1788 	vm_map_try_merge_entries(map, prev_entry, new_entry);
1789 	vm_map_try_merge_entries(map, new_entry, next_entry);
1790 
1791 	if ((cow & (MAP_PREFAULT | MAP_PREFAULT_PARTIAL)) != 0) {
1792 		vm_map_pmap_enter(map, start, prot, object, OFF_TO_IDX(offset),
1793 		    end - start, cow & MAP_PREFAULT_PARTIAL);
1794 	}
1795 
1796 	return (KERN_SUCCESS);
1797 }
1798 
1799 /*
1800  *	vm_map_findspace:
1801  *
1802  *	Find the first fit (lowest VM address) for "length" free bytes
1803  *	beginning at address >= start in the given map.
1804  *
1805  *	In a vm_map_entry, "max_free" is the maximum amount of
1806  *	contiguous free space between an entry in its subtree and a
1807  *	neighbor of that entry.  This allows finding a free region in
1808  *	one path down the tree, so O(log n) amortized with splay
1809  *	trees.
1810  *
1811  *	The map must be locked, and leaves it so.
1812  *
1813  *	Returns: starting address if sufficient space,
1814  *		 vm_map_max(map)-length+1 if insufficient space.
1815  */
1816 vm_offset_t
1817 vm_map_findspace(vm_map_t map, vm_offset_t start, vm_size_t length)
1818 {
1819 	vm_map_entry_t header, llist, rlist, root, y;
1820 	vm_size_t left_length, max_free_left, max_free_right;
1821 	vm_offset_t gap_end;
1822 
1823 	/*
1824 	 * Request must fit within min/max VM address and must avoid
1825 	 * address wrap.
1826 	 */
1827 	start = MAX(start, vm_map_min(map));
1828 	if (start >= vm_map_max(map) || length > vm_map_max(map) - start)
1829 		return (vm_map_max(map) - length + 1);
1830 
1831 	/* Empty tree means wide open address space. */
1832 	if (map->root == NULL)
1833 		return (start);
1834 
1835 	/*
1836 	 * After splay_split, if start is within an entry, push it to the start
1837 	 * of the following gap.  If rlist is at the end of the gap containing
1838 	 * start, save the end of that gap in gap_end to see if the gap is big
1839 	 * enough; otherwise set gap_end to start skip gap-checking and move
1840 	 * directly to a search of the right subtree.
1841 	 */
1842 	header = &map->header;
1843 	root = vm_map_splay_split(map, start, length, &llist, &rlist);
1844 	gap_end = rlist->start;
1845 	if (root != NULL) {
1846 		start = root->end;
1847 		if (root->right != rlist)
1848 			gap_end = start;
1849 		max_free_left = vm_map_splay_merge_left(header, root, llist);
1850 		max_free_right = vm_map_splay_merge_right(header, root, rlist);
1851 	} else if (rlist != header) {
1852 		root = rlist;
1853 		rlist = root->left;
1854 		max_free_left = vm_map_splay_merge_pred(header, root, llist);
1855 		max_free_right = vm_map_splay_merge_right(header, root, rlist);
1856 	} else {
1857 		root = llist;
1858 		llist = root->right;
1859 		max_free_left = vm_map_splay_merge_left(header, root, llist);
1860 		max_free_right = vm_map_splay_merge_succ(header, root, rlist);
1861 	}
1862 	root->max_free = vm_size_max(max_free_left, max_free_right);
1863 	map->root = root;
1864 	VM_MAP_ASSERT_CONSISTENT(map);
1865 	if (length <= gap_end - start)
1866 		return (start);
1867 
1868 	/* With max_free, can immediately tell if no solution. */
1869 	if (root->right == header || length > root->right->max_free)
1870 		return (vm_map_max(map) - length + 1);
1871 
1872 	/*
1873 	 * Splay for the least large-enough gap in the right subtree.
1874 	 */
1875 	llist = rlist = header;
1876 	for (left_length = 0;;
1877 	    left_length = vm_map_entry_max_free_left(root, llist)) {
1878 		if (length <= left_length)
1879 			SPLAY_LEFT_STEP(root, y, llist, rlist,
1880 			    length <= vm_map_entry_max_free_left(y, llist));
1881 		else
1882 			SPLAY_RIGHT_STEP(root, y, llist, rlist,
1883 			    length > vm_map_entry_max_free_left(y, root));
1884 		if (root == NULL)
1885 			break;
1886 	}
1887 	root = llist;
1888 	llist = root->right;
1889 	max_free_left = vm_map_splay_merge_left(header, root, llist);
1890 	if (rlist == header) {
1891 		root->max_free = vm_size_max(max_free_left,
1892 		    vm_map_splay_merge_succ(header, root, rlist));
1893 	} else {
1894 		y = rlist;
1895 		rlist = y->left;
1896 		y->max_free = vm_size_max(
1897 		    vm_map_splay_merge_pred(root, y, root),
1898 		    vm_map_splay_merge_right(header, y, rlist));
1899 		root->max_free = vm_size_max(max_free_left, y->max_free);
1900 	}
1901 	map->root = root;
1902 	VM_MAP_ASSERT_CONSISTENT(map);
1903 	return (root->end);
1904 }
1905 
1906 int
1907 vm_map_fixed(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
1908     vm_offset_t start, vm_size_t length, vm_prot_t prot,
1909     vm_prot_t max, int cow)
1910 {
1911 	vm_offset_t end;
1912 	int result;
1913 
1914 	end = start + length;
1915 	KASSERT((cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) == 0 ||
1916 	    object == NULL,
1917 	    ("vm_map_fixed: non-NULL backing object for stack"));
1918 	vm_map_lock(map);
1919 	VM_MAP_RANGE_CHECK(map, start, end);
1920 	if ((cow & MAP_CHECK_EXCL) == 0)
1921 		vm_map_delete(map, start, end);
1922 	if ((cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) != 0) {
1923 		result = vm_map_stack_locked(map, start, length, sgrowsiz,
1924 		    prot, max, cow);
1925 	} else {
1926 		result = vm_map_insert(map, object, offset, start, end,
1927 		    prot, max, cow);
1928 	}
1929 	vm_map_unlock(map);
1930 	return (result);
1931 }
1932 
1933 static const int aslr_pages_rnd_64[2] = {0x1000, 0x10};
1934 static const int aslr_pages_rnd_32[2] = {0x100, 0x4};
1935 
1936 static int cluster_anon = 1;
1937 SYSCTL_INT(_vm, OID_AUTO, cluster_anon, CTLFLAG_RW,
1938     &cluster_anon, 0,
1939     "Cluster anonymous mappings: 0 = no, 1 = yes if no hint, 2 = always");
1940 
1941 static bool
1942 clustering_anon_allowed(vm_offset_t addr)
1943 {
1944 
1945 	switch (cluster_anon) {
1946 	case 0:
1947 		return (false);
1948 	case 1:
1949 		return (addr == 0);
1950 	case 2:
1951 	default:
1952 		return (true);
1953 	}
1954 }
1955 
1956 static long aslr_restarts;
1957 SYSCTL_LONG(_vm, OID_AUTO, aslr_restarts, CTLFLAG_RD,
1958     &aslr_restarts, 0,
1959     "Number of aslr failures");
1960 
1961 #define	MAP_32BIT_MAX_ADDR	((vm_offset_t)1 << 31)
1962 
1963 /*
1964  * Searches for the specified amount of free space in the given map with the
1965  * specified alignment.  Performs an address-ordered, first-fit search from
1966  * the given address "*addr", with an optional upper bound "max_addr".  If the
1967  * parameter "alignment" is zero, then the alignment is computed from the
1968  * given (object, offset) pair so as to enable the greatest possible use of
1969  * superpage mappings.  Returns KERN_SUCCESS and the address of the free space
1970  * in "*addr" if successful.  Otherwise, returns KERN_NO_SPACE.
1971  *
1972  * The map must be locked.  Initially, there must be at least "length" bytes
1973  * of free space at the given address.
1974  */
1975 static int
1976 vm_map_alignspace(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
1977     vm_offset_t *addr, vm_size_t length, vm_offset_t max_addr,
1978     vm_offset_t alignment)
1979 {
1980 	vm_offset_t aligned_addr, free_addr;
1981 
1982 	VM_MAP_ASSERT_LOCKED(map);
1983 	free_addr = *addr;
1984 	KASSERT(free_addr == vm_map_findspace(map, free_addr, length),
1985 	    ("caller failed to provide space %#jx at address %p",
1986 	     (uintmax_t)length, (void *)free_addr));
1987 	for (;;) {
1988 		/*
1989 		 * At the start of every iteration, the free space at address
1990 		 * "*addr" is at least "length" bytes.
1991 		 */
1992 		if (alignment == 0)
1993 			pmap_align_superpage(object, offset, addr, length);
1994 		else if ((*addr & (alignment - 1)) != 0) {
1995 			*addr &= ~(alignment - 1);
1996 			*addr += alignment;
1997 		}
1998 		aligned_addr = *addr;
1999 		if (aligned_addr == free_addr) {
2000 			/*
2001 			 * Alignment did not change "*addr", so "*addr" must
2002 			 * still provide sufficient free space.
2003 			 */
2004 			return (KERN_SUCCESS);
2005 		}
2006 
2007 		/*
2008 		 * Test for address wrap on "*addr".  A wrapped "*addr" could
2009 		 * be a valid address, in which case vm_map_findspace() cannot
2010 		 * be relied upon to fail.
2011 		 */
2012 		if (aligned_addr < free_addr)
2013 			return (KERN_NO_SPACE);
2014 		*addr = vm_map_findspace(map, aligned_addr, length);
2015 		if (*addr + length > vm_map_max(map) ||
2016 		    (max_addr != 0 && *addr + length > max_addr))
2017 			return (KERN_NO_SPACE);
2018 		free_addr = *addr;
2019 		if (free_addr == aligned_addr) {
2020 			/*
2021 			 * If a successful call to vm_map_findspace() did not
2022 			 * change "*addr", then "*addr" must still be aligned
2023 			 * and provide sufficient free space.
2024 			 */
2025 			return (KERN_SUCCESS);
2026 		}
2027 	}
2028 }
2029 
2030 /*
2031  *	vm_map_find finds an unallocated region in the target address
2032  *	map with the given length.  The search is defined to be
2033  *	first-fit from the specified address; the region found is
2034  *	returned in the same parameter.
2035  *
2036  *	If object is non-NULL, ref count must be bumped by caller
2037  *	prior to making call to account for the new entry.
2038  */
2039 int
2040 vm_map_find(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
2041 	    vm_offset_t *addr,	/* IN/OUT */
2042 	    vm_size_t length, vm_offset_t max_addr, int find_space,
2043 	    vm_prot_t prot, vm_prot_t max, int cow)
2044 {
2045 	vm_offset_t alignment, curr_min_addr, min_addr;
2046 	int gap, pidx, rv, try;
2047 	bool cluster, en_aslr, update_anon;
2048 
2049 	KASSERT((cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) == 0 ||
2050 	    object == NULL,
2051 	    ("vm_map_find: non-NULL backing object for stack"));
2052 	MPASS((cow & MAP_REMAP) == 0 || (find_space == VMFS_NO_SPACE &&
2053 	    (cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) == 0));
2054 	if (find_space == VMFS_OPTIMAL_SPACE && (object == NULL ||
2055 	    (object->flags & OBJ_COLORED) == 0))
2056 		find_space = VMFS_ANY_SPACE;
2057 	if (find_space >> 8 != 0) {
2058 		KASSERT((find_space & 0xff) == 0, ("bad VMFS flags"));
2059 		alignment = (vm_offset_t)1 << (find_space >> 8);
2060 	} else
2061 		alignment = 0;
2062 	en_aslr = (map->flags & MAP_ASLR) != 0;
2063 	update_anon = cluster = clustering_anon_allowed(*addr) &&
2064 	    (map->flags & MAP_IS_SUB_MAP) == 0 && max_addr == 0 &&
2065 	    find_space != VMFS_NO_SPACE && object == NULL &&
2066 	    (cow & (MAP_INHERIT_SHARE | MAP_STACK_GROWS_UP |
2067 	    MAP_STACK_GROWS_DOWN)) == 0 && prot != PROT_NONE;
2068 	curr_min_addr = min_addr = *addr;
2069 	if (en_aslr && min_addr == 0 && !cluster &&
2070 	    find_space != VMFS_NO_SPACE &&
2071 	    (map->flags & MAP_ASLR_IGNSTART) != 0)
2072 		curr_min_addr = min_addr = vm_map_min(map);
2073 	try = 0;
2074 	vm_map_lock(map);
2075 	if (cluster) {
2076 		curr_min_addr = map->anon_loc;
2077 		if (curr_min_addr == 0)
2078 			cluster = false;
2079 	}
2080 	if (find_space != VMFS_NO_SPACE) {
2081 		KASSERT(find_space == VMFS_ANY_SPACE ||
2082 		    find_space == VMFS_OPTIMAL_SPACE ||
2083 		    find_space == VMFS_SUPER_SPACE ||
2084 		    alignment != 0, ("unexpected VMFS flag"));
2085 again:
2086 		/*
2087 		 * When creating an anonymous mapping, try clustering
2088 		 * with an existing anonymous mapping first.
2089 		 *
2090 		 * We make up to two attempts to find address space
2091 		 * for a given find_space value. The first attempt may
2092 		 * apply randomization or may cluster with an existing
2093 		 * anonymous mapping. If this first attempt fails,
2094 		 * perform a first-fit search of the available address
2095 		 * space.
2096 		 *
2097 		 * If all tries failed, and find_space is
2098 		 * VMFS_OPTIMAL_SPACE, fallback to VMFS_ANY_SPACE.
2099 		 * Again enable clustering and randomization.
2100 		 */
2101 		try++;
2102 		MPASS(try <= 2);
2103 
2104 		if (try == 2) {
2105 			/*
2106 			 * Second try: we failed either to find a
2107 			 * suitable region for randomizing the
2108 			 * allocation, or to cluster with an existing
2109 			 * mapping.  Retry with free run.
2110 			 */
2111 			curr_min_addr = (map->flags & MAP_ASLR_IGNSTART) != 0 ?
2112 			    vm_map_min(map) : min_addr;
2113 			atomic_add_long(&aslr_restarts, 1);
2114 		}
2115 
2116 		if (try == 1 && en_aslr && !cluster) {
2117 			/*
2118 			 * Find space for allocation, including
2119 			 * gap needed for later randomization.
2120 			 */
2121 			pidx = MAXPAGESIZES > 1 && pagesizes[1] != 0 &&
2122 			    (find_space == VMFS_SUPER_SPACE || find_space ==
2123 			    VMFS_OPTIMAL_SPACE) ? 1 : 0;
2124 			gap = vm_map_max(map) > MAP_32BIT_MAX_ADDR &&
2125 			    (max_addr == 0 || max_addr > MAP_32BIT_MAX_ADDR) ?
2126 			    aslr_pages_rnd_64[pidx] : aslr_pages_rnd_32[pidx];
2127 			*addr = vm_map_findspace(map, curr_min_addr,
2128 			    length + gap * pagesizes[pidx]);
2129 			if (*addr + length + gap * pagesizes[pidx] >
2130 			    vm_map_max(map))
2131 				goto again;
2132 			/* And randomize the start address. */
2133 			*addr += (arc4random() % gap) * pagesizes[pidx];
2134 			if (max_addr != 0 && *addr + length > max_addr)
2135 				goto again;
2136 		} else {
2137 			*addr = vm_map_findspace(map, curr_min_addr, length);
2138 			if (*addr + length > vm_map_max(map) ||
2139 			    (max_addr != 0 && *addr + length > max_addr)) {
2140 				if (cluster) {
2141 					cluster = false;
2142 					MPASS(try == 1);
2143 					goto again;
2144 				}
2145 				rv = KERN_NO_SPACE;
2146 				goto done;
2147 			}
2148 		}
2149 
2150 		if (find_space != VMFS_ANY_SPACE &&
2151 		    (rv = vm_map_alignspace(map, object, offset, addr, length,
2152 		    max_addr, alignment)) != KERN_SUCCESS) {
2153 			if (find_space == VMFS_OPTIMAL_SPACE) {
2154 				find_space = VMFS_ANY_SPACE;
2155 				curr_min_addr = min_addr;
2156 				cluster = update_anon;
2157 				try = 0;
2158 				goto again;
2159 			}
2160 			goto done;
2161 		}
2162 	} else if ((cow & MAP_REMAP) != 0) {
2163 		if (!vm_map_range_valid(map, *addr, *addr + length)) {
2164 			rv = KERN_INVALID_ADDRESS;
2165 			goto done;
2166 		}
2167 		vm_map_delete(map, *addr, *addr + length);
2168 	}
2169 	if ((cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) != 0) {
2170 		rv = vm_map_stack_locked(map, *addr, length, sgrowsiz, prot,
2171 		    max, cow);
2172 	} else {
2173 		rv = vm_map_insert(map, object, offset, *addr, *addr + length,
2174 		    prot, max, cow);
2175 	}
2176 	if (rv == KERN_SUCCESS && update_anon)
2177 		map->anon_loc = *addr + length;
2178 done:
2179 	vm_map_unlock(map);
2180 	return (rv);
2181 }
2182 
2183 /*
2184  *	vm_map_find_min() is a variant of vm_map_find() that takes an
2185  *	additional parameter (min_addr) and treats the given address
2186  *	(*addr) differently.  Specifically, it treats *addr as a hint
2187  *	and not as the minimum address where the mapping is created.
2188  *
2189  *	This function works in two phases.  First, it tries to
2190  *	allocate above the hint.  If that fails and the hint is
2191  *	greater than min_addr, it performs a second pass, replacing
2192  *	the hint with min_addr as the minimum address for the
2193  *	allocation.
2194  */
2195 int
2196 vm_map_find_min(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
2197     vm_offset_t *addr, vm_size_t length, vm_offset_t min_addr,
2198     vm_offset_t max_addr, int find_space, vm_prot_t prot, vm_prot_t max,
2199     int cow)
2200 {
2201 	vm_offset_t hint;
2202 	int rv;
2203 
2204 	hint = *addr;
2205 	for (;;) {
2206 		rv = vm_map_find(map, object, offset, addr, length, max_addr,
2207 		    find_space, prot, max, cow);
2208 		if (rv == KERN_SUCCESS || min_addr >= hint)
2209 			return (rv);
2210 		*addr = hint = min_addr;
2211 	}
2212 }
2213 
2214 /*
2215  * A map entry with any of the following flags set must not be merged with
2216  * another entry.
2217  */
2218 #define	MAP_ENTRY_NOMERGE_MASK	(MAP_ENTRY_GROWS_DOWN | MAP_ENTRY_GROWS_UP | \
2219 	    MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_IS_SUB_MAP | MAP_ENTRY_VN_EXEC)
2220 
2221 static bool
2222 vm_map_mergeable_neighbors(vm_map_entry_t prev, vm_map_entry_t entry)
2223 {
2224 
2225 	KASSERT((prev->eflags & MAP_ENTRY_NOMERGE_MASK) == 0 ||
2226 	    (entry->eflags & MAP_ENTRY_NOMERGE_MASK) == 0,
2227 	    ("vm_map_mergeable_neighbors: neither %p nor %p are mergeable",
2228 	    prev, entry));
2229 	return (prev->end == entry->start &&
2230 	    prev->object.vm_object == entry->object.vm_object &&
2231 	    (prev->object.vm_object == NULL ||
2232 	    prev->offset + (prev->end - prev->start) == entry->offset) &&
2233 	    prev->eflags == entry->eflags &&
2234 	    prev->protection == entry->protection &&
2235 	    prev->max_protection == entry->max_protection &&
2236 	    prev->inheritance == entry->inheritance &&
2237 	    prev->wired_count == entry->wired_count &&
2238 	    prev->cred == entry->cred);
2239 }
2240 
2241 static void
2242 vm_map_merged_neighbor_dispose(vm_map_t map, vm_map_entry_t entry)
2243 {
2244 
2245 	/*
2246 	 * If the backing object is a vnode object, vm_object_deallocate()
2247 	 * calls vrele().  However, vrele() does not lock the vnode because
2248 	 * the vnode has additional references.  Thus, the map lock can be
2249 	 * kept without causing a lock-order reversal with the vnode lock.
2250 	 *
2251 	 * Since we count the number of virtual page mappings in
2252 	 * object->un_pager.vnp.writemappings, the writemappings value
2253 	 * should not be adjusted when the entry is disposed of.
2254 	 */
2255 	if (entry->object.vm_object != NULL)
2256 		vm_object_deallocate(entry->object.vm_object);
2257 	if (entry->cred != NULL)
2258 		crfree(entry->cred);
2259 	vm_map_entry_dispose(map, entry);
2260 }
2261 
2262 /*
2263  *	vm_map_try_merge_entries:
2264  *
2265  *	Compare the given map entry to its predecessor, and merge its precessor
2266  *	into it if possible.  The entry remains valid, and may be extended.
2267  *	The predecessor may be deleted.
2268  *
2269  *	The map must be locked.
2270  */
2271 void
2272 vm_map_try_merge_entries(vm_map_t map, vm_map_entry_t prev_entry,
2273     vm_map_entry_t entry)
2274 {
2275 
2276 	VM_MAP_ASSERT_LOCKED(map);
2277 	if ((entry->eflags & MAP_ENTRY_NOMERGE_MASK) == 0 &&
2278 	    vm_map_mergeable_neighbors(prev_entry, entry)) {
2279 		vm_map_entry_unlink(map, prev_entry, UNLINK_MERGE_NEXT);
2280 		vm_map_merged_neighbor_dispose(map, prev_entry);
2281 	}
2282 }
2283 
2284 /*
2285  *	vm_map_entry_back:
2286  *
2287  *	Allocate an object to back a map entry.
2288  */
2289 static inline void
2290 vm_map_entry_back(vm_map_entry_t entry)
2291 {
2292 	vm_object_t object;
2293 
2294 	KASSERT(entry->object.vm_object == NULL,
2295 	    ("map entry %p has backing object", entry));
2296 	KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0,
2297 	    ("map entry %p is a submap", entry));
2298 	object = vm_object_allocate_anon(atop(entry->end - entry->start), NULL,
2299 	    entry->cred, entry->end - entry->start);
2300 	entry->object.vm_object = object;
2301 	entry->offset = 0;
2302 	entry->cred = NULL;
2303 }
2304 
2305 /*
2306  *	vm_map_entry_charge_object
2307  *
2308  *	If there is no object backing this entry, create one.  Otherwise, if
2309  *	the entry has cred, give it to the backing object.
2310  */
2311 static inline void
2312 vm_map_entry_charge_object(vm_map_t map, vm_map_entry_t entry)
2313 {
2314 
2315 	VM_MAP_ASSERT_LOCKED(map);
2316 	KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0,
2317 	    ("map entry %p is a submap", entry));
2318 	if (entry->object.vm_object == NULL && !map->system_map &&
2319 	    (entry->eflags & MAP_ENTRY_GUARD) == 0)
2320 		vm_map_entry_back(entry);
2321 	else if (entry->object.vm_object != NULL &&
2322 	    ((entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) &&
2323 	    entry->cred != NULL) {
2324 		VM_OBJECT_WLOCK(entry->object.vm_object);
2325 		KASSERT(entry->object.vm_object->cred == NULL,
2326 		    ("OVERCOMMIT: %s: both cred e %p", __func__, entry));
2327 		entry->object.vm_object->cred = entry->cred;
2328 		entry->object.vm_object->charge = entry->end - entry->start;
2329 		VM_OBJECT_WUNLOCK(entry->object.vm_object);
2330 		entry->cred = NULL;
2331 	}
2332 }
2333 
2334 /*
2335  *	vm_map_entry_clone
2336  *
2337  *	Create a duplicate map entry for clipping.
2338  */
2339 static vm_map_entry_t
2340 vm_map_entry_clone(vm_map_t map, vm_map_entry_t entry)
2341 {
2342 	vm_map_entry_t new_entry;
2343 
2344 	VM_MAP_ASSERT_LOCKED(map);
2345 
2346 	/*
2347 	 * Create a backing object now, if none exists, so that more individual
2348 	 * objects won't be created after the map entry is split.
2349 	 */
2350 	vm_map_entry_charge_object(map, entry);
2351 
2352 	/* Clone the entry. */
2353 	new_entry = vm_map_entry_create(map);
2354 	*new_entry = *entry;
2355 	if (new_entry->cred != NULL)
2356 		crhold(entry->cred);
2357 	if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
2358 		vm_object_reference(new_entry->object.vm_object);
2359 		vm_map_entry_set_vnode_text(new_entry, true);
2360 		/*
2361 		 * The object->un_pager.vnp.writemappings for the object of
2362 		 * MAP_ENTRY_WRITECNT type entry shall be kept as is here.  The
2363 		 * virtual pages are re-distributed among the clipped entries,
2364 		 * so the sum is left the same.
2365 		 */
2366 	}
2367 	return (new_entry);
2368 }
2369 
2370 /*
2371  *	vm_map_clip_start:	[ internal use only ]
2372  *
2373  *	Asserts that the given entry begins at or after
2374  *	the specified address; if necessary,
2375  *	it splits the entry into two.
2376  */
2377 static inline void
2378 vm_map_clip_start(vm_map_t map, vm_map_entry_t entry, vm_offset_t start)
2379 {
2380 	vm_map_entry_t new_entry;
2381 
2382 	if (!map->system_map)
2383 		WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
2384 		    "%s: map %p entry %p start 0x%jx", __func__, map, entry,
2385 		    (uintmax_t)start);
2386 
2387 	if (start <= entry->start)
2388 		return;
2389 
2390 	VM_MAP_ASSERT_LOCKED(map);
2391 	KASSERT(entry->end > start && entry->start < start,
2392 	    ("%s: invalid clip of entry %p", __func__, entry));
2393 
2394 	new_entry = vm_map_entry_clone(map, entry);
2395 
2396 	/*
2397 	 * Split off the front portion.  Insert the new entry BEFORE this one,
2398 	 * so that this entry has the specified starting address.
2399 	 */
2400 	new_entry->end = start;
2401 	vm_map_entry_link(map, new_entry);
2402 }
2403 
2404 /*
2405  *	vm_map_lookup_clip_start:
2406  *
2407  *	Find the entry at or just after 'start', and clip it if 'start' is in
2408  *	the interior of the entry.  Return entry after 'start', and in
2409  *	prev_entry set the entry before 'start'.
2410  */
2411 static inline vm_map_entry_t
2412 vm_map_lookup_clip_start(vm_map_t map, vm_offset_t start,
2413     vm_map_entry_t *prev_entry)
2414 {
2415 	vm_map_entry_t entry;
2416 
2417 	if (!map->system_map)
2418 		WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
2419 		    "%s: map %p start 0x%jx prev %p", __func__, map,
2420 		    (uintmax_t)start, prev_entry);
2421 
2422 	if (vm_map_lookup_entry(map, start, prev_entry)) {
2423 		entry = *prev_entry;
2424 		vm_map_clip_start(map, entry, start);
2425 		*prev_entry = vm_map_entry_pred(entry);
2426 	} else
2427 		entry = vm_map_entry_succ(*prev_entry);
2428 	return (entry);
2429 }
2430 
2431 /*
2432  *	vm_map_clip_end:	[ internal use only ]
2433  *
2434  *	Asserts that the given entry ends at or before
2435  *	the specified address; if necessary,
2436  *	it splits the entry into two.
2437  */
2438 static inline void
2439 vm_map_clip_end(vm_map_t map, vm_map_entry_t entry, vm_offset_t end)
2440 {
2441 	vm_map_entry_t new_entry;
2442 
2443 	if (!map->system_map)
2444 		WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
2445 		    "%s: map %p entry %p end 0x%jx", __func__, map, entry,
2446 		    (uintmax_t)end);
2447 
2448 	if (end >= entry->end)
2449 		return;
2450 
2451 	VM_MAP_ASSERT_LOCKED(map);
2452 	KASSERT(entry->start < end && entry->end > end,
2453 	    ("%s: invalid clip of entry %p", __func__, entry));
2454 
2455 	new_entry = vm_map_entry_clone(map, entry);
2456 
2457 	/*
2458 	 * Split off the back portion.  Insert the new entry AFTER this one,
2459 	 * so that this entry has the specified ending address.
2460 	 */
2461 	new_entry->start = end;
2462 	vm_map_entry_link(map, new_entry);
2463 }
2464 
2465 /*
2466  *	vm_map_submap:		[ kernel use only ]
2467  *
2468  *	Mark the given range as handled by a subordinate map.
2469  *
2470  *	This range must have been created with vm_map_find,
2471  *	and no other operations may have been performed on this
2472  *	range prior to calling vm_map_submap.
2473  *
2474  *	Only a limited number of operations can be performed
2475  *	within this rage after calling vm_map_submap:
2476  *		vm_fault
2477  *	[Don't try vm_map_copy!]
2478  *
2479  *	To remove a submapping, one must first remove the
2480  *	range from the superior map, and then destroy the
2481  *	submap (if desired).  [Better yet, don't try it.]
2482  */
2483 int
2484 vm_map_submap(
2485 	vm_map_t map,
2486 	vm_offset_t start,
2487 	vm_offset_t end,
2488 	vm_map_t submap)
2489 {
2490 	vm_map_entry_t entry;
2491 	int result;
2492 
2493 	result = KERN_INVALID_ARGUMENT;
2494 
2495 	vm_map_lock(submap);
2496 	submap->flags |= MAP_IS_SUB_MAP;
2497 	vm_map_unlock(submap);
2498 
2499 	vm_map_lock(map);
2500 	VM_MAP_RANGE_CHECK(map, start, end);
2501 	if (vm_map_lookup_entry(map, start, &entry) && entry->end >= end &&
2502 	    (entry->eflags & MAP_ENTRY_COW) == 0 &&
2503 	    entry->object.vm_object == NULL) {
2504 		vm_map_clip_start(map, entry, start);
2505 		vm_map_clip_end(map, entry, end);
2506 		entry->object.sub_map = submap;
2507 		entry->eflags |= MAP_ENTRY_IS_SUB_MAP;
2508 		result = KERN_SUCCESS;
2509 	}
2510 	vm_map_unlock(map);
2511 
2512 	if (result != KERN_SUCCESS) {
2513 		vm_map_lock(submap);
2514 		submap->flags &= ~MAP_IS_SUB_MAP;
2515 		vm_map_unlock(submap);
2516 	}
2517 	return (result);
2518 }
2519 
2520 /*
2521  * The maximum number of pages to map if MAP_PREFAULT_PARTIAL is specified
2522  */
2523 #define	MAX_INIT_PT	96
2524 
2525 /*
2526  *	vm_map_pmap_enter:
2527  *
2528  *	Preload the specified map's pmap with mappings to the specified
2529  *	object's memory-resident pages.  No further physical pages are
2530  *	allocated, and no further virtual pages are retrieved from secondary
2531  *	storage.  If the specified flags include MAP_PREFAULT_PARTIAL, then a
2532  *	limited number of page mappings are created at the low-end of the
2533  *	specified address range.  (For this purpose, a superpage mapping
2534  *	counts as one page mapping.)  Otherwise, all resident pages within
2535  *	the specified address range are mapped.
2536  */
2537 static void
2538 vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot,
2539     vm_object_t object, vm_pindex_t pindex, vm_size_t size, int flags)
2540 {
2541 	vm_offset_t start;
2542 	vm_page_t p, p_start;
2543 	vm_pindex_t mask, psize, threshold, tmpidx;
2544 
2545 	if ((prot & (VM_PROT_READ | VM_PROT_EXECUTE)) == 0 || object == NULL)
2546 		return;
2547 	if (object->type == OBJT_DEVICE || object->type == OBJT_SG) {
2548 		VM_OBJECT_WLOCK(object);
2549 		if (object->type == OBJT_DEVICE || object->type == OBJT_SG) {
2550 			pmap_object_init_pt(map->pmap, addr, object, pindex,
2551 			    size);
2552 			VM_OBJECT_WUNLOCK(object);
2553 			return;
2554 		}
2555 		VM_OBJECT_LOCK_DOWNGRADE(object);
2556 	} else
2557 		VM_OBJECT_RLOCK(object);
2558 
2559 	psize = atop(size);
2560 	if (psize + pindex > object->size) {
2561 		if (pindex >= object->size) {
2562 			VM_OBJECT_RUNLOCK(object);
2563 			return;
2564 		}
2565 		psize = object->size - pindex;
2566 	}
2567 
2568 	start = 0;
2569 	p_start = NULL;
2570 	threshold = MAX_INIT_PT;
2571 
2572 	p = vm_page_find_least(object, pindex);
2573 	/*
2574 	 * Assert: the variable p is either (1) the page with the
2575 	 * least pindex greater than or equal to the parameter pindex
2576 	 * or (2) NULL.
2577 	 */
2578 	for (;
2579 	     p != NULL && (tmpidx = p->pindex - pindex) < psize;
2580 	     p = TAILQ_NEXT(p, listq)) {
2581 		/*
2582 		 * don't allow an madvise to blow away our really
2583 		 * free pages allocating pv entries.
2584 		 */
2585 		if (((flags & MAP_PREFAULT_MADVISE) != 0 &&
2586 		    vm_page_count_severe()) ||
2587 		    ((flags & MAP_PREFAULT_PARTIAL) != 0 &&
2588 		    tmpidx >= threshold)) {
2589 			psize = tmpidx;
2590 			break;
2591 		}
2592 		if (vm_page_all_valid(p)) {
2593 			if (p_start == NULL) {
2594 				start = addr + ptoa(tmpidx);
2595 				p_start = p;
2596 			}
2597 			/* Jump ahead if a superpage mapping is possible. */
2598 			if (p->psind > 0 && ((addr + ptoa(tmpidx)) &
2599 			    (pagesizes[p->psind] - 1)) == 0) {
2600 				mask = atop(pagesizes[p->psind]) - 1;
2601 				if (tmpidx + mask < psize &&
2602 				    vm_page_ps_test(p, PS_ALL_VALID, NULL)) {
2603 					p += mask;
2604 					threshold += mask;
2605 				}
2606 			}
2607 		} else if (p_start != NULL) {
2608 			pmap_enter_object(map->pmap, start, addr +
2609 			    ptoa(tmpidx), p_start, prot);
2610 			p_start = NULL;
2611 		}
2612 	}
2613 	if (p_start != NULL)
2614 		pmap_enter_object(map->pmap, start, addr + ptoa(psize),
2615 		    p_start, prot);
2616 	VM_OBJECT_RUNLOCK(object);
2617 }
2618 
2619 /*
2620  *	vm_map_protect:
2621  *
2622  *	Sets the protection of the specified address
2623  *	region in the target map.  If "set_max" is
2624  *	specified, the maximum protection is to be set;
2625  *	otherwise, only the current protection is affected.
2626  */
2627 int
2628 vm_map_protect(vm_map_t map, vm_offset_t start, vm_offset_t end,
2629 	       vm_prot_t new_prot, boolean_t set_max)
2630 {
2631 	vm_map_entry_t entry, first_entry, in_tran, prev_entry;
2632 	vm_object_t obj;
2633 	struct ucred *cred;
2634 	vm_prot_t old_prot;
2635 	int rv;
2636 
2637 	if (start == end)
2638 		return (KERN_SUCCESS);
2639 
2640 again:
2641 	in_tran = NULL;
2642 	vm_map_lock(map);
2643 
2644 	/*
2645 	 * Ensure that we are not concurrently wiring pages.  vm_map_wire() may
2646 	 * need to fault pages into the map and will drop the map lock while
2647 	 * doing so, and the VM object may end up in an inconsistent state if we
2648 	 * update the protection on the map entry in between faults.
2649 	 */
2650 	vm_map_wait_busy(map);
2651 
2652 	VM_MAP_RANGE_CHECK(map, start, end);
2653 
2654 	if (!vm_map_lookup_entry(map, start, &first_entry))
2655 		first_entry = vm_map_entry_succ(first_entry);
2656 
2657 	/*
2658 	 * Make a first pass to check for protection violations.
2659 	 */
2660 	for (entry = first_entry; entry->start < end;
2661 	    entry = vm_map_entry_succ(entry)) {
2662 		if ((entry->eflags & MAP_ENTRY_GUARD) != 0)
2663 			continue;
2664 		if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0) {
2665 			vm_map_unlock(map);
2666 			return (KERN_INVALID_ARGUMENT);
2667 		}
2668 		if ((new_prot & entry->max_protection) != new_prot) {
2669 			vm_map_unlock(map);
2670 			return (KERN_PROTECTION_FAILURE);
2671 		}
2672 		if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0)
2673 			in_tran = entry;
2674 	}
2675 
2676 	/*
2677 	 * Postpone the operation until all in-transition map entries have
2678 	 * stabilized.  An in-transition entry might already have its pages
2679 	 * wired and wired_count incremented, but not yet have its
2680 	 * MAP_ENTRY_USER_WIRED flag set.  In which case, we would fail to call
2681 	 * vm_fault_copy_entry() in the final loop below.
2682 	 */
2683 	if (in_tran != NULL) {
2684 		in_tran->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
2685 		vm_map_unlock_and_wait(map, 0);
2686 		goto again;
2687 	}
2688 
2689 	/*
2690 	 * Before changing the protections, try to reserve swap space for any
2691 	 * private (i.e., copy-on-write) mappings that are transitioning from
2692 	 * read-only to read/write access.  If a reservation fails, break out
2693 	 * of this loop early and let the next loop simplify the entries, since
2694 	 * some may now be mergeable.
2695 	 */
2696 	rv = KERN_SUCCESS;
2697 	vm_map_clip_start(map, first_entry, start);
2698 	for (entry = first_entry; entry->start < end;
2699 	    entry = vm_map_entry_succ(entry)) {
2700 		vm_map_clip_end(map, entry, end);
2701 
2702 		if (set_max ||
2703 		    ((new_prot & ~entry->protection) & VM_PROT_WRITE) == 0 ||
2704 		    ENTRY_CHARGED(entry) ||
2705 		    (entry->eflags & MAP_ENTRY_GUARD) != 0) {
2706 			continue;
2707 		}
2708 
2709 		cred = curthread->td_ucred;
2710 		obj = entry->object.vm_object;
2711 
2712 		if (obj == NULL ||
2713 		    (entry->eflags & MAP_ENTRY_NEEDS_COPY) != 0) {
2714 			if (!swap_reserve(entry->end - entry->start)) {
2715 				rv = KERN_RESOURCE_SHORTAGE;
2716 				end = entry->end;
2717 				break;
2718 			}
2719 			crhold(cred);
2720 			entry->cred = cred;
2721 			continue;
2722 		}
2723 
2724 		if (obj->type != OBJT_DEFAULT && obj->type != OBJT_SWAP)
2725 			continue;
2726 		VM_OBJECT_WLOCK(obj);
2727 		if (obj->type != OBJT_DEFAULT && obj->type != OBJT_SWAP) {
2728 			VM_OBJECT_WUNLOCK(obj);
2729 			continue;
2730 		}
2731 
2732 		/*
2733 		 * Charge for the whole object allocation now, since
2734 		 * we cannot distinguish between non-charged and
2735 		 * charged clipped mapping of the same object later.
2736 		 */
2737 		KASSERT(obj->charge == 0,
2738 		    ("vm_map_protect: object %p overcharged (entry %p)",
2739 		    obj, entry));
2740 		if (!swap_reserve(ptoa(obj->size))) {
2741 			VM_OBJECT_WUNLOCK(obj);
2742 			rv = KERN_RESOURCE_SHORTAGE;
2743 			end = entry->end;
2744 			break;
2745 		}
2746 
2747 		crhold(cred);
2748 		obj->cred = cred;
2749 		obj->charge = ptoa(obj->size);
2750 		VM_OBJECT_WUNLOCK(obj);
2751 	}
2752 
2753 	/*
2754 	 * If enough swap space was available, go back and fix up protections.
2755 	 * Otherwise, just simplify entries, since some may have been modified.
2756 	 * [Note that clipping is not necessary the second time.]
2757 	 */
2758 	for (prev_entry = vm_map_entry_pred(first_entry), entry = first_entry;
2759 	    entry->start < end;
2760 	    vm_map_try_merge_entries(map, prev_entry, entry),
2761 	    prev_entry = entry, entry = vm_map_entry_succ(entry)) {
2762 		if (rv != KERN_SUCCESS ||
2763 		    (entry->eflags & MAP_ENTRY_GUARD) != 0)
2764 			continue;
2765 
2766 		old_prot = entry->protection;
2767 
2768 		if (set_max)
2769 			entry->protection =
2770 			    (entry->max_protection = new_prot) &
2771 			    old_prot;
2772 		else
2773 			entry->protection = new_prot;
2774 
2775 		/*
2776 		 * For user wired map entries, the normal lazy evaluation of
2777 		 * write access upgrades through soft page faults is
2778 		 * undesirable.  Instead, immediately copy any pages that are
2779 		 * copy-on-write and enable write access in the physical map.
2780 		 */
2781 		if ((entry->eflags & MAP_ENTRY_USER_WIRED) != 0 &&
2782 		    (entry->protection & VM_PROT_WRITE) != 0 &&
2783 		    (old_prot & VM_PROT_WRITE) == 0)
2784 			vm_fault_copy_entry(map, map, entry, entry, NULL);
2785 
2786 		/*
2787 		 * When restricting access, update the physical map.  Worry
2788 		 * about copy-on-write here.
2789 		 */
2790 		if ((old_prot & ~entry->protection) != 0) {
2791 #define MASK(entry)	(((entry)->eflags & MAP_ENTRY_COW) ? ~VM_PROT_WRITE : \
2792 							VM_PROT_ALL)
2793 			pmap_protect(map->pmap, entry->start,
2794 			    entry->end,
2795 			    entry->protection & MASK(entry));
2796 #undef	MASK
2797 		}
2798 	}
2799 	vm_map_try_merge_entries(map, prev_entry, entry);
2800 	vm_map_unlock(map);
2801 	return (rv);
2802 }
2803 
2804 /*
2805  *	vm_map_madvise:
2806  *
2807  *	This routine traverses a processes map handling the madvise
2808  *	system call.  Advisories are classified as either those effecting
2809  *	the vm_map_entry structure, or those effecting the underlying
2810  *	objects.
2811  */
2812 int
2813 vm_map_madvise(
2814 	vm_map_t map,
2815 	vm_offset_t start,
2816 	vm_offset_t end,
2817 	int behav)
2818 {
2819 	vm_map_entry_t entry, prev_entry;
2820 	bool modify_map;
2821 
2822 	/*
2823 	 * Some madvise calls directly modify the vm_map_entry, in which case
2824 	 * we need to use an exclusive lock on the map and we need to perform
2825 	 * various clipping operations.  Otherwise we only need a read-lock
2826 	 * on the map.
2827 	 */
2828 	switch(behav) {
2829 	case MADV_NORMAL:
2830 	case MADV_SEQUENTIAL:
2831 	case MADV_RANDOM:
2832 	case MADV_NOSYNC:
2833 	case MADV_AUTOSYNC:
2834 	case MADV_NOCORE:
2835 	case MADV_CORE:
2836 		if (start == end)
2837 			return (0);
2838 		modify_map = true;
2839 		vm_map_lock(map);
2840 		break;
2841 	case MADV_WILLNEED:
2842 	case MADV_DONTNEED:
2843 	case MADV_FREE:
2844 		if (start == end)
2845 			return (0);
2846 		modify_map = false;
2847 		vm_map_lock_read(map);
2848 		break;
2849 	default:
2850 		return (EINVAL);
2851 	}
2852 
2853 	/*
2854 	 * Locate starting entry and clip if necessary.
2855 	 */
2856 	VM_MAP_RANGE_CHECK(map, start, end);
2857 
2858 	if (modify_map) {
2859 		/*
2860 		 * madvise behaviors that are implemented in the vm_map_entry.
2861 		 *
2862 		 * We clip the vm_map_entry so that behavioral changes are
2863 		 * limited to the specified address range.
2864 		 */
2865 		for (entry = vm_map_lookup_clip_start(map, start, &prev_entry);
2866 		    entry->start < end;
2867 		    prev_entry = entry, entry = vm_map_entry_succ(entry)) {
2868 			if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0)
2869 				continue;
2870 
2871 			vm_map_clip_end(map, entry, end);
2872 
2873 			switch (behav) {
2874 			case MADV_NORMAL:
2875 				vm_map_entry_set_behavior(entry,
2876 				    MAP_ENTRY_BEHAV_NORMAL);
2877 				break;
2878 			case MADV_SEQUENTIAL:
2879 				vm_map_entry_set_behavior(entry,
2880 				    MAP_ENTRY_BEHAV_SEQUENTIAL);
2881 				break;
2882 			case MADV_RANDOM:
2883 				vm_map_entry_set_behavior(entry,
2884 				    MAP_ENTRY_BEHAV_RANDOM);
2885 				break;
2886 			case MADV_NOSYNC:
2887 				entry->eflags |= MAP_ENTRY_NOSYNC;
2888 				break;
2889 			case MADV_AUTOSYNC:
2890 				entry->eflags &= ~MAP_ENTRY_NOSYNC;
2891 				break;
2892 			case MADV_NOCORE:
2893 				entry->eflags |= MAP_ENTRY_NOCOREDUMP;
2894 				break;
2895 			case MADV_CORE:
2896 				entry->eflags &= ~MAP_ENTRY_NOCOREDUMP;
2897 				break;
2898 			default:
2899 				break;
2900 			}
2901 			vm_map_try_merge_entries(map, prev_entry, entry);
2902 		}
2903 		vm_map_try_merge_entries(map, prev_entry, entry);
2904 		vm_map_unlock(map);
2905 	} else {
2906 		vm_pindex_t pstart, pend;
2907 
2908 		/*
2909 		 * madvise behaviors that are implemented in the underlying
2910 		 * vm_object.
2911 		 *
2912 		 * Since we don't clip the vm_map_entry, we have to clip
2913 		 * the vm_object pindex and count.
2914 		 */
2915 		if (!vm_map_lookup_entry(map, start, &entry))
2916 			entry = vm_map_entry_succ(entry);
2917 		for (; entry->start < end;
2918 		    entry = vm_map_entry_succ(entry)) {
2919 			vm_offset_t useEnd, useStart;
2920 
2921 			if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0)
2922 				continue;
2923 
2924 			/*
2925 			 * MADV_FREE would otherwise rewind time to
2926 			 * the creation of the shadow object.  Because
2927 			 * we hold the VM map read-locked, neither the
2928 			 * entry's object nor the presence of a
2929 			 * backing object can change.
2930 			 */
2931 			if (behav == MADV_FREE &&
2932 			    entry->object.vm_object != NULL &&
2933 			    entry->object.vm_object->backing_object != NULL)
2934 				continue;
2935 
2936 			pstart = OFF_TO_IDX(entry->offset);
2937 			pend = pstart + atop(entry->end - entry->start);
2938 			useStart = entry->start;
2939 			useEnd = entry->end;
2940 
2941 			if (entry->start < start) {
2942 				pstart += atop(start - entry->start);
2943 				useStart = start;
2944 			}
2945 			if (entry->end > end) {
2946 				pend -= atop(entry->end - end);
2947 				useEnd = end;
2948 			}
2949 
2950 			if (pstart >= pend)
2951 				continue;
2952 
2953 			/*
2954 			 * Perform the pmap_advise() before clearing
2955 			 * PGA_REFERENCED in vm_page_advise().  Otherwise, a
2956 			 * concurrent pmap operation, such as pmap_remove(),
2957 			 * could clear a reference in the pmap and set
2958 			 * PGA_REFERENCED on the page before the pmap_advise()
2959 			 * had completed.  Consequently, the page would appear
2960 			 * referenced based upon an old reference that
2961 			 * occurred before this pmap_advise() ran.
2962 			 */
2963 			if (behav == MADV_DONTNEED || behav == MADV_FREE)
2964 				pmap_advise(map->pmap, useStart, useEnd,
2965 				    behav);
2966 
2967 			vm_object_madvise(entry->object.vm_object, pstart,
2968 			    pend, behav);
2969 
2970 			/*
2971 			 * Pre-populate paging structures in the
2972 			 * WILLNEED case.  For wired entries, the
2973 			 * paging structures are already populated.
2974 			 */
2975 			if (behav == MADV_WILLNEED &&
2976 			    entry->wired_count == 0) {
2977 				vm_map_pmap_enter(map,
2978 				    useStart,
2979 				    entry->protection,
2980 				    entry->object.vm_object,
2981 				    pstart,
2982 				    ptoa(pend - pstart),
2983 				    MAP_PREFAULT_MADVISE
2984 				);
2985 			}
2986 		}
2987 		vm_map_unlock_read(map);
2988 	}
2989 	return (0);
2990 }
2991 
2992 
2993 /*
2994  *	vm_map_inherit:
2995  *
2996  *	Sets the inheritance of the specified address
2997  *	range in the target map.  Inheritance
2998  *	affects how the map will be shared with
2999  *	child maps at the time of vmspace_fork.
3000  */
3001 int
3002 vm_map_inherit(vm_map_t map, vm_offset_t start, vm_offset_t end,
3003 	       vm_inherit_t new_inheritance)
3004 {
3005 	vm_map_entry_t entry, prev_entry;
3006 
3007 	switch (new_inheritance) {
3008 	case VM_INHERIT_NONE:
3009 	case VM_INHERIT_COPY:
3010 	case VM_INHERIT_SHARE:
3011 	case VM_INHERIT_ZERO:
3012 		break;
3013 	default:
3014 		return (KERN_INVALID_ARGUMENT);
3015 	}
3016 	if (start == end)
3017 		return (KERN_SUCCESS);
3018 	vm_map_lock(map);
3019 	VM_MAP_RANGE_CHECK(map, start, end);
3020 	for (entry = vm_map_lookup_clip_start(map, start, &prev_entry);
3021 	    entry->start < end;
3022 	    prev_entry = entry, entry = vm_map_entry_succ(entry)) {
3023 		vm_map_clip_end(map, entry, end);
3024 		if ((entry->eflags & MAP_ENTRY_GUARD) == 0 ||
3025 		    new_inheritance != VM_INHERIT_ZERO)
3026 			entry->inheritance = new_inheritance;
3027 		vm_map_try_merge_entries(map, prev_entry, entry);
3028 	}
3029 	vm_map_try_merge_entries(map, prev_entry, entry);
3030 	vm_map_unlock(map);
3031 	return (KERN_SUCCESS);
3032 }
3033 
3034 /*
3035  *	vm_map_entry_in_transition:
3036  *
3037  *	Release the map lock, and sleep until the entry is no longer in
3038  *	transition.  Awake and acquire the map lock.  If the map changed while
3039  *	another held the lock, lookup a possibly-changed entry at or after the
3040  *	'start' position of the old entry.
3041  */
3042 static vm_map_entry_t
3043 vm_map_entry_in_transition(vm_map_t map, vm_offset_t in_start,
3044     vm_offset_t *io_end, bool holes_ok, vm_map_entry_t in_entry)
3045 {
3046 	vm_map_entry_t entry;
3047 	vm_offset_t start;
3048 	u_int last_timestamp;
3049 
3050 	VM_MAP_ASSERT_LOCKED(map);
3051 	KASSERT((in_entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0,
3052 	    ("not in-tranition map entry %p", in_entry));
3053 	/*
3054 	 * We have not yet clipped the entry.
3055 	 */
3056 	start = MAX(in_start, in_entry->start);
3057 	in_entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
3058 	last_timestamp = map->timestamp;
3059 	if (vm_map_unlock_and_wait(map, 0)) {
3060 		/*
3061 		 * Allow interruption of user wiring/unwiring?
3062 		 */
3063 	}
3064 	vm_map_lock(map);
3065 	if (last_timestamp + 1 == map->timestamp)
3066 		return (in_entry);
3067 
3068 	/*
3069 	 * Look again for the entry because the map was modified while it was
3070 	 * unlocked.  Specifically, the entry may have been clipped, merged, or
3071 	 * deleted.
3072 	 */
3073 	if (!vm_map_lookup_entry(map, start, &entry)) {
3074 		if (!holes_ok) {
3075 			*io_end = start;
3076 			return (NULL);
3077 		}
3078 		entry = vm_map_entry_succ(entry);
3079 	}
3080 	return (entry);
3081 }
3082 
3083 /*
3084  *	vm_map_unwire:
3085  *
3086  *	Implements both kernel and user unwiring.
3087  */
3088 int
3089 vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end,
3090     int flags)
3091 {
3092 	vm_map_entry_t entry, first_entry, next_entry, prev_entry;
3093 	int rv;
3094 	bool holes_ok, need_wakeup, user_unwire;
3095 
3096 	if (start == end)
3097 		return (KERN_SUCCESS);
3098 	holes_ok = (flags & VM_MAP_WIRE_HOLESOK) != 0;
3099 	user_unwire = (flags & VM_MAP_WIRE_USER) != 0;
3100 	vm_map_lock(map);
3101 	VM_MAP_RANGE_CHECK(map, start, end);
3102 	if (!vm_map_lookup_entry(map, start, &first_entry)) {
3103 		if (holes_ok)
3104 			first_entry = vm_map_entry_succ(first_entry);
3105 		else {
3106 			vm_map_unlock(map);
3107 			return (KERN_INVALID_ADDRESS);
3108 		}
3109 	}
3110 	rv = KERN_SUCCESS;
3111 	for (entry = first_entry; entry->start < end; entry = next_entry) {
3112 		if (entry->eflags & MAP_ENTRY_IN_TRANSITION) {
3113 			/*
3114 			 * We have not yet clipped the entry.
3115 			 */
3116 			next_entry = vm_map_entry_in_transition(map, start,
3117 			    &end, holes_ok, entry);
3118 			if (next_entry == NULL) {
3119 				if (entry == first_entry) {
3120 					vm_map_unlock(map);
3121 					return (KERN_INVALID_ADDRESS);
3122 				}
3123 				rv = KERN_INVALID_ADDRESS;
3124 				break;
3125 			}
3126 			first_entry = (entry == first_entry) ?
3127 			    next_entry : NULL;
3128 			continue;
3129 		}
3130 		vm_map_clip_start(map, entry, start);
3131 		vm_map_clip_end(map, entry, end);
3132 		/*
3133 		 * Mark the entry in case the map lock is released.  (See
3134 		 * above.)
3135 		 */
3136 		KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 &&
3137 		    entry->wiring_thread == NULL,
3138 		    ("owned map entry %p", entry));
3139 		entry->eflags |= MAP_ENTRY_IN_TRANSITION;
3140 		entry->wiring_thread = curthread;
3141 		next_entry = vm_map_entry_succ(entry);
3142 		/*
3143 		 * Check the map for holes in the specified region.
3144 		 * If holes_ok, skip this check.
3145 		 */
3146 		if (!holes_ok &&
3147 		    entry->end < end && next_entry->start > entry->end) {
3148 			end = entry->end;
3149 			rv = KERN_INVALID_ADDRESS;
3150 			break;
3151 		}
3152 		/*
3153 		 * If system unwiring, require that the entry is system wired.
3154 		 */
3155 		if (!user_unwire &&
3156 		    vm_map_entry_system_wired_count(entry) == 0) {
3157 			end = entry->end;
3158 			rv = KERN_INVALID_ARGUMENT;
3159 			break;
3160 		}
3161 	}
3162 	need_wakeup = false;
3163 	if (first_entry == NULL &&
3164 	    !vm_map_lookup_entry(map, start, &first_entry)) {
3165 		KASSERT(holes_ok, ("vm_map_unwire: lookup failed"));
3166 		prev_entry = first_entry;
3167 		entry = vm_map_entry_succ(first_entry);
3168 	} else {
3169 		prev_entry = vm_map_entry_pred(first_entry);
3170 		entry = first_entry;
3171 	}
3172 	for (; entry->start < end;
3173 	    prev_entry = entry, entry = vm_map_entry_succ(entry)) {
3174 		/*
3175 		 * If holes_ok was specified, an empty
3176 		 * space in the unwired region could have been mapped
3177 		 * while the map lock was dropped for draining
3178 		 * MAP_ENTRY_IN_TRANSITION.  Moreover, another thread
3179 		 * could be simultaneously wiring this new mapping
3180 		 * entry.  Detect these cases and skip any entries
3181 		 * marked as in transition by us.
3182 		 */
3183 		if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 ||
3184 		    entry->wiring_thread != curthread) {
3185 			KASSERT(holes_ok,
3186 			    ("vm_map_unwire: !HOLESOK and new/changed entry"));
3187 			continue;
3188 		}
3189 
3190 		if (rv == KERN_SUCCESS && (!user_unwire ||
3191 		    (entry->eflags & MAP_ENTRY_USER_WIRED))) {
3192 			if (entry->wired_count == 1)
3193 				vm_map_entry_unwire(map, entry);
3194 			else
3195 				entry->wired_count--;
3196 			if (user_unwire)
3197 				entry->eflags &= ~MAP_ENTRY_USER_WIRED;
3198 		}
3199 		KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0,
3200 		    ("vm_map_unwire: in-transition flag missing %p", entry));
3201 		KASSERT(entry->wiring_thread == curthread,
3202 		    ("vm_map_unwire: alien wire %p", entry));
3203 		entry->eflags &= ~MAP_ENTRY_IN_TRANSITION;
3204 		entry->wiring_thread = NULL;
3205 		if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) {
3206 			entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP;
3207 			need_wakeup = true;
3208 		}
3209 		vm_map_try_merge_entries(map, prev_entry, entry);
3210 	}
3211 	vm_map_try_merge_entries(map, prev_entry, entry);
3212 	vm_map_unlock(map);
3213 	if (need_wakeup)
3214 		vm_map_wakeup(map);
3215 	return (rv);
3216 }
3217 
3218 static void
3219 vm_map_wire_user_count_sub(u_long npages)
3220 {
3221 
3222 	atomic_subtract_long(&vm_user_wire_count, npages);
3223 }
3224 
3225 static bool
3226 vm_map_wire_user_count_add(u_long npages)
3227 {
3228 	u_long wired;
3229 
3230 	wired = vm_user_wire_count;
3231 	do {
3232 		if (npages + wired > vm_page_max_user_wired)
3233 			return (false);
3234 	} while (!atomic_fcmpset_long(&vm_user_wire_count, &wired,
3235 	    npages + wired));
3236 
3237 	return (true);
3238 }
3239 
3240 /*
3241  *	vm_map_wire_entry_failure:
3242  *
3243  *	Handle a wiring failure on the given entry.
3244  *
3245  *	The map should be locked.
3246  */
3247 static void
3248 vm_map_wire_entry_failure(vm_map_t map, vm_map_entry_t entry,
3249     vm_offset_t failed_addr)
3250 {
3251 
3252 	VM_MAP_ASSERT_LOCKED(map);
3253 	KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0 &&
3254 	    entry->wired_count == 1,
3255 	    ("vm_map_wire_entry_failure: entry %p isn't being wired", entry));
3256 	KASSERT(failed_addr < entry->end,
3257 	    ("vm_map_wire_entry_failure: entry %p was fully wired", entry));
3258 
3259 	/*
3260 	 * If any pages at the start of this entry were successfully wired,
3261 	 * then unwire them.
3262 	 */
3263 	if (failed_addr > entry->start) {
3264 		pmap_unwire(map->pmap, entry->start, failed_addr);
3265 		vm_object_unwire(entry->object.vm_object, entry->offset,
3266 		    failed_addr - entry->start, PQ_ACTIVE);
3267 	}
3268 
3269 	/*
3270 	 * Assign an out-of-range value to represent the failure to wire this
3271 	 * entry.
3272 	 */
3273 	entry->wired_count = -1;
3274 }
3275 
3276 int
3277 vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end, int flags)
3278 {
3279 	int rv;
3280 
3281 	vm_map_lock(map);
3282 	rv = vm_map_wire_locked(map, start, end, flags);
3283 	vm_map_unlock(map);
3284 	return (rv);
3285 }
3286 
3287 
3288 /*
3289  *	vm_map_wire_locked:
3290  *
3291  *	Implements both kernel and user wiring.  Returns with the map locked,
3292  *	the map lock may be dropped.
3293  */
3294 int
3295 vm_map_wire_locked(vm_map_t map, vm_offset_t start, vm_offset_t end, int flags)
3296 {
3297 	vm_map_entry_t entry, first_entry, next_entry, prev_entry;
3298 	vm_offset_t faddr, saved_end, saved_start;
3299 	u_long npages;
3300 	u_int last_timestamp;
3301 	int rv;
3302 	bool holes_ok, need_wakeup, user_wire;
3303 	vm_prot_t prot;
3304 
3305 	VM_MAP_ASSERT_LOCKED(map);
3306 
3307 	if (start == end)
3308 		return (KERN_SUCCESS);
3309 	prot = 0;
3310 	if (flags & VM_MAP_WIRE_WRITE)
3311 		prot |= VM_PROT_WRITE;
3312 	holes_ok = (flags & VM_MAP_WIRE_HOLESOK) != 0;
3313 	user_wire = (flags & VM_MAP_WIRE_USER) != 0;
3314 	VM_MAP_RANGE_CHECK(map, start, end);
3315 	if (!vm_map_lookup_entry(map, start, &first_entry)) {
3316 		if (holes_ok)
3317 			first_entry = vm_map_entry_succ(first_entry);
3318 		else
3319 			return (KERN_INVALID_ADDRESS);
3320 	}
3321 	for (entry = first_entry; entry->start < end; entry = next_entry) {
3322 		if (entry->eflags & MAP_ENTRY_IN_TRANSITION) {
3323 			/*
3324 			 * We have not yet clipped the entry.
3325 			 */
3326 			next_entry = vm_map_entry_in_transition(map, start,
3327 			    &end, holes_ok, entry);
3328 			if (next_entry == NULL) {
3329 				if (entry == first_entry)
3330 					return (KERN_INVALID_ADDRESS);
3331 				rv = KERN_INVALID_ADDRESS;
3332 				goto done;
3333 			}
3334 			first_entry = (entry == first_entry) ?
3335 			    next_entry : NULL;
3336 			continue;
3337 		}
3338 		vm_map_clip_start(map, entry, start);
3339 		vm_map_clip_end(map, entry, end);
3340 		/*
3341 		 * Mark the entry in case the map lock is released.  (See
3342 		 * above.)
3343 		 */
3344 		KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 &&
3345 		    entry->wiring_thread == NULL,
3346 		    ("owned map entry %p", entry));
3347 		entry->eflags |= MAP_ENTRY_IN_TRANSITION;
3348 		entry->wiring_thread = curthread;
3349 		if ((entry->protection & (VM_PROT_READ | VM_PROT_EXECUTE)) == 0
3350 		    || (entry->protection & prot) != prot) {
3351 			entry->eflags |= MAP_ENTRY_WIRE_SKIPPED;
3352 			if (!holes_ok) {
3353 				end = entry->end;
3354 				rv = KERN_INVALID_ADDRESS;
3355 				goto done;
3356 			}
3357 		} else if (entry->wired_count == 0) {
3358 			entry->wired_count++;
3359 
3360 			npages = atop(entry->end - entry->start);
3361 			if (user_wire && !vm_map_wire_user_count_add(npages)) {
3362 				vm_map_wire_entry_failure(map, entry,
3363 				    entry->start);
3364 				end = entry->end;
3365 				rv = KERN_RESOURCE_SHORTAGE;
3366 				goto done;
3367 			}
3368 
3369 			/*
3370 			 * Release the map lock, relying on the in-transition
3371 			 * mark.  Mark the map busy for fork.
3372 			 */
3373 			saved_start = entry->start;
3374 			saved_end = entry->end;
3375 			last_timestamp = map->timestamp;
3376 			vm_map_busy(map);
3377 			vm_map_unlock(map);
3378 
3379 			faddr = saved_start;
3380 			do {
3381 				/*
3382 				 * Simulate a fault to get the page and enter
3383 				 * it into the physical map.
3384 				 */
3385 				if ((rv = vm_fault(map, faddr,
3386 				    VM_PROT_NONE, VM_FAULT_WIRE, NULL)) !=
3387 				    KERN_SUCCESS)
3388 					break;
3389 			} while ((faddr += PAGE_SIZE) < saved_end);
3390 			vm_map_lock(map);
3391 			vm_map_unbusy(map);
3392 			if (last_timestamp + 1 != map->timestamp) {
3393 				/*
3394 				 * Look again for the entry because the map was
3395 				 * modified while it was unlocked.  The entry
3396 				 * may have been clipped, but NOT merged or
3397 				 * deleted.
3398 				 */
3399 				if (!vm_map_lookup_entry(map, saved_start,
3400 				    &next_entry))
3401 					KASSERT(false,
3402 					    ("vm_map_wire: lookup failed"));
3403 				first_entry = (entry == first_entry) ?
3404 				    next_entry : NULL;
3405 				for (entry = next_entry; entry->end < saved_end;
3406 				    entry = vm_map_entry_succ(entry)) {
3407 					/*
3408 					 * In case of failure, handle entries
3409 					 * that were not fully wired here;
3410 					 * fully wired entries are handled
3411 					 * later.
3412 					 */
3413 					if (rv != KERN_SUCCESS &&
3414 					    faddr < entry->end)
3415 						vm_map_wire_entry_failure(map,
3416 						    entry, faddr);
3417 				}
3418 			}
3419 			if (rv != KERN_SUCCESS) {
3420 				vm_map_wire_entry_failure(map, entry, faddr);
3421 				if (user_wire)
3422 					vm_map_wire_user_count_sub(npages);
3423 				end = entry->end;
3424 				goto done;
3425 			}
3426 		} else if (!user_wire ||
3427 			   (entry->eflags & MAP_ENTRY_USER_WIRED) == 0) {
3428 			entry->wired_count++;
3429 		}
3430 		/*
3431 		 * Check the map for holes in the specified region.
3432 		 * If holes_ok was specified, skip this check.
3433 		 */
3434 		next_entry = vm_map_entry_succ(entry);
3435 		if (!holes_ok &&
3436 		    entry->end < end && next_entry->start > entry->end) {
3437 			end = entry->end;
3438 			rv = KERN_INVALID_ADDRESS;
3439 			goto done;
3440 		}
3441 	}
3442 	rv = KERN_SUCCESS;
3443 done:
3444 	need_wakeup = false;
3445 	if (first_entry == NULL &&
3446 	    !vm_map_lookup_entry(map, start, &first_entry)) {
3447 		KASSERT(holes_ok, ("vm_map_wire: lookup failed"));
3448 		prev_entry = first_entry;
3449 		entry = vm_map_entry_succ(first_entry);
3450 	} else {
3451 		prev_entry = vm_map_entry_pred(first_entry);
3452 		entry = first_entry;
3453 	}
3454 	for (; entry->start < end;
3455 	    prev_entry = entry, entry = vm_map_entry_succ(entry)) {
3456 		/*
3457 		 * If holes_ok was specified, an empty
3458 		 * space in the unwired region could have been mapped
3459 		 * while the map lock was dropped for faulting in the
3460 		 * pages or draining MAP_ENTRY_IN_TRANSITION.
3461 		 * Moreover, another thread could be simultaneously
3462 		 * wiring this new mapping entry.  Detect these cases
3463 		 * and skip any entries marked as in transition not by us.
3464 		 */
3465 		if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 ||
3466 		    entry->wiring_thread != curthread) {
3467 			KASSERT(holes_ok,
3468 			    ("vm_map_wire: !HOLESOK and new/changed entry"));
3469 			continue;
3470 		}
3471 
3472 		if ((entry->eflags & MAP_ENTRY_WIRE_SKIPPED) != 0) {
3473 			/* do nothing */
3474 		} else if (rv == KERN_SUCCESS) {
3475 			if (user_wire)
3476 				entry->eflags |= MAP_ENTRY_USER_WIRED;
3477 		} else if (entry->wired_count == -1) {
3478 			/*
3479 			 * Wiring failed on this entry.  Thus, unwiring is
3480 			 * unnecessary.
3481 			 */
3482 			entry->wired_count = 0;
3483 		} else if (!user_wire ||
3484 		    (entry->eflags & MAP_ENTRY_USER_WIRED) == 0) {
3485 			/*
3486 			 * Undo the wiring.  Wiring succeeded on this entry
3487 			 * but failed on a later entry.
3488 			 */
3489 			if (entry->wired_count == 1) {
3490 				vm_map_entry_unwire(map, entry);
3491 				if (user_wire)
3492 					vm_map_wire_user_count_sub(
3493 					    atop(entry->end - entry->start));
3494 			} else
3495 				entry->wired_count--;
3496 		}
3497 		KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0,
3498 		    ("vm_map_wire: in-transition flag missing %p", entry));
3499 		KASSERT(entry->wiring_thread == curthread,
3500 		    ("vm_map_wire: alien wire %p", entry));
3501 		entry->eflags &= ~(MAP_ENTRY_IN_TRANSITION |
3502 		    MAP_ENTRY_WIRE_SKIPPED);
3503 		entry->wiring_thread = NULL;
3504 		if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) {
3505 			entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP;
3506 			need_wakeup = true;
3507 		}
3508 		vm_map_try_merge_entries(map, prev_entry, entry);
3509 	}
3510 	vm_map_try_merge_entries(map, prev_entry, entry);
3511 	if (need_wakeup)
3512 		vm_map_wakeup(map);
3513 	return (rv);
3514 }
3515 
3516 /*
3517  * vm_map_sync
3518  *
3519  * Push any dirty cached pages in the address range to their pager.
3520  * If syncio is TRUE, dirty pages are written synchronously.
3521  * If invalidate is TRUE, any cached pages are freed as well.
3522  *
3523  * If the size of the region from start to end is zero, we are
3524  * supposed to flush all modified pages within the region containing
3525  * start.  Unfortunately, a region can be split or coalesced with
3526  * neighboring regions, making it difficult to determine what the
3527  * original region was.  Therefore, we approximate this requirement by
3528  * flushing the current region containing start.
3529  *
3530  * Returns an error if any part of the specified range is not mapped.
3531  */
3532 int
3533 vm_map_sync(
3534 	vm_map_t map,
3535 	vm_offset_t start,
3536 	vm_offset_t end,
3537 	boolean_t syncio,
3538 	boolean_t invalidate)
3539 {
3540 	vm_map_entry_t entry, first_entry, next_entry;
3541 	vm_size_t size;
3542 	vm_object_t object;
3543 	vm_ooffset_t offset;
3544 	unsigned int last_timestamp;
3545 	boolean_t failed;
3546 
3547 	vm_map_lock_read(map);
3548 	VM_MAP_RANGE_CHECK(map, start, end);
3549 	if (!vm_map_lookup_entry(map, start, &first_entry)) {
3550 		vm_map_unlock_read(map);
3551 		return (KERN_INVALID_ADDRESS);
3552 	} else if (start == end) {
3553 		start = first_entry->start;
3554 		end = first_entry->end;
3555 	}
3556 	/*
3557 	 * Make a first pass to check for user-wired memory and holes.
3558 	 */
3559 	for (entry = first_entry; entry->start < end; entry = next_entry) {
3560 		if (invalidate &&
3561 		    (entry->eflags & MAP_ENTRY_USER_WIRED) != 0) {
3562 			vm_map_unlock_read(map);
3563 			return (KERN_INVALID_ARGUMENT);
3564 		}
3565 		next_entry = vm_map_entry_succ(entry);
3566 		if (end > entry->end &&
3567 		    entry->end != next_entry->start) {
3568 			vm_map_unlock_read(map);
3569 			return (KERN_INVALID_ADDRESS);
3570 		}
3571 	}
3572 
3573 	if (invalidate)
3574 		pmap_remove(map->pmap, start, end);
3575 	failed = FALSE;
3576 
3577 	/*
3578 	 * Make a second pass, cleaning/uncaching pages from the indicated
3579 	 * objects as we go.
3580 	 */
3581 	for (entry = first_entry; entry->start < end;) {
3582 		offset = entry->offset + (start - entry->start);
3583 		size = (end <= entry->end ? end : entry->end) - start;
3584 		if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0) {
3585 			vm_map_t smap;
3586 			vm_map_entry_t tentry;
3587 			vm_size_t tsize;
3588 
3589 			smap = entry->object.sub_map;
3590 			vm_map_lock_read(smap);
3591 			(void) vm_map_lookup_entry(smap, offset, &tentry);
3592 			tsize = tentry->end - offset;
3593 			if (tsize < size)
3594 				size = tsize;
3595 			object = tentry->object.vm_object;
3596 			offset = tentry->offset + (offset - tentry->start);
3597 			vm_map_unlock_read(smap);
3598 		} else {
3599 			object = entry->object.vm_object;
3600 		}
3601 		vm_object_reference(object);
3602 		last_timestamp = map->timestamp;
3603 		vm_map_unlock_read(map);
3604 		if (!vm_object_sync(object, offset, size, syncio, invalidate))
3605 			failed = TRUE;
3606 		start += size;
3607 		vm_object_deallocate(object);
3608 		vm_map_lock_read(map);
3609 		if (last_timestamp == map->timestamp ||
3610 		    !vm_map_lookup_entry(map, start, &entry))
3611 			entry = vm_map_entry_succ(entry);
3612 	}
3613 
3614 	vm_map_unlock_read(map);
3615 	return (failed ? KERN_FAILURE : KERN_SUCCESS);
3616 }
3617 
3618 /*
3619  *	vm_map_entry_unwire:	[ internal use only ]
3620  *
3621  *	Make the region specified by this entry pageable.
3622  *
3623  *	The map in question should be locked.
3624  *	[This is the reason for this routine's existence.]
3625  */
3626 static void
3627 vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry)
3628 {
3629 	vm_size_t size;
3630 
3631 	VM_MAP_ASSERT_LOCKED(map);
3632 	KASSERT(entry->wired_count > 0,
3633 	    ("vm_map_entry_unwire: entry %p isn't wired", entry));
3634 
3635 	size = entry->end - entry->start;
3636 	if ((entry->eflags & MAP_ENTRY_USER_WIRED) != 0)
3637 		vm_map_wire_user_count_sub(atop(size));
3638 	pmap_unwire(map->pmap, entry->start, entry->end);
3639 	vm_object_unwire(entry->object.vm_object, entry->offset, size,
3640 	    PQ_ACTIVE);
3641 	entry->wired_count = 0;
3642 }
3643 
3644 static void
3645 vm_map_entry_deallocate(vm_map_entry_t entry, boolean_t system_map)
3646 {
3647 
3648 	if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0)
3649 		vm_object_deallocate(entry->object.vm_object);
3650 	uma_zfree(system_map ? kmapentzone : mapentzone, entry);
3651 }
3652 
3653 /*
3654  *	vm_map_entry_delete:	[ internal use only ]
3655  *
3656  *	Deallocate the given entry from the target map.
3657  */
3658 static void
3659 vm_map_entry_delete(vm_map_t map, vm_map_entry_t entry)
3660 {
3661 	vm_object_t object;
3662 	vm_pindex_t offidxstart, offidxend, size1;
3663 	vm_size_t size;
3664 
3665 	vm_map_entry_unlink(map, entry, UNLINK_MERGE_NONE);
3666 	object = entry->object.vm_object;
3667 
3668 	if ((entry->eflags & MAP_ENTRY_GUARD) != 0) {
3669 		MPASS(entry->cred == NULL);
3670 		MPASS((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0);
3671 		MPASS(object == NULL);
3672 		vm_map_entry_deallocate(entry, map->system_map);
3673 		return;
3674 	}
3675 
3676 	size = entry->end - entry->start;
3677 	map->size -= size;
3678 
3679 	if (entry->cred != NULL) {
3680 		swap_release_by_cred(size, entry->cred);
3681 		crfree(entry->cred);
3682 	}
3683 
3684 	if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0 || object == NULL) {
3685 		entry->object.vm_object = NULL;
3686 	} else if ((object->flags & OBJ_ANON) != 0 ||
3687 	    object == kernel_object) {
3688 		KASSERT(entry->cred == NULL || object->cred == NULL ||
3689 		    (entry->eflags & MAP_ENTRY_NEEDS_COPY),
3690 		    ("OVERCOMMIT vm_map_entry_delete: both cred %p", entry));
3691 		offidxstart = OFF_TO_IDX(entry->offset);
3692 		offidxend = offidxstart + atop(size);
3693 		VM_OBJECT_WLOCK(object);
3694 		if (object->ref_count != 1 &&
3695 		    ((object->flags & OBJ_ONEMAPPING) != 0 ||
3696 		    object == kernel_object)) {
3697 			vm_object_collapse(object);
3698 
3699 			/*
3700 			 * The option OBJPR_NOTMAPPED can be passed here
3701 			 * because vm_map_delete() already performed
3702 			 * pmap_remove() on the only mapping to this range
3703 			 * of pages.
3704 			 */
3705 			vm_object_page_remove(object, offidxstart, offidxend,
3706 			    OBJPR_NOTMAPPED);
3707 			if (offidxend >= object->size &&
3708 			    offidxstart < object->size) {
3709 				size1 = object->size;
3710 				object->size = offidxstart;
3711 				if (object->cred != NULL) {
3712 					size1 -= object->size;
3713 					KASSERT(object->charge >= ptoa(size1),
3714 					    ("object %p charge < 0", object));
3715 					swap_release_by_cred(ptoa(size1),
3716 					    object->cred);
3717 					object->charge -= ptoa(size1);
3718 				}
3719 			}
3720 		}
3721 		VM_OBJECT_WUNLOCK(object);
3722 	}
3723 	if (map->system_map)
3724 		vm_map_entry_deallocate(entry, TRUE);
3725 	else {
3726 		entry->defer_next = curthread->td_map_def_user;
3727 		curthread->td_map_def_user = entry;
3728 	}
3729 }
3730 
3731 /*
3732  *	vm_map_delete:	[ internal use only ]
3733  *
3734  *	Deallocates the given address range from the target
3735  *	map.
3736  */
3737 int
3738 vm_map_delete(vm_map_t map, vm_offset_t start, vm_offset_t end)
3739 {
3740 	vm_map_entry_t entry, next_entry;
3741 
3742 	VM_MAP_ASSERT_LOCKED(map);
3743 
3744 	if (start == end)
3745 		return (KERN_SUCCESS);
3746 
3747 	/*
3748 	 * Find the start of the region, and clip it.
3749 	 * Step through all entries in this region.
3750 	 */
3751 	for (entry = vm_map_lookup_clip_start(map, start, &entry);
3752 	    entry->start < end; entry = next_entry) {
3753 		/*
3754 		 * Wait for wiring or unwiring of an entry to complete.
3755 		 * Also wait for any system wirings to disappear on
3756 		 * user maps.
3757 		 */
3758 		if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0 ||
3759 		    (vm_map_pmap(map) != kernel_pmap &&
3760 		    vm_map_entry_system_wired_count(entry) != 0)) {
3761 			unsigned int last_timestamp;
3762 			vm_offset_t saved_start;
3763 
3764 			saved_start = entry->start;
3765 			entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
3766 			last_timestamp = map->timestamp;
3767 			(void) vm_map_unlock_and_wait(map, 0);
3768 			vm_map_lock(map);
3769 			if (last_timestamp + 1 != map->timestamp) {
3770 				/*
3771 				 * Look again for the entry because the map was
3772 				 * modified while it was unlocked.
3773 				 * Specifically, the entry may have been
3774 				 * clipped, merged, or deleted.
3775 				 */
3776 				next_entry = vm_map_lookup_clip_start(map,
3777 				    saved_start, &next_entry);
3778 			} else
3779 				next_entry = entry;
3780 			continue;
3781 		}
3782 		vm_map_clip_end(map, entry, end);
3783 		next_entry = vm_map_entry_succ(entry);
3784 
3785 		/*
3786 		 * Unwire before removing addresses from the pmap; otherwise,
3787 		 * unwiring will put the entries back in the pmap.
3788 		 */
3789 		if (entry->wired_count != 0)
3790 			vm_map_entry_unwire(map, entry);
3791 
3792 		/*
3793 		 * Remove mappings for the pages, but only if the
3794 		 * mappings could exist.  For instance, it does not
3795 		 * make sense to call pmap_remove() for guard entries.
3796 		 */
3797 		if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0 ||
3798 		    entry->object.vm_object != NULL)
3799 			pmap_remove(map->pmap, entry->start, entry->end);
3800 
3801 		if (entry->end == map->anon_loc)
3802 			map->anon_loc = entry->start;
3803 
3804 		/*
3805 		 * Delete the entry only after removing all pmap
3806 		 * entries pointing to its pages.  (Otherwise, its
3807 		 * page frames may be reallocated, and any modify bits
3808 		 * will be set in the wrong object!)
3809 		 */
3810 		vm_map_entry_delete(map, entry);
3811 	}
3812 	return (KERN_SUCCESS);
3813 }
3814 
3815 /*
3816  *	vm_map_remove:
3817  *
3818  *	Remove the given address range from the target map.
3819  *	This is the exported form of vm_map_delete.
3820  */
3821 int
3822 vm_map_remove(vm_map_t map, vm_offset_t start, vm_offset_t end)
3823 {
3824 	int result;
3825 
3826 	vm_map_lock(map);
3827 	VM_MAP_RANGE_CHECK(map, start, end);
3828 	result = vm_map_delete(map, start, end);
3829 	vm_map_unlock(map);
3830 	return (result);
3831 }
3832 
3833 /*
3834  *	vm_map_check_protection:
3835  *
3836  *	Assert that the target map allows the specified privilege on the
3837  *	entire address region given.  The entire region must be allocated.
3838  *
3839  *	WARNING!  This code does not and should not check whether the
3840  *	contents of the region is accessible.  For example a smaller file
3841  *	might be mapped into a larger address space.
3842  *
3843  *	NOTE!  This code is also called by munmap().
3844  *
3845  *	The map must be locked.  A read lock is sufficient.
3846  */
3847 boolean_t
3848 vm_map_check_protection(vm_map_t map, vm_offset_t start, vm_offset_t end,
3849 			vm_prot_t protection)
3850 {
3851 	vm_map_entry_t entry;
3852 	vm_map_entry_t tmp_entry;
3853 
3854 	if (!vm_map_lookup_entry(map, start, &tmp_entry))
3855 		return (FALSE);
3856 	entry = tmp_entry;
3857 
3858 	while (start < end) {
3859 		/*
3860 		 * No holes allowed!
3861 		 */
3862 		if (start < entry->start)
3863 			return (FALSE);
3864 		/*
3865 		 * Check protection associated with entry.
3866 		 */
3867 		if ((entry->protection & protection) != protection)
3868 			return (FALSE);
3869 		/* go to next entry */
3870 		start = entry->end;
3871 		entry = vm_map_entry_succ(entry);
3872 	}
3873 	return (TRUE);
3874 }
3875 
3876 
3877 /*
3878  *
3879  *	vm_map_copy_swap_object:
3880  *
3881  *	Copies a swap-backed object from an existing map entry to a
3882  *	new one.  Carries forward the swap charge.  May change the
3883  *	src object on return.
3884  */
3885 static void
3886 vm_map_copy_swap_object(vm_map_entry_t src_entry, vm_map_entry_t dst_entry,
3887     vm_offset_t size, vm_ooffset_t *fork_charge)
3888 {
3889 	vm_object_t src_object;
3890 	struct ucred *cred;
3891 	int charged;
3892 
3893 	src_object = src_entry->object.vm_object;
3894 	charged = ENTRY_CHARGED(src_entry);
3895 	if ((src_object->flags & OBJ_ANON) != 0) {
3896 		VM_OBJECT_WLOCK(src_object);
3897 		vm_object_collapse(src_object);
3898 		if ((src_object->flags & OBJ_ONEMAPPING) != 0) {
3899 			vm_object_split(src_entry);
3900 			src_object = src_entry->object.vm_object;
3901 		}
3902 		vm_object_reference_locked(src_object);
3903 		vm_object_clear_flag(src_object, OBJ_ONEMAPPING);
3904 		VM_OBJECT_WUNLOCK(src_object);
3905 	} else
3906 		vm_object_reference(src_object);
3907 	if (src_entry->cred != NULL &&
3908 	    !(src_entry->eflags & MAP_ENTRY_NEEDS_COPY)) {
3909 		KASSERT(src_object->cred == NULL,
3910 		    ("OVERCOMMIT: vm_map_copy_anon_entry: cred %p",
3911 		     src_object));
3912 		src_object->cred = src_entry->cred;
3913 		src_object->charge = size;
3914 	}
3915 	dst_entry->object.vm_object = src_object;
3916 	if (charged) {
3917 		cred = curthread->td_ucred;
3918 		crhold(cred);
3919 		dst_entry->cred = cred;
3920 		*fork_charge += size;
3921 		if (!(src_entry->eflags & MAP_ENTRY_NEEDS_COPY)) {
3922 			crhold(cred);
3923 			src_entry->cred = cred;
3924 			*fork_charge += size;
3925 		}
3926 	}
3927 }
3928 
3929 /*
3930  *	vm_map_copy_entry:
3931  *
3932  *	Copies the contents of the source entry to the destination
3933  *	entry.  The entries *must* be aligned properly.
3934  */
3935 static void
3936 vm_map_copy_entry(
3937 	vm_map_t src_map,
3938 	vm_map_t dst_map,
3939 	vm_map_entry_t src_entry,
3940 	vm_map_entry_t dst_entry,
3941 	vm_ooffset_t *fork_charge)
3942 {
3943 	vm_object_t src_object;
3944 	vm_map_entry_t fake_entry;
3945 	vm_offset_t size;
3946 
3947 	VM_MAP_ASSERT_LOCKED(dst_map);
3948 
3949 	if ((dst_entry->eflags|src_entry->eflags) & MAP_ENTRY_IS_SUB_MAP)
3950 		return;
3951 
3952 	if (src_entry->wired_count == 0 ||
3953 	    (src_entry->protection & VM_PROT_WRITE) == 0) {
3954 		/*
3955 		 * If the source entry is marked needs_copy, it is already
3956 		 * write-protected.
3957 		 */
3958 		if ((src_entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0 &&
3959 		    (src_entry->protection & VM_PROT_WRITE) != 0) {
3960 			pmap_protect(src_map->pmap,
3961 			    src_entry->start,
3962 			    src_entry->end,
3963 			    src_entry->protection & ~VM_PROT_WRITE);
3964 		}
3965 
3966 		/*
3967 		 * Make a copy of the object.
3968 		 */
3969 		size = src_entry->end - src_entry->start;
3970 		if ((src_object = src_entry->object.vm_object) != NULL) {
3971 			if (src_object->type == OBJT_DEFAULT ||
3972 			    src_object->type == OBJT_SWAP) {
3973 				vm_map_copy_swap_object(src_entry, dst_entry,
3974 				    size, fork_charge);
3975 				/* May have split/collapsed, reload obj. */
3976 				src_object = src_entry->object.vm_object;
3977 			} else {
3978 				vm_object_reference(src_object);
3979 				dst_entry->object.vm_object = src_object;
3980 			}
3981 			src_entry->eflags |= MAP_ENTRY_COW |
3982 			    MAP_ENTRY_NEEDS_COPY;
3983 			dst_entry->eflags |= MAP_ENTRY_COW |
3984 			    MAP_ENTRY_NEEDS_COPY;
3985 			dst_entry->offset = src_entry->offset;
3986 			if (src_entry->eflags & MAP_ENTRY_WRITECNT) {
3987 				/*
3988 				 * MAP_ENTRY_WRITECNT cannot
3989 				 * indicate write reference from
3990 				 * src_entry, since the entry is
3991 				 * marked as needs copy.  Allocate a
3992 				 * fake entry that is used to
3993 				 * decrement object->un_pager writecount
3994 				 * at the appropriate time.  Attach
3995 				 * fake_entry to the deferred list.
3996 				 */
3997 				fake_entry = vm_map_entry_create(dst_map);
3998 				fake_entry->eflags = MAP_ENTRY_WRITECNT;
3999 				src_entry->eflags &= ~MAP_ENTRY_WRITECNT;
4000 				vm_object_reference(src_object);
4001 				fake_entry->object.vm_object = src_object;
4002 				fake_entry->start = src_entry->start;
4003 				fake_entry->end = src_entry->end;
4004 				fake_entry->defer_next =
4005 				    curthread->td_map_def_user;
4006 				curthread->td_map_def_user = fake_entry;
4007 			}
4008 
4009 			pmap_copy(dst_map->pmap, src_map->pmap,
4010 			    dst_entry->start, dst_entry->end - dst_entry->start,
4011 			    src_entry->start);
4012 		} else {
4013 			dst_entry->object.vm_object = NULL;
4014 			dst_entry->offset = 0;
4015 			if (src_entry->cred != NULL) {
4016 				dst_entry->cred = curthread->td_ucred;
4017 				crhold(dst_entry->cred);
4018 				*fork_charge += size;
4019 			}
4020 		}
4021 	} else {
4022 		/*
4023 		 * We don't want to make writeable wired pages copy-on-write.
4024 		 * Immediately copy these pages into the new map by simulating
4025 		 * page faults.  The new pages are pageable.
4026 		 */
4027 		vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry,
4028 		    fork_charge);
4029 	}
4030 }
4031 
4032 /*
4033  * vmspace_map_entry_forked:
4034  * Update the newly-forked vmspace each time a map entry is inherited
4035  * or copied.  The values for vm_dsize and vm_tsize are approximate
4036  * (and mostly-obsolete ideas in the face of mmap(2) et al.)
4037  */
4038 static void
4039 vmspace_map_entry_forked(const struct vmspace *vm1, struct vmspace *vm2,
4040     vm_map_entry_t entry)
4041 {
4042 	vm_size_t entrysize;
4043 	vm_offset_t newend;
4044 
4045 	if ((entry->eflags & MAP_ENTRY_GUARD) != 0)
4046 		return;
4047 	entrysize = entry->end - entry->start;
4048 	vm2->vm_map.size += entrysize;
4049 	if (entry->eflags & (MAP_ENTRY_GROWS_DOWN | MAP_ENTRY_GROWS_UP)) {
4050 		vm2->vm_ssize += btoc(entrysize);
4051 	} else if (entry->start >= (vm_offset_t)vm1->vm_daddr &&
4052 	    entry->start < (vm_offset_t)vm1->vm_daddr + ctob(vm1->vm_dsize)) {
4053 		newend = MIN(entry->end,
4054 		    (vm_offset_t)vm1->vm_daddr + ctob(vm1->vm_dsize));
4055 		vm2->vm_dsize += btoc(newend - entry->start);
4056 	} else if (entry->start >= (vm_offset_t)vm1->vm_taddr &&
4057 	    entry->start < (vm_offset_t)vm1->vm_taddr + ctob(vm1->vm_tsize)) {
4058 		newend = MIN(entry->end,
4059 		    (vm_offset_t)vm1->vm_taddr + ctob(vm1->vm_tsize));
4060 		vm2->vm_tsize += btoc(newend - entry->start);
4061 	}
4062 }
4063 
4064 /*
4065  * vmspace_fork:
4066  * Create a new process vmspace structure and vm_map
4067  * based on those of an existing process.  The new map
4068  * is based on the old map, according to the inheritance
4069  * values on the regions in that map.
4070  *
4071  * XXX It might be worth coalescing the entries added to the new vmspace.
4072  *
4073  * The source map must not be locked.
4074  */
4075 struct vmspace *
4076 vmspace_fork(struct vmspace *vm1, vm_ooffset_t *fork_charge)
4077 {
4078 	struct vmspace *vm2;
4079 	vm_map_t new_map, old_map;
4080 	vm_map_entry_t new_entry, old_entry;
4081 	vm_object_t object;
4082 	int error, locked;
4083 	vm_inherit_t inh;
4084 
4085 	old_map = &vm1->vm_map;
4086 	/* Copy immutable fields of vm1 to vm2. */
4087 	vm2 = vmspace_alloc(vm_map_min(old_map), vm_map_max(old_map),
4088 	    pmap_pinit);
4089 	if (vm2 == NULL)
4090 		return (NULL);
4091 
4092 	vm2->vm_taddr = vm1->vm_taddr;
4093 	vm2->vm_daddr = vm1->vm_daddr;
4094 	vm2->vm_maxsaddr = vm1->vm_maxsaddr;
4095 	vm_map_lock(old_map);
4096 	if (old_map->busy)
4097 		vm_map_wait_busy(old_map);
4098 	new_map = &vm2->vm_map;
4099 	locked = vm_map_trylock(new_map); /* trylock to silence WITNESS */
4100 	KASSERT(locked, ("vmspace_fork: lock failed"));
4101 
4102 	error = pmap_vmspace_copy(new_map->pmap, old_map->pmap);
4103 	if (error != 0) {
4104 		sx_xunlock(&old_map->lock);
4105 		sx_xunlock(&new_map->lock);
4106 		vm_map_process_deferred();
4107 		vmspace_free(vm2);
4108 		return (NULL);
4109 	}
4110 
4111 	new_map->anon_loc = old_map->anon_loc;
4112 	new_map->flags |= old_map->flags & (MAP_ASLR | MAP_ASLR_IGNSTART);
4113 
4114 	VM_MAP_ENTRY_FOREACH(old_entry, old_map) {
4115 		if ((old_entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0)
4116 			panic("vm_map_fork: encountered a submap");
4117 
4118 		inh = old_entry->inheritance;
4119 		if ((old_entry->eflags & MAP_ENTRY_GUARD) != 0 &&
4120 		    inh != VM_INHERIT_NONE)
4121 			inh = VM_INHERIT_COPY;
4122 
4123 		switch (inh) {
4124 		case VM_INHERIT_NONE:
4125 			break;
4126 
4127 		case VM_INHERIT_SHARE:
4128 			/*
4129 			 * Clone the entry, creating the shared object if
4130 			 * necessary.
4131 			 */
4132 			object = old_entry->object.vm_object;
4133 			if (object == NULL) {
4134 				vm_map_entry_back(old_entry);
4135 				object = old_entry->object.vm_object;
4136 			}
4137 
4138 			/*
4139 			 * Add the reference before calling vm_object_shadow
4140 			 * to insure that a shadow object is created.
4141 			 */
4142 			vm_object_reference(object);
4143 			if (old_entry->eflags & MAP_ENTRY_NEEDS_COPY) {
4144 				vm_object_shadow(&old_entry->object.vm_object,
4145 				    &old_entry->offset,
4146 				    old_entry->end - old_entry->start,
4147 				    old_entry->cred,
4148 				    /* Transfer the second reference too. */
4149 				    true);
4150 				old_entry->eflags &= ~MAP_ENTRY_NEEDS_COPY;
4151 				old_entry->cred = NULL;
4152 
4153 				/*
4154 				 * As in vm_map_merged_neighbor_dispose(),
4155 				 * the vnode lock will not be acquired in
4156 				 * this call to vm_object_deallocate().
4157 				 */
4158 				vm_object_deallocate(object);
4159 				object = old_entry->object.vm_object;
4160 			} else {
4161 				VM_OBJECT_WLOCK(object);
4162 				vm_object_clear_flag(object, OBJ_ONEMAPPING);
4163 				if (old_entry->cred != NULL) {
4164 					KASSERT(object->cred == NULL,
4165 					    ("vmspace_fork both cred"));
4166 					object->cred = old_entry->cred;
4167 					object->charge = old_entry->end -
4168 					    old_entry->start;
4169 					old_entry->cred = NULL;
4170 				}
4171 
4172 				/*
4173 				 * Assert the correct state of the vnode
4174 				 * v_writecount while the object is locked, to
4175 				 * not relock it later for the assertion
4176 				 * correctness.
4177 				 */
4178 				if (old_entry->eflags & MAP_ENTRY_WRITECNT &&
4179 				    object->type == OBJT_VNODE) {
4180 					KASSERT(((struct vnode *)object->
4181 					    handle)->v_writecount > 0,
4182 					    ("vmspace_fork: v_writecount %p",
4183 					    object));
4184 					KASSERT(object->un_pager.vnp.
4185 					    writemappings > 0,
4186 					    ("vmspace_fork: vnp.writecount %p",
4187 					    object));
4188 				}
4189 				VM_OBJECT_WUNLOCK(object);
4190 			}
4191 
4192 			/*
4193 			 * Clone the entry, referencing the shared object.
4194 			 */
4195 			new_entry = vm_map_entry_create(new_map);
4196 			*new_entry = *old_entry;
4197 			new_entry->eflags &= ~(MAP_ENTRY_USER_WIRED |
4198 			    MAP_ENTRY_IN_TRANSITION);
4199 			new_entry->wiring_thread = NULL;
4200 			new_entry->wired_count = 0;
4201 			if (new_entry->eflags & MAP_ENTRY_WRITECNT) {
4202 				vm_pager_update_writecount(object,
4203 				    new_entry->start, new_entry->end);
4204 			}
4205 			vm_map_entry_set_vnode_text(new_entry, true);
4206 
4207 			/*
4208 			 * Insert the entry into the new map -- we know we're
4209 			 * inserting at the end of the new map.
4210 			 */
4211 			vm_map_entry_link(new_map, new_entry);
4212 			vmspace_map_entry_forked(vm1, vm2, new_entry);
4213 
4214 			/*
4215 			 * Update the physical map
4216 			 */
4217 			pmap_copy(new_map->pmap, old_map->pmap,
4218 			    new_entry->start,
4219 			    (old_entry->end - old_entry->start),
4220 			    old_entry->start);
4221 			break;
4222 
4223 		case VM_INHERIT_COPY:
4224 			/*
4225 			 * Clone the entry and link into the map.
4226 			 */
4227 			new_entry = vm_map_entry_create(new_map);
4228 			*new_entry = *old_entry;
4229 			/*
4230 			 * Copied entry is COW over the old object.
4231 			 */
4232 			new_entry->eflags &= ~(MAP_ENTRY_USER_WIRED |
4233 			    MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_WRITECNT);
4234 			new_entry->wiring_thread = NULL;
4235 			new_entry->wired_count = 0;
4236 			new_entry->object.vm_object = NULL;
4237 			new_entry->cred = NULL;
4238 			vm_map_entry_link(new_map, new_entry);
4239 			vmspace_map_entry_forked(vm1, vm2, new_entry);
4240 			vm_map_copy_entry(old_map, new_map, old_entry,
4241 			    new_entry, fork_charge);
4242 			vm_map_entry_set_vnode_text(new_entry, true);
4243 			break;
4244 
4245 		case VM_INHERIT_ZERO:
4246 			/*
4247 			 * Create a new anonymous mapping entry modelled from
4248 			 * the old one.
4249 			 */
4250 			new_entry = vm_map_entry_create(new_map);
4251 			memset(new_entry, 0, sizeof(*new_entry));
4252 
4253 			new_entry->start = old_entry->start;
4254 			new_entry->end = old_entry->end;
4255 			new_entry->eflags = old_entry->eflags &
4256 			    ~(MAP_ENTRY_USER_WIRED | MAP_ENTRY_IN_TRANSITION |
4257 			    MAP_ENTRY_WRITECNT | MAP_ENTRY_VN_EXEC);
4258 			new_entry->protection = old_entry->protection;
4259 			new_entry->max_protection = old_entry->max_protection;
4260 			new_entry->inheritance = VM_INHERIT_ZERO;
4261 
4262 			vm_map_entry_link(new_map, new_entry);
4263 			vmspace_map_entry_forked(vm1, vm2, new_entry);
4264 
4265 			new_entry->cred = curthread->td_ucred;
4266 			crhold(new_entry->cred);
4267 			*fork_charge += (new_entry->end - new_entry->start);
4268 
4269 			break;
4270 		}
4271 	}
4272 	/*
4273 	 * Use inlined vm_map_unlock() to postpone handling the deferred
4274 	 * map entries, which cannot be done until both old_map and
4275 	 * new_map locks are released.
4276 	 */
4277 	sx_xunlock(&old_map->lock);
4278 	sx_xunlock(&new_map->lock);
4279 	vm_map_process_deferred();
4280 
4281 	return (vm2);
4282 }
4283 
4284 /*
4285  * Create a process's stack for exec_new_vmspace().  This function is never
4286  * asked to wire the newly created stack.
4287  */
4288 int
4289 vm_map_stack(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize,
4290     vm_prot_t prot, vm_prot_t max, int cow)
4291 {
4292 	vm_size_t growsize, init_ssize;
4293 	rlim_t vmemlim;
4294 	int rv;
4295 
4296 	MPASS((map->flags & MAP_WIREFUTURE) == 0);
4297 	growsize = sgrowsiz;
4298 	init_ssize = (max_ssize < growsize) ? max_ssize : growsize;
4299 	vm_map_lock(map);
4300 	vmemlim = lim_cur(curthread, RLIMIT_VMEM);
4301 	/* If we would blow our VMEM resource limit, no go */
4302 	if (map->size + init_ssize > vmemlim) {
4303 		rv = KERN_NO_SPACE;
4304 		goto out;
4305 	}
4306 	rv = vm_map_stack_locked(map, addrbos, max_ssize, growsize, prot,
4307 	    max, cow);
4308 out:
4309 	vm_map_unlock(map);
4310 	return (rv);
4311 }
4312 
4313 static int stack_guard_page = 1;
4314 SYSCTL_INT(_security_bsd, OID_AUTO, stack_guard_page, CTLFLAG_RWTUN,
4315     &stack_guard_page, 0,
4316     "Specifies the number of guard pages for a stack that grows");
4317 
4318 static int
4319 vm_map_stack_locked(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize,
4320     vm_size_t growsize, vm_prot_t prot, vm_prot_t max, int cow)
4321 {
4322 	vm_map_entry_t new_entry, prev_entry;
4323 	vm_offset_t bot, gap_bot, gap_top, top;
4324 	vm_size_t init_ssize, sgp;
4325 	int orient, rv;
4326 
4327 	/*
4328 	 * The stack orientation is piggybacked with the cow argument.
4329 	 * Extract it into orient and mask the cow argument so that we
4330 	 * don't pass it around further.
4331 	 */
4332 	orient = cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP);
4333 	KASSERT(orient != 0, ("No stack grow direction"));
4334 	KASSERT(orient != (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP),
4335 	    ("bi-dir stack"));
4336 
4337 	if (max_ssize == 0 ||
4338 	    !vm_map_range_valid(map, addrbos, addrbos + max_ssize))
4339 		return (KERN_INVALID_ADDRESS);
4340 	sgp = ((curproc->p_flag2 & P2_STKGAP_DISABLE) != 0 ||
4341 	    (curproc->p_fctl0 & NT_FREEBSD_FCTL_STKGAP_DISABLE) != 0) ? 0 :
4342 	    (vm_size_t)stack_guard_page * PAGE_SIZE;
4343 	if (sgp >= max_ssize)
4344 		return (KERN_INVALID_ARGUMENT);
4345 
4346 	init_ssize = growsize;
4347 	if (max_ssize < init_ssize + sgp)
4348 		init_ssize = max_ssize - sgp;
4349 
4350 	/* If addr is already mapped, no go */
4351 	if (vm_map_lookup_entry(map, addrbos, &prev_entry))
4352 		return (KERN_NO_SPACE);
4353 
4354 	/*
4355 	 * If we can't accommodate max_ssize in the current mapping, no go.
4356 	 */
4357 	if (vm_map_entry_succ(prev_entry)->start < addrbos + max_ssize)
4358 		return (KERN_NO_SPACE);
4359 
4360 	/*
4361 	 * We initially map a stack of only init_ssize.  We will grow as
4362 	 * needed later.  Depending on the orientation of the stack (i.e.
4363 	 * the grow direction) we either map at the top of the range, the
4364 	 * bottom of the range or in the middle.
4365 	 *
4366 	 * Note: we would normally expect prot and max to be VM_PROT_ALL,
4367 	 * and cow to be 0.  Possibly we should eliminate these as input
4368 	 * parameters, and just pass these values here in the insert call.
4369 	 */
4370 	if (orient == MAP_STACK_GROWS_DOWN) {
4371 		bot = addrbos + max_ssize - init_ssize;
4372 		top = bot + init_ssize;
4373 		gap_bot = addrbos;
4374 		gap_top = bot;
4375 	} else /* if (orient == MAP_STACK_GROWS_UP) */ {
4376 		bot = addrbos;
4377 		top = bot + init_ssize;
4378 		gap_bot = top;
4379 		gap_top = addrbos + max_ssize;
4380 	}
4381 	rv = vm_map_insert(map, NULL, 0, bot, top, prot, max, cow);
4382 	if (rv != KERN_SUCCESS)
4383 		return (rv);
4384 	new_entry = vm_map_entry_succ(prev_entry);
4385 	KASSERT(new_entry->end == top || new_entry->start == bot,
4386 	    ("Bad entry start/end for new stack entry"));
4387 	KASSERT((orient & MAP_STACK_GROWS_DOWN) == 0 ||
4388 	    (new_entry->eflags & MAP_ENTRY_GROWS_DOWN) != 0,
4389 	    ("new entry lacks MAP_ENTRY_GROWS_DOWN"));
4390 	KASSERT((orient & MAP_STACK_GROWS_UP) == 0 ||
4391 	    (new_entry->eflags & MAP_ENTRY_GROWS_UP) != 0,
4392 	    ("new entry lacks MAP_ENTRY_GROWS_UP"));
4393 	if (gap_bot == gap_top)
4394 		return (KERN_SUCCESS);
4395 	rv = vm_map_insert(map, NULL, 0, gap_bot, gap_top, VM_PROT_NONE,
4396 	    VM_PROT_NONE, MAP_CREATE_GUARD | (orient == MAP_STACK_GROWS_DOWN ?
4397 	    MAP_CREATE_STACK_GAP_DN : MAP_CREATE_STACK_GAP_UP));
4398 	if (rv == KERN_SUCCESS) {
4399 		/*
4400 		 * Gap can never successfully handle a fault, so
4401 		 * read-ahead logic is never used for it.  Re-use
4402 		 * next_read of the gap entry to store
4403 		 * stack_guard_page for vm_map_growstack().
4404 		 */
4405 		if (orient == MAP_STACK_GROWS_DOWN)
4406 			vm_map_entry_pred(new_entry)->next_read = sgp;
4407 		else
4408 			vm_map_entry_succ(new_entry)->next_read = sgp;
4409 	} else {
4410 		(void)vm_map_delete(map, bot, top);
4411 	}
4412 	return (rv);
4413 }
4414 
4415 /*
4416  * Attempts to grow a vm stack entry.  Returns KERN_SUCCESS if we
4417  * successfully grow the stack.
4418  */
4419 static int
4420 vm_map_growstack(vm_map_t map, vm_offset_t addr, vm_map_entry_t gap_entry)
4421 {
4422 	vm_map_entry_t stack_entry;
4423 	struct proc *p;
4424 	struct vmspace *vm;
4425 	struct ucred *cred;
4426 	vm_offset_t gap_end, gap_start, grow_start;
4427 	vm_size_t grow_amount, guard, max_grow;
4428 	rlim_t lmemlim, stacklim, vmemlim;
4429 	int rv, rv1;
4430 	bool gap_deleted, grow_down, is_procstack;
4431 #ifdef notyet
4432 	uint64_t limit;
4433 #endif
4434 #ifdef RACCT
4435 	int error;
4436 #endif
4437 
4438 	p = curproc;
4439 	vm = p->p_vmspace;
4440 
4441 	/*
4442 	 * Disallow stack growth when the access is performed by a
4443 	 * debugger or AIO daemon.  The reason is that the wrong
4444 	 * resource limits are applied.
4445 	 */
4446 	if (p != initproc && (map != &p->p_vmspace->vm_map ||
4447 	    p->p_textvp == NULL))
4448 		return (KERN_FAILURE);
4449 
4450 	MPASS(!map->system_map);
4451 
4452 	lmemlim = lim_cur(curthread, RLIMIT_MEMLOCK);
4453 	stacklim = lim_cur(curthread, RLIMIT_STACK);
4454 	vmemlim = lim_cur(curthread, RLIMIT_VMEM);
4455 retry:
4456 	/* If addr is not in a hole for a stack grow area, no need to grow. */
4457 	if (gap_entry == NULL && !vm_map_lookup_entry(map, addr, &gap_entry))
4458 		return (KERN_FAILURE);
4459 	if ((gap_entry->eflags & MAP_ENTRY_GUARD) == 0)
4460 		return (KERN_SUCCESS);
4461 	if ((gap_entry->eflags & MAP_ENTRY_STACK_GAP_DN) != 0) {
4462 		stack_entry = vm_map_entry_succ(gap_entry);
4463 		if ((stack_entry->eflags & MAP_ENTRY_GROWS_DOWN) == 0 ||
4464 		    stack_entry->start != gap_entry->end)
4465 			return (KERN_FAILURE);
4466 		grow_amount = round_page(stack_entry->start - addr);
4467 		grow_down = true;
4468 	} else if ((gap_entry->eflags & MAP_ENTRY_STACK_GAP_UP) != 0) {
4469 		stack_entry = vm_map_entry_pred(gap_entry);
4470 		if ((stack_entry->eflags & MAP_ENTRY_GROWS_UP) == 0 ||
4471 		    stack_entry->end != gap_entry->start)
4472 			return (KERN_FAILURE);
4473 		grow_amount = round_page(addr + 1 - stack_entry->end);
4474 		grow_down = false;
4475 	} else {
4476 		return (KERN_FAILURE);
4477 	}
4478 	guard = ((curproc->p_flag2 & P2_STKGAP_DISABLE) != 0 ||
4479 	    (curproc->p_fctl0 & NT_FREEBSD_FCTL_STKGAP_DISABLE) != 0) ? 0 :
4480 	    gap_entry->next_read;
4481 	max_grow = gap_entry->end - gap_entry->start;
4482 	if (guard > max_grow)
4483 		return (KERN_NO_SPACE);
4484 	max_grow -= guard;
4485 	if (grow_amount > max_grow)
4486 		return (KERN_NO_SPACE);
4487 
4488 	/*
4489 	 * If this is the main process stack, see if we're over the stack
4490 	 * limit.
4491 	 */
4492 	is_procstack = addr >= (vm_offset_t)vm->vm_maxsaddr &&
4493 	    addr < (vm_offset_t)p->p_sysent->sv_usrstack;
4494 	if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > stacklim))
4495 		return (KERN_NO_SPACE);
4496 
4497 #ifdef RACCT
4498 	if (racct_enable) {
4499 		PROC_LOCK(p);
4500 		if (is_procstack && racct_set(p, RACCT_STACK,
4501 		    ctob(vm->vm_ssize) + grow_amount)) {
4502 			PROC_UNLOCK(p);
4503 			return (KERN_NO_SPACE);
4504 		}
4505 		PROC_UNLOCK(p);
4506 	}
4507 #endif
4508 
4509 	grow_amount = roundup(grow_amount, sgrowsiz);
4510 	if (grow_amount > max_grow)
4511 		grow_amount = max_grow;
4512 	if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > stacklim)) {
4513 		grow_amount = trunc_page((vm_size_t)stacklim) -
4514 		    ctob(vm->vm_ssize);
4515 	}
4516 
4517 #ifdef notyet
4518 	PROC_LOCK(p);
4519 	limit = racct_get_available(p, RACCT_STACK);
4520 	PROC_UNLOCK(p);
4521 	if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > limit))
4522 		grow_amount = limit - ctob(vm->vm_ssize);
4523 #endif
4524 
4525 	if (!old_mlock && (map->flags & MAP_WIREFUTURE) != 0) {
4526 		if (ptoa(pmap_wired_count(map->pmap)) + grow_amount > lmemlim) {
4527 			rv = KERN_NO_SPACE;
4528 			goto out;
4529 		}
4530 #ifdef RACCT
4531 		if (racct_enable) {
4532 			PROC_LOCK(p);
4533 			if (racct_set(p, RACCT_MEMLOCK,
4534 			    ptoa(pmap_wired_count(map->pmap)) + grow_amount)) {
4535 				PROC_UNLOCK(p);
4536 				rv = KERN_NO_SPACE;
4537 				goto out;
4538 			}
4539 			PROC_UNLOCK(p);
4540 		}
4541 #endif
4542 	}
4543 
4544 	/* If we would blow our VMEM resource limit, no go */
4545 	if (map->size + grow_amount > vmemlim) {
4546 		rv = KERN_NO_SPACE;
4547 		goto out;
4548 	}
4549 #ifdef RACCT
4550 	if (racct_enable) {
4551 		PROC_LOCK(p);
4552 		if (racct_set(p, RACCT_VMEM, map->size + grow_amount)) {
4553 			PROC_UNLOCK(p);
4554 			rv = KERN_NO_SPACE;
4555 			goto out;
4556 		}
4557 		PROC_UNLOCK(p);
4558 	}
4559 #endif
4560 
4561 	if (vm_map_lock_upgrade(map)) {
4562 		gap_entry = NULL;
4563 		vm_map_lock_read(map);
4564 		goto retry;
4565 	}
4566 
4567 	if (grow_down) {
4568 		grow_start = gap_entry->end - grow_amount;
4569 		if (gap_entry->start + grow_amount == gap_entry->end) {
4570 			gap_start = gap_entry->start;
4571 			gap_end = gap_entry->end;
4572 			vm_map_entry_delete(map, gap_entry);
4573 			gap_deleted = true;
4574 		} else {
4575 			MPASS(gap_entry->start < gap_entry->end - grow_amount);
4576 			vm_map_entry_resize(map, gap_entry, -grow_amount);
4577 			gap_deleted = false;
4578 		}
4579 		rv = vm_map_insert(map, NULL, 0, grow_start,
4580 		    grow_start + grow_amount,
4581 		    stack_entry->protection, stack_entry->max_protection,
4582 		    MAP_STACK_GROWS_DOWN);
4583 		if (rv != KERN_SUCCESS) {
4584 			if (gap_deleted) {
4585 				rv1 = vm_map_insert(map, NULL, 0, gap_start,
4586 				    gap_end, VM_PROT_NONE, VM_PROT_NONE,
4587 				    MAP_CREATE_GUARD | MAP_CREATE_STACK_GAP_DN);
4588 				MPASS(rv1 == KERN_SUCCESS);
4589 			} else
4590 				vm_map_entry_resize(map, gap_entry,
4591 				    grow_amount);
4592 		}
4593 	} else {
4594 		grow_start = stack_entry->end;
4595 		cred = stack_entry->cred;
4596 		if (cred == NULL && stack_entry->object.vm_object != NULL)
4597 			cred = stack_entry->object.vm_object->cred;
4598 		if (cred != NULL && !swap_reserve_by_cred(grow_amount, cred))
4599 			rv = KERN_NO_SPACE;
4600 		/* Grow the underlying object if applicable. */
4601 		else if (stack_entry->object.vm_object == NULL ||
4602 		    vm_object_coalesce(stack_entry->object.vm_object,
4603 		    stack_entry->offset,
4604 		    (vm_size_t)(stack_entry->end - stack_entry->start),
4605 		    grow_amount, cred != NULL)) {
4606 			if (gap_entry->start + grow_amount == gap_entry->end) {
4607 				vm_map_entry_delete(map, gap_entry);
4608 				vm_map_entry_resize(map, stack_entry,
4609 				    grow_amount);
4610 			} else {
4611 				gap_entry->start += grow_amount;
4612 				stack_entry->end += grow_amount;
4613 			}
4614 			map->size += grow_amount;
4615 			rv = KERN_SUCCESS;
4616 		} else
4617 			rv = KERN_FAILURE;
4618 	}
4619 	if (rv == KERN_SUCCESS && is_procstack)
4620 		vm->vm_ssize += btoc(grow_amount);
4621 
4622 	/*
4623 	 * Heed the MAP_WIREFUTURE flag if it was set for this process.
4624 	 */
4625 	if (rv == KERN_SUCCESS && (map->flags & MAP_WIREFUTURE) != 0) {
4626 		rv = vm_map_wire_locked(map, grow_start,
4627 		    grow_start + grow_amount,
4628 		    VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
4629 	}
4630 	vm_map_lock_downgrade(map);
4631 
4632 out:
4633 #ifdef RACCT
4634 	if (racct_enable && rv != KERN_SUCCESS) {
4635 		PROC_LOCK(p);
4636 		error = racct_set(p, RACCT_VMEM, map->size);
4637 		KASSERT(error == 0, ("decreasing RACCT_VMEM failed"));
4638 		if (!old_mlock) {
4639 			error = racct_set(p, RACCT_MEMLOCK,
4640 			    ptoa(pmap_wired_count(map->pmap)));
4641 			KASSERT(error == 0, ("decreasing RACCT_MEMLOCK failed"));
4642 		}
4643 	    	error = racct_set(p, RACCT_STACK, ctob(vm->vm_ssize));
4644 		KASSERT(error == 0, ("decreasing RACCT_STACK failed"));
4645 		PROC_UNLOCK(p);
4646 	}
4647 #endif
4648 
4649 	return (rv);
4650 }
4651 
4652 /*
4653  * Unshare the specified VM space for exec.  If other processes are
4654  * mapped to it, then create a new one.  The new vmspace is null.
4655  */
4656 int
4657 vmspace_exec(struct proc *p, vm_offset_t minuser, vm_offset_t maxuser)
4658 {
4659 	struct vmspace *oldvmspace = p->p_vmspace;
4660 	struct vmspace *newvmspace;
4661 
4662 	KASSERT((curthread->td_pflags & TDP_EXECVMSPC) == 0,
4663 	    ("vmspace_exec recursed"));
4664 	newvmspace = vmspace_alloc(minuser, maxuser, pmap_pinit);
4665 	if (newvmspace == NULL)
4666 		return (ENOMEM);
4667 	newvmspace->vm_swrss = oldvmspace->vm_swrss;
4668 	/*
4669 	 * This code is written like this for prototype purposes.  The
4670 	 * goal is to avoid running down the vmspace here, but let the
4671 	 * other process's that are still using the vmspace to finally
4672 	 * run it down.  Even though there is little or no chance of blocking
4673 	 * here, it is a good idea to keep this form for future mods.
4674 	 */
4675 	PROC_VMSPACE_LOCK(p);
4676 	p->p_vmspace = newvmspace;
4677 	PROC_VMSPACE_UNLOCK(p);
4678 	if (p == curthread->td_proc)
4679 		pmap_activate(curthread);
4680 	curthread->td_pflags |= TDP_EXECVMSPC;
4681 	return (0);
4682 }
4683 
4684 /*
4685  * Unshare the specified VM space for forcing COW.  This
4686  * is called by rfork, for the (RFMEM|RFPROC) == 0 case.
4687  */
4688 int
4689 vmspace_unshare(struct proc *p)
4690 {
4691 	struct vmspace *oldvmspace = p->p_vmspace;
4692 	struct vmspace *newvmspace;
4693 	vm_ooffset_t fork_charge;
4694 
4695 	if (oldvmspace->vm_refcnt == 1)
4696 		return (0);
4697 	fork_charge = 0;
4698 	newvmspace = vmspace_fork(oldvmspace, &fork_charge);
4699 	if (newvmspace == NULL)
4700 		return (ENOMEM);
4701 	if (!swap_reserve_by_cred(fork_charge, p->p_ucred)) {
4702 		vmspace_free(newvmspace);
4703 		return (ENOMEM);
4704 	}
4705 	PROC_VMSPACE_LOCK(p);
4706 	p->p_vmspace = newvmspace;
4707 	PROC_VMSPACE_UNLOCK(p);
4708 	if (p == curthread->td_proc)
4709 		pmap_activate(curthread);
4710 	vmspace_free(oldvmspace);
4711 	return (0);
4712 }
4713 
4714 /*
4715  *	vm_map_lookup:
4716  *
4717  *	Finds the VM object, offset, and
4718  *	protection for a given virtual address in the
4719  *	specified map, assuming a page fault of the
4720  *	type specified.
4721  *
4722  *	Leaves the map in question locked for read; return
4723  *	values are guaranteed until a vm_map_lookup_done
4724  *	call is performed.  Note that the map argument
4725  *	is in/out; the returned map must be used in
4726  *	the call to vm_map_lookup_done.
4727  *
4728  *	A handle (out_entry) is returned for use in
4729  *	vm_map_lookup_done, to make that fast.
4730  *
4731  *	If a lookup is requested with "write protection"
4732  *	specified, the map may be changed to perform virtual
4733  *	copying operations, although the data referenced will
4734  *	remain the same.
4735  */
4736 int
4737 vm_map_lookup(vm_map_t *var_map,		/* IN/OUT */
4738 	      vm_offset_t vaddr,
4739 	      vm_prot_t fault_typea,
4740 	      vm_map_entry_t *out_entry,	/* OUT */
4741 	      vm_object_t *object,		/* OUT */
4742 	      vm_pindex_t *pindex,		/* OUT */
4743 	      vm_prot_t *out_prot,		/* OUT */
4744 	      boolean_t *wired)			/* OUT */
4745 {
4746 	vm_map_entry_t entry;
4747 	vm_map_t map = *var_map;
4748 	vm_prot_t prot;
4749 	vm_prot_t fault_type;
4750 	vm_object_t eobject;
4751 	vm_size_t size;
4752 	struct ucred *cred;
4753 
4754 RetryLookup:
4755 
4756 	vm_map_lock_read(map);
4757 
4758 RetryLookupLocked:
4759 	/*
4760 	 * Lookup the faulting address.
4761 	 */
4762 	if (!vm_map_lookup_entry(map, vaddr, out_entry)) {
4763 		vm_map_unlock_read(map);
4764 		return (KERN_INVALID_ADDRESS);
4765 	}
4766 
4767 	entry = *out_entry;
4768 
4769 	/*
4770 	 * Handle submaps.
4771 	 */
4772 	if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
4773 		vm_map_t old_map = map;
4774 
4775 		*var_map = map = entry->object.sub_map;
4776 		vm_map_unlock_read(old_map);
4777 		goto RetryLookup;
4778 	}
4779 
4780 	/*
4781 	 * Check whether this task is allowed to have this page.
4782 	 */
4783 	prot = entry->protection;
4784 	if ((fault_typea & VM_PROT_FAULT_LOOKUP) != 0) {
4785 		fault_typea &= ~VM_PROT_FAULT_LOOKUP;
4786 		if (prot == VM_PROT_NONE && map != kernel_map &&
4787 		    (entry->eflags & MAP_ENTRY_GUARD) != 0 &&
4788 		    (entry->eflags & (MAP_ENTRY_STACK_GAP_DN |
4789 		    MAP_ENTRY_STACK_GAP_UP)) != 0 &&
4790 		    vm_map_growstack(map, vaddr, entry) == KERN_SUCCESS)
4791 			goto RetryLookupLocked;
4792 	}
4793 	fault_type = fault_typea & VM_PROT_ALL;
4794 	if ((fault_type & prot) != fault_type || prot == VM_PROT_NONE) {
4795 		vm_map_unlock_read(map);
4796 		return (KERN_PROTECTION_FAILURE);
4797 	}
4798 	KASSERT((prot & VM_PROT_WRITE) == 0 || (entry->eflags &
4799 	    (MAP_ENTRY_USER_WIRED | MAP_ENTRY_NEEDS_COPY)) !=
4800 	    (MAP_ENTRY_USER_WIRED | MAP_ENTRY_NEEDS_COPY),
4801 	    ("entry %p flags %x", entry, entry->eflags));
4802 	if ((fault_typea & VM_PROT_COPY) != 0 &&
4803 	    (entry->max_protection & VM_PROT_WRITE) == 0 &&
4804 	    (entry->eflags & MAP_ENTRY_COW) == 0) {
4805 		vm_map_unlock_read(map);
4806 		return (KERN_PROTECTION_FAILURE);
4807 	}
4808 
4809 	/*
4810 	 * If this page is not pageable, we have to get it for all possible
4811 	 * accesses.
4812 	 */
4813 	*wired = (entry->wired_count != 0);
4814 	if (*wired)
4815 		fault_type = entry->protection;
4816 	size = entry->end - entry->start;
4817 
4818 	/*
4819 	 * If the entry was copy-on-write, we either ...
4820 	 */
4821 	if (entry->eflags & MAP_ENTRY_NEEDS_COPY) {
4822 		/*
4823 		 * If we want to write the page, we may as well handle that
4824 		 * now since we've got the map locked.
4825 		 *
4826 		 * If we don't need to write the page, we just demote the
4827 		 * permissions allowed.
4828 		 */
4829 		if ((fault_type & VM_PROT_WRITE) != 0 ||
4830 		    (fault_typea & VM_PROT_COPY) != 0) {
4831 			/*
4832 			 * Make a new object, and place it in the object
4833 			 * chain.  Note that no new references have appeared
4834 			 * -- one just moved from the map to the new
4835 			 * object.
4836 			 */
4837 			if (vm_map_lock_upgrade(map))
4838 				goto RetryLookup;
4839 
4840 			if (entry->cred == NULL) {
4841 				/*
4842 				 * The debugger owner is charged for
4843 				 * the memory.
4844 				 */
4845 				cred = curthread->td_ucred;
4846 				crhold(cred);
4847 				if (!swap_reserve_by_cred(size, cred)) {
4848 					crfree(cred);
4849 					vm_map_unlock(map);
4850 					return (KERN_RESOURCE_SHORTAGE);
4851 				}
4852 				entry->cred = cred;
4853 			}
4854 			eobject = entry->object.vm_object;
4855 			vm_object_shadow(&entry->object.vm_object,
4856 			    &entry->offset, size, entry->cred, false);
4857 			if (eobject == entry->object.vm_object) {
4858 				/*
4859 				 * The object was not shadowed.
4860 				 */
4861 				swap_release_by_cred(size, entry->cred);
4862 				crfree(entry->cred);
4863 			}
4864 			entry->cred = NULL;
4865 			entry->eflags &= ~MAP_ENTRY_NEEDS_COPY;
4866 
4867 			vm_map_lock_downgrade(map);
4868 		} else {
4869 			/*
4870 			 * We're attempting to read a copy-on-write page --
4871 			 * don't allow writes.
4872 			 */
4873 			prot &= ~VM_PROT_WRITE;
4874 		}
4875 	}
4876 
4877 	/*
4878 	 * Create an object if necessary.
4879 	 */
4880 	if (entry->object.vm_object == NULL && !map->system_map) {
4881 		if (vm_map_lock_upgrade(map))
4882 			goto RetryLookup;
4883 		entry->object.vm_object = vm_object_allocate_anon(atop(size),
4884 		    NULL, entry->cred, entry->cred != NULL ? size : 0);
4885 		entry->offset = 0;
4886 		entry->cred = NULL;
4887 		vm_map_lock_downgrade(map);
4888 	}
4889 
4890 	/*
4891 	 * Return the object/offset from this entry.  If the entry was
4892 	 * copy-on-write or empty, it has been fixed up.
4893 	 */
4894 	*pindex = OFF_TO_IDX((vaddr - entry->start) + entry->offset);
4895 	*object = entry->object.vm_object;
4896 
4897 	*out_prot = prot;
4898 	return (KERN_SUCCESS);
4899 }
4900 
4901 /*
4902  *	vm_map_lookup_locked:
4903  *
4904  *	Lookup the faulting address.  A version of vm_map_lookup that returns
4905  *      KERN_FAILURE instead of blocking on map lock or memory allocation.
4906  */
4907 int
4908 vm_map_lookup_locked(vm_map_t *var_map,		/* IN/OUT */
4909 		     vm_offset_t vaddr,
4910 		     vm_prot_t fault_typea,
4911 		     vm_map_entry_t *out_entry,	/* OUT */
4912 		     vm_object_t *object,	/* OUT */
4913 		     vm_pindex_t *pindex,	/* OUT */
4914 		     vm_prot_t *out_prot,	/* OUT */
4915 		     boolean_t *wired)		/* OUT */
4916 {
4917 	vm_map_entry_t entry;
4918 	vm_map_t map = *var_map;
4919 	vm_prot_t prot;
4920 	vm_prot_t fault_type = fault_typea;
4921 
4922 	/*
4923 	 * Lookup the faulting address.
4924 	 */
4925 	if (!vm_map_lookup_entry(map, vaddr, out_entry))
4926 		return (KERN_INVALID_ADDRESS);
4927 
4928 	entry = *out_entry;
4929 
4930 	/*
4931 	 * Fail if the entry refers to a submap.
4932 	 */
4933 	if (entry->eflags & MAP_ENTRY_IS_SUB_MAP)
4934 		return (KERN_FAILURE);
4935 
4936 	/*
4937 	 * Check whether this task is allowed to have this page.
4938 	 */
4939 	prot = entry->protection;
4940 	fault_type &= VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE;
4941 	if ((fault_type & prot) != fault_type)
4942 		return (KERN_PROTECTION_FAILURE);
4943 
4944 	/*
4945 	 * If this page is not pageable, we have to get it for all possible
4946 	 * accesses.
4947 	 */
4948 	*wired = (entry->wired_count != 0);
4949 	if (*wired)
4950 		fault_type = entry->protection;
4951 
4952 	if (entry->eflags & MAP_ENTRY_NEEDS_COPY) {
4953 		/*
4954 		 * Fail if the entry was copy-on-write for a write fault.
4955 		 */
4956 		if (fault_type & VM_PROT_WRITE)
4957 			return (KERN_FAILURE);
4958 		/*
4959 		 * We're attempting to read a copy-on-write page --
4960 		 * don't allow writes.
4961 		 */
4962 		prot &= ~VM_PROT_WRITE;
4963 	}
4964 
4965 	/*
4966 	 * Fail if an object should be created.
4967 	 */
4968 	if (entry->object.vm_object == NULL && !map->system_map)
4969 		return (KERN_FAILURE);
4970 
4971 	/*
4972 	 * Return the object/offset from this entry.  If the entry was
4973 	 * copy-on-write or empty, it has been fixed up.
4974 	 */
4975 	*pindex = OFF_TO_IDX((vaddr - entry->start) + entry->offset);
4976 	*object = entry->object.vm_object;
4977 
4978 	*out_prot = prot;
4979 	return (KERN_SUCCESS);
4980 }
4981 
4982 /*
4983  *	vm_map_lookup_done:
4984  *
4985  *	Releases locks acquired by a vm_map_lookup
4986  *	(according to the handle returned by that lookup).
4987  */
4988 void
4989 vm_map_lookup_done(vm_map_t map, vm_map_entry_t entry)
4990 {
4991 	/*
4992 	 * Unlock the main-level map
4993 	 */
4994 	vm_map_unlock_read(map);
4995 }
4996 
4997 vm_offset_t
4998 vm_map_max_KBI(const struct vm_map *map)
4999 {
5000 
5001 	return (vm_map_max(map));
5002 }
5003 
5004 vm_offset_t
5005 vm_map_min_KBI(const struct vm_map *map)
5006 {
5007 
5008 	return (vm_map_min(map));
5009 }
5010 
5011 pmap_t
5012 vm_map_pmap_KBI(vm_map_t map)
5013 {
5014 
5015 	return (map->pmap);
5016 }
5017 
5018 bool
5019 vm_map_range_valid_KBI(vm_map_t map, vm_offset_t start, vm_offset_t end)
5020 {
5021 
5022 	return (vm_map_range_valid(map, start, end));
5023 }
5024 
5025 #ifdef INVARIANTS
5026 static void
5027 _vm_map_assert_consistent(vm_map_t map, int check)
5028 {
5029 	vm_map_entry_t entry, prev;
5030 	vm_map_entry_t cur, header, lbound, ubound;
5031 	vm_size_t max_left, max_right;
5032 
5033 #ifdef DIAGNOSTIC
5034 	++map->nupdates;
5035 #endif
5036 	if (enable_vmmap_check != check)
5037 		return;
5038 
5039 	header = prev = &map->header;
5040 	VM_MAP_ENTRY_FOREACH(entry, map) {
5041 		KASSERT(prev->end <= entry->start,
5042 		    ("map %p prev->end = %jx, start = %jx", map,
5043 		    (uintmax_t)prev->end, (uintmax_t)entry->start));
5044 		KASSERT(entry->start < entry->end,
5045 		    ("map %p start = %jx, end = %jx", map,
5046 		    (uintmax_t)entry->start, (uintmax_t)entry->end));
5047 		KASSERT(entry->left == header ||
5048 		    entry->left->start < entry->start,
5049 		    ("map %p left->start = %jx, start = %jx", map,
5050 		    (uintmax_t)entry->left->start, (uintmax_t)entry->start));
5051 		KASSERT(entry->right == header ||
5052 		    entry->start < entry->right->start,
5053 		    ("map %p start = %jx, right->start = %jx", map,
5054 		    (uintmax_t)entry->start, (uintmax_t)entry->right->start));
5055 		cur = map->root;
5056 		lbound = ubound = header;
5057 		for (;;) {
5058 			if (entry->start < cur->start) {
5059 				ubound = cur;
5060 				cur = cur->left;
5061 				KASSERT(cur != lbound,
5062 				    ("map %p cannot find %jx",
5063 				    map, (uintmax_t)entry->start));
5064 			} else if (cur->end <= entry->start) {
5065 				lbound = cur;
5066 				cur = cur->right;
5067 				KASSERT(cur != ubound,
5068 				    ("map %p cannot find %jx",
5069 				    map, (uintmax_t)entry->start));
5070 			} else {
5071 				KASSERT(cur == entry,
5072 				    ("map %p cannot find %jx",
5073 				    map, (uintmax_t)entry->start));
5074 				break;
5075 			}
5076 		}
5077 		max_left = vm_map_entry_max_free_left(entry, lbound);
5078 		max_right = vm_map_entry_max_free_right(entry, ubound);
5079 		KASSERT(entry->max_free == vm_size_max(max_left, max_right),
5080 		    ("map %p max = %jx, max_left = %jx, max_right = %jx", map,
5081 		    (uintmax_t)entry->max_free,
5082 		    (uintmax_t)max_left, (uintmax_t)max_right));
5083 		prev = entry;
5084 	}
5085 	KASSERT(prev->end <= entry->start,
5086 	    ("map %p prev->end = %jx, start = %jx", map,
5087 	    (uintmax_t)prev->end, (uintmax_t)entry->start));
5088 }
5089 #endif
5090 
5091 #include "opt_ddb.h"
5092 #ifdef DDB
5093 #include <sys/kernel.h>
5094 
5095 #include <ddb/ddb.h>
5096 
5097 static void
5098 vm_map_print(vm_map_t map)
5099 {
5100 	vm_map_entry_t entry, prev;
5101 
5102 	db_iprintf("Task map %p: pmap=%p, nentries=%d, version=%u\n",
5103 	    (void *)map,
5104 	    (void *)map->pmap, map->nentries, map->timestamp);
5105 
5106 	db_indent += 2;
5107 	prev = &map->header;
5108 	VM_MAP_ENTRY_FOREACH(entry, map) {
5109 		db_iprintf("map entry %p: start=%p, end=%p, eflags=%#x, \n",
5110 		    (void *)entry, (void *)entry->start, (void *)entry->end,
5111 		    entry->eflags);
5112 		{
5113 			static const char * const inheritance_name[4] =
5114 			{"share", "copy", "none", "donate_copy"};
5115 
5116 			db_iprintf(" prot=%x/%x/%s",
5117 			    entry->protection,
5118 			    entry->max_protection,
5119 			    inheritance_name[(int)(unsigned char)
5120 			    entry->inheritance]);
5121 			if (entry->wired_count != 0)
5122 				db_printf(", wired");
5123 		}
5124 		if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
5125 			db_printf(", share=%p, offset=0x%jx\n",
5126 			    (void *)entry->object.sub_map,
5127 			    (uintmax_t)entry->offset);
5128 			if (prev == &map->header ||
5129 			    prev->object.sub_map !=
5130 				entry->object.sub_map) {
5131 				db_indent += 2;
5132 				vm_map_print((vm_map_t)entry->object.sub_map);
5133 				db_indent -= 2;
5134 			}
5135 		} else {
5136 			if (entry->cred != NULL)
5137 				db_printf(", ruid %d", entry->cred->cr_ruid);
5138 			db_printf(", object=%p, offset=0x%jx",
5139 			    (void *)entry->object.vm_object,
5140 			    (uintmax_t)entry->offset);
5141 			if (entry->object.vm_object && entry->object.vm_object->cred)
5142 				db_printf(", obj ruid %d charge %jx",
5143 				    entry->object.vm_object->cred->cr_ruid,
5144 				    (uintmax_t)entry->object.vm_object->charge);
5145 			if (entry->eflags & MAP_ENTRY_COW)
5146 				db_printf(", copy (%s)",
5147 				    (entry->eflags & MAP_ENTRY_NEEDS_COPY) ? "needed" : "done");
5148 			db_printf("\n");
5149 
5150 			if (prev == &map->header ||
5151 			    prev->object.vm_object !=
5152 				entry->object.vm_object) {
5153 				db_indent += 2;
5154 				vm_object_print((db_expr_t)(intptr_t)
5155 						entry->object.vm_object,
5156 						0, 0, (char *)0);
5157 				db_indent -= 2;
5158 			}
5159 		}
5160 		prev = entry;
5161 	}
5162 	db_indent -= 2;
5163 }
5164 
5165 DB_SHOW_COMMAND(map, map)
5166 {
5167 
5168 	if (!have_addr) {
5169 		db_printf("usage: show map <addr>\n");
5170 		return;
5171 	}
5172 	vm_map_print((vm_map_t)addr);
5173 }
5174 
5175 DB_SHOW_COMMAND(procvm, procvm)
5176 {
5177 	struct proc *p;
5178 
5179 	if (have_addr) {
5180 		p = db_lookup_proc(addr);
5181 	} else {
5182 		p = curproc;
5183 	}
5184 
5185 	db_printf("p = %p, vmspace = %p, map = %p, pmap = %p\n",
5186 	    (void *)p, (void *)p->p_vmspace, (void *)&p->p_vmspace->vm_map,
5187 	    (void *)vmspace_pmap(p->p_vmspace));
5188 
5189 	vm_map_print((vm_map_t)&p->p_vmspace->vm_map);
5190 }
5191 
5192 #endif /* DDB */
5193