xref: /freebsd/sys/vm/vm_mmap.c (revision 3c6e15bceeab4470243c60c9a4b5b9cafca9abaa)
1 /*-
2  * Copyright (c) 1988 University of Utah.
3  * Copyright (c) 1991, 1993
4  *	The Regents of the University of California.  All rights reserved.
5  *
6  * This code is derived from software contributed to Berkeley by
7  * the Systems Programming Group of the University of Utah Computer
8  * Science Department.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 4. Neither the name of the University nor the names of its contributors
19  *    may be used to endorse or promote products derived from this software
20  *    without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  * from: Utah $Hdr: vm_mmap.c 1.6 91/10/21$
35  *
36  *	@(#)vm_mmap.c	8.4 (Berkeley) 1/12/94
37  */
38 
39 /*
40  * Mapped file (mmap) interface to VM
41  */
42 
43 #include <sys/cdefs.h>
44 __FBSDID("$FreeBSD$");
45 
46 #include "opt_compat.h"
47 #include "opt_hwpmc_hooks.h"
48 #include "opt_mac.h"
49 
50 #include <sys/param.h>
51 #include <sys/systm.h>
52 #include <sys/kernel.h>
53 #include <sys/lock.h>
54 #include <sys/mutex.h>
55 #include <sys/sysproto.h>
56 #include <sys/filedesc.h>
57 #include <sys/priv.h>
58 #include <sys/proc.h>
59 #include <sys/resource.h>
60 #include <sys/resourcevar.h>
61 #include <sys/vnode.h>
62 #include <sys/fcntl.h>
63 #include <sys/file.h>
64 #include <sys/mman.h>
65 #include <sys/mount.h>
66 #include <sys/conf.h>
67 #include <sys/stat.h>
68 #include <sys/vmmeter.h>
69 #include <sys/sysctl.h>
70 
71 #include <security/mac/mac_framework.h>
72 
73 #include <vm/vm.h>
74 #include <vm/vm_param.h>
75 #include <vm/pmap.h>
76 #include <vm/vm_map.h>
77 #include <vm/vm_object.h>
78 #include <vm/vm_page.h>
79 #include <vm/vm_pager.h>
80 #include <vm/vm_pageout.h>
81 #include <vm/vm_extern.h>
82 #include <vm/vm_page.h>
83 #include <vm/vm_kern.h>
84 
85 #ifdef HWPMC_HOOKS
86 #include <sys/pmckern.h>
87 #endif
88 
89 #ifndef _SYS_SYSPROTO_H_
90 struct sbrk_args {
91 	int incr;
92 };
93 #endif
94 
95 static int max_proc_mmap;
96 SYSCTL_INT(_vm, OID_AUTO, max_proc_mmap, CTLFLAG_RW, &max_proc_mmap, 0,
97     "Maximum number of memory-mapped files per process");
98 
99 /*
100  * Set the maximum number of vm_map_entry structures per process.  Roughly
101  * speaking vm_map_entry structures are tiny, so allowing them to eat 1/100
102  * of our KVM malloc space still results in generous limits.  We want a
103  * default that is good enough to prevent the kernel running out of resources
104  * if attacked from compromised user account but generous enough such that
105  * multi-threaded processes are not unduly inconvenienced.
106  */
107 static void vmmapentry_rsrc_init(void *);
108 SYSINIT(vmmersrc, SI_SUB_KVM_RSRC, SI_ORDER_FIRST, vmmapentry_rsrc_init,
109     NULL);
110 
111 static void
112 vmmapentry_rsrc_init(dummy)
113         void *dummy;
114 {
115     max_proc_mmap = vm_kmem_size / sizeof(struct vm_map_entry);
116     max_proc_mmap /= 100;
117 }
118 
119 static int vm_mmap_vnode(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *,
120     int *, struct vnode *, vm_ooffset_t, vm_object_t *);
121 static int vm_mmap_cdev(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *,
122     int *, struct cdev *, vm_ooffset_t, vm_object_t *);
123 static int vm_mmap_shm(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *,
124     int *, struct shmfd *, vm_ooffset_t, vm_object_t *);
125 
126 /*
127  * MPSAFE
128  */
129 /* ARGSUSED */
130 int
131 sbrk(td, uap)
132 	struct thread *td;
133 	struct sbrk_args *uap;
134 {
135 	/* Not yet implemented */
136 	return (EOPNOTSUPP);
137 }
138 
139 #ifndef _SYS_SYSPROTO_H_
140 struct sstk_args {
141 	int incr;
142 };
143 #endif
144 
145 /*
146  * MPSAFE
147  */
148 /* ARGSUSED */
149 int
150 sstk(td, uap)
151 	struct thread *td;
152 	struct sstk_args *uap;
153 {
154 	/* Not yet implemented */
155 	return (EOPNOTSUPP);
156 }
157 
158 #if defined(COMPAT_43)
159 #ifndef _SYS_SYSPROTO_H_
160 struct getpagesize_args {
161 	int dummy;
162 };
163 #endif
164 
165 /* ARGSUSED */
166 int
167 ogetpagesize(td, uap)
168 	struct thread *td;
169 	struct getpagesize_args *uap;
170 {
171 	/* MP SAFE */
172 	td->td_retval[0] = PAGE_SIZE;
173 	return (0);
174 }
175 #endif				/* COMPAT_43 */
176 
177 
178 /*
179  * Memory Map (mmap) system call.  Note that the file offset
180  * and address are allowed to be NOT page aligned, though if
181  * the MAP_FIXED flag it set, both must have the same remainder
182  * modulo the PAGE_SIZE (POSIX 1003.1b).  If the address is not
183  * page-aligned, the actual mapping starts at trunc_page(addr)
184  * and the return value is adjusted up by the page offset.
185  *
186  * Generally speaking, only character devices which are themselves
187  * memory-based, such as a video framebuffer, can be mmap'd.  Otherwise
188  * there would be no cache coherency between a descriptor and a VM mapping
189  * both to the same character device.
190  */
191 #ifndef _SYS_SYSPROTO_H_
192 struct mmap_args {
193 	void *addr;
194 	size_t len;
195 	int prot;
196 	int flags;
197 	int fd;
198 	long pad;
199 	off_t pos;
200 };
201 #endif
202 
203 /*
204  * MPSAFE
205  */
206 int
207 mmap(td, uap)
208 	struct thread *td;
209 	struct mmap_args *uap;
210 {
211 #ifdef HWPMC_HOOKS
212 	struct pmckern_map_in pkm;
213 #endif
214 	struct file *fp;
215 	struct vnode *vp;
216 	vm_offset_t addr;
217 	vm_size_t size, pageoff;
218 	vm_prot_t prot, maxprot;
219 	void *handle;
220 	objtype_t handle_type;
221 	int flags, error;
222 	off_t pos;
223 	struct vmspace *vms = td->td_proc->p_vmspace;
224 
225 	addr = (vm_offset_t) uap->addr;
226 	size = uap->len;
227 	prot = uap->prot & VM_PROT_ALL;
228 	flags = uap->flags;
229 	pos = uap->pos;
230 
231 	fp = NULL;
232 	/* make sure mapping fits into numeric range etc */
233 	if ((ssize_t) uap->len < 0 ||
234 	    ((flags & MAP_ANON) && uap->fd != -1))
235 		return (EINVAL);
236 
237 	if (flags & MAP_STACK) {
238 		if ((uap->fd != -1) ||
239 		    ((prot & (PROT_READ | PROT_WRITE)) != (PROT_READ | PROT_WRITE)))
240 			return (EINVAL);
241 		flags |= MAP_ANON;
242 		pos = 0;
243 	}
244 
245 	/*
246 	 * Align the file position to a page boundary,
247 	 * and save its page offset component.
248 	 */
249 	pageoff = (pos & PAGE_MASK);
250 	pos -= pageoff;
251 
252 	/* Adjust size for rounding (on both ends). */
253 	size += pageoff;			/* low end... */
254 	size = (vm_size_t) round_page(size);	/* hi end */
255 
256 	/*
257 	 * Check for illegal addresses.  Watch out for address wrap... Note
258 	 * that VM_*_ADDRESS are not constants due to casts (argh).
259 	 */
260 	if (flags & MAP_FIXED) {
261 		/*
262 		 * The specified address must have the same remainder
263 		 * as the file offset taken modulo PAGE_SIZE, so it
264 		 * should be aligned after adjustment by pageoff.
265 		 */
266 		addr -= pageoff;
267 		if (addr & PAGE_MASK)
268 			return (EINVAL);
269 		/* Address range must be all in user VM space. */
270 		if (addr < vm_map_min(&vms->vm_map) ||
271 		    addr + size > vm_map_max(&vms->vm_map))
272 			return (EINVAL);
273 		if (addr + size < addr)
274 			return (EINVAL);
275 	} else {
276 	/*
277 	 * XXX for non-fixed mappings where no hint is provided or
278 	 * the hint would fall in the potential heap space,
279 	 * place it after the end of the largest possible heap.
280 	 *
281 	 * There should really be a pmap call to determine a reasonable
282 	 * location.
283 	 */
284 		PROC_LOCK(td->td_proc);
285 		if (addr == 0 ||
286 		    (addr >= round_page((vm_offset_t)vms->vm_taddr) &&
287 		    addr < round_page((vm_offset_t)vms->vm_daddr +
288 		    lim_max(td->td_proc, RLIMIT_DATA))))
289 			addr = round_page((vm_offset_t)vms->vm_daddr +
290 			    lim_max(td->td_proc, RLIMIT_DATA));
291 		PROC_UNLOCK(td->td_proc);
292 	}
293 	if (flags & MAP_ANON) {
294 		/*
295 		 * Mapping blank space is trivial.
296 		 */
297 		handle = NULL;
298 		handle_type = OBJT_DEFAULT;
299 		maxprot = VM_PROT_ALL;
300 		pos = 0;
301 	} else {
302 		/*
303 		 * Mapping file, get fp for validation and
304 		 * don't let the descriptor disappear on us if we block.
305 		 */
306 		if ((error = fget(td, uap->fd, &fp)) != 0)
307 			goto done;
308 		if (fp->f_type == DTYPE_SHM) {
309 			handle = fp->f_data;
310 			handle_type = OBJT_SWAP;
311 			maxprot = VM_PROT_NONE;
312 
313 			/* FREAD should always be set. */
314 			if (fp->f_flag & FREAD)
315 				maxprot |= VM_PROT_EXECUTE | VM_PROT_READ;
316 			if (fp->f_flag & FWRITE)
317 				maxprot |= VM_PROT_WRITE;
318 			goto map;
319 		}
320 		if (fp->f_type != DTYPE_VNODE) {
321 			error = ENODEV;
322 			goto done;
323 		}
324 #if defined(COMPAT_FREEBSD7) || defined(COMPAT_FREEBSD6) || \
325     defined(COMPAT_FREEBSD5) || defined(COMPAT_FREEBSD4)
326 		/*
327 		 * POSIX shared-memory objects are defined to have
328 		 * kernel persistence, and are not defined to support
329 		 * read(2)/write(2) -- or even open(2).  Thus, we can
330 		 * use MAP_ASYNC to trade on-disk coherence for speed.
331 		 * The shm_open(3) library routine turns on the FPOSIXSHM
332 		 * flag to request this behavior.
333 		 */
334 		if (fp->f_flag & FPOSIXSHM)
335 			flags |= MAP_NOSYNC;
336 #endif
337 		vp = fp->f_vnode;
338 		/*
339 		 * Ensure that file and memory protections are
340 		 * compatible.  Note that we only worry about
341 		 * writability if mapping is shared; in this case,
342 		 * current and max prot are dictated by the open file.
343 		 * XXX use the vnode instead?  Problem is: what
344 		 * credentials do we use for determination? What if
345 		 * proc does a setuid?
346 		 */
347 		if (vp->v_mount != NULL && vp->v_mount->mnt_flag & MNT_NOEXEC)
348 			maxprot = VM_PROT_NONE;
349 		else
350 			maxprot = VM_PROT_EXECUTE;
351 		if (fp->f_flag & FREAD) {
352 			maxprot |= VM_PROT_READ;
353 		} else if (prot & PROT_READ) {
354 			error = EACCES;
355 			goto done;
356 		}
357 		/*
358 		 * If we are sharing potential changes (either via
359 		 * MAP_SHARED or via the implicit sharing of character
360 		 * device mappings), and we are trying to get write
361 		 * permission although we opened it without asking
362 		 * for it, bail out.
363 		 */
364 		if ((flags & MAP_SHARED) != 0) {
365 			if ((fp->f_flag & FWRITE) != 0) {
366 				maxprot |= VM_PROT_WRITE;
367 			} else if ((prot & PROT_WRITE) != 0) {
368 				error = EACCES;
369 				goto done;
370 			}
371 		} else if (vp->v_type != VCHR || (fp->f_flag & FWRITE) != 0) {
372 			maxprot |= VM_PROT_WRITE;
373 		}
374 		handle = (void *)vp;
375 		handle_type = OBJT_VNODE;
376 	}
377 map:
378 
379 	/*
380 	 * Do not allow more then a certain number of vm_map_entry structures
381 	 * per process.  Scale with the number of rforks sharing the map
382 	 * to make the limit reasonable for threads.
383 	 */
384 	if (max_proc_mmap &&
385 	    vms->vm_map.nentries >= max_proc_mmap * vms->vm_refcnt) {
386 		error = ENOMEM;
387 		goto done;
388 	}
389 
390 	td->td_fpop = fp;
391 	error = vm_mmap(&vms->vm_map, &addr, size, prot, maxprot,
392 	    flags, handle_type, handle, pos);
393 	td->td_fpop = NULL;
394 #ifdef HWPMC_HOOKS
395 	/* inform hwpmc(4) if an executable is being mapped */
396 	if (error == 0 && handle_type == OBJT_VNODE &&
397 	    (prot & PROT_EXEC)) {
398 		pkm.pm_file = handle;
399 		pkm.pm_address = (uintptr_t) addr;
400 		PMC_CALL_HOOK(td, PMC_FN_MMAP, (void *) &pkm);
401 	}
402 #endif
403 	if (error == 0)
404 		td->td_retval[0] = (register_t) (addr + pageoff);
405 done:
406 	if (fp)
407 		fdrop(fp, td);
408 
409 	return (error);
410 }
411 
412 int
413 freebsd6_mmap(struct thread *td, struct freebsd6_mmap_args *uap)
414 {
415 	struct mmap_args oargs;
416 
417 	oargs.addr = uap->addr;
418 	oargs.len = uap->len;
419 	oargs.prot = uap->prot;
420 	oargs.flags = uap->flags;
421 	oargs.fd = uap->fd;
422 	oargs.pos = uap->pos;
423 	return (mmap(td, &oargs));
424 }
425 
426 #ifdef COMPAT_43
427 #ifndef _SYS_SYSPROTO_H_
428 struct ommap_args {
429 	caddr_t addr;
430 	int len;
431 	int prot;
432 	int flags;
433 	int fd;
434 	long pos;
435 };
436 #endif
437 int
438 ommap(td, uap)
439 	struct thread *td;
440 	struct ommap_args *uap;
441 {
442 	struct mmap_args nargs;
443 	static const char cvtbsdprot[8] = {
444 		0,
445 		PROT_EXEC,
446 		PROT_WRITE,
447 		PROT_EXEC | PROT_WRITE,
448 		PROT_READ,
449 		PROT_EXEC | PROT_READ,
450 		PROT_WRITE | PROT_READ,
451 		PROT_EXEC | PROT_WRITE | PROT_READ,
452 	};
453 
454 #define	OMAP_ANON	0x0002
455 #define	OMAP_COPY	0x0020
456 #define	OMAP_SHARED	0x0010
457 #define	OMAP_FIXED	0x0100
458 
459 	nargs.addr = uap->addr;
460 	nargs.len = uap->len;
461 	nargs.prot = cvtbsdprot[uap->prot & 0x7];
462 	nargs.flags = 0;
463 	if (uap->flags & OMAP_ANON)
464 		nargs.flags |= MAP_ANON;
465 	if (uap->flags & OMAP_COPY)
466 		nargs.flags |= MAP_COPY;
467 	if (uap->flags & OMAP_SHARED)
468 		nargs.flags |= MAP_SHARED;
469 	else
470 		nargs.flags |= MAP_PRIVATE;
471 	if (uap->flags & OMAP_FIXED)
472 		nargs.flags |= MAP_FIXED;
473 	nargs.fd = uap->fd;
474 	nargs.pos = uap->pos;
475 	return (mmap(td, &nargs));
476 }
477 #endif				/* COMPAT_43 */
478 
479 
480 #ifndef _SYS_SYSPROTO_H_
481 struct msync_args {
482 	void *addr;
483 	size_t len;
484 	int flags;
485 };
486 #endif
487 /*
488  * MPSAFE
489  */
490 int
491 msync(td, uap)
492 	struct thread *td;
493 	struct msync_args *uap;
494 {
495 	vm_offset_t addr;
496 	vm_size_t size, pageoff;
497 	int flags;
498 	vm_map_t map;
499 	int rv;
500 
501 	addr = (vm_offset_t) uap->addr;
502 	size = uap->len;
503 	flags = uap->flags;
504 
505 	pageoff = (addr & PAGE_MASK);
506 	addr -= pageoff;
507 	size += pageoff;
508 	size = (vm_size_t) round_page(size);
509 	if (addr + size < addr)
510 		return (EINVAL);
511 
512 	if ((flags & (MS_ASYNC|MS_INVALIDATE)) == (MS_ASYNC|MS_INVALIDATE))
513 		return (EINVAL);
514 
515 	map = &td->td_proc->p_vmspace->vm_map;
516 
517 	/*
518 	 * Clean the pages and interpret the return value.
519 	 */
520 	rv = vm_map_sync(map, addr, addr + size, (flags & MS_ASYNC) == 0,
521 	    (flags & MS_INVALIDATE) != 0);
522 	switch (rv) {
523 	case KERN_SUCCESS:
524 		return (0);
525 	case KERN_INVALID_ADDRESS:
526 		return (EINVAL);	/* Sun returns ENOMEM? */
527 	case KERN_INVALID_ARGUMENT:
528 		return (EBUSY);
529 	default:
530 		return (EINVAL);
531 	}
532 }
533 
534 #ifndef _SYS_SYSPROTO_H_
535 struct munmap_args {
536 	void *addr;
537 	size_t len;
538 };
539 #endif
540 /*
541  * MPSAFE
542  */
543 int
544 munmap(td, uap)
545 	struct thread *td;
546 	struct munmap_args *uap;
547 {
548 #ifdef HWPMC_HOOKS
549 	struct pmckern_map_out pkm;
550 	vm_map_entry_t entry;
551 #endif
552 	vm_offset_t addr;
553 	vm_size_t size, pageoff;
554 	vm_map_t map;
555 
556 	addr = (vm_offset_t) uap->addr;
557 	size = uap->len;
558 	if (size == 0)
559 		return (EINVAL);
560 
561 	pageoff = (addr & PAGE_MASK);
562 	addr -= pageoff;
563 	size += pageoff;
564 	size = (vm_size_t) round_page(size);
565 	if (addr + size < addr)
566 		return (EINVAL);
567 
568 	/*
569 	 * Check for illegal addresses.  Watch out for address wrap...
570 	 */
571 	map = &td->td_proc->p_vmspace->vm_map;
572 	if (addr < vm_map_min(map) || addr + size > vm_map_max(map))
573 		return (EINVAL);
574 	vm_map_lock(map);
575 #ifdef HWPMC_HOOKS
576 	/*
577 	 * Inform hwpmc if the address range being unmapped contains
578 	 * an executable region.
579 	 */
580 	if (vm_map_lookup_entry(map, addr, &entry)) {
581 		for (;
582 		     entry != &map->header && entry->start < addr + size;
583 		     entry = entry->next) {
584 			if (vm_map_check_protection(map, entry->start,
585 				entry->end, VM_PROT_EXECUTE) == TRUE) {
586 				pkm.pm_address = (uintptr_t) addr;
587 				pkm.pm_size = (size_t) size;
588 				PMC_CALL_HOOK(td, PMC_FN_MUNMAP,
589 				    (void *) &pkm);
590 				break;
591 			}
592 		}
593 	}
594 #endif
595 	/* returns nothing but KERN_SUCCESS anyway */
596 	vm_map_delete(map, addr, addr + size);
597 	vm_map_unlock(map);
598 	return (0);
599 }
600 
601 #ifndef _SYS_SYSPROTO_H_
602 struct mprotect_args {
603 	const void *addr;
604 	size_t len;
605 	int prot;
606 };
607 #endif
608 /*
609  * MPSAFE
610  */
611 int
612 mprotect(td, uap)
613 	struct thread *td;
614 	struct mprotect_args *uap;
615 {
616 	vm_offset_t addr;
617 	vm_size_t size, pageoff;
618 	vm_prot_t prot;
619 
620 	addr = (vm_offset_t) uap->addr;
621 	size = uap->len;
622 	prot = uap->prot & VM_PROT_ALL;
623 #if defined(VM_PROT_READ_IS_EXEC)
624 	if (prot & VM_PROT_READ)
625 		prot |= VM_PROT_EXECUTE;
626 #endif
627 
628 	pageoff = (addr & PAGE_MASK);
629 	addr -= pageoff;
630 	size += pageoff;
631 	size = (vm_size_t) round_page(size);
632 	if (addr + size < addr)
633 		return (EINVAL);
634 
635 	switch (vm_map_protect(&td->td_proc->p_vmspace->vm_map, addr,
636 	    addr + size, prot, FALSE)) {
637 	case KERN_SUCCESS:
638 		return (0);
639 	case KERN_PROTECTION_FAILURE:
640 		return (EACCES);
641 	}
642 	return (EINVAL);
643 }
644 
645 #ifndef _SYS_SYSPROTO_H_
646 struct minherit_args {
647 	void *addr;
648 	size_t len;
649 	int inherit;
650 };
651 #endif
652 /*
653  * MPSAFE
654  */
655 int
656 minherit(td, uap)
657 	struct thread *td;
658 	struct minherit_args *uap;
659 {
660 	vm_offset_t addr;
661 	vm_size_t size, pageoff;
662 	vm_inherit_t inherit;
663 
664 	addr = (vm_offset_t)uap->addr;
665 	size = uap->len;
666 	inherit = uap->inherit;
667 
668 	pageoff = (addr & PAGE_MASK);
669 	addr -= pageoff;
670 	size += pageoff;
671 	size = (vm_size_t) round_page(size);
672 	if (addr + size < addr)
673 		return (EINVAL);
674 
675 	switch (vm_map_inherit(&td->td_proc->p_vmspace->vm_map, addr,
676 	    addr + size, inherit)) {
677 	case KERN_SUCCESS:
678 		return (0);
679 	case KERN_PROTECTION_FAILURE:
680 		return (EACCES);
681 	}
682 	return (EINVAL);
683 }
684 
685 #ifndef _SYS_SYSPROTO_H_
686 struct madvise_args {
687 	void *addr;
688 	size_t len;
689 	int behav;
690 };
691 #endif
692 
693 /*
694  * MPSAFE
695  */
696 /* ARGSUSED */
697 int
698 madvise(td, uap)
699 	struct thread *td;
700 	struct madvise_args *uap;
701 {
702 	vm_offset_t start, end;
703 	vm_map_t map;
704 	struct proc *p;
705 	int error;
706 
707 	/*
708 	 * Check for our special case, advising the swap pager we are
709 	 * "immortal."
710 	 */
711 	if (uap->behav == MADV_PROTECT) {
712 		error = priv_check(td, PRIV_VM_MADV_PROTECT);
713 		if (error == 0) {
714 			p = td->td_proc;
715 			PROC_LOCK(p);
716 			p->p_flag |= P_PROTECTED;
717 			PROC_UNLOCK(p);
718 		}
719 		return (error);
720 	}
721 	/*
722 	 * Check for illegal behavior
723 	 */
724 	if (uap->behav < 0 || uap->behav > MADV_CORE)
725 		return (EINVAL);
726 	/*
727 	 * Check for illegal addresses.  Watch out for address wrap... Note
728 	 * that VM_*_ADDRESS are not constants due to casts (argh).
729 	 */
730 	map = &td->td_proc->p_vmspace->vm_map;
731 	if ((vm_offset_t)uap->addr < vm_map_min(map) ||
732 	    (vm_offset_t)uap->addr + uap->len > vm_map_max(map))
733 		return (EINVAL);
734 	if (((vm_offset_t) uap->addr + uap->len) < (vm_offset_t) uap->addr)
735 		return (EINVAL);
736 
737 	/*
738 	 * Since this routine is only advisory, we default to conservative
739 	 * behavior.
740 	 */
741 	start = trunc_page((vm_offset_t) uap->addr);
742 	end = round_page((vm_offset_t) uap->addr + uap->len);
743 
744 	if (vm_map_madvise(map, start, end, uap->behav))
745 		return (EINVAL);
746 	return (0);
747 }
748 
749 #ifndef _SYS_SYSPROTO_H_
750 struct mincore_args {
751 	const void *addr;
752 	size_t len;
753 	char *vec;
754 };
755 #endif
756 
757 /*
758  * MPSAFE
759  */
760 /* ARGSUSED */
761 int
762 mincore(td, uap)
763 	struct thread *td;
764 	struct mincore_args *uap;
765 {
766 	vm_offset_t addr, first_addr;
767 	vm_offset_t end, cend;
768 	pmap_t pmap;
769 	vm_map_t map;
770 	char *vec;
771 	int error = 0;
772 	int vecindex, lastvecindex;
773 	vm_map_entry_t current;
774 	vm_map_entry_t entry;
775 	int mincoreinfo;
776 	unsigned int timestamp;
777 
778 	/*
779 	 * Make sure that the addresses presented are valid for user
780 	 * mode.
781 	 */
782 	first_addr = addr = trunc_page((vm_offset_t) uap->addr);
783 	end = addr + (vm_size_t)round_page(uap->len);
784 	map = &td->td_proc->p_vmspace->vm_map;
785 	if (end > vm_map_max(map) || end < addr)
786 		return (ENOMEM);
787 
788 	/*
789 	 * Address of byte vector
790 	 */
791 	vec = uap->vec;
792 
793 	pmap = vmspace_pmap(td->td_proc->p_vmspace);
794 
795 	vm_map_lock_read(map);
796 RestartScan:
797 	timestamp = map->timestamp;
798 
799 	if (!vm_map_lookup_entry(map, addr, &entry)) {
800 		vm_map_unlock_read(map);
801 		return (ENOMEM);
802 	}
803 
804 	/*
805 	 * Do this on a map entry basis so that if the pages are not
806 	 * in the current processes address space, we can easily look
807 	 * up the pages elsewhere.
808 	 */
809 	lastvecindex = -1;
810 	for (current = entry;
811 	    (current != &map->header) && (current->start < end);
812 	    current = current->next) {
813 
814 		/*
815 		 * check for contiguity
816 		 */
817 		if (current->end < end &&
818 		    (entry->next == &map->header ||
819 		     current->next->start > current->end)) {
820 			vm_map_unlock_read(map);
821 			return (ENOMEM);
822 		}
823 
824 		/*
825 		 * ignore submaps (for now) or null objects
826 		 */
827 		if ((current->eflags & MAP_ENTRY_IS_SUB_MAP) ||
828 			current->object.vm_object == NULL)
829 			continue;
830 
831 		/*
832 		 * limit this scan to the current map entry and the
833 		 * limits for the mincore call
834 		 */
835 		if (addr < current->start)
836 			addr = current->start;
837 		cend = current->end;
838 		if (cend > end)
839 			cend = end;
840 
841 		/*
842 		 * scan this entry one page at a time
843 		 */
844 		while (addr < cend) {
845 			/*
846 			 * Check pmap first, it is likely faster, also
847 			 * it can provide info as to whether we are the
848 			 * one referencing or modifying the page.
849 			 */
850 			mincoreinfo = pmap_mincore(pmap, addr);
851 			if (!mincoreinfo) {
852 				vm_pindex_t pindex;
853 				vm_ooffset_t offset;
854 				vm_page_t m;
855 				/*
856 				 * calculate the page index into the object
857 				 */
858 				offset = current->offset + (addr - current->start);
859 				pindex = OFF_TO_IDX(offset);
860 				VM_OBJECT_LOCK(current->object.vm_object);
861 				m = vm_page_lookup(current->object.vm_object,
862 					pindex);
863 				/*
864 				 * if the page is resident, then gather information about
865 				 * it.
866 				 */
867 				if (m != NULL && m->valid != 0) {
868 					mincoreinfo = MINCORE_INCORE;
869 					vm_page_lock_queues();
870 					if (m->dirty ||
871 						pmap_is_modified(m))
872 						mincoreinfo |= MINCORE_MODIFIED_OTHER;
873 					if ((m->flags & PG_REFERENCED) ||
874 						pmap_ts_referenced(m)) {
875 						vm_page_flag_set(m, PG_REFERENCED);
876 						mincoreinfo |= MINCORE_REFERENCED_OTHER;
877 					}
878 					vm_page_unlock_queues();
879 				}
880 				VM_OBJECT_UNLOCK(current->object.vm_object);
881 			}
882 
883 			/*
884 			 * subyte may page fault.  In case it needs to modify
885 			 * the map, we release the lock.
886 			 */
887 			vm_map_unlock_read(map);
888 
889 			/*
890 			 * calculate index into user supplied byte vector
891 			 */
892 			vecindex = OFF_TO_IDX(addr - first_addr);
893 
894 			/*
895 			 * If we have skipped map entries, we need to make sure that
896 			 * the byte vector is zeroed for those skipped entries.
897 			 */
898 			while ((lastvecindex + 1) < vecindex) {
899 				error = subyte(vec + lastvecindex, 0);
900 				if (error) {
901 					error = EFAULT;
902 					goto done2;
903 				}
904 				++lastvecindex;
905 			}
906 
907 			/*
908 			 * Pass the page information to the user
909 			 */
910 			error = subyte(vec + vecindex, mincoreinfo);
911 			if (error) {
912 				error = EFAULT;
913 				goto done2;
914 			}
915 
916 			/*
917 			 * If the map has changed, due to the subyte, the previous
918 			 * output may be invalid.
919 			 */
920 			vm_map_lock_read(map);
921 			if (timestamp != map->timestamp)
922 				goto RestartScan;
923 
924 			lastvecindex = vecindex;
925 			addr += PAGE_SIZE;
926 		}
927 	}
928 
929 	/*
930 	 * subyte may page fault.  In case it needs to modify
931 	 * the map, we release the lock.
932 	 */
933 	vm_map_unlock_read(map);
934 
935 	/*
936 	 * Zero the last entries in the byte vector.
937 	 */
938 	vecindex = OFF_TO_IDX(end - first_addr);
939 	while ((lastvecindex + 1) < vecindex) {
940 		error = subyte(vec + lastvecindex, 0);
941 		if (error) {
942 			error = EFAULT;
943 			goto done2;
944 		}
945 		++lastvecindex;
946 	}
947 
948 	/*
949 	 * If the map has changed, due to the subyte, the previous
950 	 * output may be invalid.
951 	 */
952 	vm_map_lock_read(map);
953 	if (timestamp != map->timestamp)
954 		goto RestartScan;
955 	vm_map_unlock_read(map);
956 done2:
957 	return (error);
958 }
959 
960 #ifndef _SYS_SYSPROTO_H_
961 struct mlock_args {
962 	const void *addr;
963 	size_t len;
964 };
965 #endif
966 /*
967  * MPSAFE
968  */
969 int
970 mlock(td, uap)
971 	struct thread *td;
972 	struct mlock_args *uap;
973 {
974 	struct proc *proc;
975 	vm_offset_t addr, end, last, start;
976 	vm_size_t npages, size;
977 	int error;
978 
979 	error = priv_check(td, PRIV_VM_MLOCK);
980 	if (error)
981 		return (error);
982 	addr = (vm_offset_t)uap->addr;
983 	size = uap->len;
984 	last = addr + size;
985 	start = trunc_page(addr);
986 	end = round_page(last);
987 	if (last < addr || end < addr)
988 		return (EINVAL);
989 	npages = atop(end - start);
990 	if (npages > vm_page_max_wired)
991 		return (ENOMEM);
992 	proc = td->td_proc;
993 	PROC_LOCK(proc);
994 	if (ptoa(npages +
995 	    pmap_wired_count(vm_map_pmap(&proc->p_vmspace->vm_map))) >
996 	    lim_cur(proc, RLIMIT_MEMLOCK)) {
997 		PROC_UNLOCK(proc);
998 		return (ENOMEM);
999 	}
1000 	PROC_UNLOCK(proc);
1001 	if (npages + cnt.v_wire_count > vm_page_max_wired)
1002 		return (EAGAIN);
1003 	error = vm_map_wire(&proc->p_vmspace->vm_map, start, end,
1004 	    VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
1005 	return (error == KERN_SUCCESS ? 0 : ENOMEM);
1006 }
1007 
1008 #ifndef _SYS_SYSPROTO_H_
1009 struct mlockall_args {
1010 	int	how;
1011 };
1012 #endif
1013 
1014 /*
1015  * MPSAFE
1016  */
1017 int
1018 mlockall(td, uap)
1019 	struct thread *td;
1020 	struct mlockall_args *uap;
1021 {
1022 	vm_map_t map;
1023 	int error;
1024 
1025 	map = &td->td_proc->p_vmspace->vm_map;
1026 	error = 0;
1027 
1028 	if ((uap->how == 0) || ((uap->how & ~(MCL_CURRENT|MCL_FUTURE)) != 0))
1029 		return (EINVAL);
1030 
1031 #if 0
1032 	/*
1033 	 * If wiring all pages in the process would cause it to exceed
1034 	 * a hard resource limit, return ENOMEM.
1035 	 */
1036 	PROC_LOCK(td->td_proc);
1037 	if (map->size - ptoa(pmap_wired_count(vm_map_pmap(map)) >
1038 		lim_cur(td->td_proc, RLIMIT_MEMLOCK))) {
1039 		PROC_UNLOCK(td->td_proc);
1040 		return (ENOMEM);
1041 	}
1042 	PROC_UNLOCK(td->td_proc);
1043 #else
1044 	error = priv_check(td, PRIV_VM_MLOCK);
1045 	if (error)
1046 		return (error);
1047 #endif
1048 
1049 	if (uap->how & MCL_FUTURE) {
1050 		vm_map_lock(map);
1051 		vm_map_modflags(map, MAP_WIREFUTURE, 0);
1052 		vm_map_unlock(map);
1053 		error = 0;
1054 	}
1055 
1056 	if (uap->how & MCL_CURRENT) {
1057 		/*
1058 		 * P1003.1-2001 mandates that all currently mapped pages
1059 		 * will be memory resident and locked (wired) upon return
1060 		 * from mlockall(). vm_map_wire() will wire pages, by
1061 		 * calling vm_fault_wire() for each page in the region.
1062 		 */
1063 		error = vm_map_wire(map, vm_map_min(map), vm_map_max(map),
1064 		    VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK);
1065 		error = (error == KERN_SUCCESS ? 0 : EAGAIN);
1066 	}
1067 
1068 	return (error);
1069 }
1070 
1071 #ifndef _SYS_SYSPROTO_H_
1072 struct munlockall_args {
1073 	register_t dummy;
1074 };
1075 #endif
1076 
1077 /*
1078  * MPSAFE
1079  */
1080 int
1081 munlockall(td, uap)
1082 	struct thread *td;
1083 	struct munlockall_args *uap;
1084 {
1085 	vm_map_t map;
1086 	int error;
1087 
1088 	map = &td->td_proc->p_vmspace->vm_map;
1089 	error = priv_check(td, PRIV_VM_MUNLOCK);
1090 	if (error)
1091 		return (error);
1092 
1093 	/* Clear the MAP_WIREFUTURE flag from this vm_map. */
1094 	vm_map_lock(map);
1095 	vm_map_modflags(map, 0, MAP_WIREFUTURE);
1096 	vm_map_unlock(map);
1097 
1098 	/* Forcibly unwire all pages. */
1099 	error = vm_map_unwire(map, vm_map_min(map), vm_map_max(map),
1100 	    VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK);
1101 
1102 	return (error);
1103 }
1104 
1105 #ifndef _SYS_SYSPROTO_H_
1106 struct munlock_args {
1107 	const void *addr;
1108 	size_t len;
1109 };
1110 #endif
1111 /*
1112  * MPSAFE
1113  */
1114 int
1115 munlock(td, uap)
1116 	struct thread *td;
1117 	struct munlock_args *uap;
1118 {
1119 	vm_offset_t addr, end, last, start;
1120 	vm_size_t size;
1121 	int error;
1122 
1123 	error = priv_check(td, PRIV_VM_MUNLOCK);
1124 	if (error)
1125 		return (error);
1126 	addr = (vm_offset_t)uap->addr;
1127 	size = uap->len;
1128 	last = addr + size;
1129 	start = trunc_page(addr);
1130 	end = round_page(last);
1131 	if (last < addr || end < addr)
1132 		return (EINVAL);
1133 	error = vm_map_unwire(&td->td_proc->p_vmspace->vm_map, start, end,
1134 	    VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
1135 	return (error == KERN_SUCCESS ? 0 : ENOMEM);
1136 }
1137 
1138 /*
1139  * vm_mmap_vnode()
1140  *
1141  * MPSAFE
1142  *
1143  * Helper function for vm_mmap.  Perform sanity check specific for mmap
1144  * operations on vnodes.
1145  */
1146 int
1147 vm_mmap_vnode(struct thread *td, vm_size_t objsize,
1148     vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp,
1149     struct vnode *vp, vm_ooffset_t foff, vm_object_t *objp)
1150 {
1151 	struct vattr va;
1152 	void *handle;
1153 	vm_object_t obj;
1154 	struct mount *mp;
1155 	struct cdevsw *dsw;
1156 	struct ucred *cred;
1157 	int error, flags, type;
1158 	int vfslocked;
1159 
1160 	mp = vp->v_mount;
1161 	cred = td->td_ucred;
1162 	vfslocked = VFS_LOCK_GIANT(mp);
1163 	if ((error = vget(vp, LK_EXCLUSIVE, td)) != 0) {
1164 		VFS_UNLOCK_GIANT(vfslocked);
1165 		return (error);
1166 	}
1167 	flags = *flagsp;
1168 	obj = vp->v_object;
1169 	if (vp->v_type == VREG) {
1170 		/*
1171 		 * Get the proper underlying object
1172 		 */
1173 		if (obj == NULL) {
1174 			error = EINVAL;
1175 			goto done;
1176 		}
1177 		if (obj->handle != vp) {
1178 			vput(vp);
1179 			vp = (struct vnode*)obj->handle;
1180 			vget(vp, LK_EXCLUSIVE, td);
1181 		}
1182 		type = OBJT_VNODE;
1183 		handle = vp;
1184 	} else if (vp->v_type == VCHR) {
1185 		type = OBJT_DEVICE;
1186 		handle = vp->v_rdev;
1187 
1188 		dsw = dev_refthread(handle);
1189 		if (dsw == NULL) {
1190 			error = ENXIO;
1191 			goto done;
1192 		}
1193 		if (dsw->d_flags & D_MMAP_ANON) {
1194 			dev_relthread(handle);
1195 			*maxprotp = VM_PROT_ALL;
1196 			*flagsp |= MAP_ANON;
1197 			error = 0;
1198 			goto done;
1199 		}
1200 		dev_relthread(handle);
1201 		/*
1202 		 * cdevs does not provide private mappings of any kind.
1203 		 */
1204 		if ((*maxprotp & VM_PROT_WRITE) == 0 &&
1205 		    (prot & PROT_WRITE) != 0) {
1206 			error = EACCES;
1207 			goto done;
1208 		}
1209 		if (flags & (MAP_PRIVATE|MAP_COPY)) {
1210 			error = EINVAL;
1211 			goto done;
1212 		}
1213 		/*
1214 		 * Force device mappings to be shared.
1215 		 */
1216 		flags |= MAP_SHARED;
1217 	} else {
1218 		error = EINVAL;
1219 		goto done;
1220 	}
1221 	if ((error = VOP_GETATTR(vp, &va, cred)))
1222 		goto done;
1223 #ifdef MAC
1224 	error = mac_vnode_check_mmap(cred, vp, prot, flags);
1225 	if (error != 0)
1226 		goto done;
1227 #endif
1228 	if ((flags & MAP_SHARED) != 0) {
1229 		if ((va.va_flags & (SF_SNAPSHOT|IMMUTABLE|APPEND)) != 0) {
1230 			if (prot & PROT_WRITE) {
1231 				error = EPERM;
1232 				goto done;
1233 			}
1234 			*maxprotp &= ~VM_PROT_WRITE;
1235 		}
1236 	}
1237 	/*
1238 	 * If it is a regular file without any references
1239 	 * we do not need to sync it.
1240 	 * Adjust object size to be the size of actual file.
1241 	 */
1242 	if (vp->v_type == VREG) {
1243 		objsize = round_page(va.va_size);
1244 		if (va.va_nlink == 0)
1245 			flags |= MAP_NOSYNC;
1246 	}
1247 	obj = vm_pager_allocate(type, handle, objsize, prot, foff);
1248 	if (obj == NULL) {
1249 		error = (type == OBJT_DEVICE ? EINVAL : ENOMEM);
1250 		goto done;
1251 	}
1252 	*objp = obj;
1253 	*flagsp = flags;
1254 	vfs_mark_atime(vp, cred);
1255 
1256 done:
1257 	vput(vp);
1258 	VFS_UNLOCK_GIANT(vfslocked);
1259 	return (error);
1260 }
1261 
1262 /*
1263  * vm_mmap_cdev()
1264  *
1265  * MPSAFE
1266  *
1267  * Helper function for vm_mmap.  Perform sanity check specific for mmap
1268  * operations on cdevs.
1269  */
1270 int
1271 vm_mmap_cdev(struct thread *td, vm_size_t objsize,
1272     vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp,
1273     struct cdev *cdev, vm_ooffset_t foff, vm_object_t *objp)
1274 {
1275 	vm_object_t obj;
1276 	struct cdevsw *dsw;
1277 	int flags;
1278 
1279 	flags = *flagsp;
1280 
1281 	dsw = dev_refthread(cdev);
1282 	if (dsw == NULL)
1283 		return (ENXIO);
1284 	if (dsw->d_flags & D_MMAP_ANON) {
1285 		dev_relthread(cdev);
1286 		*maxprotp = VM_PROT_ALL;
1287 		*flagsp |= MAP_ANON;
1288 		return (0);
1289 	}
1290 	dev_relthread(cdev);
1291 	/*
1292 	 * cdevs does not provide private mappings of any kind.
1293 	 */
1294 	if ((*maxprotp & VM_PROT_WRITE) == 0 &&
1295 	    (prot & PROT_WRITE) != 0)
1296 		return (EACCES);
1297 	if (flags & (MAP_PRIVATE|MAP_COPY))
1298 		return (EINVAL);
1299 	/*
1300 	 * Force device mappings to be shared.
1301 	 */
1302 	flags |= MAP_SHARED;
1303 #ifdef MAC_XXX
1304 	error = mac_check_cdev_mmap(td->td_ucred, cdev, prot);
1305 	if (error != 0)
1306 		return (error);
1307 #endif
1308 	obj = vm_pager_allocate(OBJT_DEVICE, cdev, objsize, prot, foff);
1309 	if (obj == NULL)
1310 		return (EINVAL);
1311 	*objp = obj;
1312 	*flagsp = flags;
1313 	return (0);
1314 }
1315 
1316 /*
1317  * vm_mmap_shm()
1318  *
1319  * MPSAFE
1320  *
1321  * Helper function for vm_mmap.  Perform sanity check specific for mmap
1322  * operations on shm file descriptors.
1323  */
1324 int
1325 vm_mmap_shm(struct thread *td, vm_size_t objsize,
1326     vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp,
1327     struct shmfd *shmfd, vm_ooffset_t foff, vm_object_t *objp)
1328 {
1329 	int error;
1330 
1331 	if ((*maxprotp & VM_PROT_WRITE) == 0 &&
1332 	    (prot & PROT_WRITE) != 0)
1333 		return (EACCES);
1334 #ifdef MAC
1335 	error = mac_posixshm_check_mmap(td->td_ucred, shmfd, prot, *flagsp);
1336 	if (error != 0)
1337 		return (error);
1338 #endif
1339 	error = shm_mmap(shmfd, objsize, foff, objp);
1340 	if (error)
1341 		return (error);
1342 	return (0);
1343 }
1344 
1345 /*
1346  * vm_mmap()
1347  *
1348  * MPSAFE
1349  *
1350  * Internal version of mmap.  Currently used by mmap, exec, and sys5
1351  * shared memory.  Handle is either a vnode pointer or NULL for MAP_ANON.
1352  */
1353 int
1354 vm_mmap(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot,
1355 	vm_prot_t maxprot, int flags,
1356 	objtype_t handle_type, void *handle,
1357 	vm_ooffset_t foff)
1358 {
1359 	boolean_t fitit;
1360 	vm_object_t object = NULL;
1361 	int rv = KERN_SUCCESS;
1362 	int docow, error;
1363 	struct thread *td = curthread;
1364 
1365 	if (size == 0)
1366 		return (0);
1367 
1368 	size = round_page(size);
1369 
1370 	PROC_LOCK(td->td_proc);
1371 	if (td->td_proc->p_vmspace->vm_map.size + size >
1372 	    lim_cur(td->td_proc, RLIMIT_VMEM)) {
1373 		PROC_UNLOCK(td->td_proc);
1374 		return(ENOMEM);
1375 	}
1376 	PROC_UNLOCK(td->td_proc);
1377 
1378 	/*
1379 	 * We currently can only deal with page aligned file offsets.
1380 	 * The check is here rather than in the syscall because the
1381 	 * kernel calls this function internally for other mmaping
1382 	 * operations (such as in exec) and non-aligned offsets will
1383 	 * cause pmap inconsistencies...so we want to be sure to
1384 	 * disallow this in all cases.
1385 	 */
1386 	if (foff & PAGE_MASK)
1387 		return (EINVAL);
1388 
1389 	if ((flags & MAP_FIXED) == 0) {
1390 		fitit = TRUE;
1391 		*addr = round_page(*addr);
1392 	} else {
1393 		if (*addr != trunc_page(*addr))
1394 			return (EINVAL);
1395 		fitit = FALSE;
1396 	}
1397 	/*
1398 	 * Lookup/allocate object.
1399 	 */
1400 	switch (handle_type) {
1401 	case OBJT_DEVICE:
1402 		error = vm_mmap_cdev(td, size, prot, &maxprot, &flags,
1403 		    handle, foff, &object);
1404 		break;
1405 	case OBJT_VNODE:
1406 		error = vm_mmap_vnode(td, size, prot, &maxprot, &flags,
1407 		    handle, foff, &object);
1408 		break;
1409 	case OBJT_SWAP:
1410 		error = vm_mmap_shm(td, size, prot, &maxprot, &flags,
1411 		    handle, foff, &object);
1412 		break;
1413 	case OBJT_DEFAULT:
1414 		if (handle == NULL) {
1415 			error = 0;
1416 			break;
1417 		}
1418 		/* FALLTHROUGH */
1419 	default:
1420 		error = EINVAL;
1421 		break;
1422 	}
1423 	if (error)
1424 		return (error);
1425 	if (flags & MAP_ANON) {
1426 		object = NULL;
1427 		docow = 0;
1428 		/*
1429 		 * Unnamed anonymous regions always start at 0.
1430 		 */
1431 		if (handle == 0)
1432 			foff = 0;
1433 	} else {
1434 		docow = MAP_PREFAULT_PARTIAL;
1435 	}
1436 
1437 	if ((flags & (MAP_ANON|MAP_SHARED)) == 0)
1438 		docow |= MAP_COPY_ON_WRITE;
1439 	if (flags & MAP_NOSYNC)
1440 		docow |= MAP_DISABLE_SYNCER;
1441 	if (flags & MAP_NOCORE)
1442 		docow |= MAP_DISABLE_COREDUMP;
1443 
1444 #if defined(VM_PROT_READ_IS_EXEC)
1445 	if (prot & VM_PROT_READ)
1446 		prot |= VM_PROT_EXECUTE;
1447 
1448 	if (maxprot & VM_PROT_READ)
1449 		maxprot |= VM_PROT_EXECUTE;
1450 #endif
1451 
1452 	if (flags & MAP_STACK)
1453 		rv = vm_map_stack(map, *addr, size, prot, maxprot,
1454 		    docow | MAP_STACK_GROWS_DOWN);
1455 	else if (fitit)
1456 		rv = vm_map_find(map, object, foff, addr, size,
1457 		    object != NULL && object->type == OBJT_DEVICE ?
1458 		    VMFS_ALIGNED_SPACE : VMFS_ANY_SPACE, prot, maxprot, docow);
1459 	else
1460 		rv = vm_map_fixed(map, object, foff, *addr, size,
1461 				 prot, maxprot, docow);
1462 
1463 	if (rv != KERN_SUCCESS) {
1464 		/*
1465 		 * Lose the object reference. Will destroy the
1466 		 * object if it's an unnamed anonymous mapping
1467 		 * or named anonymous without other references.
1468 		 */
1469 		vm_object_deallocate(object);
1470 	} else if (flags & MAP_SHARED) {
1471 		/*
1472 		 * Shared memory is also shared with children.
1473 		 */
1474 		rv = vm_map_inherit(map, *addr, *addr + size, VM_INHERIT_SHARE);
1475 		if (rv != KERN_SUCCESS)
1476 			(void) vm_map_remove(map, *addr, *addr + size);
1477 	}
1478 
1479 	/*
1480 	 * If the process has requested that all future mappings
1481 	 * be wired, then heed this.
1482 	 */
1483 	if ((rv == KERN_SUCCESS) && (map->flags & MAP_WIREFUTURE))
1484 		vm_map_wire(map, *addr, *addr + size,
1485 		    VM_MAP_WIRE_USER|VM_MAP_WIRE_NOHOLES);
1486 
1487 	switch (rv) {
1488 	case KERN_SUCCESS:
1489 		return (0);
1490 	case KERN_INVALID_ADDRESS:
1491 	case KERN_NO_SPACE:
1492 		return (ENOMEM);
1493 	case KERN_PROTECTION_FAILURE:
1494 		return (EACCES);
1495 	default:
1496 		return (EINVAL);
1497 	}
1498 }
1499