xref: /freebsd/sys/vm/vm_mmap.c (revision 57f804675e65951d630a38d94c07be4a27ae4053)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1988 University of Utah.
5  * Copyright (c) 1991, 1993
6  *	The Regents of the University of California.  All rights reserved.
7  *
8  * This code is derived from software contributed to Berkeley by
9  * the Systems Programming Group of the University of Utah Computer
10  * Science Department.
11  *
12  * Redistribution and use in source and binary forms, with or without
13  * modification, are permitted provided that the following conditions
14  * are met:
15  * 1. Redistributions of source code must retain the above copyright
16  *    notice, this list of conditions and the following disclaimer.
17  * 2. Redistributions in binary form must reproduce the above copyright
18  *    notice, this list of conditions and the following disclaimer in the
19  *    documentation and/or other materials provided with the distribution.
20  * 3. Neither the name of the University nor the names of its contributors
21  *    may be used to endorse or promote products derived from this software
22  *    without specific prior written permission.
23  *
24  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34  * SUCH DAMAGE.
35  *
36  * from: Utah $Hdr: vm_mmap.c 1.6 91/10/21$
37  *
38  *	@(#)vm_mmap.c	8.4 (Berkeley) 1/12/94
39  */
40 
41 /*
42  * Mapped file (mmap) interface to VM
43  */
44 
45 #include <sys/cdefs.h>
46 __FBSDID("$FreeBSD$");
47 
48 #include "opt_hwpmc_hooks.h"
49 #include "opt_vm.h"
50 
51 #include <sys/param.h>
52 #include <sys/systm.h>
53 #include <sys/capsicum.h>
54 #include <sys/kernel.h>
55 #include <sys/lock.h>
56 #include <sys/mutex.h>
57 #include <sys/sysproto.h>
58 #include <sys/elf.h>
59 #include <sys/filedesc.h>
60 #include <sys/priv.h>
61 #include <sys/proc.h>
62 #include <sys/procctl.h>
63 #include <sys/racct.h>
64 #include <sys/resource.h>
65 #include <sys/resourcevar.h>
66 #include <sys/rwlock.h>
67 #include <sys/sysctl.h>
68 #include <sys/vnode.h>
69 #include <sys/fcntl.h>
70 #include <sys/file.h>
71 #include <sys/mman.h>
72 #include <sys/mount.h>
73 #include <sys/conf.h>
74 #include <sys/stat.h>
75 #include <sys/syscallsubr.h>
76 #include <sys/sysent.h>
77 #include <sys/vmmeter.h>
78 #if defined(__amd64__) || defined(__i386__) /* for i386_read_exec */
79 #include <machine/md_var.h>
80 #endif
81 
82 #include <security/audit/audit.h>
83 #include <security/mac/mac_framework.h>
84 
85 #include <vm/vm.h>
86 #include <vm/vm_param.h>
87 #include <vm/pmap.h>
88 #include <vm/vm_map.h>
89 #include <vm/vm_object.h>
90 #include <vm/vm_page.h>
91 #include <vm/vm_pager.h>
92 #include <vm/vm_pageout.h>
93 #include <vm/vm_extern.h>
94 #include <vm/vm_page.h>
95 #include <vm/vnode_pager.h>
96 
97 #ifdef HWPMC_HOOKS
98 #include <sys/pmckern.h>
99 #endif
100 
101 int old_mlock = 0;
102 SYSCTL_INT(_vm, OID_AUTO, old_mlock, CTLFLAG_RWTUN, &old_mlock, 0,
103     "Do not apply RLIMIT_MEMLOCK on mlockall");
104 static int mincore_mapped = 1;
105 SYSCTL_INT(_vm, OID_AUTO, mincore_mapped, CTLFLAG_RWTUN, &mincore_mapped, 0,
106     "mincore reports mappings, not residency");
107 static int imply_prot_max = 0;
108 SYSCTL_INT(_vm, OID_AUTO, imply_prot_max, CTLFLAG_RWTUN, &imply_prot_max, 0,
109     "Imply maximum page permissions in mmap() when none are specified");
110 
111 #ifdef MAP_32BIT
112 #define	MAP_32BIT_MAX_ADDR	((vm_offset_t)1 << 31)
113 #endif
114 
115 #ifndef _SYS_SYSPROTO_H_
116 struct sbrk_args {
117 	int incr;
118 };
119 #endif
120 
121 int
122 sys_sbrk(struct thread *td, struct sbrk_args *uap)
123 {
124 	/* Not yet implemented */
125 	return (EOPNOTSUPP);
126 }
127 
128 #ifndef _SYS_SYSPROTO_H_
129 struct sstk_args {
130 	int incr;
131 };
132 #endif
133 
134 int
135 sys_sstk(struct thread *td, struct sstk_args *uap)
136 {
137 	/* Not yet implemented */
138 	return (EOPNOTSUPP);
139 }
140 
141 #if defined(COMPAT_43)
142 int
143 ogetpagesize(struct thread *td, struct ogetpagesize_args *uap)
144 {
145 
146 	td->td_retval[0] = PAGE_SIZE;
147 	return (0);
148 }
149 #endif				/* COMPAT_43 */
150 
151 
152 /*
153  * Memory Map (mmap) system call.  Note that the file offset
154  * and address are allowed to be NOT page aligned, though if
155  * the MAP_FIXED flag it set, both must have the same remainder
156  * modulo the PAGE_SIZE (POSIX 1003.1b).  If the address is not
157  * page-aligned, the actual mapping starts at trunc_page(addr)
158  * and the return value is adjusted up by the page offset.
159  *
160  * Generally speaking, only character devices which are themselves
161  * memory-based, such as a video framebuffer, can be mmap'd.  Otherwise
162  * there would be no cache coherency between a descriptor and a VM mapping
163  * both to the same character device.
164  */
165 #ifndef _SYS_SYSPROTO_H_
166 struct mmap_args {
167 	void *addr;
168 	size_t len;
169 	int prot;
170 	int flags;
171 	int fd;
172 	long pad;
173 	off_t pos;
174 };
175 #endif
176 
177 int
178 sys_mmap(struct thread *td, struct mmap_args *uap)
179 {
180 
181 	return (kern_mmap(td, (uintptr_t)uap->addr, uap->len, uap->prot,
182 	    uap->flags, uap->fd, uap->pos));
183 }
184 
185 int
186 kern_mmap_maxprot(struct proc *p, int prot)
187 {
188 
189 	if ((p->p_flag2 & P2_PROTMAX_DISABLE) != 0 ||
190 	    (p->p_fctl0 & NT_FREEBSD_FCTL_PROTMAX_DISABLE) != 0)
191 		return (_PROT_ALL);
192 	if (((p->p_flag2 & P2_PROTMAX_ENABLE) != 0 || imply_prot_max) &&
193 	    prot != PROT_NONE)
194 		 return (prot);
195 	return (_PROT_ALL);
196 }
197 
198 int
199 kern_mmap(struct thread *td, uintptr_t addr0, size_t len, int prot, int flags,
200     int fd, off_t pos)
201 {
202 
203 	return (kern_mmap_fpcheck(td, addr0, len, prot, flags, fd, pos, NULL));
204 }
205 
206 /*
207  * When mmap'ing a file, check_fp_fn may be used for the caller to do any
208  * last-minute validation based on the referenced file in a non-racy way.
209  */
210 int
211 kern_mmap_fpcheck(struct thread *td, uintptr_t addr0, size_t len, int prot,
212     int flags, int fd, off_t pos, mmap_check_fp_fn check_fp_fn)
213 {
214 	struct vmspace *vms;
215 	struct file *fp;
216 	struct proc *p;
217 	vm_offset_t addr;
218 	vm_size_t pageoff, size;
219 	vm_prot_t cap_maxprot;
220 	int align, error, max_prot;
221 	cap_rights_t rights;
222 
223 	if ((prot & ~(_PROT_ALL | PROT_MAX(_PROT_ALL))) != 0)
224 		return (EINVAL);
225 	max_prot = PROT_MAX_EXTRACT(prot);
226 	prot = PROT_EXTRACT(prot);
227 	if (max_prot != 0 && (max_prot & prot) != prot)
228 		return (ENOTSUP);
229 
230 	p = td->td_proc;
231 
232 	/*
233 	 * Always honor PROT_MAX if set.  If not, default to all
234 	 * permissions unless we're implying maximum permissions.
235 	 */
236 	if (max_prot == 0)
237 		max_prot = kern_mmap_maxprot(p, prot);
238 
239 	vms = p->p_vmspace;
240 	fp = NULL;
241 	AUDIT_ARG_FD(fd);
242 	addr = addr0;
243 
244 	/*
245 	 * Ignore old flags that used to be defined but did not do anything.
246 	 */
247 	flags &= ~(MAP_RESERVED0020 | MAP_RESERVED0040);
248 
249 	/*
250 	 * Enforce the constraints.
251 	 * Mapping of length 0 is only allowed for old binaries.
252 	 * Anonymous mapping shall specify -1 as filedescriptor and
253 	 * zero position for new code. Be nice to ancient a.out
254 	 * binaries and correct pos for anonymous mapping, since old
255 	 * ld.so sometimes issues anonymous map requests with non-zero
256 	 * pos.
257 	 */
258 	if (!SV_CURPROC_FLAG(SV_AOUT)) {
259 		if ((len == 0 && p->p_osrel >= P_OSREL_MAP_ANON) ||
260 		    ((flags & MAP_ANON) != 0 && (fd != -1 || pos != 0)))
261 			return (EINVAL);
262 	} else {
263 		if ((flags & MAP_ANON) != 0)
264 			pos = 0;
265 	}
266 
267 	if (flags & MAP_STACK) {
268 		if ((fd != -1) ||
269 		    ((prot & (PROT_READ | PROT_WRITE)) != (PROT_READ | PROT_WRITE)))
270 			return (EINVAL);
271 		flags |= MAP_ANON;
272 		pos = 0;
273 	}
274 	if ((flags & ~(MAP_SHARED | MAP_PRIVATE | MAP_FIXED | MAP_HASSEMAPHORE |
275 	    MAP_STACK | MAP_NOSYNC | MAP_ANON | MAP_EXCL | MAP_NOCORE |
276 	    MAP_PREFAULT_READ | MAP_GUARD |
277 #ifdef MAP_32BIT
278 	    MAP_32BIT |
279 #endif
280 	    MAP_ALIGNMENT_MASK)) != 0)
281 		return (EINVAL);
282 	if ((flags & (MAP_EXCL | MAP_FIXED)) == MAP_EXCL)
283 		return (EINVAL);
284 	if ((flags & (MAP_SHARED | MAP_PRIVATE)) == (MAP_SHARED | MAP_PRIVATE))
285 		return (EINVAL);
286 	if (prot != PROT_NONE &&
287 	    (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC)) != 0)
288 		return (EINVAL);
289 	if ((flags & MAP_GUARD) != 0 && (prot != PROT_NONE || fd != -1 ||
290 	    pos != 0 || (flags & ~(MAP_FIXED | MAP_GUARD | MAP_EXCL |
291 #ifdef MAP_32BIT
292 	    MAP_32BIT |
293 #endif
294 	    MAP_ALIGNMENT_MASK)) != 0))
295 		return (EINVAL);
296 
297 	/*
298 	 * Align the file position to a page boundary,
299 	 * and save its page offset component.
300 	 */
301 	pageoff = (pos & PAGE_MASK);
302 	pos -= pageoff;
303 
304 	/* Compute size from len by rounding (on both ends). */
305 	size = len + pageoff;			/* low end... */
306 	size = round_page(size);		/* hi end */
307 	/* Check for rounding up to zero. */
308 	if (len > size)
309 		return (ENOMEM);
310 
311 	/* Ensure alignment is at least a page and fits in a pointer. */
312 	align = flags & MAP_ALIGNMENT_MASK;
313 	if (align != 0 && align != MAP_ALIGNED_SUPER &&
314 	    (align >> MAP_ALIGNMENT_SHIFT >= sizeof(void *) * NBBY ||
315 	    align >> MAP_ALIGNMENT_SHIFT < PAGE_SHIFT))
316 		return (EINVAL);
317 
318 	/*
319 	 * Check for illegal addresses.  Watch out for address wrap... Note
320 	 * that VM_*_ADDRESS are not constants due to casts (argh).
321 	 */
322 	if (flags & MAP_FIXED) {
323 		/*
324 		 * The specified address must have the same remainder
325 		 * as the file offset taken modulo PAGE_SIZE, so it
326 		 * should be aligned after adjustment by pageoff.
327 		 */
328 		addr -= pageoff;
329 		if (addr & PAGE_MASK)
330 			return (EINVAL);
331 
332 		/* Address range must be all in user VM space. */
333 		if (addr < vm_map_min(&vms->vm_map) ||
334 		    addr + size > vm_map_max(&vms->vm_map))
335 			return (EINVAL);
336 		if (addr + size < addr)
337 			return (EINVAL);
338 #ifdef MAP_32BIT
339 		if (flags & MAP_32BIT && addr + size > MAP_32BIT_MAX_ADDR)
340 			return (EINVAL);
341 	} else if (flags & MAP_32BIT) {
342 		/*
343 		 * For MAP_32BIT, override the hint if it is too high and
344 		 * do not bother moving the mapping past the heap (since
345 		 * the heap is usually above 2GB).
346 		 */
347 		if (addr + size > MAP_32BIT_MAX_ADDR)
348 			addr = 0;
349 #endif
350 	} else {
351 		/*
352 		 * XXX for non-fixed mappings where no hint is provided or
353 		 * the hint would fall in the potential heap space,
354 		 * place it after the end of the largest possible heap.
355 		 *
356 		 * There should really be a pmap call to determine a reasonable
357 		 * location.
358 		 */
359 		if (addr == 0 ||
360 		    (addr >= round_page((vm_offset_t)vms->vm_taddr) &&
361 		    addr < round_page((vm_offset_t)vms->vm_daddr +
362 		    lim_max(td, RLIMIT_DATA))))
363 			addr = round_page((vm_offset_t)vms->vm_daddr +
364 			    lim_max(td, RLIMIT_DATA));
365 	}
366 	if (len == 0) {
367 		/*
368 		 * Return success without mapping anything for old
369 		 * binaries that request a page-aligned mapping of
370 		 * length 0.  For modern binaries, this function
371 		 * returns an error earlier.
372 		 */
373 		error = 0;
374 	} else if ((flags & MAP_GUARD) != 0) {
375 		error = vm_mmap_object(&vms->vm_map, &addr, size, VM_PROT_NONE,
376 		    VM_PROT_NONE, flags, NULL, pos, FALSE, td);
377 	} else if ((flags & MAP_ANON) != 0) {
378 		/*
379 		 * Mapping blank space is trivial.
380 		 *
381 		 * This relies on VM_PROT_* matching PROT_*.
382 		 */
383 		error = vm_mmap_object(&vms->vm_map, &addr, size, prot,
384 		    max_prot, flags, NULL, pos, FALSE, td);
385 	} else {
386 		/*
387 		 * Mapping file, get fp for validation and don't let the
388 		 * descriptor disappear on us if we block. Check capability
389 		 * rights, but also return the maximum rights to be combined
390 		 * with maxprot later.
391 		 */
392 		cap_rights_init_one(&rights, CAP_MMAP);
393 		if (prot & PROT_READ)
394 			cap_rights_set_one(&rights, CAP_MMAP_R);
395 		if ((flags & MAP_SHARED) != 0) {
396 			if (prot & PROT_WRITE)
397 				cap_rights_set_one(&rights, CAP_MMAP_W);
398 		}
399 		if (prot & PROT_EXEC)
400 			cap_rights_set_one(&rights, CAP_MMAP_X);
401 		error = fget_mmap(td, fd, &rights, &cap_maxprot, &fp);
402 		if (error != 0)
403 			goto done;
404 		if ((flags & (MAP_SHARED | MAP_PRIVATE)) == 0 &&
405 		    p->p_osrel >= P_OSREL_MAP_FSTRICT) {
406 			error = EINVAL;
407 			goto done;
408 		}
409 		if (check_fp_fn != NULL) {
410 			error = check_fp_fn(fp, prot, max_prot & cap_maxprot,
411 			    flags);
412 			if (error != 0)
413 				goto done;
414 		}
415 		/* This relies on VM_PROT_* matching PROT_*. */
416 		error = fo_mmap(fp, &vms->vm_map, &addr, size, prot,
417 		    max_prot & cap_maxprot, flags, pos, td);
418 	}
419 
420 	if (error == 0)
421 		td->td_retval[0] = (register_t) (addr + pageoff);
422 done:
423 	if (fp)
424 		fdrop(fp, td);
425 
426 	return (error);
427 }
428 
429 #if defined(COMPAT_FREEBSD6)
430 int
431 freebsd6_mmap(struct thread *td, struct freebsd6_mmap_args *uap)
432 {
433 
434 	return (kern_mmap(td, (uintptr_t)uap->addr, uap->len, uap->prot,
435 	    uap->flags, uap->fd, uap->pos));
436 }
437 #endif
438 
439 #ifdef COMPAT_43
440 #ifndef _SYS_SYSPROTO_H_
441 struct ommap_args {
442 	caddr_t addr;
443 	int len;
444 	int prot;
445 	int flags;
446 	int fd;
447 	long pos;
448 };
449 #endif
450 int
451 ommap(struct thread *td, struct ommap_args *uap)
452 {
453 	static const char cvtbsdprot[8] = {
454 		0,
455 		PROT_EXEC,
456 		PROT_WRITE,
457 		PROT_EXEC | PROT_WRITE,
458 		PROT_READ,
459 		PROT_EXEC | PROT_READ,
460 		PROT_WRITE | PROT_READ,
461 		PROT_EXEC | PROT_WRITE | PROT_READ,
462 	};
463 	int flags, prot;
464 
465 #define	OMAP_ANON	0x0002
466 #define	OMAP_COPY	0x0020
467 #define	OMAP_SHARED	0x0010
468 #define	OMAP_FIXED	0x0100
469 
470 	prot = cvtbsdprot[uap->prot & 0x7];
471 #if (defined(COMPAT_FREEBSD32) && defined(__amd64__)) || defined(__i386__)
472 	if (i386_read_exec && SV_PROC_FLAG(td->td_proc, SV_ILP32) &&
473 	    prot != 0)
474 		prot |= PROT_EXEC;
475 #endif
476 	flags = 0;
477 	if (uap->flags & OMAP_ANON)
478 		flags |= MAP_ANON;
479 	if (uap->flags & OMAP_COPY)
480 		flags |= MAP_COPY;
481 	if (uap->flags & OMAP_SHARED)
482 		flags |= MAP_SHARED;
483 	else
484 		flags |= MAP_PRIVATE;
485 	if (uap->flags & OMAP_FIXED)
486 		flags |= MAP_FIXED;
487 	return (kern_mmap(td, (uintptr_t)uap->addr, uap->len, prot, flags,
488 	    uap->fd, uap->pos));
489 }
490 #endif				/* COMPAT_43 */
491 
492 
493 #ifndef _SYS_SYSPROTO_H_
494 struct msync_args {
495 	void *addr;
496 	size_t len;
497 	int flags;
498 };
499 #endif
500 int
501 sys_msync(struct thread *td, struct msync_args *uap)
502 {
503 
504 	return (kern_msync(td, (uintptr_t)uap->addr, uap->len, uap->flags));
505 }
506 
507 int
508 kern_msync(struct thread *td, uintptr_t addr0, size_t size, int flags)
509 {
510 	vm_offset_t addr;
511 	vm_size_t pageoff;
512 	vm_map_t map;
513 	int rv;
514 
515 	addr = addr0;
516 	pageoff = (addr & PAGE_MASK);
517 	addr -= pageoff;
518 	size += pageoff;
519 	size = (vm_size_t) round_page(size);
520 	if (addr + size < addr)
521 		return (EINVAL);
522 
523 	if ((flags & (MS_ASYNC|MS_INVALIDATE)) == (MS_ASYNC|MS_INVALIDATE))
524 		return (EINVAL);
525 
526 	map = &td->td_proc->p_vmspace->vm_map;
527 
528 	/*
529 	 * Clean the pages and interpret the return value.
530 	 */
531 	rv = vm_map_sync(map, addr, addr + size, (flags & MS_ASYNC) == 0,
532 	    (flags & MS_INVALIDATE) != 0);
533 	switch (rv) {
534 	case KERN_SUCCESS:
535 		return (0);
536 	case KERN_INVALID_ADDRESS:
537 		return (ENOMEM);
538 	case KERN_INVALID_ARGUMENT:
539 		return (EBUSY);
540 	case KERN_FAILURE:
541 		return (EIO);
542 	default:
543 		return (EINVAL);
544 	}
545 }
546 
547 #ifndef _SYS_SYSPROTO_H_
548 struct munmap_args {
549 	void *addr;
550 	size_t len;
551 };
552 #endif
553 int
554 sys_munmap(struct thread *td, struct munmap_args *uap)
555 {
556 
557 	return (kern_munmap(td, (uintptr_t)uap->addr, uap->len));
558 }
559 
560 int
561 kern_munmap(struct thread *td, uintptr_t addr0, size_t size)
562 {
563 #ifdef HWPMC_HOOKS
564 	struct pmckern_map_out pkm;
565 	vm_map_entry_t entry;
566 	bool pmc_handled;
567 #endif
568 	vm_offset_t addr;
569 	vm_size_t pageoff;
570 	vm_map_t map;
571 
572 	if (size == 0)
573 		return (EINVAL);
574 
575 	addr = addr0;
576 	pageoff = (addr & PAGE_MASK);
577 	addr -= pageoff;
578 	size += pageoff;
579 	size = (vm_size_t) round_page(size);
580 	if (addr + size < addr)
581 		return (EINVAL);
582 
583 	/*
584 	 * Check for illegal addresses.  Watch out for address wrap...
585 	 */
586 	map = &td->td_proc->p_vmspace->vm_map;
587 	if (addr < vm_map_min(map) || addr + size > vm_map_max(map))
588 		return (EINVAL);
589 	vm_map_lock(map);
590 #ifdef HWPMC_HOOKS
591 	pmc_handled = false;
592 	if (PMC_HOOK_INSTALLED(PMC_FN_MUNMAP)) {
593 		pmc_handled = true;
594 		/*
595 		 * Inform hwpmc if the address range being unmapped contains
596 		 * an executable region.
597 		 */
598 		pkm.pm_address = (uintptr_t) NULL;
599 		if (vm_map_lookup_entry(map, addr, &entry)) {
600 			for (; entry->start < addr + size;
601 			    entry = vm_map_entry_succ(entry)) {
602 				if (vm_map_check_protection(map, entry->start,
603 					entry->end, VM_PROT_EXECUTE) == TRUE) {
604 					pkm.pm_address = (uintptr_t) addr;
605 					pkm.pm_size = (size_t) size;
606 					break;
607 				}
608 			}
609 		}
610 	}
611 #endif
612 	vm_map_delete(map, addr, addr + size);
613 
614 #ifdef HWPMC_HOOKS
615 	if (__predict_false(pmc_handled)) {
616 		/* downgrade the lock to prevent a LOR with the pmc-sx lock */
617 		vm_map_lock_downgrade(map);
618 		if (pkm.pm_address != (uintptr_t) NULL)
619 			PMC_CALL_HOOK(td, PMC_FN_MUNMAP, (void *) &pkm);
620 		vm_map_unlock_read(map);
621 	} else
622 #endif
623 		vm_map_unlock(map);
624 
625 	/* vm_map_delete returns nothing but KERN_SUCCESS anyway */
626 	return (0);
627 }
628 
629 #ifndef _SYS_SYSPROTO_H_
630 struct mprotect_args {
631 	const void *addr;
632 	size_t len;
633 	int prot;
634 };
635 #endif
636 int
637 sys_mprotect(struct thread *td, struct mprotect_args *uap)
638 {
639 
640 	return (kern_mprotect(td, (uintptr_t)uap->addr, uap->len, uap->prot));
641 }
642 
643 int
644 kern_mprotect(struct thread *td, uintptr_t addr0, size_t size, int prot)
645 {
646 	vm_offset_t addr;
647 	vm_size_t pageoff;
648 	int vm_error, max_prot;
649 
650 	addr = addr0;
651 	if ((prot & ~(_PROT_ALL | PROT_MAX(_PROT_ALL))) != 0)
652 		return (EINVAL);
653 	max_prot = PROT_MAX_EXTRACT(prot);
654 	prot = PROT_EXTRACT(prot);
655 	pageoff = (addr & PAGE_MASK);
656 	addr -= pageoff;
657 	size += pageoff;
658 	size = (vm_size_t) round_page(size);
659 #ifdef COMPAT_FREEBSD32
660 	if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) {
661 		if (((addr + size) & 0xffffffff) < addr)
662 			return (EINVAL);
663 	} else
664 #endif
665 	if (addr + size < addr)
666 		return (EINVAL);
667 
668 	vm_error = KERN_SUCCESS;
669 	if (max_prot != 0) {
670 		if ((max_prot & prot) != prot)
671 			return (ENOTSUP);
672 		vm_error = vm_map_protect(&td->td_proc->p_vmspace->vm_map,
673 		    addr, addr + size, max_prot, TRUE);
674 	}
675 	if (vm_error == KERN_SUCCESS)
676 		vm_error = vm_map_protect(&td->td_proc->p_vmspace->vm_map,
677 		    addr, addr + size, prot, FALSE);
678 
679 	switch (vm_error) {
680 	case KERN_SUCCESS:
681 		return (0);
682 	case KERN_PROTECTION_FAILURE:
683 		return (EACCES);
684 	case KERN_RESOURCE_SHORTAGE:
685 		return (ENOMEM);
686 	}
687 	return (EINVAL);
688 }
689 
690 #ifndef _SYS_SYSPROTO_H_
691 struct minherit_args {
692 	void *addr;
693 	size_t len;
694 	int inherit;
695 };
696 #endif
697 int
698 sys_minherit(struct thread *td, struct minherit_args *uap)
699 {
700 	vm_offset_t addr;
701 	vm_size_t size, pageoff;
702 	vm_inherit_t inherit;
703 
704 	addr = (vm_offset_t)uap->addr;
705 	size = uap->len;
706 	inherit = uap->inherit;
707 
708 	pageoff = (addr & PAGE_MASK);
709 	addr -= pageoff;
710 	size += pageoff;
711 	size = (vm_size_t) round_page(size);
712 	if (addr + size < addr)
713 		return (EINVAL);
714 
715 	switch (vm_map_inherit(&td->td_proc->p_vmspace->vm_map, addr,
716 	    addr + size, inherit)) {
717 	case KERN_SUCCESS:
718 		return (0);
719 	case KERN_PROTECTION_FAILURE:
720 		return (EACCES);
721 	}
722 	return (EINVAL);
723 }
724 
725 #ifndef _SYS_SYSPROTO_H_
726 struct madvise_args {
727 	void *addr;
728 	size_t len;
729 	int behav;
730 };
731 #endif
732 
733 int
734 sys_madvise(struct thread *td, struct madvise_args *uap)
735 {
736 
737 	return (kern_madvise(td, (uintptr_t)uap->addr, uap->len, uap->behav));
738 }
739 
740 int
741 kern_madvise(struct thread *td, uintptr_t addr0, size_t len, int behav)
742 {
743 	vm_map_t map;
744 	vm_offset_t addr, end, start;
745 	int flags;
746 
747 	/*
748 	 * Check for our special case, advising the swap pager we are
749 	 * "immortal."
750 	 */
751 	if (behav == MADV_PROTECT) {
752 		flags = PPROT_SET;
753 		return (kern_procctl(td, P_PID, td->td_proc->p_pid,
754 		    PROC_SPROTECT, &flags));
755 	}
756 
757 	/*
758 	 * Check for illegal addresses.  Watch out for address wrap... Note
759 	 * that VM_*_ADDRESS are not constants due to casts (argh).
760 	 */
761 	map = &td->td_proc->p_vmspace->vm_map;
762 	addr = addr0;
763 	if (addr < vm_map_min(map) || addr + len > vm_map_max(map))
764 		return (EINVAL);
765 	if ((addr + len) < addr)
766 		return (EINVAL);
767 
768 	/*
769 	 * Since this routine is only advisory, we default to conservative
770 	 * behavior.
771 	 */
772 	start = trunc_page(addr);
773 	end = round_page(addr + len);
774 
775 	/*
776 	 * vm_map_madvise() checks for illegal values of behav.
777 	 */
778 	return (vm_map_madvise(map, start, end, behav));
779 }
780 
781 #ifndef _SYS_SYSPROTO_H_
782 struct mincore_args {
783 	const void *addr;
784 	size_t len;
785 	char *vec;
786 };
787 #endif
788 
789 int
790 sys_mincore(struct thread *td, struct mincore_args *uap)
791 {
792 
793 	return (kern_mincore(td, (uintptr_t)uap->addr, uap->len, uap->vec));
794 }
795 
796 int
797 kern_mincore(struct thread *td, uintptr_t addr0, size_t len, char *vec)
798 {
799 	pmap_t pmap;
800 	vm_map_t map;
801 	vm_map_entry_t current, entry;
802 	vm_object_t object;
803 	vm_offset_t addr, cend, end, first_addr;
804 	vm_paddr_t pa;
805 	vm_page_t m;
806 	vm_pindex_t pindex;
807 	int error, lastvecindex, mincoreinfo, vecindex;
808 	unsigned int timestamp;
809 
810 	/*
811 	 * Make sure that the addresses presented are valid for user
812 	 * mode.
813 	 */
814 	first_addr = addr = trunc_page(addr0);
815 	end = round_page(addr0 + len);
816 	map = &td->td_proc->p_vmspace->vm_map;
817 	if (end > vm_map_max(map) || end < addr)
818 		return (ENOMEM);
819 
820 	pmap = vmspace_pmap(td->td_proc->p_vmspace);
821 
822 	vm_map_lock_read(map);
823 RestartScan:
824 	timestamp = map->timestamp;
825 
826 	if (!vm_map_lookup_entry(map, addr, &entry)) {
827 		vm_map_unlock_read(map);
828 		return (ENOMEM);
829 	}
830 
831 	/*
832 	 * Do this on a map entry basis so that if the pages are not
833 	 * in the current processes address space, we can easily look
834 	 * up the pages elsewhere.
835 	 */
836 	lastvecindex = -1;
837 	while (entry->start < end) {
838 
839 		/*
840 		 * check for contiguity
841 		 */
842 		current = entry;
843 		entry = vm_map_entry_succ(current);
844 		if (current->end < end &&
845 		    entry->start > current->end) {
846 			vm_map_unlock_read(map);
847 			return (ENOMEM);
848 		}
849 
850 		/*
851 		 * ignore submaps (for now) or null objects
852 		 */
853 		if ((current->eflags & MAP_ENTRY_IS_SUB_MAP) ||
854 		    current->object.vm_object == NULL)
855 			continue;
856 
857 		/*
858 		 * limit this scan to the current map entry and the
859 		 * limits for the mincore call
860 		 */
861 		if (addr < current->start)
862 			addr = current->start;
863 		cend = current->end;
864 		if (cend > end)
865 			cend = end;
866 
867 		for (; addr < cend; addr += PAGE_SIZE) {
868 			/*
869 			 * Check pmap first, it is likely faster, also
870 			 * it can provide info as to whether we are the
871 			 * one referencing or modifying the page.
872 			 */
873 			m = NULL;
874 			object = NULL;
875 retry:
876 			pa = 0;
877 			mincoreinfo = pmap_mincore(pmap, addr, &pa);
878 			if (mincore_mapped) {
879 				/*
880 				 * We only care about this pmap's
881 				 * mapping of the page, if any.
882 				 */
883 				;
884 			} else if (pa != 0) {
885 				/*
886 				 * The page is mapped by this process but not
887 				 * both accessed and modified.  It is also
888 				 * managed.  Acquire the object lock so that
889 				 * other mappings might be examined.  The page's
890 				 * identity may change at any point before its
891 				 * object lock is acquired, so re-validate if
892 				 * necessary.
893 				 */
894 				m = PHYS_TO_VM_PAGE(pa);
895 				while (object == NULL || m->object != object) {
896 					if (object != NULL)
897 						VM_OBJECT_WUNLOCK(object);
898 					object = atomic_load_ptr(&m->object);
899 					if (object == NULL)
900 						goto retry;
901 					VM_OBJECT_WLOCK(object);
902 				}
903 				if (pa != pmap_extract(pmap, addr))
904 					goto retry;
905 				KASSERT(vm_page_all_valid(m),
906 				    ("mincore: page %p is mapped but invalid",
907 				    m));
908 			} else if (mincoreinfo == 0) {
909 				/*
910 				 * The page is not mapped by this process.  If
911 				 * the object implements managed pages, then
912 				 * determine if the page is resident so that
913 				 * the mappings might be examined.
914 				 */
915 				if (current->object.vm_object != object) {
916 					if (object != NULL)
917 						VM_OBJECT_WUNLOCK(object);
918 					object = current->object.vm_object;
919 					VM_OBJECT_WLOCK(object);
920 				}
921 				if (object->type == OBJT_DEFAULT ||
922 				    object->type == OBJT_SWAP ||
923 				    object->type == OBJT_VNODE) {
924 					pindex = OFF_TO_IDX(current->offset +
925 					    (addr - current->start));
926 					m = vm_page_lookup(object, pindex);
927 					if (m != NULL && vm_page_none_valid(m))
928 						m = NULL;
929 					if (m != NULL)
930 						mincoreinfo = MINCORE_INCORE;
931 				}
932 			}
933 			if (m != NULL) {
934 				VM_OBJECT_ASSERT_WLOCKED(m->object);
935 
936 				/* Examine other mappings of the page. */
937 				if (m->dirty == 0 && pmap_is_modified(m))
938 					vm_page_dirty(m);
939 				if (m->dirty != 0)
940 					mincoreinfo |= MINCORE_MODIFIED_OTHER;
941 
942 				/*
943 				 * The first test for PGA_REFERENCED is an
944 				 * optimization.  The second test is
945 				 * required because a concurrent pmap
946 				 * operation could clear the last reference
947 				 * and set PGA_REFERENCED before the call to
948 				 * pmap_is_referenced().
949 				 */
950 				if ((m->a.flags & PGA_REFERENCED) != 0 ||
951 				    pmap_is_referenced(m) ||
952 				    (m->a.flags & PGA_REFERENCED) != 0)
953 					mincoreinfo |= MINCORE_REFERENCED_OTHER;
954 			}
955 			if (object != NULL)
956 				VM_OBJECT_WUNLOCK(object);
957 
958 			/*
959 			 * subyte may page fault.  In case it needs to modify
960 			 * the map, we release the lock.
961 			 */
962 			vm_map_unlock_read(map);
963 
964 			/*
965 			 * calculate index into user supplied byte vector
966 			 */
967 			vecindex = atop(addr - first_addr);
968 
969 			/*
970 			 * If we have skipped map entries, we need to make sure that
971 			 * the byte vector is zeroed for those skipped entries.
972 			 */
973 			while ((lastvecindex + 1) < vecindex) {
974 				++lastvecindex;
975 				error = subyte(vec + lastvecindex, 0);
976 				if (error) {
977 					error = EFAULT;
978 					goto done2;
979 				}
980 			}
981 
982 			/*
983 			 * Pass the page information to the user
984 			 */
985 			error = subyte(vec + vecindex, mincoreinfo);
986 			if (error) {
987 				error = EFAULT;
988 				goto done2;
989 			}
990 
991 			/*
992 			 * If the map has changed, due to the subyte, the previous
993 			 * output may be invalid.
994 			 */
995 			vm_map_lock_read(map);
996 			if (timestamp != map->timestamp)
997 				goto RestartScan;
998 
999 			lastvecindex = vecindex;
1000 		}
1001 	}
1002 
1003 	/*
1004 	 * subyte may page fault.  In case it needs to modify
1005 	 * the map, we release the lock.
1006 	 */
1007 	vm_map_unlock_read(map);
1008 
1009 	/*
1010 	 * Zero the last entries in the byte vector.
1011 	 */
1012 	vecindex = atop(end - first_addr);
1013 	while ((lastvecindex + 1) < vecindex) {
1014 		++lastvecindex;
1015 		error = subyte(vec + lastvecindex, 0);
1016 		if (error) {
1017 			error = EFAULT;
1018 			goto done2;
1019 		}
1020 	}
1021 
1022 	/*
1023 	 * If the map has changed, due to the subyte, the previous
1024 	 * output may be invalid.
1025 	 */
1026 	vm_map_lock_read(map);
1027 	if (timestamp != map->timestamp)
1028 		goto RestartScan;
1029 	vm_map_unlock_read(map);
1030 done2:
1031 	return (error);
1032 }
1033 
1034 #ifndef _SYS_SYSPROTO_H_
1035 struct mlock_args {
1036 	const void *addr;
1037 	size_t len;
1038 };
1039 #endif
1040 int
1041 sys_mlock(struct thread *td, struct mlock_args *uap)
1042 {
1043 
1044 	return (kern_mlock(td->td_proc, td->td_ucred,
1045 	    __DECONST(uintptr_t, uap->addr), uap->len));
1046 }
1047 
1048 int
1049 kern_mlock(struct proc *proc, struct ucred *cred, uintptr_t addr0, size_t len)
1050 {
1051 	vm_offset_t addr, end, last, start;
1052 	vm_size_t npages, size;
1053 	vm_map_t map;
1054 	unsigned long nsize;
1055 	int error;
1056 
1057 	error = priv_check_cred(cred, PRIV_VM_MLOCK);
1058 	if (error)
1059 		return (error);
1060 	addr = addr0;
1061 	size = len;
1062 	last = addr + size;
1063 	start = trunc_page(addr);
1064 	end = round_page(last);
1065 	if (last < addr || end < addr)
1066 		return (EINVAL);
1067 	npages = atop(end - start);
1068 	if (npages > vm_page_max_user_wired)
1069 		return (ENOMEM);
1070 	map = &proc->p_vmspace->vm_map;
1071 	PROC_LOCK(proc);
1072 	nsize = ptoa(npages + pmap_wired_count(map->pmap));
1073 	if (nsize > lim_cur_proc(proc, RLIMIT_MEMLOCK)) {
1074 		PROC_UNLOCK(proc);
1075 		return (ENOMEM);
1076 	}
1077 	PROC_UNLOCK(proc);
1078 #ifdef RACCT
1079 	if (racct_enable) {
1080 		PROC_LOCK(proc);
1081 		error = racct_set(proc, RACCT_MEMLOCK, nsize);
1082 		PROC_UNLOCK(proc);
1083 		if (error != 0)
1084 			return (ENOMEM);
1085 	}
1086 #endif
1087 	error = vm_map_wire(map, start, end,
1088 	    VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
1089 #ifdef RACCT
1090 	if (racct_enable && error != KERN_SUCCESS) {
1091 		PROC_LOCK(proc);
1092 		racct_set(proc, RACCT_MEMLOCK,
1093 		    ptoa(pmap_wired_count(map->pmap)));
1094 		PROC_UNLOCK(proc);
1095 	}
1096 #endif
1097 	return (error == KERN_SUCCESS ? 0 : ENOMEM);
1098 }
1099 
1100 #ifndef _SYS_SYSPROTO_H_
1101 struct mlockall_args {
1102 	int	how;
1103 };
1104 #endif
1105 
1106 int
1107 sys_mlockall(struct thread *td, struct mlockall_args *uap)
1108 {
1109 	vm_map_t map;
1110 	int error;
1111 
1112 	map = &td->td_proc->p_vmspace->vm_map;
1113 	error = priv_check(td, PRIV_VM_MLOCK);
1114 	if (error)
1115 		return (error);
1116 
1117 	if ((uap->how == 0) || ((uap->how & ~(MCL_CURRENT|MCL_FUTURE)) != 0))
1118 		return (EINVAL);
1119 
1120 	/*
1121 	 * If wiring all pages in the process would cause it to exceed
1122 	 * a hard resource limit, return ENOMEM.
1123 	 */
1124 	if (!old_mlock && uap->how & MCL_CURRENT) {
1125 		if (map->size > lim_cur(td, RLIMIT_MEMLOCK))
1126 			return (ENOMEM);
1127 	}
1128 #ifdef RACCT
1129 	if (racct_enable) {
1130 		PROC_LOCK(td->td_proc);
1131 		error = racct_set(td->td_proc, RACCT_MEMLOCK, map->size);
1132 		PROC_UNLOCK(td->td_proc);
1133 		if (error != 0)
1134 			return (ENOMEM);
1135 	}
1136 #endif
1137 
1138 	if (uap->how & MCL_FUTURE) {
1139 		vm_map_lock(map);
1140 		vm_map_modflags(map, MAP_WIREFUTURE, 0);
1141 		vm_map_unlock(map);
1142 		error = 0;
1143 	}
1144 
1145 	if (uap->how & MCL_CURRENT) {
1146 		/*
1147 		 * P1003.1-2001 mandates that all currently mapped pages
1148 		 * will be memory resident and locked (wired) upon return
1149 		 * from mlockall(). vm_map_wire() will wire pages, by
1150 		 * calling vm_fault_wire() for each page in the region.
1151 		 */
1152 		error = vm_map_wire(map, vm_map_min(map), vm_map_max(map),
1153 		    VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK);
1154 		if (error == KERN_SUCCESS)
1155 			error = 0;
1156 		else if (error == KERN_RESOURCE_SHORTAGE)
1157 			error = ENOMEM;
1158 		else
1159 			error = EAGAIN;
1160 	}
1161 #ifdef RACCT
1162 	if (racct_enable && error != KERN_SUCCESS) {
1163 		PROC_LOCK(td->td_proc);
1164 		racct_set(td->td_proc, RACCT_MEMLOCK,
1165 		    ptoa(pmap_wired_count(map->pmap)));
1166 		PROC_UNLOCK(td->td_proc);
1167 	}
1168 #endif
1169 
1170 	return (error);
1171 }
1172 
1173 #ifndef _SYS_SYSPROTO_H_
1174 struct munlockall_args {
1175 	register_t dummy;
1176 };
1177 #endif
1178 
1179 int
1180 sys_munlockall(struct thread *td, struct munlockall_args *uap)
1181 {
1182 	vm_map_t map;
1183 	int error;
1184 
1185 	map = &td->td_proc->p_vmspace->vm_map;
1186 	error = priv_check(td, PRIV_VM_MUNLOCK);
1187 	if (error)
1188 		return (error);
1189 
1190 	/* Clear the MAP_WIREFUTURE flag from this vm_map. */
1191 	vm_map_lock(map);
1192 	vm_map_modflags(map, 0, MAP_WIREFUTURE);
1193 	vm_map_unlock(map);
1194 
1195 	/* Forcibly unwire all pages. */
1196 	error = vm_map_unwire(map, vm_map_min(map), vm_map_max(map),
1197 	    VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK);
1198 #ifdef RACCT
1199 	if (racct_enable && error == KERN_SUCCESS) {
1200 		PROC_LOCK(td->td_proc);
1201 		racct_set(td->td_proc, RACCT_MEMLOCK, 0);
1202 		PROC_UNLOCK(td->td_proc);
1203 	}
1204 #endif
1205 
1206 	return (error);
1207 }
1208 
1209 #ifndef _SYS_SYSPROTO_H_
1210 struct munlock_args {
1211 	const void *addr;
1212 	size_t len;
1213 };
1214 #endif
1215 int
1216 sys_munlock(struct thread *td, struct munlock_args *uap)
1217 {
1218 
1219 	return (kern_munlock(td, (uintptr_t)uap->addr, uap->len));
1220 }
1221 
1222 int
1223 kern_munlock(struct thread *td, uintptr_t addr0, size_t size)
1224 {
1225 	vm_offset_t addr, end, last, start;
1226 #ifdef RACCT
1227 	vm_map_t map;
1228 #endif
1229 	int error;
1230 
1231 	error = priv_check(td, PRIV_VM_MUNLOCK);
1232 	if (error)
1233 		return (error);
1234 	addr = addr0;
1235 	last = addr + size;
1236 	start = trunc_page(addr);
1237 	end = round_page(last);
1238 	if (last < addr || end < addr)
1239 		return (EINVAL);
1240 	error = vm_map_unwire(&td->td_proc->p_vmspace->vm_map, start, end,
1241 	    VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
1242 #ifdef RACCT
1243 	if (racct_enable && error == KERN_SUCCESS) {
1244 		PROC_LOCK(td->td_proc);
1245 		map = &td->td_proc->p_vmspace->vm_map;
1246 		racct_set(td->td_proc, RACCT_MEMLOCK,
1247 		    ptoa(pmap_wired_count(map->pmap)));
1248 		PROC_UNLOCK(td->td_proc);
1249 	}
1250 #endif
1251 	return (error == KERN_SUCCESS ? 0 : ENOMEM);
1252 }
1253 
1254 /*
1255  * vm_mmap_vnode()
1256  *
1257  * Helper function for vm_mmap.  Perform sanity check specific for mmap
1258  * operations on vnodes.
1259  */
1260 int
1261 vm_mmap_vnode(struct thread *td, vm_size_t objsize,
1262     vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp,
1263     struct vnode *vp, vm_ooffset_t *foffp, vm_object_t *objp,
1264     boolean_t *writecounted)
1265 {
1266 	struct vattr va;
1267 	vm_object_t obj;
1268 	vm_ooffset_t foff;
1269 	struct ucred *cred;
1270 	int error, flags;
1271 	bool writex;
1272 
1273 	cred = td->td_ucred;
1274 	writex = (*maxprotp & VM_PROT_WRITE) != 0 &&
1275 	    (*flagsp & MAP_SHARED) != 0;
1276 	if ((error = vget(vp, LK_SHARED, td)) != 0)
1277 		return (error);
1278 	AUDIT_ARG_VNODE1(vp);
1279 	foff = *foffp;
1280 	flags = *flagsp;
1281 	obj = vp->v_object;
1282 	if (vp->v_type == VREG) {
1283 		/*
1284 		 * Get the proper underlying object
1285 		 */
1286 		if (obj == NULL) {
1287 			error = EINVAL;
1288 			goto done;
1289 		}
1290 		if (obj->type == OBJT_VNODE && obj->handle != vp) {
1291 			vput(vp);
1292 			vp = (struct vnode *)obj->handle;
1293 			/*
1294 			 * Bypass filesystems obey the mpsafety of the
1295 			 * underlying fs.  Tmpfs never bypasses.
1296 			 */
1297 			error = vget(vp, LK_SHARED, td);
1298 			if (error != 0)
1299 				return (error);
1300 		}
1301 		if (writex) {
1302 			*writecounted = TRUE;
1303 			vm_pager_update_writecount(obj, 0, objsize);
1304 		}
1305 	} else {
1306 		error = EINVAL;
1307 		goto done;
1308 	}
1309 	if ((error = VOP_GETATTR(vp, &va, cred)))
1310 		goto done;
1311 #ifdef MAC
1312 	/* This relies on VM_PROT_* matching PROT_*. */
1313 	error = mac_vnode_check_mmap(cred, vp, (int)prot, flags);
1314 	if (error != 0)
1315 		goto done;
1316 #endif
1317 	if ((flags & MAP_SHARED) != 0) {
1318 		if ((va.va_flags & (SF_SNAPSHOT|IMMUTABLE|APPEND)) != 0) {
1319 			if (prot & VM_PROT_WRITE) {
1320 				error = EPERM;
1321 				goto done;
1322 			}
1323 			*maxprotp &= ~VM_PROT_WRITE;
1324 		}
1325 	}
1326 	/*
1327 	 * If it is a regular file without any references
1328 	 * we do not need to sync it.
1329 	 * Adjust object size to be the size of actual file.
1330 	 */
1331 	objsize = round_page(va.va_size);
1332 	if (va.va_nlink == 0)
1333 		flags |= MAP_NOSYNC;
1334 	if (obj->type == OBJT_VNODE) {
1335 		obj = vm_pager_allocate(OBJT_VNODE, vp, objsize, prot, foff,
1336 		    cred);
1337 		if (obj == NULL) {
1338 			error = ENOMEM;
1339 			goto done;
1340 		}
1341 	} else {
1342 		KASSERT(obj->type == OBJT_DEFAULT || obj->type == OBJT_SWAP,
1343 		    ("wrong object type"));
1344 		vm_object_reference(obj);
1345 #if VM_NRESERVLEVEL > 0
1346 		if ((obj->flags & OBJ_COLORED) == 0) {
1347 			VM_OBJECT_WLOCK(obj);
1348 			vm_object_color(obj, 0);
1349 			VM_OBJECT_WUNLOCK(obj);
1350 		}
1351 #endif
1352 	}
1353 	*objp = obj;
1354 	*flagsp = flags;
1355 
1356 	VOP_MMAPPED(vp);
1357 
1358 done:
1359 	if (error != 0 && *writecounted) {
1360 		*writecounted = FALSE;
1361 		vm_pager_update_writecount(obj, objsize, 0);
1362 	}
1363 	vput(vp);
1364 	return (error);
1365 }
1366 
1367 /*
1368  * vm_mmap_cdev()
1369  *
1370  * Helper function for vm_mmap.  Perform sanity check specific for mmap
1371  * operations on cdevs.
1372  */
1373 int
1374 vm_mmap_cdev(struct thread *td, vm_size_t objsize, vm_prot_t prot,
1375     vm_prot_t *maxprotp, int *flagsp, struct cdev *cdev, struct cdevsw *dsw,
1376     vm_ooffset_t *foff, vm_object_t *objp)
1377 {
1378 	vm_object_t obj;
1379 	int error, flags;
1380 
1381 	flags = *flagsp;
1382 
1383 	if (dsw->d_flags & D_MMAP_ANON) {
1384 		*objp = NULL;
1385 		*foff = 0;
1386 		*maxprotp = VM_PROT_ALL;
1387 		*flagsp |= MAP_ANON;
1388 		return (0);
1389 	}
1390 	/*
1391 	 * cdevs do not provide private mappings of any kind.
1392 	 */
1393 	if ((*maxprotp & VM_PROT_WRITE) == 0 &&
1394 	    (prot & VM_PROT_WRITE) != 0)
1395 		return (EACCES);
1396 	if (flags & (MAP_PRIVATE|MAP_COPY))
1397 		return (EINVAL);
1398 	/*
1399 	 * Force device mappings to be shared.
1400 	 */
1401 	flags |= MAP_SHARED;
1402 #ifdef MAC_XXX
1403 	error = mac_cdev_check_mmap(td->td_ucred, cdev, (int)prot);
1404 	if (error != 0)
1405 		return (error);
1406 #endif
1407 	/*
1408 	 * First, try d_mmap_single().  If that is not implemented
1409 	 * (returns ENODEV), fall back to using the device pager.
1410 	 * Note that d_mmap_single() must return a reference to the
1411 	 * object (it needs to bump the reference count of the object
1412 	 * it returns somehow).
1413 	 *
1414 	 * XXX assumes VM_PROT_* == PROT_*
1415 	 */
1416 	error = dsw->d_mmap_single(cdev, foff, objsize, objp, (int)prot);
1417 	if (error != ENODEV)
1418 		return (error);
1419 	obj = vm_pager_allocate(OBJT_DEVICE, cdev, objsize, prot, *foff,
1420 	    td->td_ucred);
1421 	if (obj == NULL)
1422 		return (EINVAL);
1423 	*objp = obj;
1424 	*flagsp = flags;
1425 	return (0);
1426 }
1427 
1428 /*
1429  * vm_mmap()
1430  *
1431  * Internal version of mmap used by exec, sys5 shared memory, and
1432  * various device drivers.  Handle is either a vnode pointer, a
1433  * character device, or NULL for MAP_ANON.
1434  */
1435 int
1436 vm_mmap(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot,
1437 	vm_prot_t maxprot, int flags,
1438 	objtype_t handle_type, void *handle,
1439 	vm_ooffset_t foff)
1440 {
1441 	vm_object_t object;
1442 	struct thread *td = curthread;
1443 	int error;
1444 	boolean_t writecounted;
1445 
1446 	if (size == 0)
1447 		return (EINVAL);
1448 
1449 	size = round_page(size);
1450 	object = NULL;
1451 	writecounted = FALSE;
1452 
1453 	/*
1454 	 * Lookup/allocate object.
1455 	 */
1456 	switch (handle_type) {
1457 	case OBJT_DEVICE: {
1458 		struct cdevsw *dsw;
1459 		struct cdev *cdev;
1460 		int ref;
1461 
1462 		cdev = handle;
1463 		dsw = dev_refthread(cdev, &ref);
1464 		if (dsw == NULL)
1465 			return (ENXIO);
1466 		error = vm_mmap_cdev(td, size, prot, &maxprot, &flags, cdev,
1467 		    dsw, &foff, &object);
1468 		dev_relthread(cdev, ref);
1469 		break;
1470 	}
1471 	case OBJT_VNODE:
1472 		error = vm_mmap_vnode(td, size, prot, &maxprot, &flags,
1473 		    handle, &foff, &object, &writecounted);
1474 		break;
1475 	case OBJT_DEFAULT:
1476 		if (handle == NULL) {
1477 			error = 0;
1478 			break;
1479 		}
1480 		/* FALLTHROUGH */
1481 	default:
1482 		error = EINVAL;
1483 		break;
1484 	}
1485 	if (error)
1486 		return (error);
1487 
1488 	error = vm_mmap_object(map, addr, size, prot, maxprot, flags, object,
1489 	    foff, writecounted, td);
1490 	if (error != 0 && object != NULL) {
1491 		/*
1492 		 * If this mapping was accounted for in the vnode's
1493 		 * writecount, then undo that now.
1494 		 */
1495 		if (writecounted)
1496 			vm_pager_release_writecount(object, 0, size);
1497 		vm_object_deallocate(object);
1498 	}
1499 	return (error);
1500 }
1501 
1502 /*
1503  * Internal version of mmap that maps a specific VM object into an
1504  * map.  Called by mmap for MAP_ANON, vm_mmap, shm_mmap, and vn_mmap.
1505  */
1506 int
1507 vm_mmap_object(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot,
1508     vm_prot_t maxprot, int flags, vm_object_t object, vm_ooffset_t foff,
1509     boolean_t writecounted, struct thread *td)
1510 {
1511 	boolean_t curmap, fitit;
1512 	vm_offset_t max_addr;
1513 	int docow, error, findspace, rv;
1514 
1515 	curmap = map == &td->td_proc->p_vmspace->vm_map;
1516 	if (curmap) {
1517 		RACCT_PROC_LOCK(td->td_proc);
1518 		if (map->size + size > lim_cur(td, RLIMIT_VMEM)) {
1519 			RACCT_PROC_UNLOCK(td->td_proc);
1520 			return (ENOMEM);
1521 		}
1522 		if (racct_set(td->td_proc, RACCT_VMEM, map->size + size)) {
1523 			RACCT_PROC_UNLOCK(td->td_proc);
1524 			return (ENOMEM);
1525 		}
1526 		if (!old_mlock && map->flags & MAP_WIREFUTURE) {
1527 			if (ptoa(pmap_wired_count(map->pmap)) + size >
1528 			    lim_cur(td, RLIMIT_MEMLOCK)) {
1529 				racct_set_force(td->td_proc, RACCT_VMEM,
1530 				    map->size);
1531 				RACCT_PROC_UNLOCK(td->td_proc);
1532 				return (ENOMEM);
1533 			}
1534 			error = racct_set(td->td_proc, RACCT_MEMLOCK,
1535 			    ptoa(pmap_wired_count(map->pmap)) + size);
1536 			if (error != 0) {
1537 				racct_set_force(td->td_proc, RACCT_VMEM,
1538 				    map->size);
1539 				RACCT_PROC_UNLOCK(td->td_proc);
1540 				return (error);
1541 			}
1542 		}
1543 		RACCT_PROC_UNLOCK(td->td_proc);
1544 	}
1545 
1546 	/*
1547 	 * We currently can only deal with page aligned file offsets.
1548 	 * The mmap() system call already enforces this by subtracting
1549 	 * the page offset from the file offset, but checking here
1550 	 * catches errors in device drivers (e.g. d_single_mmap()
1551 	 * callbacks) and other internal mapping requests (such as in
1552 	 * exec).
1553 	 */
1554 	if (foff & PAGE_MASK)
1555 		return (EINVAL);
1556 
1557 	if ((flags & MAP_FIXED) == 0) {
1558 		fitit = TRUE;
1559 		*addr = round_page(*addr);
1560 	} else {
1561 		if (*addr != trunc_page(*addr))
1562 			return (EINVAL);
1563 		fitit = FALSE;
1564 	}
1565 
1566 	if (flags & MAP_ANON) {
1567 		if (object != NULL || foff != 0)
1568 			return (EINVAL);
1569 		docow = 0;
1570 	} else if (flags & MAP_PREFAULT_READ)
1571 		docow = MAP_PREFAULT;
1572 	else
1573 		docow = MAP_PREFAULT_PARTIAL;
1574 
1575 	if ((flags & (MAP_ANON|MAP_SHARED)) == 0)
1576 		docow |= MAP_COPY_ON_WRITE;
1577 	if (flags & MAP_NOSYNC)
1578 		docow |= MAP_DISABLE_SYNCER;
1579 	if (flags & MAP_NOCORE)
1580 		docow |= MAP_DISABLE_COREDUMP;
1581 	/* Shared memory is also shared with children. */
1582 	if (flags & MAP_SHARED)
1583 		docow |= MAP_INHERIT_SHARE;
1584 	if (writecounted)
1585 		docow |= MAP_WRITECOUNT;
1586 	if (flags & MAP_STACK) {
1587 		if (object != NULL)
1588 			return (EINVAL);
1589 		docow |= MAP_STACK_GROWS_DOWN;
1590 	}
1591 	if ((flags & MAP_EXCL) != 0)
1592 		docow |= MAP_CHECK_EXCL;
1593 	if ((flags & MAP_GUARD) != 0)
1594 		docow |= MAP_CREATE_GUARD;
1595 
1596 	if (fitit) {
1597 		if ((flags & MAP_ALIGNMENT_MASK) == MAP_ALIGNED_SUPER)
1598 			findspace = VMFS_SUPER_SPACE;
1599 		else if ((flags & MAP_ALIGNMENT_MASK) != 0)
1600 			findspace = VMFS_ALIGNED_SPACE(flags >>
1601 			    MAP_ALIGNMENT_SHIFT);
1602 		else
1603 			findspace = VMFS_OPTIMAL_SPACE;
1604 		max_addr = 0;
1605 #ifdef MAP_32BIT
1606 		if ((flags & MAP_32BIT) != 0)
1607 			max_addr = MAP_32BIT_MAX_ADDR;
1608 #endif
1609 		if (curmap) {
1610 			rv = vm_map_find_min(map, object, foff, addr, size,
1611 			    round_page((vm_offset_t)td->td_proc->p_vmspace->
1612 			    vm_daddr + lim_max(td, RLIMIT_DATA)), max_addr,
1613 			    findspace, prot, maxprot, docow);
1614 		} else {
1615 			rv = vm_map_find(map, object, foff, addr, size,
1616 			    max_addr, findspace, prot, maxprot, docow);
1617 		}
1618 	} else {
1619 		rv = vm_map_fixed(map, object, foff, *addr, size,
1620 		    prot, maxprot, docow);
1621 	}
1622 
1623 	if (rv == KERN_SUCCESS) {
1624 		/*
1625 		 * If the process has requested that all future mappings
1626 		 * be wired, then heed this.
1627 		 */
1628 		if ((map->flags & MAP_WIREFUTURE) != 0) {
1629 			vm_map_lock(map);
1630 			if ((map->flags & MAP_WIREFUTURE) != 0)
1631 				(void)vm_map_wire_locked(map, *addr,
1632 				    *addr + size, VM_MAP_WIRE_USER |
1633 				    ((flags & MAP_STACK) ? VM_MAP_WIRE_HOLESOK :
1634 				    VM_MAP_WIRE_NOHOLES));
1635 			vm_map_unlock(map);
1636 		}
1637 	}
1638 	return (vm_mmap_to_errno(rv));
1639 }
1640 
1641 /*
1642  * Translate a Mach VM return code to zero on success or the appropriate errno
1643  * on failure.
1644  */
1645 int
1646 vm_mmap_to_errno(int rv)
1647 {
1648 
1649 	switch (rv) {
1650 	case KERN_SUCCESS:
1651 		return (0);
1652 	case KERN_INVALID_ADDRESS:
1653 	case KERN_NO_SPACE:
1654 		return (ENOMEM);
1655 	case KERN_PROTECTION_FAILURE:
1656 		return (EACCES);
1657 	default:
1658 		return (EINVAL);
1659 	}
1660 }
1661