xref: /freebsd/sys/vm/vm_mmap.c (revision 2f513db72b034fd5ef7f080b11be5c711c15186a)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1988 University of Utah.
5  * Copyright (c) 1991, 1993
6  *	The Regents of the University of California.  All rights reserved.
7  *
8  * This code is derived from software contributed to Berkeley by
9  * the Systems Programming Group of the University of Utah Computer
10  * Science Department.
11  *
12  * Redistribution and use in source and binary forms, with or without
13  * modification, are permitted provided that the following conditions
14  * are met:
15  * 1. Redistributions of source code must retain the above copyright
16  *    notice, this list of conditions and the following disclaimer.
17  * 2. Redistributions in binary form must reproduce the above copyright
18  *    notice, this list of conditions and the following disclaimer in the
19  *    documentation and/or other materials provided with the distribution.
20  * 3. Neither the name of the University nor the names of its contributors
21  *    may be used to endorse or promote products derived from this software
22  *    without specific prior written permission.
23  *
24  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34  * SUCH DAMAGE.
35  *
36  * from: Utah $Hdr: vm_mmap.c 1.6 91/10/21$
37  *
38  *	@(#)vm_mmap.c	8.4 (Berkeley) 1/12/94
39  */
40 
41 /*
42  * Mapped file (mmap) interface to VM
43  */
44 
45 #include <sys/cdefs.h>
46 __FBSDID("$FreeBSD$");
47 
48 #include "opt_hwpmc_hooks.h"
49 #include "opt_vm.h"
50 
51 #include <sys/param.h>
52 #include <sys/systm.h>
53 #include <sys/capsicum.h>
54 #include <sys/kernel.h>
55 #include <sys/lock.h>
56 #include <sys/mutex.h>
57 #include <sys/sysproto.h>
58 #include <sys/elf.h>
59 #include <sys/filedesc.h>
60 #include <sys/priv.h>
61 #include <sys/proc.h>
62 #include <sys/procctl.h>
63 #include <sys/racct.h>
64 #include <sys/resource.h>
65 #include <sys/resourcevar.h>
66 #include <sys/rwlock.h>
67 #include <sys/sysctl.h>
68 #include <sys/vnode.h>
69 #include <sys/fcntl.h>
70 #include <sys/file.h>
71 #include <sys/mman.h>
72 #include <sys/mount.h>
73 #include <sys/conf.h>
74 #include <sys/stat.h>
75 #include <sys/syscallsubr.h>
76 #include <sys/sysent.h>
77 #include <sys/vmmeter.h>
78 #if defined(__amd64__) || defined(__i386__) /* for i386_read_exec */
79 #include <machine/md_var.h>
80 #endif
81 
82 #include <security/audit/audit.h>
83 #include <security/mac/mac_framework.h>
84 
85 #include <vm/vm.h>
86 #include <vm/vm_param.h>
87 #include <vm/pmap.h>
88 #include <vm/vm_map.h>
89 #include <vm/vm_object.h>
90 #include <vm/vm_page.h>
91 #include <vm/vm_pager.h>
92 #include <vm/vm_pageout.h>
93 #include <vm/vm_extern.h>
94 #include <vm/vm_page.h>
95 #include <vm/vnode_pager.h>
96 
97 #ifdef HWPMC_HOOKS
98 #include <sys/pmckern.h>
99 #endif
100 
101 int old_mlock = 0;
102 SYSCTL_INT(_vm, OID_AUTO, old_mlock, CTLFLAG_RWTUN, &old_mlock, 0,
103     "Do not apply RLIMIT_MEMLOCK on mlockall");
104 static int mincore_mapped = 1;
105 SYSCTL_INT(_vm, OID_AUTO, mincore_mapped, CTLFLAG_RWTUN, &mincore_mapped, 0,
106     "mincore reports mappings, not residency");
107 static int imply_prot_max = 0;
108 SYSCTL_INT(_vm, OID_AUTO, imply_prot_max, CTLFLAG_RWTUN, &imply_prot_max, 0,
109     "Imply maximum page permissions in mmap() when none are specified");
110 
111 #ifdef MAP_32BIT
112 #define	MAP_32BIT_MAX_ADDR	((vm_offset_t)1 << 31)
113 #endif
114 
115 #ifndef _SYS_SYSPROTO_H_
116 struct sbrk_args {
117 	int incr;
118 };
119 #endif
120 
121 int
122 sys_sbrk(struct thread *td, struct sbrk_args *uap)
123 {
124 	/* Not yet implemented */
125 	return (EOPNOTSUPP);
126 }
127 
128 #ifndef _SYS_SYSPROTO_H_
129 struct sstk_args {
130 	int incr;
131 };
132 #endif
133 
134 int
135 sys_sstk(struct thread *td, struct sstk_args *uap)
136 {
137 	/* Not yet implemented */
138 	return (EOPNOTSUPP);
139 }
140 
141 #if defined(COMPAT_43)
142 int
143 ogetpagesize(struct thread *td, struct ogetpagesize_args *uap)
144 {
145 
146 	td->td_retval[0] = PAGE_SIZE;
147 	return (0);
148 }
149 #endif				/* COMPAT_43 */
150 
151 
152 /*
153  * Memory Map (mmap) system call.  Note that the file offset
154  * and address are allowed to be NOT page aligned, though if
155  * the MAP_FIXED flag it set, both must have the same remainder
156  * modulo the PAGE_SIZE (POSIX 1003.1b).  If the address is not
157  * page-aligned, the actual mapping starts at trunc_page(addr)
158  * and the return value is adjusted up by the page offset.
159  *
160  * Generally speaking, only character devices which are themselves
161  * memory-based, such as a video framebuffer, can be mmap'd.  Otherwise
162  * there would be no cache coherency between a descriptor and a VM mapping
163  * both to the same character device.
164  */
165 #ifndef _SYS_SYSPROTO_H_
166 struct mmap_args {
167 	void *addr;
168 	size_t len;
169 	int prot;
170 	int flags;
171 	int fd;
172 	long pad;
173 	off_t pos;
174 };
175 #endif
176 
177 int
178 sys_mmap(struct thread *td, struct mmap_args *uap)
179 {
180 
181 	return (kern_mmap(td, (uintptr_t)uap->addr, uap->len, uap->prot,
182 	    uap->flags, uap->fd, uap->pos));
183 }
184 
185 int
186 kern_mmap_maxprot(struct proc *p, int prot)
187 {
188 
189 	if ((p->p_flag2 & P2_PROTMAX_DISABLE) != 0 ||
190 	    (p->p_fctl0 & NT_FREEBSD_FCTL_PROTMAX_DISABLE) != 0)
191 		return (_PROT_ALL);
192 	if (((p->p_flag2 & P2_PROTMAX_ENABLE) != 0 || imply_prot_max) &&
193 	    prot != PROT_NONE)
194 		 return (prot);
195 	return (_PROT_ALL);
196 }
197 
198 int
199 kern_mmap(struct thread *td, uintptr_t addr0, size_t len, int prot, int flags,
200     int fd, off_t pos)
201 {
202 
203 	return (kern_mmap_fpcheck(td, addr0, len, prot, flags, fd, pos, NULL));
204 }
205 
206 /*
207  * When mmap'ing a file, check_fp_fn may be used for the caller to do any
208  * last-minute validation based on the referenced file in a non-racy way.
209  */
210 int
211 kern_mmap_fpcheck(struct thread *td, uintptr_t addr0, size_t len, int prot,
212     int flags, int fd, off_t pos, mmap_check_fp_fn check_fp_fn)
213 {
214 	struct vmspace *vms;
215 	struct file *fp;
216 	struct proc *p;
217 	vm_offset_t addr;
218 	vm_size_t pageoff, size;
219 	vm_prot_t cap_maxprot;
220 	int align, error, max_prot;
221 	cap_rights_t rights;
222 
223 	if ((prot & ~(_PROT_ALL | PROT_MAX(_PROT_ALL))) != 0)
224 		return (EINVAL);
225 	max_prot = PROT_MAX_EXTRACT(prot);
226 	prot = PROT_EXTRACT(prot);
227 	if (max_prot != 0 && (max_prot & prot) != prot)
228 		return (EINVAL);
229 
230 	p = td->td_proc;
231 
232 	/*
233 	 * Always honor PROT_MAX if set.  If not, default to all
234 	 * permissions unless we're implying maximum permissions.
235 	 */
236 	if (max_prot == 0)
237 		max_prot = kern_mmap_maxprot(p, prot);
238 
239 	vms = p->p_vmspace;
240 	fp = NULL;
241 	AUDIT_ARG_FD(fd);
242 	addr = addr0;
243 
244 	/*
245 	 * Ignore old flags that used to be defined but did not do anything.
246 	 */
247 	flags &= ~(MAP_RESERVED0020 | MAP_RESERVED0040);
248 
249 	/*
250 	 * Enforce the constraints.
251 	 * Mapping of length 0 is only allowed for old binaries.
252 	 * Anonymous mapping shall specify -1 as filedescriptor and
253 	 * zero position for new code. Be nice to ancient a.out
254 	 * binaries and correct pos for anonymous mapping, since old
255 	 * ld.so sometimes issues anonymous map requests with non-zero
256 	 * pos.
257 	 */
258 	if (!SV_CURPROC_FLAG(SV_AOUT)) {
259 		if ((len == 0 && p->p_osrel >= P_OSREL_MAP_ANON) ||
260 		    ((flags & MAP_ANON) != 0 && (fd != -1 || pos != 0)))
261 			return (EINVAL);
262 	} else {
263 		if ((flags & MAP_ANON) != 0)
264 			pos = 0;
265 	}
266 
267 	if (flags & MAP_STACK) {
268 		if ((fd != -1) ||
269 		    ((prot & (PROT_READ | PROT_WRITE)) != (PROT_READ | PROT_WRITE)))
270 			return (EINVAL);
271 		flags |= MAP_ANON;
272 		pos = 0;
273 	}
274 	if ((flags & ~(MAP_SHARED | MAP_PRIVATE | MAP_FIXED | MAP_HASSEMAPHORE |
275 	    MAP_STACK | MAP_NOSYNC | MAP_ANON | MAP_EXCL | MAP_NOCORE |
276 	    MAP_PREFAULT_READ | MAP_GUARD |
277 #ifdef MAP_32BIT
278 	    MAP_32BIT |
279 #endif
280 	    MAP_ALIGNMENT_MASK)) != 0)
281 		return (EINVAL);
282 	if ((flags & (MAP_EXCL | MAP_FIXED)) == MAP_EXCL)
283 		return (EINVAL);
284 	if ((flags & (MAP_SHARED | MAP_PRIVATE)) == (MAP_SHARED | MAP_PRIVATE))
285 		return (EINVAL);
286 	if (prot != PROT_NONE &&
287 	    (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC)) != 0)
288 		return (EINVAL);
289 	if ((flags & MAP_GUARD) != 0 && (prot != PROT_NONE || fd != -1 ||
290 	    pos != 0 || (flags & ~(MAP_FIXED | MAP_GUARD | MAP_EXCL |
291 #ifdef MAP_32BIT
292 	    MAP_32BIT |
293 #endif
294 	    MAP_ALIGNMENT_MASK)) != 0))
295 		return (EINVAL);
296 
297 	/*
298 	 * Align the file position to a page boundary,
299 	 * and save its page offset component.
300 	 */
301 	pageoff = (pos & PAGE_MASK);
302 	pos -= pageoff;
303 
304 	/* Compute size from len by rounding (on both ends). */
305 	size = len + pageoff;			/* low end... */
306 	size = round_page(size);		/* hi end */
307 	/* Check for rounding up to zero. */
308 	if (len > size)
309 		return (ENOMEM);
310 
311 	/* Ensure alignment is at least a page and fits in a pointer. */
312 	align = flags & MAP_ALIGNMENT_MASK;
313 	if (align != 0 && align != MAP_ALIGNED_SUPER &&
314 	    (align >> MAP_ALIGNMENT_SHIFT >= sizeof(void *) * NBBY ||
315 	    align >> MAP_ALIGNMENT_SHIFT < PAGE_SHIFT))
316 		return (EINVAL);
317 
318 	/*
319 	 * Check for illegal addresses.  Watch out for address wrap... Note
320 	 * that VM_*_ADDRESS are not constants due to casts (argh).
321 	 */
322 	if (flags & MAP_FIXED) {
323 		/*
324 		 * The specified address must have the same remainder
325 		 * as the file offset taken modulo PAGE_SIZE, so it
326 		 * should be aligned after adjustment by pageoff.
327 		 */
328 		addr -= pageoff;
329 		if (addr & PAGE_MASK)
330 			return (EINVAL);
331 
332 		/* Address range must be all in user VM space. */
333 		if (addr < vm_map_min(&vms->vm_map) ||
334 		    addr + size > vm_map_max(&vms->vm_map))
335 			return (EINVAL);
336 		if (addr + size < addr)
337 			return (EINVAL);
338 #ifdef MAP_32BIT
339 		if (flags & MAP_32BIT && addr + size > MAP_32BIT_MAX_ADDR)
340 			return (EINVAL);
341 	} else if (flags & MAP_32BIT) {
342 		/*
343 		 * For MAP_32BIT, override the hint if it is too high and
344 		 * do not bother moving the mapping past the heap (since
345 		 * the heap is usually above 2GB).
346 		 */
347 		if (addr + size > MAP_32BIT_MAX_ADDR)
348 			addr = 0;
349 #endif
350 	} else {
351 		/*
352 		 * XXX for non-fixed mappings where no hint is provided or
353 		 * the hint would fall in the potential heap space,
354 		 * place it after the end of the largest possible heap.
355 		 *
356 		 * There should really be a pmap call to determine a reasonable
357 		 * location.
358 		 */
359 		if (addr == 0 ||
360 		    (addr >= round_page((vm_offset_t)vms->vm_taddr) &&
361 		    addr < round_page((vm_offset_t)vms->vm_daddr +
362 		    lim_max(td, RLIMIT_DATA))))
363 			addr = round_page((vm_offset_t)vms->vm_daddr +
364 			    lim_max(td, RLIMIT_DATA));
365 	}
366 	if (len == 0) {
367 		/*
368 		 * Return success without mapping anything for old
369 		 * binaries that request a page-aligned mapping of
370 		 * length 0.  For modern binaries, this function
371 		 * returns an error earlier.
372 		 */
373 		error = 0;
374 	} else if ((flags & MAP_GUARD) != 0) {
375 		error = vm_mmap_object(&vms->vm_map, &addr, size, VM_PROT_NONE,
376 		    VM_PROT_NONE, flags, NULL, pos, FALSE, td);
377 	} else if ((flags & MAP_ANON) != 0) {
378 		/*
379 		 * Mapping blank space is trivial.
380 		 *
381 		 * This relies on VM_PROT_* matching PROT_*.
382 		 */
383 		error = vm_mmap_object(&vms->vm_map, &addr, size, prot,
384 		    max_prot, flags, NULL, pos, FALSE, td);
385 	} else {
386 		/*
387 		 * Mapping file, get fp for validation and don't let the
388 		 * descriptor disappear on us if we block. Check capability
389 		 * rights, but also return the maximum rights to be combined
390 		 * with maxprot later.
391 		 */
392 		cap_rights_init(&rights, CAP_MMAP);
393 		if (prot & PROT_READ)
394 			cap_rights_set(&rights, CAP_MMAP_R);
395 		if ((flags & MAP_SHARED) != 0) {
396 			if (prot & PROT_WRITE)
397 				cap_rights_set(&rights, CAP_MMAP_W);
398 		}
399 		if (prot & PROT_EXEC)
400 			cap_rights_set(&rights, CAP_MMAP_X);
401 		error = fget_mmap(td, fd, &rights, &cap_maxprot, &fp);
402 		if (error != 0)
403 			goto done;
404 		if ((flags & (MAP_SHARED | MAP_PRIVATE)) == 0 &&
405 		    p->p_osrel >= P_OSREL_MAP_FSTRICT) {
406 			error = EINVAL;
407 			goto done;
408 		}
409 		if (check_fp_fn != NULL) {
410 			error = check_fp_fn(fp, prot, max_prot & cap_maxprot,
411 			    flags);
412 			if (error != 0)
413 				goto done;
414 		}
415 		/* This relies on VM_PROT_* matching PROT_*. */
416 		error = fo_mmap(fp, &vms->vm_map, &addr, size, prot,
417 		    max_prot & cap_maxprot, flags, pos, td);
418 	}
419 
420 	if (error == 0)
421 		td->td_retval[0] = (register_t) (addr + pageoff);
422 done:
423 	if (fp)
424 		fdrop(fp, td);
425 
426 	return (error);
427 }
428 
429 #if defined(COMPAT_FREEBSD6)
430 int
431 freebsd6_mmap(struct thread *td, struct freebsd6_mmap_args *uap)
432 {
433 
434 	return (kern_mmap(td, (uintptr_t)uap->addr, uap->len, uap->prot,
435 	    uap->flags, uap->fd, uap->pos));
436 }
437 #endif
438 
439 #ifdef COMPAT_43
440 #ifndef _SYS_SYSPROTO_H_
441 struct ommap_args {
442 	caddr_t addr;
443 	int len;
444 	int prot;
445 	int flags;
446 	int fd;
447 	long pos;
448 };
449 #endif
450 int
451 ommap(struct thread *td, struct ommap_args *uap)
452 {
453 	static const char cvtbsdprot[8] = {
454 		0,
455 		PROT_EXEC,
456 		PROT_WRITE,
457 		PROT_EXEC | PROT_WRITE,
458 		PROT_READ,
459 		PROT_EXEC | PROT_READ,
460 		PROT_WRITE | PROT_READ,
461 		PROT_EXEC | PROT_WRITE | PROT_READ,
462 	};
463 	int flags, prot;
464 
465 #define	OMAP_ANON	0x0002
466 #define	OMAP_COPY	0x0020
467 #define	OMAP_SHARED	0x0010
468 #define	OMAP_FIXED	0x0100
469 
470 	prot = cvtbsdprot[uap->prot & 0x7];
471 #if (defined(COMPAT_FREEBSD32) && defined(__amd64__)) || defined(__i386__)
472 	if (i386_read_exec && SV_PROC_FLAG(td->td_proc, SV_ILP32) &&
473 	    prot != 0)
474 		prot |= PROT_EXEC;
475 #endif
476 	flags = 0;
477 	if (uap->flags & OMAP_ANON)
478 		flags |= MAP_ANON;
479 	if (uap->flags & OMAP_COPY)
480 		flags |= MAP_COPY;
481 	if (uap->flags & OMAP_SHARED)
482 		flags |= MAP_SHARED;
483 	else
484 		flags |= MAP_PRIVATE;
485 	if (uap->flags & OMAP_FIXED)
486 		flags |= MAP_FIXED;
487 	return (kern_mmap(td, (uintptr_t)uap->addr, uap->len, prot, flags,
488 	    uap->fd, uap->pos));
489 }
490 #endif				/* COMPAT_43 */
491 
492 
493 #ifndef _SYS_SYSPROTO_H_
494 struct msync_args {
495 	void *addr;
496 	size_t len;
497 	int flags;
498 };
499 #endif
500 int
501 sys_msync(struct thread *td, struct msync_args *uap)
502 {
503 
504 	return (kern_msync(td, (uintptr_t)uap->addr, uap->len, uap->flags));
505 }
506 
507 int
508 kern_msync(struct thread *td, uintptr_t addr0, size_t size, int flags)
509 {
510 	vm_offset_t addr;
511 	vm_size_t pageoff;
512 	vm_map_t map;
513 	int rv;
514 
515 	addr = addr0;
516 	pageoff = (addr & PAGE_MASK);
517 	addr -= pageoff;
518 	size += pageoff;
519 	size = (vm_size_t) round_page(size);
520 	if (addr + size < addr)
521 		return (EINVAL);
522 
523 	if ((flags & (MS_ASYNC|MS_INVALIDATE)) == (MS_ASYNC|MS_INVALIDATE))
524 		return (EINVAL);
525 
526 	map = &td->td_proc->p_vmspace->vm_map;
527 
528 	/*
529 	 * Clean the pages and interpret the return value.
530 	 */
531 	rv = vm_map_sync(map, addr, addr + size, (flags & MS_ASYNC) == 0,
532 	    (flags & MS_INVALIDATE) != 0);
533 	switch (rv) {
534 	case KERN_SUCCESS:
535 		return (0);
536 	case KERN_INVALID_ADDRESS:
537 		return (ENOMEM);
538 	case KERN_INVALID_ARGUMENT:
539 		return (EBUSY);
540 	case KERN_FAILURE:
541 		return (EIO);
542 	default:
543 		return (EINVAL);
544 	}
545 }
546 
547 #ifndef _SYS_SYSPROTO_H_
548 struct munmap_args {
549 	void *addr;
550 	size_t len;
551 };
552 #endif
553 int
554 sys_munmap(struct thread *td, struct munmap_args *uap)
555 {
556 
557 	return (kern_munmap(td, (uintptr_t)uap->addr, uap->len));
558 }
559 
560 int
561 kern_munmap(struct thread *td, uintptr_t addr0, size_t size)
562 {
563 #ifdef HWPMC_HOOKS
564 	struct pmckern_map_out pkm;
565 	vm_map_entry_t entry;
566 	bool pmc_handled;
567 #endif
568 	vm_offset_t addr;
569 	vm_size_t pageoff;
570 	vm_map_t map;
571 
572 	if (size == 0)
573 		return (EINVAL);
574 
575 	addr = addr0;
576 	pageoff = (addr & PAGE_MASK);
577 	addr -= pageoff;
578 	size += pageoff;
579 	size = (vm_size_t) round_page(size);
580 	if (addr + size < addr)
581 		return (EINVAL);
582 
583 	/*
584 	 * Check for illegal addresses.  Watch out for address wrap...
585 	 */
586 	map = &td->td_proc->p_vmspace->vm_map;
587 	if (addr < vm_map_min(map) || addr + size > vm_map_max(map))
588 		return (EINVAL);
589 	vm_map_lock(map);
590 #ifdef HWPMC_HOOKS
591 	pmc_handled = false;
592 	if (PMC_HOOK_INSTALLED(PMC_FN_MUNMAP)) {
593 		pmc_handled = true;
594 		/*
595 		 * Inform hwpmc if the address range being unmapped contains
596 		 * an executable region.
597 		 */
598 		pkm.pm_address = (uintptr_t) NULL;
599 		if (vm_map_lookup_entry(map, addr, &entry)) {
600 			for (; entry->start < addr + size;
601 			    entry = vm_map_entry_succ(entry)) {
602 				if (vm_map_check_protection(map, entry->start,
603 					entry->end, VM_PROT_EXECUTE) == TRUE) {
604 					pkm.pm_address = (uintptr_t) addr;
605 					pkm.pm_size = (size_t) size;
606 					break;
607 				}
608 			}
609 		}
610 	}
611 #endif
612 	vm_map_delete(map, addr, addr + size);
613 
614 #ifdef HWPMC_HOOKS
615 	if (__predict_false(pmc_handled)) {
616 		/* downgrade the lock to prevent a LOR with the pmc-sx lock */
617 		vm_map_lock_downgrade(map);
618 		if (pkm.pm_address != (uintptr_t) NULL)
619 			PMC_CALL_HOOK(td, PMC_FN_MUNMAP, (void *) &pkm);
620 		vm_map_unlock_read(map);
621 	} else
622 #endif
623 		vm_map_unlock(map);
624 
625 	/* vm_map_delete returns nothing but KERN_SUCCESS anyway */
626 	return (0);
627 }
628 
629 #ifndef _SYS_SYSPROTO_H_
630 struct mprotect_args {
631 	const void *addr;
632 	size_t len;
633 	int prot;
634 };
635 #endif
636 int
637 sys_mprotect(struct thread *td, struct mprotect_args *uap)
638 {
639 
640 	return (kern_mprotect(td, (uintptr_t)uap->addr, uap->len, uap->prot));
641 }
642 
643 int
644 kern_mprotect(struct thread *td, uintptr_t addr0, size_t size, int prot)
645 {
646 	vm_offset_t addr;
647 	vm_size_t pageoff;
648 	int vm_error, max_prot;
649 
650 	addr = addr0;
651 	if ((prot & ~(_PROT_ALL | PROT_MAX(_PROT_ALL))) != 0)
652 		return (EINVAL);
653 	max_prot = PROT_MAX_EXTRACT(prot);
654 	prot = PROT_EXTRACT(prot);
655 	pageoff = (addr & PAGE_MASK);
656 	addr -= pageoff;
657 	size += pageoff;
658 	size = (vm_size_t) round_page(size);
659 #ifdef COMPAT_FREEBSD32
660 	if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) {
661 		if (((addr + size) & 0xffffffff) < addr)
662 			return (EINVAL);
663 	} else
664 #endif
665 	if (addr + size < addr)
666 		return (EINVAL);
667 
668 	vm_error = KERN_SUCCESS;
669 	if (max_prot != 0) {
670 		if ((max_prot & prot) != prot)
671 			return (EINVAL);
672 		vm_error = vm_map_protect(&td->td_proc->p_vmspace->vm_map,
673 		    addr, addr + size, max_prot, TRUE);
674 	}
675 	if (vm_error == KERN_SUCCESS)
676 		vm_error = vm_map_protect(&td->td_proc->p_vmspace->vm_map,
677 		    addr, addr + size, prot, FALSE);
678 
679 	switch (vm_error) {
680 	case KERN_SUCCESS:
681 		return (0);
682 	case KERN_PROTECTION_FAILURE:
683 		return (EACCES);
684 	case KERN_RESOURCE_SHORTAGE:
685 		return (ENOMEM);
686 	}
687 	return (EINVAL);
688 }
689 
690 #ifndef _SYS_SYSPROTO_H_
691 struct minherit_args {
692 	void *addr;
693 	size_t len;
694 	int inherit;
695 };
696 #endif
697 int
698 sys_minherit(struct thread *td, struct minherit_args *uap)
699 {
700 	vm_offset_t addr;
701 	vm_size_t size, pageoff;
702 	vm_inherit_t inherit;
703 
704 	addr = (vm_offset_t)uap->addr;
705 	size = uap->len;
706 	inherit = uap->inherit;
707 
708 	pageoff = (addr & PAGE_MASK);
709 	addr -= pageoff;
710 	size += pageoff;
711 	size = (vm_size_t) round_page(size);
712 	if (addr + size < addr)
713 		return (EINVAL);
714 
715 	switch (vm_map_inherit(&td->td_proc->p_vmspace->vm_map, addr,
716 	    addr + size, inherit)) {
717 	case KERN_SUCCESS:
718 		return (0);
719 	case KERN_PROTECTION_FAILURE:
720 		return (EACCES);
721 	}
722 	return (EINVAL);
723 }
724 
725 #ifndef _SYS_SYSPROTO_H_
726 struct madvise_args {
727 	void *addr;
728 	size_t len;
729 	int behav;
730 };
731 #endif
732 
733 int
734 sys_madvise(struct thread *td, struct madvise_args *uap)
735 {
736 
737 	return (kern_madvise(td, (uintptr_t)uap->addr, uap->len, uap->behav));
738 }
739 
740 int
741 kern_madvise(struct thread *td, uintptr_t addr0, size_t len, int behav)
742 {
743 	vm_map_t map;
744 	vm_offset_t addr, end, start;
745 	int flags;
746 
747 	/*
748 	 * Check for our special case, advising the swap pager we are
749 	 * "immortal."
750 	 */
751 	if (behav == MADV_PROTECT) {
752 		flags = PPROT_SET;
753 		return (kern_procctl(td, P_PID, td->td_proc->p_pid,
754 		    PROC_SPROTECT, &flags));
755 	}
756 
757 	/*
758 	 * Check for illegal addresses.  Watch out for address wrap... Note
759 	 * that VM_*_ADDRESS are not constants due to casts (argh).
760 	 */
761 	map = &td->td_proc->p_vmspace->vm_map;
762 	addr = addr0;
763 	if (addr < vm_map_min(map) || addr + len > vm_map_max(map))
764 		return (EINVAL);
765 	if ((addr + len) < addr)
766 		return (EINVAL);
767 
768 	/*
769 	 * Since this routine is only advisory, we default to conservative
770 	 * behavior.
771 	 */
772 	start = trunc_page(addr);
773 	end = round_page(addr + len);
774 
775 	/*
776 	 * vm_map_madvise() checks for illegal values of behav.
777 	 */
778 	return (vm_map_madvise(map, start, end, behav));
779 }
780 
781 #ifndef _SYS_SYSPROTO_H_
782 struct mincore_args {
783 	const void *addr;
784 	size_t len;
785 	char *vec;
786 };
787 #endif
788 
789 int
790 sys_mincore(struct thread *td, struct mincore_args *uap)
791 {
792 
793 	return (kern_mincore(td, (uintptr_t)uap->addr, uap->len, uap->vec));
794 }
795 
796 int
797 kern_mincore(struct thread *td, uintptr_t addr0, size_t len, char *vec)
798 {
799 	pmap_t pmap;
800 	vm_map_t map;
801 	vm_map_entry_t current, entry;
802 	vm_object_t object;
803 	vm_offset_t addr, cend, end, first_addr;
804 	vm_paddr_t pa;
805 	vm_page_t m;
806 	vm_pindex_t pindex;
807 	int error, lastvecindex, mincoreinfo, vecindex;
808 	unsigned int timestamp;
809 
810 	/*
811 	 * Make sure that the addresses presented are valid for user
812 	 * mode.
813 	 */
814 	first_addr = addr = trunc_page(addr0);
815 	end = round_page(addr0 + len);
816 	map = &td->td_proc->p_vmspace->vm_map;
817 	if (end > vm_map_max(map) || end < addr)
818 		return (ENOMEM);
819 
820 	pmap = vmspace_pmap(td->td_proc->p_vmspace);
821 
822 	vm_map_lock_read(map);
823 RestartScan:
824 	timestamp = map->timestamp;
825 
826 	if (!vm_map_lookup_entry(map, addr, &entry)) {
827 		vm_map_unlock_read(map);
828 		return (ENOMEM);
829 	}
830 
831 	/*
832 	 * Do this on a map entry basis so that if the pages are not
833 	 * in the current processes address space, we can easily look
834 	 * up the pages elsewhere.
835 	 */
836 	lastvecindex = -1;
837 	while (entry->start < end) {
838 
839 		/*
840 		 * check for contiguity
841 		 */
842 		current = entry;
843 		entry = vm_map_entry_succ(current);
844 		if (current->end < end &&
845 		    entry->start > current->end) {
846 			vm_map_unlock_read(map);
847 			return (ENOMEM);
848 		}
849 
850 		/*
851 		 * ignore submaps (for now) or null objects
852 		 */
853 		if ((current->eflags & MAP_ENTRY_IS_SUB_MAP) ||
854 		    current->object.vm_object == NULL)
855 			continue;
856 
857 		/*
858 		 * limit this scan to the current map entry and the
859 		 * limits for the mincore call
860 		 */
861 		if (addr < current->start)
862 			addr = current->start;
863 		cend = current->end;
864 		if (cend > end)
865 			cend = end;
866 
867 		for (; addr < cend; addr += PAGE_SIZE) {
868 			/*
869 			 * Check pmap first, it is likely faster, also
870 			 * it can provide info as to whether we are the
871 			 * one referencing or modifying the page.
872 			 */
873 			m = NULL;
874 			object = NULL;
875 retry:
876 			pa = 0;
877 			mincoreinfo = pmap_mincore(pmap, addr, &pa);
878 			if (mincore_mapped) {
879 				/*
880 				 * We only care about this pmap's
881 				 * mapping of the page, if any.
882 				 */
883 				;
884 			} else if (pa != 0) {
885 				/*
886 				 * The page is mapped by this process but not
887 				 * both accessed and modified.  It is also
888 				 * managed.  Acquire the object lock so that
889 				 * other mappings might be examined.  The page's
890 				 * identity may change at any point before its
891 				 * object lock is acquired, so re-validate if
892 				 * necessary.
893 				 */
894 				m = PHYS_TO_VM_PAGE(pa);
895 				while (object == NULL || m->object != object) {
896 					if (object != NULL)
897 						VM_OBJECT_WUNLOCK(object);
898 					object = (vm_object_t)atomic_load_ptr(
899 					    &m->object);
900 					if (object == NULL)
901 						goto retry;
902 					VM_OBJECT_WLOCK(object);
903 				}
904 				if (pa != pmap_extract(pmap, addr))
905 					goto retry;
906 				KASSERT(vm_page_all_valid(m),
907 				    ("mincore: page %p is mapped but invalid",
908 				    m));
909 			} else if (mincoreinfo == 0) {
910 				/*
911 				 * The page is not mapped by this process.  If
912 				 * the object implements managed pages, then
913 				 * determine if the page is resident so that
914 				 * the mappings might be examined.
915 				 */
916 				if (current->object.vm_object != object) {
917 					if (object != NULL)
918 						VM_OBJECT_WUNLOCK(object);
919 					object = current->object.vm_object;
920 					VM_OBJECT_WLOCK(object);
921 				}
922 				if (object->type == OBJT_DEFAULT ||
923 				    object->type == OBJT_SWAP ||
924 				    object->type == OBJT_VNODE) {
925 					pindex = OFF_TO_IDX(current->offset +
926 					    (addr - current->start));
927 					m = vm_page_lookup(object, pindex);
928 					if (m != NULL && vm_page_none_valid(m))
929 						m = NULL;
930 					if (m != NULL)
931 						mincoreinfo = MINCORE_INCORE;
932 				}
933 			}
934 			if (m != NULL) {
935 				VM_OBJECT_ASSERT_WLOCKED(m->object);
936 
937 				/* Examine other mappings of the page. */
938 				if (m->dirty == 0 && pmap_is_modified(m))
939 					vm_page_dirty(m);
940 				if (m->dirty != 0)
941 					mincoreinfo |= MINCORE_MODIFIED_OTHER;
942 
943 				/*
944 				 * The first test for PGA_REFERENCED is an
945 				 * optimization.  The second test is
946 				 * required because a concurrent pmap
947 				 * operation could clear the last reference
948 				 * and set PGA_REFERENCED before the call to
949 				 * pmap_is_referenced().
950 				 */
951 				if ((m->a.flags & PGA_REFERENCED) != 0 ||
952 				    pmap_is_referenced(m) ||
953 				    (m->a.flags & PGA_REFERENCED) != 0)
954 					mincoreinfo |= MINCORE_REFERENCED_OTHER;
955 			}
956 			if (object != NULL)
957 				VM_OBJECT_WUNLOCK(object);
958 
959 			/*
960 			 * subyte may page fault.  In case it needs to modify
961 			 * the map, we release the lock.
962 			 */
963 			vm_map_unlock_read(map);
964 
965 			/*
966 			 * calculate index into user supplied byte vector
967 			 */
968 			vecindex = atop(addr - first_addr);
969 
970 			/*
971 			 * If we have skipped map entries, we need to make sure that
972 			 * the byte vector is zeroed for those skipped entries.
973 			 */
974 			while ((lastvecindex + 1) < vecindex) {
975 				++lastvecindex;
976 				error = subyte(vec + lastvecindex, 0);
977 				if (error) {
978 					error = EFAULT;
979 					goto done2;
980 				}
981 			}
982 
983 			/*
984 			 * Pass the page information to the user
985 			 */
986 			error = subyte(vec + vecindex, mincoreinfo);
987 			if (error) {
988 				error = EFAULT;
989 				goto done2;
990 			}
991 
992 			/*
993 			 * If the map has changed, due to the subyte, the previous
994 			 * output may be invalid.
995 			 */
996 			vm_map_lock_read(map);
997 			if (timestamp != map->timestamp)
998 				goto RestartScan;
999 
1000 			lastvecindex = vecindex;
1001 		}
1002 	}
1003 
1004 	/*
1005 	 * subyte may page fault.  In case it needs to modify
1006 	 * the map, we release the lock.
1007 	 */
1008 	vm_map_unlock_read(map);
1009 
1010 	/*
1011 	 * Zero the last entries in the byte vector.
1012 	 */
1013 	vecindex = atop(end - first_addr);
1014 	while ((lastvecindex + 1) < vecindex) {
1015 		++lastvecindex;
1016 		error = subyte(vec + lastvecindex, 0);
1017 		if (error) {
1018 			error = EFAULT;
1019 			goto done2;
1020 		}
1021 	}
1022 
1023 	/*
1024 	 * If the map has changed, due to the subyte, the previous
1025 	 * output may be invalid.
1026 	 */
1027 	vm_map_lock_read(map);
1028 	if (timestamp != map->timestamp)
1029 		goto RestartScan;
1030 	vm_map_unlock_read(map);
1031 done2:
1032 	return (error);
1033 }
1034 
1035 #ifndef _SYS_SYSPROTO_H_
1036 struct mlock_args {
1037 	const void *addr;
1038 	size_t len;
1039 };
1040 #endif
1041 int
1042 sys_mlock(struct thread *td, struct mlock_args *uap)
1043 {
1044 
1045 	return (kern_mlock(td->td_proc, td->td_ucred,
1046 	    __DECONST(uintptr_t, uap->addr), uap->len));
1047 }
1048 
1049 int
1050 kern_mlock(struct proc *proc, struct ucred *cred, uintptr_t addr0, size_t len)
1051 {
1052 	vm_offset_t addr, end, last, start;
1053 	vm_size_t npages, size;
1054 	vm_map_t map;
1055 	unsigned long nsize;
1056 	int error;
1057 
1058 	error = priv_check_cred(cred, PRIV_VM_MLOCK);
1059 	if (error)
1060 		return (error);
1061 	addr = addr0;
1062 	size = len;
1063 	last = addr + size;
1064 	start = trunc_page(addr);
1065 	end = round_page(last);
1066 	if (last < addr || end < addr)
1067 		return (EINVAL);
1068 	npages = atop(end - start);
1069 	if (npages > vm_page_max_user_wired)
1070 		return (ENOMEM);
1071 	map = &proc->p_vmspace->vm_map;
1072 	PROC_LOCK(proc);
1073 	nsize = ptoa(npages + pmap_wired_count(map->pmap));
1074 	if (nsize > lim_cur_proc(proc, RLIMIT_MEMLOCK)) {
1075 		PROC_UNLOCK(proc);
1076 		return (ENOMEM);
1077 	}
1078 	PROC_UNLOCK(proc);
1079 #ifdef RACCT
1080 	if (racct_enable) {
1081 		PROC_LOCK(proc);
1082 		error = racct_set(proc, RACCT_MEMLOCK, nsize);
1083 		PROC_UNLOCK(proc);
1084 		if (error != 0)
1085 			return (ENOMEM);
1086 	}
1087 #endif
1088 	error = vm_map_wire(map, start, end,
1089 	    VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
1090 #ifdef RACCT
1091 	if (racct_enable && error != KERN_SUCCESS) {
1092 		PROC_LOCK(proc);
1093 		racct_set(proc, RACCT_MEMLOCK,
1094 		    ptoa(pmap_wired_count(map->pmap)));
1095 		PROC_UNLOCK(proc);
1096 	}
1097 #endif
1098 	return (error == KERN_SUCCESS ? 0 : ENOMEM);
1099 }
1100 
1101 #ifndef _SYS_SYSPROTO_H_
1102 struct mlockall_args {
1103 	int	how;
1104 };
1105 #endif
1106 
1107 int
1108 sys_mlockall(struct thread *td, struct mlockall_args *uap)
1109 {
1110 	vm_map_t map;
1111 	int error;
1112 
1113 	map = &td->td_proc->p_vmspace->vm_map;
1114 	error = priv_check(td, PRIV_VM_MLOCK);
1115 	if (error)
1116 		return (error);
1117 
1118 	if ((uap->how == 0) || ((uap->how & ~(MCL_CURRENT|MCL_FUTURE)) != 0))
1119 		return (EINVAL);
1120 
1121 	/*
1122 	 * If wiring all pages in the process would cause it to exceed
1123 	 * a hard resource limit, return ENOMEM.
1124 	 */
1125 	if (!old_mlock && uap->how & MCL_CURRENT) {
1126 		if (map->size > lim_cur(td, RLIMIT_MEMLOCK))
1127 			return (ENOMEM);
1128 	}
1129 #ifdef RACCT
1130 	if (racct_enable) {
1131 		PROC_LOCK(td->td_proc);
1132 		error = racct_set(td->td_proc, RACCT_MEMLOCK, map->size);
1133 		PROC_UNLOCK(td->td_proc);
1134 		if (error != 0)
1135 			return (ENOMEM);
1136 	}
1137 #endif
1138 
1139 	if (uap->how & MCL_FUTURE) {
1140 		vm_map_lock(map);
1141 		vm_map_modflags(map, MAP_WIREFUTURE, 0);
1142 		vm_map_unlock(map);
1143 		error = 0;
1144 	}
1145 
1146 	if (uap->how & MCL_CURRENT) {
1147 		/*
1148 		 * P1003.1-2001 mandates that all currently mapped pages
1149 		 * will be memory resident and locked (wired) upon return
1150 		 * from mlockall(). vm_map_wire() will wire pages, by
1151 		 * calling vm_fault_wire() for each page in the region.
1152 		 */
1153 		error = vm_map_wire(map, vm_map_min(map), vm_map_max(map),
1154 		    VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK);
1155 		if (error == KERN_SUCCESS)
1156 			error = 0;
1157 		else if (error == KERN_RESOURCE_SHORTAGE)
1158 			error = ENOMEM;
1159 		else
1160 			error = EAGAIN;
1161 	}
1162 #ifdef RACCT
1163 	if (racct_enable && error != KERN_SUCCESS) {
1164 		PROC_LOCK(td->td_proc);
1165 		racct_set(td->td_proc, RACCT_MEMLOCK,
1166 		    ptoa(pmap_wired_count(map->pmap)));
1167 		PROC_UNLOCK(td->td_proc);
1168 	}
1169 #endif
1170 
1171 	return (error);
1172 }
1173 
1174 #ifndef _SYS_SYSPROTO_H_
1175 struct munlockall_args {
1176 	register_t dummy;
1177 };
1178 #endif
1179 
1180 int
1181 sys_munlockall(struct thread *td, struct munlockall_args *uap)
1182 {
1183 	vm_map_t map;
1184 	int error;
1185 
1186 	map = &td->td_proc->p_vmspace->vm_map;
1187 	error = priv_check(td, PRIV_VM_MUNLOCK);
1188 	if (error)
1189 		return (error);
1190 
1191 	/* Clear the MAP_WIREFUTURE flag from this vm_map. */
1192 	vm_map_lock(map);
1193 	vm_map_modflags(map, 0, MAP_WIREFUTURE);
1194 	vm_map_unlock(map);
1195 
1196 	/* Forcibly unwire all pages. */
1197 	error = vm_map_unwire(map, vm_map_min(map), vm_map_max(map),
1198 	    VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK);
1199 #ifdef RACCT
1200 	if (racct_enable && error == KERN_SUCCESS) {
1201 		PROC_LOCK(td->td_proc);
1202 		racct_set(td->td_proc, RACCT_MEMLOCK, 0);
1203 		PROC_UNLOCK(td->td_proc);
1204 	}
1205 #endif
1206 
1207 	return (error);
1208 }
1209 
1210 #ifndef _SYS_SYSPROTO_H_
1211 struct munlock_args {
1212 	const void *addr;
1213 	size_t len;
1214 };
1215 #endif
1216 int
1217 sys_munlock(struct thread *td, struct munlock_args *uap)
1218 {
1219 
1220 	return (kern_munlock(td, (uintptr_t)uap->addr, uap->len));
1221 }
1222 
1223 int
1224 kern_munlock(struct thread *td, uintptr_t addr0, size_t size)
1225 {
1226 	vm_offset_t addr, end, last, start;
1227 #ifdef RACCT
1228 	vm_map_t map;
1229 #endif
1230 	int error;
1231 
1232 	error = priv_check(td, PRIV_VM_MUNLOCK);
1233 	if (error)
1234 		return (error);
1235 	addr = addr0;
1236 	last = addr + size;
1237 	start = trunc_page(addr);
1238 	end = round_page(last);
1239 	if (last < addr || end < addr)
1240 		return (EINVAL);
1241 	error = vm_map_unwire(&td->td_proc->p_vmspace->vm_map, start, end,
1242 	    VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
1243 #ifdef RACCT
1244 	if (racct_enable && error == KERN_SUCCESS) {
1245 		PROC_LOCK(td->td_proc);
1246 		map = &td->td_proc->p_vmspace->vm_map;
1247 		racct_set(td->td_proc, RACCT_MEMLOCK,
1248 		    ptoa(pmap_wired_count(map->pmap)));
1249 		PROC_UNLOCK(td->td_proc);
1250 	}
1251 #endif
1252 	return (error == KERN_SUCCESS ? 0 : ENOMEM);
1253 }
1254 
1255 /*
1256  * vm_mmap_vnode()
1257  *
1258  * Helper function for vm_mmap.  Perform sanity check specific for mmap
1259  * operations on vnodes.
1260  */
1261 int
1262 vm_mmap_vnode(struct thread *td, vm_size_t objsize,
1263     vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp,
1264     struct vnode *vp, vm_ooffset_t *foffp, vm_object_t *objp,
1265     boolean_t *writecounted)
1266 {
1267 	struct vattr va;
1268 	vm_object_t obj;
1269 	vm_ooffset_t foff;
1270 	struct ucred *cred;
1271 	int error, flags;
1272 	bool writex;
1273 
1274 	cred = td->td_ucred;
1275 	writex = (*maxprotp & VM_PROT_WRITE) != 0 &&
1276 	    (*flagsp & MAP_SHARED) != 0;
1277 	if ((error = vget(vp, LK_SHARED, td)) != 0)
1278 		return (error);
1279 	AUDIT_ARG_VNODE1(vp);
1280 	foff = *foffp;
1281 	flags = *flagsp;
1282 	obj = vp->v_object;
1283 	if (vp->v_type == VREG) {
1284 		/*
1285 		 * Get the proper underlying object
1286 		 */
1287 		if (obj == NULL) {
1288 			error = EINVAL;
1289 			goto done;
1290 		}
1291 		if (obj->type == OBJT_VNODE && obj->handle != vp) {
1292 			vput(vp);
1293 			vp = (struct vnode *)obj->handle;
1294 			/*
1295 			 * Bypass filesystems obey the mpsafety of the
1296 			 * underlying fs.  Tmpfs never bypasses.
1297 			 */
1298 			error = vget(vp, LK_SHARED, td);
1299 			if (error != 0)
1300 				return (error);
1301 		}
1302 		if (writex) {
1303 			*writecounted = TRUE;
1304 			vm_pager_update_writecount(obj, 0, objsize);
1305 		}
1306 	} else {
1307 		error = EINVAL;
1308 		goto done;
1309 	}
1310 	if ((error = VOP_GETATTR(vp, &va, cred)))
1311 		goto done;
1312 #ifdef MAC
1313 	/* This relies on VM_PROT_* matching PROT_*. */
1314 	error = mac_vnode_check_mmap(cred, vp, (int)prot, flags);
1315 	if (error != 0)
1316 		goto done;
1317 #endif
1318 	if ((flags & MAP_SHARED) != 0) {
1319 		if ((va.va_flags & (SF_SNAPSHOT|IMMUTABLE|APPEND)) != 0) {
1320 			if (prot & VM_PROT_WRITE) {
1321 				error = EPERM;
1322 				goto done;
1323 			}
1324 			*maxprotp &= ~VM_PROT_WRITE;
1325 		}
1326 	}
1327 	/*
1328 	 * If it is a regular file without any references
1329 	 * we do not need to sync it.
1330 	 * Adjust object size to be the size of actual file.
1331 	 */
1332 	objsize = round_page(va.va_size);
1333 	if (va.va_nlink == 0)
1334 		flags |= MAP_NOSYNC;
1335 	if (obj->type == OBJT_VNODE) {
1336 		obj = vm_pager_allocate(OBJT_VNODE, vp, objsize, prot, foff,
1337 		    cred);
1338 		if (obj == NULL) {
1339 			error = ENOMEM;
1340 			goto done;
1341 		}
1342 	} else {
1343 		KASSERT(obj->type == OBJT_DEFAULT || obj->type == OBJT_SWAP,
1344 		    ("wrong object type"));
1345 		vm_object_reference(obj);
1346 #if VM_NRESERVLEVEL > 0
1347 		if ((obj->flags & OBJ_COLORED) == 0) {
1348 			VM_OBJECT_WLOCK(obj);
1349 			vm_object_color(obj, 0);
1350 			VM_OBJECT_WUNLOCK(obj);
1351 		}
1352 #endif
1353 	}
1354 	*objp = obj;
1355 	*flagsp = flags;
1356 
1357 	VOP_MMAPPED(vp);
1358 
1359 done:
1360 	if (error != 0 && *writecounted) {
1361 		*writecounted = FALSE;
1362 		vm_pager_update_writecount(obj, objsize, 0);
1363 	}
1364 	vput(vp);
1365 	return (error);
1366 }
1367 
1368 /*
1369  * vm_mmap_cdev()
1370  *
1371  * Helper function for vm_mmap.  Perform sanity check specific for mmap
1372  * operations on cdevs.
1373  */
1374 int
1375 vm_mmap_cdev(struct thread *td, vm_size_t objsize, vm_prot_t prot,
1376     vm_prot_t *maxprotp, int *flagsp, struct cdev *cdev, struct cdevsw *dsw,
1377     vm_ooffset_t *foff, vm_object_t *objp)
1378 {
1379 	vm_object_t obj;
1380 	int error, flags;
1381 
1382 	flags = *flagsp;
1383 
1384 	if (dsw->d_flags & D_MMAP_ANON) {
1385 		*objp = NULL;
1386 		*foff = 0;
1387 		*maxprotp = VM_PROT_ALL;
1388 		*flagsp |= MAP_ANON;
1389 		return (0);
1390 	}
1391 	/*
1392 	 * cdevs do not provide private mappings of any kind.
1393 	 */
1394 	if ((*maxprotp & VM_PROT_WRITE) == 0 &&
1395 	    (prot & VM_PROT_WRITE) != 0)
1396 		return (EACCES);
1397 	if (flags & (MAP_PRIVATE|MAP_COPY))
1398 		return (EINVAL);
1399 	/*
1400 	 * Force device mappings to be shared.
1401 	 */
1402 	flags |= MAP_SHARED;
1403 #ifdef MAC_XXX
1404 	error = mac_cdev_check_mmap(td->td_ucred, cdev, (int)prot);
1405 	if (error != 0)
1406 		return (error);
1407 #endif
1408 	/*
1409 	 * First, try d_mmap_single().  If that is not implemented
1410 	 * (returns ENODEV), fall back to using the device pager.
1411 	 * Note that d_mmap_single() must return a reference to the
1412 	 * object (it needs to bump the reference count of the object
1413 	 * it returns somehow).
1414 	 *
1415 	 * XXX assumes VM_PROT_* == PROT_*
1416 	 */
1417 	error = dsw->d_mmap_single(cdev, foff, objsize, objp, (int)prot);
1418 	if (error != ENODEV)
1419 		return (error);
1420 	obj = vm_pager_allocate(OBJT_DEVICE, cdev, objsize, prot, *foff,
1421 	    td->td_ucred);
1422 	if (obj == NULL)
1423 		return (EINVAL);
1424 	*objp = obj;
1425 	*flagsp = flags;
1426 	return (0);
1427 }
1428 
1429 /*
1430  * vm_mmap()
1431  *
1432  * Internal version of mmap used by exec, sys5 shared memory, and
1433  * various device drivers.  Handle is either a vnode pointer, a
1434  * character device, or NULL for MAP_ANON.
1435  */
1436 int
1437 vm_mmap(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot,
1438 	vm_prot_t maxprot, int flags,
1439 	objtype_t handle_type, void *handle,
1440 	vm_ooffset_t foff)
1441 {
1442 	vm_object_t object;
1443 	struct thread *td = curthread;
1444 	int error;
1445 	boolean_t writecounted;
1446 
1447 	if (size == 0)
1448 		return (EINVAL);
1449 
1450 	size = round_page(size);
1451 	object = NULL;
1452 	writecounted = FALSE;
1453 
1454 	/*
1455 	 * Lookup/allocate object.
1456 	 */
1457 	switch (handle_type) {
1458 	case OBJT_DEVICE: {
1459 		struct cdevsw *dsw;
1460 		struct cdev *cdev;
1461 		int ref;
1462 
1463 		cdev = handle;
1464 		dsw = dev_refthread(cdev, &ref);
1465 		if (dsw == NULL)
1466 			return (ENXIO);
1467 		error = vm_mmap_cdev(td, size, prot, &maxprot, &flags, cdev,
1468 		    dsw, &foff, &object);
1469 		dev_relthread(cdev, ref);
1470 		break;
1471 	}
1472 	case OBJT_VNODE:
1473 		error = vm_mmap_vnode(td, size, prot, &maxprot, &flags,
1474 		    handle, &foff, &object, &writecounted);
1475 		break;
1476 	case OBJT_DEFAULT:
1477 		if (handle == NULL) {
1478 			error = 0;
1479 			break;
1480 		}
1481 		/* FALLTHROUGH */
1482 	default:
1483 		error = EINVAL;
1484 		break;
1485 	}
1486 	if (error)
1487 		return (error);
1488 
1489 	error = vm_mmap_object(map, addr, size, prot, maxprot, flags, object,
1490 	    foff, writecounted, td);
1491 	if (error != 0 && object != NULL) {
1492 		/*
1493 		 * If this mapping was accounted for in the vnode's
1494 		 * writecount, then undo that now.
1495 		 */
1496 		if (writecounted)
1497 			vm_pager_release_writecount(object, 0, size);
1498 		vm_object_deallocate(object);
1499 	}
1500 	return (error);
1501 }
1502 
1503 /*
1504  * Internal version of mmap that maps a specific VM object into an
1505  * map.  Called by mmap for MAP_ANON, vm_mmap, shm_mmap, and vn_mmap.
1506  */
1507 int
1508 vm_mmap_object(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot,
1509     vm_prot_t maxprot, int flags, vm_object_t object, vm_ooffset_t foff,
1510     boolean_t writecounted, struct thread *td)
1511 {
1512 	boolean_t curmap, fitit;
1513 	vm_offset_t max_addr;
1514 	int docow, error, findspace, rv;
1515 
1516 	curmap = map == &td->td_proc->p_vmspace->vm_map;
1517 	if (curmap) {
1518 		RACCT_PROC_LOCK(td->td_proc);
1519 		if (map->size + size > lim_cur(td, RLIMIT_VMEM)) {
1520 			RACCT_PROC_UNLOCK(td->td_proc);
1521 			return (ENOMEM);
1522 		}
1523 		if (racct_set(td->td_proc, RACCT_VMEM, map->size + size)) {
1524 			RACCT_PROC_UNLOCK(td->td_proc);
1525 			return (ENOMEM);
1526 		}
1527 		if (!old_mlock && map->flags & MAP_WIREFUTURE) {
1528 			if (ptoa(pmap_wired_count(map->pmap)) + size >
1529 			    lim_cur(td, RLIMIT_MEMLOCK)) {
1530 				racct_set_force(td->td_proc, RACCT_VMEM,
1531 				    map->size);
1532 				RACCT_PROC_UNLOCK(td->td_proc);
1533 				return (ENOMEM);
1534 			}
1535 			error = racct_set(td->td_proc, RACCT_MEMLOCK,
1536 			    ptoa(pmap_wired_count(map->pmap)) + size);
1537 			if (error != 0) {
1538 				racct_set_force(td->td_proc, RACCT_VMEM,
1539 				    map->size);
1540 				RACCT_PROC_UNLOCK(td->td_proc);
1541 				return (error);
1542 			}
1543 		}
1544 		RACCT_PROC_UNLOCK(td->td_proc);
1545 	}
1546 
1547 	/*
1548 	 * We currently can only deal with page aligned file offsets.
1549 	 * The mmap() system call already enforces this by subtracting
1550 	 * the page offset from the file offset, but checking here
1551 	 * catches errors in device drivers (e.g. d_single_mmap()
1552 	 * callbacks) and other internal mapping requests (such as in
1553 	 * exec).
1554 	 */
1555 	if (foff & PAGE_MASK)
1556 		return (EINVAL);
1557 
1558 	if ((flags & MAP_FIXED) == 0) {
1559 		fitit = TRUE;
1560 		*addr = round_page(*addr);
1561 	} else {
1562 		if (*addr != trunc_page(*addr))
1563 			return (EINVAL);
1564 		fitit = FALSE;
1565 	}
1566 
1567 	if (flags & MAP_ANON) {
1568 		if (object != NULL || foff != 0)
1569 			return (EINVAL);
1570 		docow = 0;
1571 	} else if (flags & MAP_PREFAULT_READ)
1572 		docow = MAP_PREFAULT;
1573 	else
1574 		docow = MAP_PREFAULT_PARTIAL;
1575 
1576 	if ((flags & (MAP_ANON|MAP_SHARED)) == 0)
1577 		docow |= MAP_COPY_ON_WRITE;
1578 	if (flags & MAP_NOSYNC)
1579 		docow |= MAP_DISABLE_SYNCER;
1580 	if (flags & MAP_NOCORE)
1581 		docow |= MAP_DISABLE_COREDUMP;
1582 	/* Shared memory is also shared with children. */
1583 	if (flags & MAP_SHARED)
1584 		docow |= MAP_INHERIT_SHARE;
1585 	if (writecounted)
1586 		docow |= MAP_WRITECOUNT;
1587 	if (flags & MAP_STACK) {
1588 		if (object != NULL)
1589 			return (EINVAL);
1590 		docow |= MAP_STACK_GROWS_DOWN;
1591 	}
1592 	if ((flags & MAP_EXCL) != 0)
1593 		docow |= MAP_CHECK_EXCL;
1594 	if ((flags & MAP_GUARD) != 0)
1595 		docow |= MAP_CREATE_GUARD;
1596 
1597 	if (fitit) {
1598 		if ((flags & MAP_ALIGNMENT_MASK) == MAP_ALIGNED_SUPER)
1599 			findspace = VMFS_SUPER_SPACE;
1600 		else if ((flags & MAP_ALIGNMENT_MASK) != 0)
1601 			findspace = VMFS_ALIGNED_SPACE(flags >>
1602 			    MAP_ALIGNMENT_SHIFT);
1603 		else
1604 			findspace = VMFS_OPTIMAL_SPACE;
1605 		max_addr = 0;
1606 #ifdef MAP_32BIT
1607 		if ((flags & MAP_32BIT) != 0)
1608 			max_addr = MAP_32BIT_MAX_ADDR;
1609 #endif
1610 		if (curmap) {
1611 			rv = vm_map_find_min(map, object, foff, addr, size,
1612 			    round_page((vm_offset_t)td->td_proc->p_vmspace->
1613 			    vm_daddr + lim_max(td, RLIMIT_DATA)), max_addr,
1614 			    findspace, prot, maxprot, docow);
1615 		} else {
1616 			rv = vm_map_find(map, object, foff, addr, size,
1617 			    max_addr, findspace, prot, maxprot, docow);
1618 		}
1619 	} else {
1620 		rv = vm_map_fixed(map, object, foff, *addr, size,
1621 		    prot, maxprot, docow);
1622 	}
1623 
1624 	if (rv == KERN_SUCCESS) {
1625 		/*
1626 		 * If the process has requested that all future mappings
1627 		 * be wired, then heed this.
1628 		 */
1629 		if ((map->flags & MAP_WIREFUTURE) != 0) {
1630 			vm_map_lock(map);
1631 			if ((map->flags & MAP_WIREFUTURE) != 0)
1632 				(void)vm_map_wire_locked(map, *addr,
1633 				    *addr + size, VM_MAP_WIRE_USER |
1634 				    ((flags & MAP_STACK) ? VM_MAP_WIRE_HOLESOK :
1635 				    VM_MAP_WIRE_NOHOLES));
1636 			vm_map_unlock(map);
1637 		}
1638 	}
1639 	return (vm_mmap_to_errno(rv));
1640 }
1641 
1642 /*
1643  * Translate a Mach VM return code to zero on success or the appropriate errno
1644  * on failure.
1645  */
1646 int
1647 vm_mmap_to_errno(int rv)
1648 {
1649 
1650 	switch (rv) {
1651 	case KERN_SUCCESS:
1652 		return (0);
1653 	case KERN_INVALID_ADDRESS:
1654 	case KERN_NO_SPACE:
1655 		return (ENOMEM);
1656 	case KERN_PROTECTION_FAILURE:
1657 		return (EACCES);
1658 	default:
1659 		return (EINVAL);
1660 	}
1661 }
1662