1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1988 University of Utah. 5 * Copyright (c) 1991, 1993 6 * The Regents of the University of California. All rights reserved. 7 * 8 * This code is derived from software contributed to Berkeley by 9 * the Systems Programming Group of the University of Utah Computer 10 * Science Department. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * from: Utah $Hdr: vm_mmap.c 1.6 91/10/21$ 37 * 38 * @(#)vm_mmap.c 8.4 (Berkeley) 1/12/94 39 */ 40 41 /* 42 * Mapped file (mmap) interface to VM 43 */ 44 45 #include <sys/cdefs.h> 46 __FBSDID("$FreeBSD$"); 47 48 #include "opt_hwpmc_hooks.h" 49 #include "opt_vm.h" 50 51 #include <sys/param.h> 52 #include <sys/systm.h> 53 #include <sys/capsicum.h> 54 #include <sys/kernel.h> 55 #include <sys/lock.h> 56 #include <sys/mutex.h> 57 #include <sys/sysproto.h> 58 #include <sys/elf.h> 59 #include <sys/filedesc.h> 60 #include <sys/priv.h> 61 #include <sys/proc.h> 62 #include <sys/procctl.h> 63 #include <sys/racct.h> 64 #include <sys/resource.h> 65 #include <sys/resourcevar.h> 66 #include <sys/rwlock.h> 67 #include <sys/sysctl.h> 68 #include <sys/vnode.h> 69 #include <sys/fcntl.h> 70 #include <sys/file.h> 71 #include <sys/mman.h> 72 #include <sys/mount.h> 73 #include <sys/conf.h> 74 #include <sys/stat.h> 75 #include <sys/syscallsubr.h> 76 #include <sys/sysent.h> 77 #include <sys/vmmeter.h> 78 #if defined(__amd64__) || defined(__i386__) /* for i386_read_exec */ 79 #include <machine/md_var.h> 80 #endif 81 82 #include <security/audit/audit.h> 83 #include <security/mac/mac_framework.h> 84 85 #include <vm/vm.h> 86 #include <vm/vm_param.h> 87 #include <vm/pmap.h> 88 #include <vm/vm_map.h> 89 #include <vm/vm_object.h> 90 #include <vm/vm_page.h> 91 #include <vm/vm_pager.h> 92 #include <vm/vm_pageout.h> 93 #include <vm/vm_extern.h> 94 #include <vm/vm_page.h> 95 #include <vm/vnode_pager.h> 96 97 #ifdef HWPMC_HOOKS 98 #include <sys/pmckern.h> 99 #endif 100 101 int old_mlock = 0; 102 SYSCTL_INT(_vm, OID_AUTO, old_mlock, CTLFLAG_RWTUN, &old_mlock, 0, 103 "Do not apply RLIMIT_MEMLOCK on mlockall"); 104 static int mincore_mapped = 1; 105 SYSCTL_INT(_vm, OID_AUTO, mincore_mapped, CTLFLAG_RWTUN, &mincore_mapped, 0, 106 "mincore reports mappings, not residency"); 107 static int imply_prot_max = 0; 108 SYSCTL_INT(_vm, OID_AUTO, imply_prot_max, CTLFLAG_RWTUN, &imply_prot_max, 0, 109 "Imply maximum page permissions in mmap() when none are specified"); 110 111 #ifdef MAP_32BIT 112 #define MAP_32BIT_MAX_ADDR ((vm_offset_t)1 << 31) 113 #endif 114 115 #ifndef _SYS_SYSPROTO_H_ 116 struct sbrk_args { 117 int incr; 118 }; 119 #endif 120 121 int 122 sys_sbrk(struct thread *td, struct sbrk_args *uap) 123 { 124 /* Not yet implemented */ 125 return (EOPNOTSUPP); 126 } 127 128 #ifndef _SYS_SYSPROTO_H_ 129 struct sstk_args { 130 int incr; 131 }; 132 #endif 133 134 int 135 sys_sstk(struct thread *td, struct sstk_args *uap) 136 { 137 /* Not yet implemented */ 138 return (EOPNOTSUPP); 139 } 140 141 #if defined(COMPAT_43) 142 int 143 ogetpagesize(struct thread *td, struct ogetpagesize_args *uap) 144 { 145 146 td->td_retval[0] = PAGE_SIZE; 147 return (0); 148 } 149 #endif /* COMPAT_43 */ 150 151 152 /* 153 * Memory Map (mmap) system call. Note that the file offset 154 * and address are allowed to be NOT page aligned, though if 155 * the MAP_FIXED flag it set, both must have the same remainder 156 * modulo the PAGE_SIZE (POSIX 1003.1b). If the address is not 157 * page-aligned, the actual mapping starts at trunc_page(addr) 158 * and the return value is adjusted up by the page offset. 159 * 160 * Generally speaking, only character devices which are themselves 161 * memory-based, such as a video framebuffer, can be mmap'd. Otherwise 162 * there would be no cache coherency between a descriptor and a VM mapping 163 * both to the same character device. 164 */ 165 #ifndef _SYS_SYSPROTO_H_ 166 struct mmap_args { 167 void *addr; 168 size_t len; 169 int prot; 170 int flags; 171 int fd; 172 long pad; 173 off_t pos; 174 }; 175 #endif 176 177 int 178 sys_mmap(struct thread *td, struct mmap_args *uap) 179 { 180 181 return (kern_mmap(td, (uintptr_t)uap->addr, uap->len, uap->prot, 182 uap->flags, uap->fd, uap->pos)); 183 } 184 185 int 186 kern_mmap_maxprot(struct proc *p, int prot) 187 { 188 189 if ((p->p_flag2 & P2_PROTMAX_DISABLE) != 0 || 190 (p->p_fctl0 & NT_FREEBSD_FCTL_PROTMAX_DISABLE) != 0) 191 return (_PROT_ALL); 192 if (((p->p_flag2 & P2_PROTMAX_ENABLE) != 0 || imply_prot_max) && 193 prot != PROT_NONE) 194 return (prot); 195 return (_PROT_ALL); 196 } 197 198 int 199 kern_mmap(struct thread *td, uintptr_t addr0, size_t len, int prot, int flags, 200 int fd, off_t pos) 201 { 202 struct vmspace *vms; 203 struct file *fp; 204 struct proc *p; 205 vm_offset_t addr; 206 vm_size_t pageoff, size; 207 vm_prot_t cap_maxprot; 208 int align, error, max_prot; 209 cap_rights_t rights; 210 211 if ((prot & ~(_PROT_ALL | PROT_MAX(_PROT_ALL))) != 0) 212 return (EINVAL); 213 max_prot = PROT_MAX_EXTRACT(prot); 214 prot = PROT_EXTRACT(prot); 215 if (max_prot != 0 && (max_prot & prot) != prot) 216 return (EINVAL); 217 218 p = td->td_proc; 219 220 /* 221 * Always honor PROT_MAX if set. If not, default to all 222 * permissions unless we're implying maximum permissions. 223 */ 224 if (max_prot == 0) 225 max_prot = kern_mmap_maxprot(p, prot); 226 227 vms = p->p_vmspace; 228 fp = NULL; 229 AUDIT_ARG_FD(fd); 230 addr = addr0; 231 232 /* 233 * Ignore old flags that used to be defined but did not do anything. 234 */ 235 flags &= ~(MAP_RESERVED0020 | MAP_RESERVED0040); 236 237 /* 238 * Enforce the constraints. 239 * Mapping of length 0 is only allowed for old binaries. 240 * Anonymous mapping shall specify -1 as filedescriptor and 241 * zero position for new code. Be nice to ancient a.out 242 * binaries and correct pos for anonymous mapping, since old 243 * ld.so sometimes issues anonymous map requests with non-zero 244 * pos. 245 */ 246 if (!SV_CURPROC_FLAG(SV_AOUT)) { 247 if ((len == 0 && p->p_osrel >= P_OSREL_MAP_ANON) || 248 ((flags & MAP_ANON) != 0 && (fd != -1 || pos != 0))) 249 return (EINVAL); 250 } else { 251 if ((flags & MAP_ANON) != 0) 252 pos = 0; 253 } 254 255 if (flags & MAP_STACK) { 256 if ((fd != -1) || 257 ((prot & (PROT_READ | PROT_WRITE)) != (PROT_READ | PROT_WRITE))) 258 return (EINVAL); 259 flags |= MAP_ANON; 260 pos = 0; 261 } 262 if ((flags & ~(MAP_SHARED | MAP_PRIVATE | MAP_FIXED | MAP_HASSEMAPHORE | 263 MAP_STACK | MAP_NOSYNC | MAP_ANON | MAP_EXCL | MAP_NOCORE | 264 MAP_PREFAULT_READ | MAP_GUARD | 265 #ifdef MAP_32BIT 266 MAP_32BIT | 267 #endif 268 MAP_ALIGNMENT_MASK)) != 0) 269 return (EINVAL); 270 if ((flags & (MAP_EXCL | MAP_FIXED)) == MAP_EXCL) 271 return (EINVAL); 272 if ((flags & (MAP_SHARED | MAP_PRIVATE)) == (MAP_SHARED | MAP_PRIVATE)) 273 return (EINVAL); 274 if (prot != PROT_NONE && 275 (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC)) != 0) 276 return (EINVAL); 277 if ((flags & MAP_GUARD) != 0 && (prot != PROT_NONE || fd != -1 || 278 pos != 0 || (flags & ~(MAP_FIXED | MAP_GUARD | MAP_EXCL | 279 #ifdef MAP_32BIT 280 MAP_32BIT | 281 #endif 282 MAP_ALIGNMENT_MASK)) != 0)) 283 return (EINVAL); 284 285 /* 286 * Align the file position to a page boundary, 287 * and save its page offset component. 288 */ 289 pageoff = (pos & PAGE_MASK); 290 pos -= pageoff; 291 292 /* Compute size from len by rounding (on both ends). */ 293 size = len + pageoff; /* low end... */ 294 size = round_page(size); /* hi end */ 295 /* Check for rounding up to zero. */ 296 if (len > size) 297 return (ENOMEM); 298 299 /* Ensure alignment is at least a page and fits in a pointer. */ 300 align = flags & MAP_ALIGNMENT_MASK; 301 if (align != 0 && align != MAP_ALIGNED_SUPER && 302 (align >> MAP_ALIGNMENT_SHIFT >= sizeof(void *) * NBBY || 303 align >> MAP_ALIGNMENT_SHIFT < PAGE_SHIFT)) 304 return (EINVAL); 305 306 /* 307 * Check for illegal addresses. Watch out for address wrap... Note 308 * that VM_*_ADDRESS are not constants due to casts (argh). 309 */ 310 if (flags & MAP_FIXED) { 311 /* 312 * The specified address must have the same remainder 313 * as the file offset taken modulo PAGE_SIZE, so it 314 * should be aligned after adjustment by pageoff. 315 */ 316 addr -= pageoff; 317 if (addr & PAGE_MASK) 318 return (EINVAL); 319 320 /* Address range must be all in user VM space. */ 321 if (addr < vm_map_min(&vms->vm_map) || 322 addr + size > vm_map_max(&vms->vm_map)) 323 return (EINVAL); 324 if (addr + size < addr) 325 return (EINVAL); 326 #ifdef MAP_32BIT 327 if (flags & MAP_32BIT && addr + size > MAP_32BIT_MAX_ADDR) 328 return (EINVAL); 329 } else if (flags & MAP_32BIT) { 330 /* 331 * For MAP_32BIT, override the hint if it is too high and 332 * do not bother moving the mapping past the heap (since 333 * the heap is usually above 2GB). 334 */ 335 if (addr + size > MAP_32BIT_MAX_ADDR) 336 addr = 0; 337 #endif 338 } else { 339 /* 340 * XXX for non-fixed mappings where no hint is provided or 341 * the hint would fall in the potential heap space, 342 * place it after the end of the largest possible heap. 343 * 344 * There should really be a pmap call to determine a reasonable 345 * location. 346 */ 347 if (addr == 0 || 348 (addr >= round_page((vm_offset_t)vms->vm_taddr) && 349 addr < round_page((vm_offset_t)vms->vm_daddr + 350 lim_max(td, RLIMIT_DATA)))) 351 addr = round_page((vm_offset_t)vms->vm_daddr + 352 lim_max(td, RLIMIT_DATA)); 353 } 354 if (len == 0) { 355 /* 356 * Return success without mapping anything for old 357 * binaries that request a page-aligned mapping of 358 * length 0. For modern binaries, this function 359 * returns an error earlier. 360 */ 361 error = 0; 362 } else if ((flags & MAP_GUARD) != 0) { 363 error = vm_mmap_object(&vms->vm_map, &addr, size, VM_PROT_NONE, 364 VM_PROT_NONE, flags, NULL, pos, FALSE, td); 365 } else if ((flags & MAP_ANON) != 0) { 366 /* 367 * Mapping blank space is trivial. 368 * 369 * This relies on VM_PROT_* matching PROT_*. 370 */ 371 error = vm_mmap_object(&vms->vm_map, &addr, size, prot, 372 max_prot, flags, NULL, pos, FALSE, td); 373 } else { 374 /* 375 * Mapping file, get fp for validation and don't let the 376 * descriptor disappear on us if we block. Check capability 377 * rights, but also return the maximum rights to be combined 378 * with maxprot later. 379 */ 380 cap_rights_init(&rights, CAP_MMAP); 381 if (prot & PROT_READ) 382 cap_rights_set(&rights, CAP_MMAP_R); 383 if ((flags & MAP_SHARED) != 0) { 384 if (prot & PROT_WRITE) 385 cap_rights_set(&rights, CAP_MMAP_W); 386 } 387 if (prot & PROT_EXEC) 388 cap_rights_set(&rights, CAP_MMAP_X); 389 error = fget_mmap(td, fd, &rights, &cap_maxprot, &fp); 390 if (error != 0) 391 goto done; 392 if ((flags & (MAP_SHARED | MAP_PRIVATE)) == 0 && 393 p->p_osrel >= P_OSREL_MAP_FSTRICT) { 394 error = EINVAL; 395 goto done; 396 } 397 398 /* This relies on VM_PROT_* matching PROT_*. */ 399 error = fo_mmap(fp, &vms->vm_map, &addr, size, prot, 400 max_prot & cap_maxprot, flags, pos, td); 401 } 402 403 if (error == 0) 404 td->td_retval[0] = (register_t) (addr + pageoff); 405 done: 406 if (fp) 407 fdrop(fp, td); 408 409 return (error); 410 } 411 412 #if defined(COMPAT_FREEBSD6) 413 int 414 freebsd6_mmap(struct thread *td, struct freebsd6_mmap_args *uap) 415 { 416 417 return (kern_mmap(td, (uintptr_t)uap->addr, uap->len, uap->prot, 418 uap->flags, uap->fd, uap->pos)); 419 } 420 #endif 421 422 #ifdef COMPAT_43 423 #ifndef _SYS_SYSPROTO_H_ 424 struct ommap_args { 425 caddr_t addr; 426 int len; 427 int prot; 428 int flags; 429 int fd; 430 long pos; 431 }; 432 #endif 433 int 434 ommap(struct thread *td, struct ommap_args *uap) 435 { 436 static const char cvtbsdprot[8] = { 437 0, 438 PROT_EXEC, 439 PROT_WRITE, 440 PROT_EXEC | PROT_WRITE, 441 PROT_READ, 442 PROT_EXEC | PROT_READ, 443 PROT_WRITE | PROT_READ, 444 PROT_EXEC | PROT_WRITE | PROT_READ, 445 }; 446 int flags, prot; 447 448 #define OMAP_ANON 0x0002 449 #define OMAP_COPY 0x0020 450 #define OMAP_SHARED 0x0010 451 #define OMAP_FIXED 0x0100 452 453 prot = cvtbsdprot[uap->prot & 0x7]; 454 #if (defined(COMPAT_FREEBSD32) && defined(__amd64__)) || defined(__i386__) 455 if (i386_read_exec && SV_PROC_FLAG(td->td_proc, SV_ILP32) && 456 prot != 0) 457 prot |= PROT_EXEC; 458 #endif 459 flags = 0; 460 if (uap->flags & OMAP_ANON) 461 flags |= MAP_ANON; 462 if (uap->flags & OMAP_COPY) 463 flags |= MAP_COPY; 464 if (uap->flags & OMAP_SHARED) 465 flags |= MAP_SHARED; 466 else 467 flags |= MAP_PRIVATE; 468 if (uap->flags & OMAP_FIXED) 469 flags |= MAP_FIXED; 470 return (kern_mmap(td, (uintptr_t)uap->addr, uap->len, prot, flags, 471 uap->fd, uap->pos)); 472 } 473 #endif /* COMPAT_43 */ 474 475 476 #ifndef _SYS_SYSPROTO_H_ 477 struct msync_args { 478 void *addr; 479 size_t len; 480 int flags; 481 }; 482 #endif 483 int 484 sys_msync(struct thread *td, struct msync_args *uap) 485 { 486 487 return (kern_msync(td, (uintptr_t)uap->addr, uap->len, uap->flags)); 488 } 489 490 int 491 kern_msync(struct thread *td, uintptr_t addr0, size_t size, int flags) 492 { 493 vm_offset_t addr; 494 vm_size_t pageoff; 495 vm_map_t map; 496 int rv; 497 498 addr = addr0; 499 pageoff = (addr & PAGE_MASK); 500 addr -= pageoff; 501 size += pageoff; 502 size = (vm_size_t) round_page(size); 503 if (addr + size < addr) 504 return (EINVAL); 505 506 if ((flags & (MS_ASYNC|MS_INVALIDATE)) == (MS_ASYNC|MS_INVALIDATE)) 507 return (EINVAL); 508 509 map = &td->td_proc->p_vmspace->vm_map; 510 511 /* 512 * Clean the pages and interpret the return value. 513 */ 514 rv = vm_map_sync(map, addr, addr + size, (flags & MS_ASYNC) == 0, 515 (flags & MS_INVALIDATE) != 0); 516 switch (rv) { 517 case KERN_SUCCESS: 518 return (0); 519 case KERN_INVALID_ADDRESS: 520 return (ENOMEM); 521 case KERN_INVALID_ARGUMENT: 522 return (EBUSY); 523 case KERN_FAILURE: 524 return (EIO); 525 default: 526 return (EINVAL); 527 } 528 } 529 530 #ifndef _SYS_SYSPROTO_H_ 531 struct munmap_args { 532 void *addr; 533 size_t len; 534 }; 535 #endif 536 int 537 sys_munmap(struct thread *td, struct munmap_args *uap) 538 { 539 540 return (kern_munmap(td, (uintptr_t)uap->addr, uap->len)); 541 } 542 543 int 544 kern_munmap(struct thread *td, uintptr_t addr0, size_t size) 545 { 546 #ifdef HWPMC_HOOKS 547 struct pmckern_map_out pkm; 548 vm_map_entry_t entry; 549 bool pmc_handled; 550 #endif 551 vm_offset_t addr; 552 vm_size_t pageoff; 553 vm_map_t map; 554 555 if (size == 0) 556 return (EINVAL); 557 558 addr = addr0; 559 pageoff = (addr & PAGE_MASK); 560 addr -= pageoff; 561 size += pageoff; 562 size = (vm_size_t) round_page(size); 563 if (addr + size < addr) 564 return (EINVAL); 565 566 /* 567 * Check for illegal addresses. Watch out for address wrap... 568 */ 569 map = &td->td_proc->p_vmspace->vm_map; 570 if (addr < vm_map_min(map) || addr + size > vm_map_max(map)) 571 return (EINVAL); 572 vm_map_lock(map); 573 #ifdef HWPMC_HOOKS 574 pmc_handled = false; 575 if (PMC_HOOK_INSTALLED(PMC_FN_MUNMAP)) { 576 pmc_handled = true; 577 /* 578 * Inform hwpmc if the address range being unmapped contains 579 * an executable region. 580 */ 581 pkm.pm_address = (uintptr_t) NULL; 582 if (vm_map_lookup_entry(map, addr, &entry)) { 583 for (; entry->start < addr + size; 584 entry = vm_map_entry_succ(entry)) { 585 if (vm_map_check_protection(map, entry->start, 586 entry->end, VM_PROT_EXECUTE) == TRUE) { 587 pkm.pm_address = (uintptr_t) addr; 588 pkm.pm_size = (size_t) size; 589 break; 590 } 591 } 592 } 593 } 594 #endif 595 vm_map_delete(map, addr, addr + size); 596 597 #ifdef HWPMC_HOOKS 598 if (__predict_false(pmc_handled)) { 599 /* downgrade the lock to prevent a LOR with the pmc-sx lock */ 600 vm_map_lock_downgrade(map); 601 if (pkm.pm_address != (uintptr_t) NULL) 602 PMC_CALL_HOOK(td, PMC_FN_MUNMAP, (void *) &pkm); 603 vm_map_unlock_read(map); 604 } else 605 #endif 606 vm_map_unlock(map); 607 608 /* vm_map_delete returns nothing but KERN_SUCCESS anyway */ 609 return (0); 610 } 611 612 #ifndef _SYS_SYSPROTO_H_ 613 struct mprotect_args { 614 const void *addr; 615 size_t len; 616 int prot; 617 }; 618 #endif 619 int 620 sys_mprotect(struct thread *td, struct mprotect_args *uap) 621 { 622 623 return (kern_mprotect(td, (uintptr_t)uap->addr, uap->len, uap->prot)); 624 } 625 626 int 627 kern_mprotect(struct thread *td, uintptr_t addr0, size_t size, int prot) 628 { 629 vm_offset_t addr; 630 vm_size_t pageoff; 631 int vm_error, max_prot; 632 633 addr = addr0; 634 if ((prot & ~(_PROT_ALL | PROT_MAX(_PROT_ALL))) != 0) 635 return (EINVAL); 636 max_prot = PROT_MAX_EXTRACT(prot); 637 prot = PROT_EXTRACT(prot); 638 pageoff = (addr & PAGE_MASK); 639 addr -= pageoff; 640 size += pageoff; 641 size = (vm_size_t) round_page(size); 642 #ifdef COMPAT_FREEBSD32 643 if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) { 644 if (((addr + size) & 0xffffffff) < addr) 645 return (EINVAL); 646 } else 647 #endif 648 if (addr + size < addr) 649 return (EINVAL); 650 651 vm_error = KERN_SUCCESS; 652 if (max_prot != 0) { 653 if ((max_prot & prot) != prot) 654 return (EINVAL); 655 vm_error = vm_map_protect(&td->td_proc->p_vmspace->vm_map, 656 addr, addr + size, max_prot, TRUE); 657 } 658 if (vm_error == KERN_SUCCESS) 659 vm_error = vm_map_protect(&td->td_proc->p_vmspace->vm_map, 660 addr, addr + size, prot, FALSE); 661 662 switch (vm_error) { 663 case KERN_SUCCESS: 664 return (0); 665 case KERN_PROTECTION_FAILURE: 666 return (EACCES); 667 case KERN_RESOURCE_SHORTAGE: 668 return (ENOMEM); 669 } 670 return (EINVAL); 671 } 672 673 #ifndef _SYS_SYSPROTO_H_ 674 struct minherit_args { 675 void *addr; 676 size_t len; 677 int inherit; 678 }; 679 #endif 680 int 681 sys_minherit(struct thread *td, struct minherit_args *uap) 682 { 683 vm_offset_t addr; 684 vm_size_t size, pageoff; 685 vm_inherit_t inherit; 686 687 addr = (vm_offset_t)uap->addr; 688 size = uap->len; 689 inherit = uap->inherit; 690 691 pageoff = (addr & PAGE_MASK); 692 addr -= pageoff; 693 size += pageoff; 694 size = (vm_size_t) round_page(size); 695 if (addr + size < addr) 696 return (EINVAL); 697 698 switch (vm_map_inherit(&td->td_proc->p_vmspace->vm_map, addr, 699 addr + size, inherit)) { 700 case KERN_SUCCESS: 701 return (0); 702 case KERN_PROTECTION_FAILURE: 703 return (EACCES); 704 } 705 return (EINVAL); 706 } 707 708 #ifndef _SYS_SYSPROTO_H_ 709 struct madvise_args { 710 void *addr; 711 size_t len; 712 int behav; 713 }; 714 #endif 715 716 int 717 sys_madvise(struct thread *td, struct madvise_args *uap) 718 { 719 720 return (kern_madvise(td, (uintptr_t)uap->addr, uap->len, uap->behav)); 721 } 722 723 int 724 kern_madvise(struct thread *td, uintptr_t addr0, size_t len, int behav) 725 { 726 vm_map_t map; 727 vm_offset_t addr, end, start; 728 int flags; 729 730 /* 731 * Check for our special case, advising the swap pager we are 732 * "immortal." 733 */ 734 if (behav == MADV_PROTECT) { 735 flags = PPROT_SET; 736 return (kern_procctl(td, P_PID, td->td_proc->p_pid, 737 PROC_SPROTECT, &flags)); 738 } 739 740 /* 741 * Check for illegal addresses. Watch out for address wrap... Note 742 * that VM_*_ADDRESS are not constants due to casts (argh). 743 */ 744 map = &td->td_proc->p_vmspace->vm_map; 745 addr = addr0; 746 if (addr < vm_map_min(map) || addr + len > vm_map_max(map)) 747 return (EINVAL); 748 if ((addr + len) < addr) 749 return (EINVAL); 750 751 /* 752 * Since this routine is only advisory, we default to conservative 753 * behavior. 754 */ 755 start = trunc_page(addr); 756 end = round_page(addr + len); 757 758 /* 759 * vm_map_madvise() checks for illegal values of behav. 760 */ 761 return (vm_map_madvise(map, start, end, behav)); 762 } 763 764 #ifndef _SYS_SYSPROTO_H_ 765 struct mincore_args { 766 const void *addr; 767 size_t len; 768 char *vec; 769 }; 770 #endif 771 772 int 773 sys_mincore(struct thread *td, struct mincore_args *uap) 774 { 775 776 return (kern_mincore(td, (uintptr_t)uap->addr, uap->len, uap->vec)); 777 } 778 779 int 780 kern_mincore(struct thread *td, uintptr_t addr0, size_t len, char *vec) 781 { 782 pmap_t pmap; 783 vm_map_t map; 784 vm_map_entry_t current, entry; 785 vm_object_t object; 786 vm_offset_t addr, cend, end, first_addr; 787 vm_paddr_t pa; 788 vm_page_t m; 789 vm_pindex_t pindex; 790 int error, lastvecindex, mincoreinfo, vecindex; 791 unsigned int timestamp; 792 793 /* 794 * Make sure that the addresses presented are valid for user 795 * mode. 796 */ 797 first_addr = addr = trunc_page(addr0); 798 end = round_page(addr0 + len); 799 map = &td->td_proc->p_vmspace->vm_map; 800 if (end > vm_map_max(map) || end < addr) 801 return (ENOMEM); 802 803 pmap = vmspace_pmap(td->td_proc->p_vmspace); 804 805 vm_map_lock_read(map); 806 RestartScan: 807 timestamp = map->timestamp; 808 809 if (!vm_map_lookup_entry(map, addr, &entry)) { 810 vm_map_unlock_read(map); 811 return (ENOMEM); 812 } 813 814 /* 815 * Do this on a map entry basis so that if the pages are not 816 * in the current processes address space, we can easily look 817 * up the pages elsewhere. 818 */ 819 lastvecindex = -1; 820 while (entry->start < end) { 821 822 /* 823 * check for contiguity 824 */ 825 current = entry; 826 entry = vm_map_entry_succ(current); 827 if (current->end < end && 828 entry->start > current->end) { 829 vm_map_unlock_read(map); 830 return (ENOMEM); 831 } 832 833 /* 834 * ignore submaps (for now) or null objects 835 */ 836 if ((current->eflags & MAP_ENTRY_IS_SUB_MAP) || 837 current->object.vm_object == NULL) 838 continue; 839 840 /* 841 * limit this scan to the current map entry and the 842 * limits for the mincore call 843 */ 844 if (addr < current->start) 845 addr = current->start; 846 cend = current->end; 847 if (cend > end) 848 cend = end; 849 850 for (; addr < cend; addr += PAGE_SIZE) { 851 /* 852 * Check pmap first, it is likely faster, also 853 * it can provide info as to whether we are the 854 * one referencing or modifying the page. 855 */ 856 m = NULL; 857 object = NULL; 858 retry: 859 pa = 0; 860 mincoreinfo = pmap_mincore(pmap, addr, &pa); 861 if (mincore_mapped) { 862 /* 863 * We only care about this pmap's 864 * mapping of the page, if any. 865 */ 866 ; 867 } else if (pa != 0) { 868 /* 869 * The page is mapped by this process but not 870 * both accessed and modified. It is also 871 * managed. Acquire the object lock so that 872 * other mappings might be examined. The page's 873 * identity may change at any point before its 874 * object lock is acquired, so re-validate if 875 * necessary. 876 */ 877 m = PHYS_TO_VM_PAGE(pa); 878 while (object == NULL || m->object != object) { 879 if (object != NULL) 880 VM_OBJECT_WUNLOCK(object); 881 object = (vm_object_t)atomic_load_ptr( 882 &m->object); 883 if (object == NULL) 884 goto retry; 885 VM_OBJECT_WLOCK(object); 886 } 887 if (pa != pmap_extract(pmap, addr)) 888 goto retry; 889 KASSERT(vm_page_all_valid(m), 890 ("mincore: page %p is mapped but invalid", 891 m)); 892 } else if (mincoreinfo == 0) { 893 /* 894 * The page is not mapped by this process. If 895 * the object implements managed pages, then 896 * determine if the page is resident so that 897 * the mappings might be examined. 898 */ 899 if (current->object.vm_object != object) { 900 if (object != NULL) 901 VM_OBJECT_WUNLOCK(object); 902 object = current->object.vm_object; 903 VM_OBJECT_WLOCK(object); 904 } 905 if (object->type == OBJT_DEFAULT || 906 object->type == OBJT_SWAP || 907 object->type == OBJT_VNODE) { 908 pindex = OFF_TO_IDX(current->offset + 909 (addr - current->start)); 910 m = vm_page_lookup(object, pindex); 911 if (m != NULL && vm_page_none_valid(m)) 912 m = NULL; 913 if (m != NULL) 914 mincoreinfo = MINCORE_INCORE; 915 } 916 } 917 if (m != NULL) { 918 VM_OBJECT_ASSERT_WLOCKED(m->object); 919 920 /* Examine other mappings of the page. */ 921 if (m->dirty == 0 && pmap_is_modified(m)) 922 vm_page_dirty(m); 923 if (m->dirty != 0) 924 mincoreinfo |= MINCORE_MODIFIED_OTHER; 925 926 /* 927 * The first test for PGA_REFERENCED is an 928 * optimization. The second test is 929 * required because a concurrent pmap 930 * operation could clear the last reference 931 * and set PGA_REFERENCED before the call to 932 * pmap_is_referenced(). 933 */ 934 if ((m->a.flags & PGA_REFERENCED) != 0 || 935 pmap_is_referenced(m) || 936 (m->a.flags & PGA_REFERENCED) != 0) 937 mincoreinfo |= MINCORE_REFERENCED_OTHER; 938 } 939 if (object != NULL) 940 VM_OBJECT_WUNLOCK(object); 941 942 /* 943 * subyte may page fault. In case it needs to modify 944 * the map, we release the lock. 945 */ 946 vm_map_unlock_read(map); 947 948 /* 949 * calculate index into user supplied byte vector 950 */ 951 vecindex = atop(addr - first_addr); 952 953 /* 954 * If we have skipped map entries, we need to make sure that 955 * the byte vector is zeroed for those skipped entries. 956 */ 957 while ((lastvecindex + 1) < vecindex) { 958 ++lastvecindex; 959 error = subyte(vec + lastvecindex, 0); 960 if (error) { 961 error = EFAULT; 962 goto done2; 963 } 964 } 965 966 /* 967 * Pass the page information to the user 968 */ 969 error = subyte(vec + vecindex, mincoreinfo); 970 if (error) { 971 error = EFAULT; 972 goto done2; 973 } 974 975 /* 976 * If the map has changed, due to the subyte, the previous 977 * output may be invalid. 978 */ 979 vm_map_lock_read(map); 980 if (timestamp != map->timestamp) 981 goto RestartScan; 982 983 lastvecindex = vecindex; 984 } 985 } 986 987 /* 988 * subyte may page fault. In case it needs to modify 989 * the map, we release the lock. 990 */ 991 vm_map_unlock_read(map); 992 993 /* 994 * Zero the last entries in the byte vector. 995 */ 996 vecindex = atop(end - first_addr); 997 while ((lastvecindex + 1) < vecindex) { 998 ++lastvecindex; 999 error = subyte(vec + lastvecindex, 0); 1000 if (error) { 1001 error = EFAULT; 1002 goto done2; 1003 } 1004 } 1005 1006 /* 1007 * If the map has changed, due to the subyte, the previous 1008 * output may be invalid. 1009 */ 1010 vm_map_lock_read(map); 1011 if (timestamp != map->timestamp) 1012 goto RestartScan; 1013 vm_map_unlock_read(map); 1014 done2: 1015 return (error); 1016 } 1017 1018 #ifndef _SYS_SYSPROTO_H_ 1019 struct mlock_args { 1020 const void *addr; 1021 size_t len; 1022 }; 1023 #endif 1024 int 1025 sys_mlock(struct thread *td, struct mlock_args *uap) 1026 { 1027 1028 return (kern_mlock(td->td_proc, td->td_ucred, 1029 __DECONST(uintptr_t, uap->addr), uap->len)); 1030 } 1031 1032 int 1033 kern_mlock(struct proc *proc, struct ucred *cred, uintptr_t addr0, size_t len) 1034 { 1035 vm_offset_t addr, end, last, start; 1036 vm_size_t npages, size; 1037 vm_map_t map; 1038 unsigned long nsize; 1039 int error; 1040 1041 error = priv_check_cred(cred, PRIV_VM_MLOCK); 1042 if (error) 1043 return (error); 1044 addr = addr0; 1045 size = len; 1046 last = addr + size; 1047 start = trunc_page(addr); 1048 end = round_page(last); 1049 if (last < addr || end < addr) 1050 return (EINVAL); 1051 npages = atop(end - start); 1052 if (npages > vm_page_max_user_wired) 1053 return (ENOMEM); 1054 map = &proc->p_vmspace->vm_map; 1055 PROC_LOCK(proc); 1056 nsize = ptoa(npages + pmap_wired_count(map->pmap)); 1057 if (nsize > lim_cur_proc(proc, RLIMIT_MEMLOCK)) { 1058 PROC_UNLOCK(proc); 1059 return (ENOMEM); 1060 } 1061 PROC_UNLOCK(proc); 1062 #ifdef RACCT 1063 if (racct_enable) { 1064 PROC_LOCK(proc); 1065 error = racct_set(proc, RACCT_MEMLOCK, nsize); 1066 PROC_UNLOCK(proc); 1067 if (error != 0) 1068 return (ENOMEM); 1069 } 1070 #endif 1071 error = vm_map_wire(map, start, end, 1072 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 1073 #ifdef RACCT 1074 if (racct_enable && error != KERN_SUCCESS) { 1075 PROC_LOCK(proc); 1076 racct_set(proc, RACCT_MEMLOCK, 1077 ptoa(pmap_wired_count(map->pmap))); 1078 PROC_UNLOCK(proc); 1079 } 1080 #endif 1081 return (error == KERN_SUCCESS ? 0 : ENOMEM); 1082 } 1083 1084 #ifndef _SYS_SYSPROTO_H_ 1085 struct mlockall_args { 1086 int how; 1087 }; 1088 #endif 1089 1090 int 1091 sys_mlockall(struct thread *td, struct mlockall_args *uap) 1092 { 1093 vm_map_t map; 1094 int error; 1095 1096 map = &td->td_proc->p_vmspace->vm_map; 1097 error = priv_check(td, PRIV_VM_MLOCK); 1098 if (error) 1099 return (error); 1100 1101 if ((uap->how == 0) || ((uap->how & ~(MCL_CURRENT|MCL_FUTURE)) != 0)) 1102 return (EINVAL); 1103 1104 /* 1105 * If wiring all pages in the process would cause it to exceed 1106 * a hard resource limit, return ENOMEM. 1107 */ 1108 if (!old_mlock && uap->how & MCL_CURRENT) { 1109 if (map->size > lim_cur(td, RLIMIT_MEMLOCK)) 1110 return (ENOMEM); 1111 } 1112 #ifdef RACCT 1113 if (racct_enable) { 1114 PROC_LOCK(td->td_proc); 1115 error = racct_set(td->td_proc, RACCT_MEMLOCK, map->size); 1116 PROC_UNLOCK(td->td_proc); 1117 if (error != 0) 1118 return (ENOMEM); 1119 } 1120 #endif 1121 1122 if (uap->how & MCL_FUTURE) { 1123 vm_map_lock(map); 1124 vm_map_modflags(map, MAP_WIREFUTURE, 0); 1125 vm_map_unlock(map); 1126 error = 0; 1127 } 1128 1129 if (uap->how & MCL_CURRENT) { 1130 /* 1131 * P1003.1-2001 mandates that all currently mapped pages 1132 * will be memory resident and locked (wired) upon return 1133 * from mlockall(). vm_map_wire() will wire pages, by 1134 * calling vm_fault_wire() for each page in the region. 1135 */ 1136 error = vm_map_wire(map, vm_map_min(map), vm_map_max(map), 1137 VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK); 1138 if (error == KERN_SUCCESS) 1139 error = 0; 1140 else if (error == KERN_RESOURCE_SHORTAGE) 1141 error = ENOMEM; 1142 else 1143 error = EAGAIN; 1144 } 1145 #ifdef RACCT 1146 if (racct_enable && error != KERN_SUCCESS) { 1147 PROC_LOCK(td->td_proc); 1148 racct_set(td->td_proc, RACCT_MEMLOCK, 1149 ptoa(pmap_wired_count(map->pmap))); 1150 PROC_UNLOCK(td->td_proc); 1151 } 1152 #endif 1153 1154 return (error); 1155 } 1156 1157 #ifndef _SYS_SYSPROTO_H_ 1158 struct munlockall_args { 1159 register_t dummy; 1160 }; 1161 #endif 1162 1163 int 1164 sys_munlockall(struct thread *td, struct munlockall_args *uap) 1165 { 1166 vm_map_t map; 1167 int error; 1168 1169 map = &td->td_proc->p_vmspace->vm_map; 1170 error = priv_check(td, PRIV_VM_MUNLOCK); 1171 if (error) 1172 return (error); 1173 1174 /* Clear the MAP_WIREFUTURE flag from this vm_map. */ 1175 vm_map_lock(map); 1176 vm_map_modflags(map, 0, MAP_WIREFUTURE); 1177 vm_map_unlock(map); 1178 1179 /* Forcibly unwire all pages. */ 1180 error = vm_map_unwire(map, vm_map_min(map), vm_map_max(map), 1181 VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK); 1182 #ifdef RACCT 1183 if (racct_enable && error == KERN_SUCCESS) { 1184 PROC_LOCK(td->td_proc); 1185 racct_set(td->td_proc, RACCT_MEMLOCK, 0); 1186 PROC_UNLOCK(td->td_proc); 1187 } 1188 #endif 1189 1190 return (error); 1191 } 1192 1193 #ifndef _SYS_SYSPROTO_H_ 1194 struct munlock_args { 1195 const void *addr; 1196 size_t len; 1197 }; 1198 #endif 1199 int 1200 sys_munlock(struct thread *td, struct munlock_args *uap) 1201 { 1202 1203 return (kern_munlock(td, (uintptr_t)uap->addr, uap->len)); 1204 } 1205 1206 int 1207 kern_munlock(struct thread *td, uintptr_t addr0, size_t size) 1208 { 1209 vm_offset_t addr, end, last, start; 1210 #ifdef RACCT 1211 vm_map_t map; 1212 #endif 1213 int error; 1214 1215 error = priv_check(td, PRIV_VM_MUNLOCK); 1216 if (error) 1217 return (error); 1218 addr = addr0; 1219 last = addr + size; 1220 start = trunc_page(addr); 1221 end = round_page(last); 1222 if (last < addr || end < addr) 1223 return (EINVAL); 1224 error = vm_map_unwire(&td->td_proc->p_vmspace->vm_map, start, end, 1225 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 1226 #ifdef RACCT 1227 if (racct_enable && error == KERN_SUCCESS) { 1228 PROC_LOCK(td->td_proc); 1229 map = &td->td_proc->p_vmspace->vm_map; 1230 racct_set(td->td_proc, RACCT_MEMLOCK, 1231 ptoa(pmap_wired_count(map->pmap))); 1232 PROC_UNLOCK(td->td_proc); 1233 } 1234 #endif 1235 return (error == KERN_SUCCESS ? 0 : ENOMEM); 1236 } 1237 1238 /* 1239 * vm_mmap_vnode() 1240 * 1241 * Helper function for vm_mmap. Perform sanity check specific for mmap 1242 * operations on vnodes. 1243 */ 1244 int 1245 vm_mmap_vnode(struct thread *td, vm_size_t objsize, 1246 vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp, 1247 struct vnode *vp, vm_ooffset_t *foffp, vm_object_t *objp, 1248 boolean_t *writecounted) 1249 { 1250 struct vattr va; 1251 vm_object_t obj; 1252 vm_ooffset_t foff; 1253 struct ucred *cred; 1254 int error, flags; 1255 bool writex; 1256 1257 cred = td->td_ucred; 1258 writex = (*maxprotp & VM_PROT_WRITE) != 0 && 1259 (*flagsp & MAP_SHARED) != 0; 1260 if ((error = vget(vp, LK_SHARED, td)) != 0) 1261 return (error); 1262 AUDIT_ARG_VNODE1(vp); 1263 foff = *foffp; 1264 flags = *flagsp; 1265 obj = vp->v_object; 1266 if (vp->v_type == VREG) { 1267 /* 1268 * Get the proper underlying object 1269 */ 1270 if (obj == NULL) { 1271 error = EINVAL; 1272 goto done; 1273 } 1274 if (obj->type == OBJT_VNODE && obj->handle != vp) { 1275 vput(vp); 1276 vp = (struct vnode *)obj->handle; 1277 /* 1278 * Bypass filesystems obey the mpsafety of the 1279 * underlying fs. Tmpfs never bypasses. 1280 */ 1281 error = vget(vp, LK_SHARED, td); 1282 if (error != 0) 1283 return (error); 1284 } 1285 if (writex) { 1286 *writecounted = TRUE; 1287 vm_pager_update_writecount(obj, 0, objsize); 1288 } 1289 } else { 1290 error = EINVAL; 1291 goto done; 1292 } 1293 if ((error = VOP_GETATTR(vp, &va, cred))) 1294 goto done; 1295 #ifdef MAC 1296 /* This relies on VM_PROT_* matching PROT_*. */ 1297 error = mac_vnode_check_mmap(cred, vp, (int)prot, flags); 1298 if (error != 0) 1299 goto done; 1300 #endif 1301 if ((flags & MAP_SHARED) != 0) { 1302 if ((va.va_flags & (SF_SNAPSHOT|IMMUTABLE|APPEND)) != 0) { 1303 if (prot & VM_PROT_WRITE) { 1304 error = EPERM; 1305 goto done; 1306 } 1307 *maxprotp &= ~VM_PROT_WRITE; 1308 } 1309 } 1310 /* 1311 * If it is a regular file without any references 1312 * we do not need to sync it. 1313 * Adjust object size to be the size of actual file. 1314 */ 1315 objsize = round_page(va.va_size); 1316 if (va.va_nlink == 0) 1317 flags |= MAP_NOSYNC; 1318 if (obj->type == OBJT_VNODE) { 1319 obj = vm_pager_allocate(OBJT_VNODE, vp, objsize, prot, foff, 1320 cred); 1321 if (obj == NULL) { 1322 error = ENOMEM; 1323 goto done; 1324 } 1325 } else { 1326 KASSERT(obj->type == OBJT_DEFAULT || obj->type == OBJT_SWAP, 1327 ("wrong object type")); 1328 vm_object_reference(obj); 1329 #if VM_NRESERVLEVEL > 0 1330 if ((obj->flags & OBJ_COLORED) == 0) { 1331 VM_OBJECT_WLOCK(obj); 1332 vm_object_color(obj, 0); 1333 VM_OBJECT_WUNLOCK(obj); 1334 } 1335 #endif 1336 } 1337 *objp = obj; 1338 *flagsp = flags; 1339 1340 vfs_mark_atime(vp, cred); 1341 1342 done: 1343 if (error != 0 && *writecounted) { 1344 *writecounted = FALSE; 1345 vm_pager_update_writecount(obj, objsize, 0); 1346 } 1347 vput(vp); 1348 return (error); 1349 } 1350 1351 /* 1352 * vm_mmap_cdev() 1353 * 1354 * Helper function for vm_mmap. Perform sanity check specific for mmap 1355 * operations on cdevs. 1356 */ 1357 int 1358 vm_mmap_cdev(struct thread *td, vm_size_t objsize, vm_prot_t prot, 1359 vm_prot_t *maxprotp, int *flagsp, struct cdev *cdev, struct cdevsw *dsw, 1360 vm_ooffset_t *foff, vm_object_t *objp) 1361 { 1362 vm_object_t obj; 1363 int error, flags; 1364 1365 flags = *flagsp; 1366 1367 if (dsw->d_flags & D_MMAP_ANON) { 1368 *objp = NULL; 1369 *foff = 0; 1370 *maxprotp = VM_PROT_ALL; 1371 *flagsp |= MAP_ANON; 1372 return (0); 1373 } 1374 /* 1375 * cdevs do not provide private mappings of any kind. 1376 */ 1377 if ((*maxprotp & VM_PROT_WRITE) == 0 && 1378 (prot & VM_PROT_WRITE) != 0) 1379 return (EACCES); 1380 if (flags & (MAP_PRIVATE|MAP_COPY)) 1381 return (EINVAL); 1382 /* 1383 * Force device mappings to be shared. 1384 */ 1385 flags |= MAP_SHARED; 1386 #ifdef MAC_XXX 1387 error = mac_cdev_check_mmap(td->td_ucred, cdev, (int)prot); 1388 if (error != 0) 1389 return (error); 1390 #endif 1391 /* 1392 * First, try d_mmap_single(). If that is not implemented 1393 * (returns ENODEV), fall back to using the device pager. 1394 * Note that d_mmap_single() must return a reference to the 1395 * object (it needs to bump the reference count of the object 1396 * it returns somehow). 1397 * 1398 * XXX assumes VM_PROT_* == PROT_* 1399 */ 1400 error = dsw->d_mmap_single(cdev, foff, objsize, objp, (int)prot); 1401 if (error != ENODEV) 1402 return (error); 1403 obj = vm_pager_allocate(OBJT_DEVICE, cdev, objsize, prot, *foff, 1404 td->td_ucred); 1405 if (obj == NULL) 1406 return (EINVAL); 1407 *objp = obj; 1408 *flagsp = flags; 1409 return (0); 1410 } 1411 1412 /* 1413 * vm_mmap() 1414 * 1415 * Internal version of mmap used by exec, sys5 shared memory, and 1416 * various device drivers. Handle is either a vnode pointer, a 1417 * character device, or NULL for MAP_ANON. 1418 */ 1419 int 1420 vm_mmap(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot, 1421 vm_prot_t maxprot, int flags, 1422 objtype_t handle_type, void *handle, 1423 vm_ooffset_t foff) 1424 { 1425 vm_object_t object; 1426 struct thread *td = curthread; 1427 int error; 1428 boolean_t writecounted; 1429 1430 if (size == 0) 1431 return (EINVAL); 1432 1433 size = round_page(size); 1434 object = NULL; 1435 writecounted = FALSE; 1436 1437 /* 1438 * Lookup/allocate object. 1439 */ 1440 switch (handle_type) { 1441 case OBJT_DEVICE: { 1442 struct cdevsw *dsw; 1443 struct cdev *cdev; 1444 int ref; 1445 1446 cdev = handle; 1447 dsw = dev_refthread(cdev, &ref); 1448 if (dsw == NULL) 1449 return (ENXIO); 1450 error = vm_mmap_cdev(td, size, prot, &maxprot, &flags, cdev, 1451 dsw, &foff, &object); 1452 dev_relthread(cdev, ref); 1453 break; 1454 } 1455 case OBJT_VNODE: 1456 error = vm_mmap_vnode(td, size, prot, &maxprot, &flags, 1457 handle, &foff, &object, &writecounted); 1458 break; 1459 case OBJT_DEFAULT: 1460 if (handle == NULL) { 1461 error = 0; 1462 break; 1463 } 1464 /* FALLTHROUGH */ 1465 default: 1466 error = EINVAL; 1467 break; 1468 } 1469 if (error) 1470 return (error); 1471 1472 error = vm_mmap_object(map, addr, size, prot, maxprot, flags, object, 1473 foff, writecounted, td); 1474 if (error != 0 && object != NULL) { 1475 /* 1476 * If this mapping was accounted for in the vnode's 1477 * writecount, then undo that now. 1478 */ 1479 if (writecounted) 1480 vm_pager_release_writecount(object, 0, size); 1481 vm_object_deallocate(object); 1482 } 1483 return (error); 1484 } 1485 1486 /* 1487 * Internal version of mmap that maps a specific VM object into an 1488 * map. Called by mmap for MAP_ANON, vm_mmap, shm_mmap, and vn_mmap. 1489 */ 1490 int 1491 vm_mmap_object(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot, 1492 vm_prot_t maxprot, int flags, vm_object_t object, vm_ooffset_t foff, 1493 boolean_t writecounted, struct thread *td) 1494 { 1495 boolean_t curmap, fitit; 1496 vm_offset_t max_addr; 1497 int docow, error, findspace, rv; 1498 1499 curmap = map == &td->td_proc->p_vmspace->vm_map; 1500 if (curmap) { 1501 RACCT_PROC_LOCK(td->td_proc); 1502 if (map->size + size > lim_cur(td, RLIMIT_VMEM)) { 1503 RACCT_PROC_UNLOCK(td->td_proc); 1504 return (ENOMEM); 1505 } 1506 if (racct_set(td->td_proc, RACCT_VMEM, map->size + size)) { 1507 RACCT_PROC_UNLOCK(td->td_proc); 1508 return (ENOMEM); 1509 } 1510 if (!old_mlock && map->flags & MAP_WIREFUTURE) { 1511 if (ptoa(pmap_wired_count(map->pmap)) + size > 1512 lim_cur(td, RLIMIT_MEMLOCK)) { 1513 racct_set_force(td->td_proc, RACCT_VMEM, 1514 map->size); 1515 RACCT_PROC_UNLOCK(td->td_proc); 1516 return (ENOMEM); 1517 } 1518 error = racct_set(td->td_proc, RACCT_MEMLOCK, 1519 ptoa(pmap_wired_count(map->pmap)) + size); 1520 if (error != 0) { 1521 racct_set_force(td->td_proc, RACCT_VMEM, 1522 map->size); 1523 RACCT_PROC_UNLOCK(td->td_proc); 1524 return (error); 1525 } 1526 } 1527 RACCT_PROC_UNLOCK(td->td_proc); 1528 } 1529 1530 /* 1531 * We currently can only deal with page aligned file offsets. 1532 * The mmap() system call already enforces this by subtracting 1533 * the page offset from the file offset, but checking here 1534 * catches errors in device drivers (e.g. d_single_mmap() 1535 * callbacks) and other internal mapping requests (such as in 1536 * exec). 1537 */ 1538 if (foff & PAGE_MASK) 1539 return (EINVAL); 1540 1541 if ((flags & MAP_FIXED) == 0) { 1542 fitit = TRUE; 1543 *addr = round_page(*addr); 1544 } else { 1545 if (*addr != trunc_page(*addr)) 1546 return (EINVAL); 1547 fitit = FALSE; 1548 } 1549 1550 if (flags & MAP_ANON) { 1551 if (object != NULL || foff != 0) 1552 return (EINVAL); 1553 docow = 0; 1554 } else if (flags & MAP_PREFAULT_READ) 1555 docow = MAP_PREFAULT; 1556 else 1557 docow = MAP_PREFAULT_PARTIAL; 1558 1559 if ((flags & (MAP_ANON|MAP_SHARED)) == 0) 1560 docow |= MAP_COPY_ON_WRITE; 1561 if (flags & MAP_NOSYNC) 1562 docow |= MAP_DISABLE_SYNCER; 1563 if (flags & MAP_NOCORE) 1564 docow |= MAP_DISABLE_COREDUMP; 1565 /* Shared memory is also shared with children. */ 1566 if (flags & MAP_SHARED) 1567 docow |= MAP_INHERIT_SHARE; 1568 if (writecounted) 1569 docow |= MAP_WRITECOUNT; 1570 if (flags & MAP_STACK) { 1571 if (object != NULL) 1572 return (EINVAL); 1573 docow |= MAP_STACK_GROWS_DOWN; 1574 } 1575 if ((flags & MAP_EXCL) != 0) 1576 docow |= MAP_CHECK_EXCL; 1577 if ((flags & MAP_GUARD) != 0) 1578 docow |= MAP_CREATE_GUARD; 1579 1580 if (fitit) { 1581 if ((flags & MAP_ALIGNMENT_MASK) == MAP_ALIGNED_SUPER) 1582 findspace = VMFS_SUPER_SPACE; 1583 else if ((flags & MAP_ALIGNMENT_MASK) != 0) 1584 findspace = VMFS_ALIGNED_SPACE(flags >> 1585 MAP_ALIGNMENT_SHIFT); 1586 else 1587 findspace = VMFS_OPTIMAL_SPACE; 1588 max_addr = 0; 1589 #ifdef MAP_32BIT 1590 if ((flags & MAP_32BIT) != 0) 1591 max_addr = MAP_32BIT_MAX_ADDR; 1592 #endif 1593 if (curmap) { 1594 rv = vm_map_find_min(map, object, foff, addr, size, 1595 round_page((vm_offset_t)td->td_proc->p_vmspace-> 1596 vm_daddr + lim_max(td, RLIMIT_DATA)), max_addr, 1597 findspace, prot, maxprot, docow); 1598 } else { 1599 rv = vm_map_find(map, object, foff, addr, size, 1600 max_addr, findspace, prot, maxprot, docow); 1601 } 1602 } else { 1603 rv = vm_map_fixed(map, object, foff, *addr, size, 1604 prot, maxprot, docow); 1605 } 1606 1607 if (rv == KERN_SUCCESS) { 1608 /* 1609 * If the process has requested that all future mappings 1610 * be wired, then heed this. 1611 */ 1612 if ((map->flags & MAP_WIREFUTURE) != 0) { 1613 vm_map_lock(map); 1614 if ((map->flags & MAP_WIREFUTURE) != 0) 1615 (void)vm_map_wire_locked(map, *addr, 1616 *addr + size, VM_MAP_WIRE_USER | 1617 ((flags & MAP_STACK) ? VM_MAP_WIRE_HOLESOK : 1618 VM_MAP_WIRE_NOHOLES)); 1619 vm_map_unlock(map); 1620 } 1621 } 1622 return (vm_mmap_to_errno(rv)); 1623 } 1624 1625 /* 1626 * Translate a Mach VM return code to zero on success or the appropriate errno 1627 * on failure. 1628 */ 1629 int 1630 vm_mmap_to_errno(int rv) 1631 { 1632 1633 switch (rv) { 1634 case KERN_SUCCESS: 1635 return (0); 1636 case KERN_INVALID_ADDRESS: 1637 case KERN_NO_SPACE: 1638 return (ENOMEM); 1639 case KERN_PROTECTION_FAILURE: 1640 return (EACCES); 1641 default: 1642 return (EINVAL); 1643 } 1644 } 1645