1 /*- 2 * Copyright (c) 1988 University of Utah. 3 * Copyright (c) 1991, 1993 4 * The Regents of the University of California. All rights reserved. 5 * 6 * This code is derived from software contributed to Berkeley by 7 * the Systems Programming Group of the University of Utah Computer 8 * Science Department. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 4. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * from: Utah $Hdr: vm_mmap.c 1.6 91/10/21$ 35 * 36 * @(#)vm_mmap.c 8.4 (Berkeley) 1/12/94 37 */ 38 39 /* 40 * Mapped file (mmap) interface to VM 41 */ 42 43 #include <sys/cdefs.h> 44 __FBSDID("$FreeBSD$"); 45 46 #include "opt_compat.h" 47 #include "opt_hwpmc_hooks.h" 48 49 #include <sys/param.h> 50 #include <sys/systm.h> 51 #include <sys/capability.h> 52 #include <sys/kernel.h> 53 #include <sys/lock.h> 54 #include <sys/mutex.h> 55 #include <sys/sysproto.h> 56 #include <sys/filedesc.h> 57 #include <sys/priv.h> 58 #include <sys/proc.h> 59 #include <sys/racct.h> 60 #include <sys/resource.h> 61 #include <sys/resourcevar.h> 62 #include <sys/rwlock.h> 63 #include <sys/sysctl.h> 64 #include <sys/vnode.h> 65 #include <sys/fcntl.h> 66 #include <sys/file.h> 67 #include <sys/mman.h> 68 #include <sys/mount.h> 69 #include <sys/conf.h> 70 #include <sys/stat.h> 71 #include <sys/sysent.h> 72 #include <sys/vmmeter.h> 73 74 #include <security/mac/mac_framework.h> 75 76 #include <vm/vm.h> 77 #include <vm/vm_param.h> 78 #include <vm/pmap.h> 79 #include <vm/vm_map.h> 80 #include <vm/vm_object.h> 81 #include <vm/vm_page.h> 82 #include <vm/vm_pager.h> 83 #include <vm/vm_pageout.h> 84 #include <vm/vm_extern.h> 85 #include <vm/vm_page.h> 86 #include <vm/vnode_pager.h> 87 88 #ifdef HWPMC_HOOKS 89 #include <sys/pmckern.h> 90 #endif 91 92 int old_mlock = 0; 93 SYSCTL_INT(_vm, OID_AUTO, old_mlock, CTLFLAG_RW | CTLFLAG_TUN, &old_mlock, 0, 94 "Do not apply RLIMIT_MEMLOCK on mlockall"); 95 TUNABLE_INT("vm.old_mlock", &old_mlock); 96 97 #ifndef _SYS_SYSPROTO_H_ 98 struct sbrk_args { 99 int incr; 100 }; 101 #endif 102 103 static int vm_mmap_vnode(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *, 104 int *, struct vnode *, vm_ooffset_t *, vm_object_t *, boolean_t *); 105 static int vm_mmap_cdev(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *, 106 int *, struct cdev *, vm_ooffset_t *, vm_object_t *); 107 static int vm_mmap_shm(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *, 108 int *, struct shmfd *, vm_ooffset_t, vm_object_t *); 109 110 /* 111 * MPSAFE 112 */ 113 /* ARGSUSED */ 114 int 115 sys_sbrk(td, uap) 116 struct thread *td; 117 struct sbrk_args *uap; 118 { 119 /* Not yet implemented */ 120 return (EOPNOTSUPP); 121 } 122 123 #ifndef _SYS_SYSPROTO_H_ 124 struct sstk_args { 125 int incr; 126 }; 127 #endif 128 129 /* 130 * MPSAFE 131 */ 132 /* ARGSUSED */ 133 int 134 sys_sstk(td, uap) 135 struct thread *td; 136 struct sstk_args *uap; 137 { 138 /* Not yet implemented */ 139 return (EOPNOTSUPP); 140 } 141 142 #if defined(COMPAT_43) 143 #ifndef _SYS_SYSPROTO_H_ 144 struct getpagesize_args { 145 int dummy; 146 }; 147 #endif 148 149 int 150 ogetpagesize(td, uap) 151 struct thread *td; 152 struct getpagesize_args *uap; 153 { 154 /* MP SAFE */ 155 td->td_retval[0] = PAGE_SIZE; 156 return (0); 157 } 158 #endif /* COMPAT_43 */ 159 160 161 /* 162 * Memory Map (mmap) system call. Note that the file offset 163 * and address are allowed to be NOT page aligned, though if 164 * the MAP_FIXED flag it set, both must have the same remainder 165 * modulo the PAGE_SIZE (POSIX 1003.1b). If the address is not 166 * page-aligned, the actual mapping starts at trunc_page(addr) 167 * and the return value is adjusted up by the page offset. 168 * 169 * Generally speaking, only character devices which are themselves 170 * memory-based, such as a video framebuffer, can be mmap'd. Otherwise 171 * there would be no cache coherency between a descriptor and a VM mapping 172 * both to the same character device. 173 */ 174 #ifndef _SYS_SYSPROTO_H_ 175 struct mmap_args { 176 void *addr; 177 size_t len; 178 int prot; 179 int flags; 180 int fd; 181 long pad; 182 off_t pos; 183 }; 184 #endif 185 186 /* 187 * MPSAFE 188 */ 189 int 190 sys_mmap(td, uap) 191 struct thread *td; 192 struct mmap_args *uap; 193 { 194 #ifdef HWPMC_HOOKS 195 struct pmckern_map_in pkm; 196 #endif 197 struct file *fp; 198 struct vnode *vp; 199 vm_offset_t addr; 200 vm_size_t size, pageoff; 201 vm_prot_t cap_maxprot, prot, maxprot; 202 void *handle; 203 objtype_t handle_type; 204 int flags, error; 205 off_t pos; 206 struct vmspace *vms = td->td_proc->p_vmspace; 207 cap_rights_t rights; 208 209 addr = (vm_offset_t) uap->addr; 210 size = uap->len; 211 prot = uap->prot & VM_PROT_ALL; 212 flags = uap->flags; 213 pos = uap->pos; 214 215 fp = NULL; 216 217 /* 218 * Enforce the constraints. 219 * Mapping of length 0 is only allowed for old binaries. 220 * Anonymous mapping shall specify -1 as filedescriptor and 221 * zero position for new code. Be nice to ancient a.out 222 * binaries and correct pos for anonymous mapping, since old 223 * ld.so sometimes issues anonymous map requests with non-zero 224 * pos. 225 */ 226 if (!SV_CURPROC_FLAG(SV_AOUT)) { 227 if ((uap->len == 0 && curproc->p_osrel >= P_OSREL_MAP_ANON) || 228 ((flags & MAP_ANON) != 0 && (uap->fd != -1 || pos != 0))) 229 return (EINVAL); 230 } else { 231 if ((flags & MAP_ANON) != 0) 232 pos = 0; 233 } 234 235 if (flags & MAP_STACK) { 236 if ((uap->fd != -1) || 237 ((prot & (PROT_READ | PROT_WRITE)) != (PROT_READ | PROT_WRITE))) 238 return (EINVAL); 239 flags |= MAP_ANON; 240 pos = 0; 241 } 242 243 /* 244 * Align the file position to a page boundary, 245 * and save its page offset component. 246 */ 247 pageoff = (pos & PAGE_MASK); 248 pos -= pageoff; 249 250 /* Adjust size for rounding (on both ends). */ 251 size += pageoff; /* low end... */ 252 size = (vm_size_t) round_page(size); /* hi end */ 253 254 /* 255 * Check for illegal addresses. Watch out for address wrap... Note 256 * that VM_*_ADDRESS are not constants due to casts (argh). 257 */ 258 if (flags & MAP_FIXED) { 259 /* 260 * The specified address must have the same remainder 261 * as the file offset taken modulo PAGE_SIZE, so it 262 * should be aligned after adjustment by pageoff. 263 */ 264 addr -= pageoff; 265 if (addr & PAGE_MASK) 266 return (EINVAL); 267 268 /* Address range must be all in user VM space. */ 269 if (addr < vm_map_min(&vms->vm_map) || 270 addr + size > vm_map_max(&vms->vm_map)) 271 return (EINVAL); 272 if (addr + size < addr) 273 return (EINVAL); 274 } else { 275 /* 276 * XXX for non-fixed mappings where no hint is provided or 277 * the hint would fall in the potential heap space, 278 * place it after the end of the largest possible heap. 279 * 280 * There should really be a pmap call to determine a reasonable 281 * location. 282 */ 283 PROC_LOCK(td->td_proc); 284 if (addr == 0 || 285 (addr >= round_page((vm_offset_t)vms->vm_taddr) && 286 addr < round_page((vm_offset_t)vms->vm_daddr + 287 lim_max(td->td_proc, RLIMIT_DATA)))) 288 addr = round_page((vm_offset_t)vms->vm_daddr + 289 lim_max(td->td_proc, RLIMIT_DATA)); 290 PROC_UNLOCK(td->td_proc); 291 } 292 if (flags & MAP_ANON) { 293 /* 294 * Mapping blank space is trivial. 295 */ 296 handle = NULL; 297 handle_type = OBJT_DEFAULT; 298 maxprot = VM_PROT_ALL; 299 cap_maxprot = VM_PROT_ALL; 300 } else { 301 /* 302 * Mapping file, get fp for validation and don't let the 303 * descriptor disappear on us if we block. Check capability 304 * rights, but also return the maximum rights to be combined 305 * with maxprot later. 306 */ 307 rights = CAP_MMAP; 308 if (prot & PROT_READ) 309 rights |= CAP_MMAP_R; 310 if ((flags & MAP_SHARED) != 0) { 311 if (prot & PROT_WRITE) 312 rights |= CAP_MMAP_W; 313 } 314 if (prot & PROT_EXEC) 315 rights |= CAP_MMAP_X; 316 if ((error = fget_mmap(td, uap->fd, rights, &cap_maxprot, 317 &fp)) != 0) 318 goto done; 319 if (fp->f_type == DTYPE_SHM) { 320 handle = fp->f_data; 321 handle_type = OBJT_SWAP; 322 maxprot = VM_PROT_NONE; 323 324 /* FREAD should always be set. */ 325 if (fp->f_flag & FREAD) 326 maxprot |= VM_PROT_EXECUTE | VM_PROT_READ; 327 if (fp->f_flag & FWRITE) 328 maxprot |= VM_PROT_WRITE; 329 goto map; 330 } 331 if (fp->f_type != DTYPE_VNODE) { 332 error = ENODEV; 333 goto done; 334 } 335 #if defined(COMPAT_FREEBSD7) || defined(COMPAT_FREEBSD6) || \ 336 defined(COMPAT_FREEBSD5) || defined(COMPAT_FREEBSD4) 337 /* 338 * POSIX shared-memory objects are defined to have 339 * kernel persistence, and are not defined to support 340 * read(2)/write(2) -- or even open(2). Thus, we can 341 * use MAP_ASYNC to trade on-disk coherence for speed. 342 * The shm_open(3) library routine turns on the FPOSIXSHM 343 * flag to request this behavior. 344 */ 345 if (fp->f_flag & FPOSIXSHM) 346 flags |= MAP_NOSYNC; 347 #endif 348 vp = fp->f_vnode; 349 /* 350 * Ensure that file and memory protections are 351 * compatible. Note that we only worry about 352 * writability if mapping is shared; in this case, 353 * current and max prot are dictated by the open file. 354 * XXX use the vnode instead? Problem is: what 355 * credentials do we use for determination? What if 356 * proc does a setuid? 357 */ 358 if (vp->v_mount != NULL && vp->v_mount->mnt_flag & MNT_NOEXEC) 359 maxprot = VM_PROT_NONE; 360 else 361 maxprot = VM_PROT_EXECUTE; 362 if (fp->f_flag & FREAD) { 363 maxprot |= VM_PROT_READ; 364 } else if (prot & PROT_READ) { 365 error = EACCES; 366 goto done; 367 } 368 /* 369 * If we are sharing potential changes (either via 370 * MAP_SHARED or via the implicit sharing of character 371 * device mappings), and we are trying to get write 372 * permission although we opened it without asking 373 * for it, bail out. 374 */ 375 if ((flags & MAP_SHARED) != 0) { 376 if ((fp->f_flag & FWRITE) != 0) { 377 maxprot |= VM_PROT_WRITE; 378 } else if ((prot & PROT_WRITE) != 0) { 379 error = EACCES; 380 goto done; 381 } 382 } else if (vp->v_type != VCHR || (fp->f_flag & FWRITE) != 0) { 383 maxprot |= VM_PROT_WRITE; 384 cap_maxprot |= VM_PROT_WRITE; 385 } 386 handle = (void *)vp; 387 handle_type = OBJT_VNODE; 388 } 389 map: 390 td->td_fpop = fp; 391 maxprot &= cap_maxprot; 392 error = vm_mmap(&vms->vm_map, &addr, size, prot, maxprot, 393 flags, handle_type, handle, pos); 394 td->td_fpop = NULL; 395 #ifdef HWPMC_HOOKS 396 /* inform hwpmc(4) if an executable is being mapped */ 397 if (error == 0 && handle_type == OBJT_VNODE && 398 (prot & PROT_EXEC)) { 399 pkm.pm_file = handle; 400 pkm.pm_address = (uintptr_t) addr; 401 PMC_CALL_HOOK(td, PMC_FN_MMAP, (void *) &pkm); 402 } 403 #endif 404 if (error == 0) 405 td->td_retval[0] = (register_t) (addr + pageoff); 406 done: 407 if (fp) 408 fdrop(fp, td); 409 410 return (error); 411 } 412 413 int 414 freebsd6_mmap(struct thread *td, struct freebsd6_mmap_args *uap) 415 { 416 struct mmap_args oargs; 417 418 oargs.addr = uap->addr; 419 oargs.len = uap->len; 420 oargs.prot = uap->prot; 421 oargs.flags = uap->flags; 422 oargs.fd = uap->fd; 423 oargs.pos = uap->pos; 424 return (sys_mmap(td, &oargs)); 425 } 426 427 #ifdef COMPAT_43 428 #ifndef _SYS_SYSPROTO_H_ 429 struct ommap_args { 430 caddr_t addr; 431 int len; 432 int prot; 433 int flags; 434 int fd; 435 long pos; 436 }; 437 #endif 438 int 439 ommap(td, uap) 440 struct thread *td; 441 struct ommap_args *uap; 442 { 443 struct mmap_args nargs; 444 static const char cvtbsdprot[8] = { 445 0, 446 PROT_EXEC, 447 PROT_WRITE, 448 PROT_EXEC | PROT_WRITE, 449 PROT_READ, 450 PROT_EXEC | PROT_READ, 451 PROT_WRITE | PROT_READ, 452 PROT_EXEC | PROT_WRITE | PROT_READ, 453 }; 454 455 #define OMAP_ANON 0x0002 456 #define OMAP_COPY 0x0020 457 #define OMAP_SHARED 0x0010 458 #define OMAP_FIXED 0x0100 459 460 nargs.addr = uap->addr; 461 nargs.len = uap->len; 462 nargs.prot = cvtbsdprot[uap->prot & 0x7]; 463 #ifdef COMPAT_FREEBSD32 464 #if defined(__amd64__) || defined(__ia64__) 465 if (i386_read_exec && SV_PROC_FLAG(td->td_proc, SV_ILP32) && 466 nargs.prot != 0) 467 nargs.prot |= PROT_EXEC; 468 #endif 469 #endif 470 nargs.flags = 0; 471 if (uap->flags & OMAP_ANON) 472 nargs.flags |= MAP_ANON; 473 if (uap->flags & OMAP_COPY) 474 nargs.flags |= MAP_COPY; 475 if (uap->flags & OMAP_SHARED) 476 nargs.flags |= MAP_SHARED; 477 else 478 nargs.flags |= MAP_PRIVATE; 479 if (uap->flags & OMAP_FIXED) 480 nargs.flags |= MAP_FIXED; 481 nargs.fd = uap->fd; 482 nargs.pos = uap->pos; 483 return (sys_mmap(td, &nargs)); 484 } 485 #endif /* COMPAT_43 */ 486 487 488 #ifndef _SYS_SYSPROTO_H_ 489 struct msync_args { 490 void *addr; 491 size_t len; 492 int flags; 493 }; 494 #endif 495 /* 496 * MPSAFE 497 */ 498 int 499 sys_msync(td, uap) 500 struct thread *td; 501 struct msync_args *uap; 502 { 503 vm_offset_t addr; 504 vm_size_t size, pageoff; 505 int flags; 506 vm_map_t map; 507 int rv; 508 509 addr = (vm_offset_t) uap->addr; 510 size = uap->len; 511 flags = uap->flags; 512 513 pageoff = (addr & PAGE_MASK); 514 addr -= pageoff; 515 size += pageoff; 516 size = (vm_size_t) round_page(size); 517 if (addr + size < addr) 518 return (EINVAL); 519 520 if ((flags & (MS_ASYNC|MS_INVALIDATE)) == (MS_ASYNC|MS_INVALIDATE)) 521 return (EINVAL); 522 523 map = &td->td_proc->p_vmspace->vm_map; 524 525 /* 526 * Clean the pages and interpret the return value. 527 */ 528 rv = vm_map_sync(map, addr, addr + size, (flags & MS_ASYNC) == 0, 529 (flags & MS_INVALIDATE) != 0); 530 switch (rv) { 531 case KERN_SUCCESS: 532 return (0); 533 case KERN_INVALID_ADDRESS: 534 return (EINVAL); /* Sun returns ENOMEM? */ 535 case KERN_INVALID_ARGUMENT: 536 return (EBUSY); 537 case KERN_FAILURE: 538 return (EIO); 539 default: 540 return (EINVAL); 541 } 542 } 543 544 #ifndef _SYS_SYSPROTO_H_ 545 struct munmap_args { 546 void *addr; 547 size_t len; 548 }; 549 #endif 550 /* 551 * MPSAFE 552 */ 553 int 554 sys_munmap(td, uap) 555 struct thread *td; 556 struct munmap_args *uap; 557 { 558 #ifdef HWPMC_HOOKS 559 struct pmckern_map_out pkm; 560 vm_map_entry_t entry; 561 #endif 562 vm_offset_t addr; 563 vm_size_t size, pageoff; 564 vm_map_t map; 565 566 addr = (vm_offset_t) uap->addr; 567 size = uap->len; 568 if (size == 0) 569 return (EINVAL); 570 571 pageoff = (addr & PAGE_MASK); 572 addr -= pageoff; 573 size += pageoff; 574 size = (vm_size_t) round_page(size); 575 if (addr + size < addr) 576 return (EINVAL); 577 578 /* 579 * Check for illegal addresses. Watch out for address wrap... 580 */ 581 map = &td->td_proc->p_vmspace->vm_map; 582 if (addr < vm_map_min(map) || addr + size > vm_map_max(map)) 583 return (EINVAL); 584 vm_map_lock(map); 585 #ifdef HWPMC_HOOKS 586 /* 587 * Inform hwpmc if the address range being unmapped contains 588 * an executable region. 589 */ 590 pkm.pm_address = (uintptr_t) NULL; 591 if (vm_map_lookup_entry(map, addr, &entry)) { 592 for (; 593 entry != &map->header && entry->start < addr + size; 594 entry = entry->next) { 595 if (vm_map_check_protection(map, entry->start, 596 entry->end, VM_PROT_EXECUTE) == TRUE) { 597 pkm.pm_address = (uintptr_t) addr; 598 pkm.pm_size = (size_t) size; 599 break; 600 } 601 } 602 } 603 #endif 604 vm_map_delete(map, addr, addr + size); 605 606 #ifdef HWPMC_HOOKS 607 /* downgrade the lock to prevent a LOR with the pmc-sx lock */ 608 vm_map_lock_downgrade(map); 609 if (pkm.pm_address != (uintptr_t) NULL) 610 PMC_CALL_HOOK(td, PMC_FN_MUNMAP, (void *) &pkm); 611 vm_map_unlock_read(map); 612 #else 613 vm_map_unlock(map); 614 #endif 615 /* vm_map_delete returns nothing but KERN_SUCCESS anyway */ 616 return (0); 617 } 618 619 #ifndef _SYS_SYSPROTO_H_ 620 struct mprotect_args { 621 const void *addr; 622 size_t len; 623 int prot; 624 }; 625 #endif 626 /* 627 * MPSAFE 628 */ 629 int 630 sys_mprotect(td, uap) 631 struct thread *td; 632 struct mprotect_args *uap; 633 { 634 vm_offset_t addr; 635 vm_size_t size, pageoff; 636 vm_prot_t prot; 637 638 addr = (vm_offset_t) uap->addr; 639 size = uap->len; 640 prot = uap->prot & VM_PROT_ALL; 641 642 pageoff = (addr & PAGE_MASK); 643 addr -= pageoff; 644 size += pageoff; 645 size = (vm_size_t) round_page(size); 646 if (addr + size < addr) 647 return (EINVAL); 648 649 switch (vm_map_protect(&td->td_proc->p_vmspace->vm_map, addr, 650 addr + size, prot, FALSE)) { 651 case KERN_SUCCESS: 652 return (0); 653 case KERN_PROTECTION_FAILURE: 654 return (EACCES); 655 case KERN_RESOURCE_SHORTAGE: 656 return (ENOMEM); 657 } 658 return (EINVAL); 659 } 660 661 #ifndef _SYS_SYSPROTO_H_ 662 struct minherit_args { 663 void *addr; 664 size_t len; 665 int inherit; 666 }; 667 #endif 668 /* 669 * MPSAFE 670 */ 671 int 672 sys_minherit(td, uap) 673 struct thread *td; 674 struct minherit_args *uap; 675 { 676 vm_offset_t addr; 677 vm_size_t size, pageoff; 678 vm_inherit_t inherit; 679 680 addr = (vm_offset_t)uap->addr; 681 size = uap->len; 682 inherit = uap->inherit; 683 684 pageoff = (addr & PAGE_MASK); 685 addr -= pageoff; 686 size += pageoff; 687 size = (vm_size_t) round_page(size); 688 if (addr + size < addr) 689 return (EINVAL); 690 691 switch (vm_map_inherit(&td->td_proc->p_vmspace->vm_map, addr, 692 addr + size, inherit)) { 693 case KERN_SUCCESS: 694 return (0); 695 case KERN_PROTECTION_FAILURE: 696 return (EACCES); 697 } 698 return (EINVAL); 699 } 700 701 #ifndef _SYS_SYSPROTO_H_ 702 struct madvise_args { 703 void *addr; 704 size_t len; 705 int behav; 706 }; 707 #endif 708 709 /* 710 * MPSAFE 711 */ 712 int 713 sys_madvise(td, uap) 714 struct thread *td; 715 struct madvise_args *uap; 716 { 717 vm_offset_t start, end; 718 vm_map_t map; 719 struct proc *p; 720 int error; 721 722 /* 723 * Check for our special case, advising the swap pager we are 724 * "immortal." 725 */ 726 if (uap->behav == MADV_PROTECT) { 727 error = priv_check(td, PRIV_VM_MADV_PROTECT); 728 if (error == 0) { 729 p = td->td_proc; 730 PROC_LOCK(p); 731 p->p_flag |= P_PROTECTED; 732 PROC_UNLOCK(p); 733 } 734 return (error); 735 } 736 /* 737 * Check for illegal behavior 738 */ 739 if (uap->behav < 0 || uap->behav > MADV_CORE) 740 return (EINVAL); 741 /* 742 * Check for illegal addresses. Watch out for address wrap... Note 743 * that VM_*_ADDRESS are not constants due to casts (argh). 744 */ 745 map = &td->td_proc->p_vmspace->vm_map; 746 if ((vm_offset_t)uap->addr < vm_map_min(map) || 747 (vm_offset_t)uap->addr + uap->len > vm_map_max(map)) 748 return (EINVAL); 749 if (((vm_offset_t) uap->addr + uap->len) < (vm_offset_t) uap->addr) 750 return (EINVAL); 751 752 /* 753 * Since this routine is only advisory, we default to conservative 754 * behavior. 755 */ 756 start = trunc_page((vm_offset_t) uap->addr); 757 end = round_page((vm_offset_t) uap->addr + uap->len); 758 759 if (vm_map_madvise(map, start, end, uap->behav)) 760 return (EINVAL); 761 return (0); 762 } 763 764 #ifndef _SYS_SYSPROTO_H_ 765 struct mincore_args { 766 const void *addr; 767 size_t len; 768 char *vec; 769 }; 770 #endif 771 772 /* 773 * MPSAFE 774 */ 775 int 776 sys_mincore(td, uap) 777 struct thread *td; 778 struct mincore_args *uap; 779 { 780 vm_offset_t addr, first_addr; 781 vm_offset_t end, cend; 782 pmap_t pmap; 783 vm_map_t map; 784 char *vec; 785 int error = 0; 786 int vecindex, lastvecindex; 787 vm_map_entry_t current; 788 vm_map_entry_t entry; 789 vm_object_t object; 790 vm_paddr_t locked_pa; 791 vm_page_t m; 792 vm_pindex_t pindex; 793 int mincoreinfo; 794 unsigned int timestamp; 795 boolean_t locked; 796 797 /* 798 * Make sure that the addresses presented are valid for user 799 * mode. 800 */ 801 first_addr = addr = trunc_page((vm_offset_t) uap->addr); 802 end = addr + (vm_size_t)round_page(uap->len); 803 map = &td->td_proc->p_vmspace->vm_map; 804 if (end > vm_map_max(map) || end < addr) 805 return (ENOMEM); 806 807 /* 808 * Address of byte vector 809 */ 810 vec = uap->vec; 811 812 pmap = vmspace_pmap(td->td_proc->p_vmspace); 813 814 vm_map_lock_read(map); 815 RestartScan: 816 timestamp = map->timestamp; 817 818 if (!vm_map_lookup_entry(map, addr, &entry)) { 819 vm_map_unlock_read(map); 820 return (ENOMEM); 821 } 822 823 /* 824 * Do this on a map entry basis so that if the pages are not 825 * in the current processes address space, we can easily look 826 * up the pages elsewhere. 827 */ 828 lastvecindex = -1; 829 for (current = entry; 830 (current != &map->header) && (current->start < end); 831 current = current->next) { 832 833 /* 834 * check for contiguity 835 */ 836 if (current->end < end && 837 (entry->next == &map->header || 838 current->next->start > current->end)) { 839 vm_map_unlock_read(map); 840 return (ENOMEM); 841 } 842 843 /* 844 * ignore submaps (for now) or null objects 845 */ 846 if ((current->eflags & MAP_ENTRY_IS_SUB_MAP) || 847 current->object.vm_object == NULL) 848 continue; 849 850 /* 851 * limit this scan to the current map entry and the 852 * limits for the mincore call 853 */ 854 if (addr < current->start) 855 addr = current->start; 856 cend = current->end; 857 if (cend > end) 858 cend = end; 859 860 /* 861 * scan this entry one page at a time 862 */ 863 while (addr < cend) { 864 /* 865 * Check pmap first, it is likely faster, also 866 * it can provide info as to whether we are the 867 * one referencing or modifying the page. 868 */ 869 object = NULL; 870 locked_pa = 0; 871 retry: 872 m = NULL; 873 mincoreinfo = pmap_mincore(pmap, addr, &locked_pa); 874 if (locked_pa != 0) { 875 /* 876 * The page is mapped by this process but not 877 * both accessed and modified. It is also 878 * managed. Acquire the object lock so that 879 * other mappings might be examined. 880 */ 881 m = PHYS_TO_VM_PAGE(locked_pa); 882 if (m->object != object) { 883 if (object != NULL) 884 VM_OBJECT_WUNLOCK(object); 885 object = m->object; 886 locked = VM_OBJECT_TRYWLOCK(object); 887 vm_page_unlock(m); 888 if (!locked) { 889 VM_OBJECT_WLOCK(object); 890 vm_page_lock(m); 891 goto retry; 892 } 893 } else 894 vm_page_unlock(m); 895 KASSERT(m->valid == VM_PAGE_BITS_ALL, 896 ("mincore: page %p is mapped but invalid", 897 m)); 898 } else if (mincoreinfo == 0) { 899 /* 900 * The page is not mapped by this process. If 901 * the object implements managed pages, then 902 * determine if the page is resident so that 903 * the mappings might be examined. 904 */ 905 if (current->object.vm_object != object) { 906 if (object != NULL) 907 VM_OBJECT_WUNLOCK(object); 908 object = current->object.vm_object; 909 VM_OBJECT_WLOCK(object); 910 } 911 if (object->type == OBJT_DEFAULT || 912 object->type == OBJT_SWAP || 913 object->type == OBJT_VNODE) { 914 pindex = OFF_TO_IDX(current->offset + 915 (addr - current->start)); 916 m = vm_page_lookup(object, pindex); 917 if (m == NULL && 918 vm_page_is_cached(object, pindex)) 919 mincoreinfo = MINCORE_INCORE; 920 if (m != NULL && m->valid == 0) 921 m = NULL; 922 if (m != NULL) 923 mincoreinfo = MINCORE_INCORE; 924 } 925 } 926 if (m != NULL) { 927 /* Examine other mappings to the page. */ 928 if (m->dirty == 0 && pmap_is_modified(m)) 929 vm_page_dirty(m); 930 if (m->dirty != 0) 931 mincoreinfo |= MINCORE_MODIFIED_OTHER; 932 /* 933 * The first test for PGA_REFERENCED is an 934 * optimization. The second test is 935 * required because a concurrent pmap 936 * operation could clear the last reference 937 * and set PGA_REFERENCED before the call to 938 * pmap_is_referenced(). 939 */ 940 if ((m->aflags & PGA_REFERENCED) != 0 || 941 pmap_is_referenced(m) || 942 (m->aflags & PGA_REFERENCED) != 0) 943 mincoreinfo |= MINCORE_REFERENCED_OTHER; 944 } 945 if (object != NULL) 946 VM_OBJECT_WUNLOCK(object); 947 948 /* 949 * subyte may page fault. In case it needs to modify 950 * the map, we release the lock. 951 */ 952 vm_map_unlock_read(map); 953 954 /* 955 * calculate index into user supplied byte vector 956 */ 957 vecindex = OFF_TO_IDX(addr - first_addr); 958 959 /* 960 * If we have skipped map entries, we need to make sure that 961 * the byte vector is zeroed for those skipped entries. 962 */ 963 while ((lastvecindex + 1) < vecindex) { 964 error = subyte(vec + lastvecindex, 0); 965 if (error) { 966 error = EFAULT; 967 goto done2; 968 } 969 ++lastvecindex; 970 } 971 972 /* 973 * Pass the page information to the user 974 */ 975 error = subyte(vec + vecindex, mincoreinfo); 976 if (error) { 977 error = EFAULT; 978 goto done2; 979 } 980 981 /* 982 * If the map has changed, due to the subyte, the previous 983 * output may be invalid. 984 */ 985 vm_map_lock_read(map); 986 if (timestamp != map->timestamp) 987 goto RestartScan; 988 989 lastvecindex = vecindex; 990 addr += PAGE_SIZE; 991 } 992 } 993 994 /* 995 * subyte may page fault. In case it needs to modify 996 * the map, we release the lock. 997 */ 998 vm_map_unlock_read(map); 999 1000 /* 1001 * Zero the last entries in the byte vector. 1002 */ 1003 vecindex = OFF_TO_IDX(end - first_addr); 1004 while ((lastvecindex + 1) < vecindex) { 1005 error = subyte(vec + lastvecindex, 0); 1006 if (error) { 1007 error = EFAULT; 1008 goto done2; 1009 } 1010 ++lastvecindex; 1011 } 1012 1013 /* 1014 * If the map has changed, due to the subyte, the previous 1015 * output may be invalid. 1016 */ 1017 vm_map_lock_read(map); 1018 if (timestamp != map->timestamp) 1019 goto RestartScan; 1020 vm_map_unlock_read(map); 1021 done2: 1022 return (error); 1023 } 1024 1025 #ifndef _SYS_SYSPROTO_H_ 1026 struct mlock_args { 1027 const void *addr; 1028 size_t len; 1029 }; 1030 #endif 1031 /* 1032 * MPSAFE 1033 */ 1034 int 1035 sys_mlock(td, uap) 1036 struct thread *td; 1037 struct mlock_args *uap; 1038 { 1039 1040 return (vm_mlock(td->td_proc, td->td_ucred, uap->addr, uap->len)); 1041 } 1042 1043 int 1044 vm_mlock(struct proc *proc, struct ucred *cred, const void *addr0, size_t len) 1045 { 1046 vm_offset_t addr, end, last, start; 1047 vm_size_t npages, size; 1048 vm_map_t map; 1049 unsigned long nsize; 1050 int error; 1051 1052 error = priv_check_cred(cred, PRIV_VM_MLOCK, 0); 1053 if (error) 1054 return (error); 1055 addr = (vm_offset_t)addr0; 1056 size = len; 1057 last = addr + size; 1058 start = trunc_page(addr); 1059 end = round_page(last); 1060 if (last < addr || end < addr) 1061 return (EINVAL); 1062 npages = atop(end - start); 1063 if (npages > vm_page_max_wired) 1064 return (ENOMEM); 1065 map = &proc->p_vmspace->vm_map; 1066 PROC_LOCK(proc); 1067 nsize = ptoa(npages + pmap_wired_count(map->pmap)); 1068 if (nsize > lim_cur(proc, RLIMIT_MEMLOCK)) { 1069 PROC_UNLOCK(proc); 1070 return (ENOMEM); 1071 } 1072 PROC_UNLOCK(proc); 1073 if (npages + cnt.v_wire_count > vm_page_max_wired) 1074 return (EAGAIN); 1075 #ifdef RACCT 1076 PROC_LOCK(proc); 1077 error = racct_set(proc, RACCT_MEMLOCK, nsize); 1078 PROC_UNLOCK(proc); 1079 if (error != 0) 1080 return (ENOMEM); 1081 #endif 1082 error = vm_map_wire(map, start, end, 1083 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 1084 #ifdef RACCT 1085 if (error != KERN_SUCCESS) { 1086 PROC_LOCK(proc); 1087 racct_set(proc, RACCT_MEMLOCK, 1088 ptoa(pmap_wired_count(map->pmap))); 1089 PROC_UNLOCK(proc); 1090 } 1091 #endif 1092 return (error == KERN_SUCCESS ? 0 : ENOMEM); 1093 } 1094 1095 #ifndef _SYS_SYSPROTO_H_ 1096 struct mlockall_args { 1097 int how; 1098 }; 1099 #endif 1100 1101 /* 1102 * MPSAFE 1103 */ 1104 int 1105 sys_mlockall(td, uap) 1106 struct thread *td; 1107 struct mlockall_args *uap; 1108 { 1109 vm_map_t map; 1110 int error; 1111 1112 map = &td->td_proc->p_vmspace->vm_map; 1113 error = priv_check(td, PRIV_VM_MLOCK); 1114 if (error) 1115 return (error); 1116 1117 if ((uap->how == 0) || ((uap->how & ~(MCL_CURRENT|MCL_FUTURE)) != 0)) 1118 return (EINVAL); 1119 1120 /* 1121 * If wiring all pages in the process would cause it to exceed 1122 * a hard resource limit, return ENOMEM. 1123 */ 1124 if (!old_mlock && uap->how & MCL_CURRENT) { 1125 PROC_LOCK(td->td_proc); 1126 if (map->size > lim_cur(td->td_proc, RLIMIT_MEMLOCK)) { 1127 PROC_UNLOCK(td->td_proc); 1128 return (ENOMEM); 1129 } 1130 PROC_UNLOCK(td->td_proc); 1131 } 1132 #ifdef RACCT 1133 PROC_LOCK(td->td_proc); 1134 error = racct_set(td->td_proc, RACCT_MEMLOCK, map->size); 1135 PROC_UNLOCK(td->td_proc); 1136 if (error != 0) 1137 return (ENOMEM); 1138 #endif 1139 1140 if (uap->how & MCL_FUTURE) { 1141 vm_map_lock(map); 1142 vm_map_modflags(map, MAP_WIREFUTURE, 0); 1143 vm_map_unlock(map); 1144 error = 0; 1145 } 1146 1147 if (uap->how & MCL_CURRENT) { 1148 /* 1149 * P1003.1-2001 mandates that all currently mapped pages 1150 * will be memory resident and locked (wired) upon return 1151 * from mlockall(). vm_map_wire() will wire pages, by 1152 * calling vm_fault_wire() for each page in the region. 1153 */ 1154 error = vm_map_wire(map, vm_map_min(map), vm_map_max(map), 1155 VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK); 1156 error = (error == KERN_SUCCESS ? 0 : EAGAIN); 1157 } 1158 #ifdef RACCT 1159 if (error != KERN_SUCCESS) { 1160 PROC_LOCK(td->td_proc); 1161 racct_set(td->td_proc, RACCT_MEMLOCK, 1162 ptoa(pmap_wired_count(map->pmap))); 1163 PROC_UNLOCK(td->td_proc); 1164 } 1165 #endif 1166 1167 return (error); 1168 } 1169 1170 #ifndef _SYS_SYSPROTO_H_ 1171 struct munlockall_args { 1172 register_t dummy; 1173 }; 1174 #endif 1175 1176 /* 1177 * MPSAFE 1178 */ 1179 int 1180 sys_munlockall(td, uap) 1181 struct thread *td; 1182 struct munlockall_args *uap; 1183 { 1184 vm_map_t map; 1185 int error; 1186 1187 map = &td->td_proc->p_vmspace->vm_map; 1188 error = priv_check(td, PRIV_VM_MUNLOCK); 1189 if (error) 1190 return (error); 1191 1192 /* Clear the MAP_WIREFUTURE flag from this vm_map. */ 1193 vm_map_lock(map); 1194 vm_map_modflags(map, 0, MAP_WIREFUTURE); 1195 vm_map_unlock(map); 1196 1197 /* Forcibly unwire all pages. */ 1198 error = vm_map_unwire(map, vm_map_min(map), vm_map_max(map), 1199 VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK); 1200 #ifdef RACCT 1201 if (error == KERN_SUCCESS) { 1202 PROC_LOCK(td->td_proc); 1203 racct_set(td->td_proc, RACCT_MEMLOCK, 0); 1204 PROC_UNLOCK(td->td_proc); 1205 } 1206 #endif 1207 1208 return (error); 1209 } 1210 1211 #ifndef _SYS_SYSPROTO_H_ 1212 struct munlock_args { 1213 const void *addr; 1214 size_t len; 1215 }; 1216 #endif 1217 /* 1218 * MPSAFE 1219 */ 1220 int 1221 sys_munlock(td, uap) 1222 struct thread *td; 1223 struct munlock_args *uap; 1224 { 1225 vm_offset_t addr, end, last, start; 1226 vm_size_t size; 1227 #ifdef RACCT 1228 vm_map_t map; 1229 #endif 1230 int error; 1231 1232 error = priv_check(td, PRIV_VM_MUNLOCK); 1233 if (error) 1234 return (error); 1235 addr = (vm_offset_t)uap->addr; 1236 size = uap->len; 1237 last = addr + size; 1238 start = trunc_page(addr); 1239 end = round_page(last); 1240 if (last < addr || end < addr) 1241 return (EINVAL); 1242 error = vm_map_unwire(&td->td_proc->p_vmspace->vm_map, start, end, 1243 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 1244 #ifdef RACCT 1245 if (error == KERN_SUCCESS) { 1246 PROC_LOCK(td->td_proc); 1247 map = &td->td_proc->p_vmspace->vm_map; 1248 racct_set(td->td_proc, RACCT_MEMLOCK, 1249 ptoa(pmap_wired_count(map->pmap))); 1250 PROC_UNLOCK(td->td_proc); 1251 } 1252 #endif 1253 return (error == KERN_SUCCESS ? 0 : ENOMEM); 1254 } 1255 1256 /* 1257 * vm_mmap_vnode() 1258 * 1259 * Helper function for vm_mmap. Perform sanity check specific for mmap 1260 * operations on vnodes. 1261 * 1262 * For VCHR vnodes, the vnode lock is held over the call to 1263 * vm_mmap_cdev() to keep vp->v_rdev valid. 1264 */ 1265 int 1266 vm_mmap_vnode(struct thread *td, vm_size_t objsize, 1267 vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp, 1268 struct vnode *vp, vm_ooffset_t *foffp, vm_object_t *objp, 1269 boolean_t *writecounted) 1270 { 1271 struct vattr va; 1272 vm_object_t obj; 1273 vm_offset_t foff; 1274 struct mount *mp; 1275 struct ucred *cred; 1276 int error, flags, locktype; 1277 1278 mp = vp->v_mount; 1279 cred = td->td_ucred; 1280 if ((*maxprotp & VM_PROT_WRITE) && (*flagsp & MAP_SHARED)) 1281 locktype = LK_EXCLUSIVE; 1282 else 1283 locktype = LK_SHARED; 1284 if ((error = vget(vp, locktype, td)) != 0) 1285 return (error); 1286 foff = *foffp; 1287 flags = *flagsp; 1288 obj = vp->v_object; 1289 if (vp->v_type == VREG) { 1290 /* 1291 * Get the proper underlying object 1292 */ 1293 if (obj == NULL) { 1294 error = EINVAL; 1295 goto done; 1296 } 1297 if (obj->type == OBJT_VNODE && obj->handle != vp) { 1298 vput(vp); 1299 vp = (struct vnode *)obj->handle; 1300 /* 1301 * Bypass filesystems obey the mpsafety of the 1302 * underlying fs. Tmpfs never bypasses. 1303 */ 1304 error = vget(vp, locktype, td); 1305 if (error != 0) 1306 return (error); 1307 } 1308 if (locktype == LK_EXCLUSIVE) { 1309 *writecounted = TRUE; 1310 vnode_pager_update_writecount(obj, 0, objsize); 1311 } 1312 } else if (vp->v_type == VCHR) { 1313 error = vm_mmap_cdev(td, objsize, prot, maxprotp, flagsp, 1314 vp->v_rdev, foffp, objp); 1315 if (error == 0) 1316 goto mark_atime; 1317 goto done; 1318 } else { 1319 error = EINVAL; 1320 goto done; 1321 } 1322 if ((error = VOP_GETATTR(vp, &va, cred))) 1323 goto done; 1324 #ifdef MAC 1325 error = mac_vnode_check_mmap(cred, vp, prot, flags); 1326 if (error != 0) 1327 goto done; 1328 #endif 1329 if ((flags & MAP_SHARED) != 0) { 1330 if ((va.va_flags & (SF_SNAPSHOT|IMMUTABLE|APPEND)) != 0) { 1331 if (prot & PROT_WRITE) { 1332 error = EPERM; 1333 goto done; 1334 } 1335 *maxprotp &= ~VM_PROT_WRITE; 1336 } 1337 } 1338 /* 1339 * If it is a regular file without any references 1340 * we do not need to sync it. 1341 * Adjust object size to be the size of actual file. 1342 */ 1343 objsize = round_page(va.va_size); 1344 if (va.va_nlink == 0) 1345 flags |= MAP_NOSYNC; 1346 if (obj->type == OBJT_VNODE) 1347 obj = vm_pager_allocate(OBJT_VNODE, vp, objsize, prot, foff, 1348 cred); 1349 else { 1350 KASSERT(obj->type == OBJT_DEFAULT || obj->type == OBJT_SWAP, 1351 ("wrong object type")); 1352 vm_object_reference(obj); 1353 } 1354 if (obj == NULL) { 1355 error = ENOMEM; 1356 goto done; 1357 } 1358 *objp = obj; 1359 *flagsp = flags; 1360 1361 mark_atime: 1362 vfs_mark_atime(vp, cred); 1363 1364 done: 1365 if (error != 0 && *writecounted) { 1366 *writecounted = FALSE; 1367 vnode_pager_update_writecount(obj, objsize, 0); 1368 } 1369 vput(vp); 1370 return (error); 1371 } 1372 1373 /* 1374 * vm_mmap_cdev() 1375 * 1376 * MPSAFE 1377 * 1378 * Helper function for vm_mmap. Perform sanity check specific for mmap 1379 * operations on cdevs. 1380 */ 1381 int 1382 vm_mmap_cdev(struct thread *td, vm_size_t objsize, 1383 vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp, 1384 struct cdev *cdev, vm_ooffset_t *foff, vm_object_t *objp) 1385 { 1386 vm_object_t obj; 1387 struct cdevsw *dsw; 1388 int error, flags, ref; 1389 1390 flags = *flagsp; 1391 1392 dsw = dev_refthread(cdev, &ref); 1393 if (dsw == NULL) 1394 return (ENXIO); 1395 if (dsw->d_flags & D_MMAP_ANON) { 1396 dev_relthread(cdev, ref); 1397 *maxprotp = VM_PROT_ALL; 1398 *flagsp |= MAP_ANON; 1399 return (0); 1400 } 1401 /* 1402 * cdevs do not provide private mappings of any kind. 1403 */ 1404 if ((*maxprotp & VM_PROT_WRITE) == 0 && 1405 (prot & PROT_WRITE) != 0) { 1406 dev_relthread(cdev, ref); 1407 return (EACCES); 1408 } 1409 if (flags & (MAP_PRIVATE|MAP_COPY)) { 1410 dev_relthread(cdev, ref); 1411 return (EINVAL); 1412 } 1413 /* 1414 * Force device mappings to be shared. 1415 */ 1416 flags |= MAP_SHARED; 1417 #ifdef MAC_XXX 1418 error = mac_cdev_check_mmap(td->td_ucred, cdev, prot); 1419 if (error != 0) { 1420 dev_relthread(cdev, ref); 1421 return (error); 1422 } 1423 #endif 1424 /* 1425 * First, try d_mmap_single(). If that is not implemented 1426 * (returns ENODEV), fall back to using the device pager. 1427 * Note that d_mmap_single() must return a reference to the 1428 * object (it needs to bump the reference count of the object 1429 * it returns somehow). 1430 * 1431 * XXX assumes VM_PROT_* == PROT_* 1432 */ 1433 error = dsw->d_mmap_single(cdev, foff, objsize, objp, (int)prot); 1434 dev_relthread(cdev, ref); 1435 if (error != ENODEV) 1436 return (error); 1437 obj = vm_pager_allocate(OBJT_DEVICE, cdev, objsize, prot, *foff, 1438 td->td_ucred); 1439 if (obj == NULL) 1440 return (EINVAL); 1441 *objp = obj; 1442 *flagsp = flags; 1443 return (0); 1444 } 1445 1446 /* 1447 * vm_mmap_shm() 1448 * 1449 * MPSAFE 1450 * 1451 * Helper function for vm_mmap. Perform sanity check specific for mmap 1452 * operations on shm file descriptors. 1453 */ 1454 int 1455 vm_mmap_shm(struct thread *td, vm_size_t objsize, 1456 vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp, 1457 struct shmfd *shmfd, vm_ooffset_t foff, vm_object_t *objp) 1458 { 1459 int error; 1460 1461 if ((*flagsp & MAP_SHARED) != 0 && 1462 (*maxprotp & VM_PROT_WRITE) == 0 && 1463 (prot & PROT_WRITE) != 0) 1464 return (EACCES); 1465 #ifdef MAC 1466 error = mac_posixshm_check_mmap(td->td_ucred, shmfd, prot, *flagsp); 1467 if (error != 0) 1468 return (error); 1469 #endif 1470 error = shm_mmap(shmfd, objsize, foff, objp); 1471 if (error) 1472 return (error); 1473 return (0); 1474 } 1475 1476 /* 1477 * vm_mmap() 1478 * 1479 * MPSAFE 1480 * 1481 * Internal version of mmap. Currently used by mmap, exec, and sys5 1482 * shared memory. Handle is either a vnode pointer or NULL for MAP_ANON. 1483 */ 1484 int 1485 vm_mmap(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot, 1486 vm_prot_t maxprot, int flags, 1487 objtype_t handle_type, void *handle, 1488 vm_ooffset_t foff) 1489 { 1490 boolean_t fitit; 1491 vm_object_t object = NULL; 1492 struct thread *td = curthread; 1493 int docow, error, rv; 1494 boolean_t writecounted; 1495 1496 if (size == 0) 1497 return (0); 1498 1499 size = round_page(size); 1500 1501 if (map == &td->td_proc->p_vmspace->vm_map) { 1502 PROC_LOCK(td->td_proc); 1503 if (map->size + size > lim_cur(td->td_proc, RLIMIT_VMEM)) { 1504 PROC_UNLOCK(td->td_proc); 1505 return (ENOMEM); 1506 } 1507 if (racct_set(td->td_proc, RACCT_VMEM, map->size + size)) { 1508 PROC_UNLOCK(td->td_proc); 1509 return (ENOMEM); 1510 } 1511 if (!old_mlock && map->flags & MAP_WIREFUTURE) { 1512 if (ptoa(pmap_wired_count(map->pmap)) + size > 1513 lim_cur(td->td_proc, RLIMIT_MEMLOCK)) { 1514 racct_set_force(td->td_proc, RACCT_VMEM, 1515 map->size); 1516 PROC_UNLOCK(td->td_proc); 1517 return (ENOMEM); 1518 } 1519 error = racct_set(td->td_proc, RACCT_MEMLOCK, 1520 ptoa(pmap_wired_count(map->pmap)) + size); 1521 if (error != 0) { 1522 racct_set_force(td->td_proc, RACCT_VMEM, 1523 map->size); 1524 PROC_UNLOCK(td->td_proc); 1525 return (error); 1526 } 1527 } 1528 PROC_UNLOCK(td->td_proc); 1529 } 1530 1531 /* 1532 * We currently can only deal with page aligned file offsets. 1533 * The check is here rather than in the syscall because the 1534 * kernel calls this function internally for other mmaping 1535 * operations (such as in exec) and non-aligned offsets will 1536 * cause pmap inconsistencies...so we want to be sure to 1537 * disallow this in all cases. 1538 */ 1539 if (foff & PAGE_MASK) 1540 return (EINVAL); 1541 1542 if ((flags & MAP_FIXED) == 0) { 1543 fitit = TRUE; 1544 *addr = round_page(*addr); 1545 } else { 1546 if (*addr != trunc_page(*addr)) 1547 return (EINVAL); 1548 fitit = FALSE; 1549 } 1550 writecounted = FALSE; 1551 1552 /* 1553 * Lookup/allocate object. 1554 */ 1555 switch (handle_type) { 1556 case OBJT_DEVICE: 1557 error = vm_mmap_cdev(td, size, prot, &maxprot, &flags, 1558 handle, &foff, &object); 1559 break; 1560 case OBJT_VNODE: 1561 error = vm_mmap_vnode(td, size, prot, &maxprot, &flags, 1562 handle, &foff, &object, &writecounted); 1563 break; 1564 case OBJT_SWAP: 1565 error = vm_mmap_shm(td, size, prot, &maxprot, &flags, 1566 handle, foff, &object); 1567 break; 1568 case OBJT_DEFAULT: 1569 if (handle == NULL) { 1570 error = 0; 1571 break; 1572 } 1573 /* FALLTHROUGH */ 1574 default: 1575 error = EINVAL; 1576 break; 1577 } 1578 if (error) 1579 return (error); 1580 if (flags & MAP_ANON) { 1581 object = NULL; 1582 docow = 0; 1583 /* 1584 * Unnamed anonymous regions always start at 0. 1585 */ 1586 if (handle == 0) 1587 foff = 0; 1588 } else if (flags & MAP_PREFAULT_READ) 1589 docow = MAP_PREFAULT; 1590 else 1591 docow = MAP_PREFAULT_PARTIAL; 1592 1593 if ((flags & (MAP_ANON|MAP_SHARED)) == 0) 1594 docow |= MAP_COPY_ON_WRITE; 1595 if (flags & MAP_NOSYNC) 1596 docow |= MAP_DISABLE_SYNCER; 1597 if (flags & MAP_NOCORE) 1598 docow |= MAP_DISABLE_COREDUMP; 1599 /* Shared memory is also shared with children. */ 1600 if (flags & MAP_SHARED) 1601 docow |= MAP_INHERIT_SHARE; 1602 if (writecounted) 1603 docow |= MAP_VN_WRITECOUNT; 1604 1605 if (flags & MAP_STACK) 1606 rv = vm_map_stack(map, *addr, size, prot, maxprot, 1607 docow | MAP_STACK_GROWS_DOWN); 1608 else if (fitit) 1609 rv = vm_map_find(map, object, foff, addr, size, 1610 object != NULL && object->type == OBJT_DEVICE ? 1611 VMFS_ALIGNED_SPACE : VMFS_OPTIMAL_SPACE, prot, maxprot, 1612 docow); 1613 else 1614 rv = vm_map_fixed(map, object, foff, *addr, size, 1615 prot, maxprot, docow); 1616 1617 if (rv == KERN_SUCCESS) { 1618 /* 1619 * If the process has requested that all future mappings 1620 * be wired, then heed this. 1621 */ 1622 if (map->flags & MAP_WIREFUTURE) { 1623 vm_map_wire(map, *addr, *addr + size, 1624 VM_MAP_WIRE_USER | ((flags & MAP_STACK) ? 1625 VM_MAP_WIRE_HOLESOK : VM_MAP_WIRE_NOHOLES)); 1626 } 1627 } else { 1628 /* 1629 * If this mapping was accounted for in the vnode's 1630 * writecount, then undo that now. 1631 */ 1632 if (writecounted) 1633 vnode_pager_release_writecount(object, 0, size); 1634 /* 1635 * Lose the object reference. Will destroy the 1636 * object if it's an unnamed anonymous mapping 1637 * or named anonymous without other references. 1638 */ 1639 vm_object_deallocate(object); 1640 } 1641 return (vm_mmap_to_errno(rv)); 1642 } 1643 1644 /* 1645 * Translate a Mach VM return code to zero on success or the appropriate errno 1646 * on failure. 1647 */ 1648 int 1649 vm_mmap_to_errno(int rv) 1650 { 1651 1652 switch (rv) { 1653 case KERN_SUCCESS: 1654 return (0); 1655 case KERN_INVALID_ADDRESS: 1656 case KERN_NO_SPACE: 1657 return (ENOMEM); 1658 case KERN_PROTECTION_FAILURE: 1659 return (EACCES); 1660 default: 1661 return (EINVAL); 1662 } 1663 } 1664