1 /*- 2 * Copyright (c) 1988 University of Utah. 3 * Copyright (c) 1991, 1993 4 * The Regents of the University of California. All rights reserved. 5 * 6 * This code is derived from software contributed to Berkeley by 7 * the Systems Programming Group of the University of Utah Computer 8 * Science Department. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 4. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * from: Utah $Hdr: vm_mmap.c 1.6 91/10/21$ 35 * 36 * @(#)vm_mmap.c 8.4 (Berkeley) 1/12/94 37 */ 38 39 /* 40 * Mapped file (mmap) interface to VM 41 */ 42 43 #include <sys/cdefs.h> 44 __FBSDID("$FreeBSD$"); 45 46 #include "opt_compat.h" 47 #include "opt_hwpmc_hooks.h" 48 49 #include <sys/param.h> 50 #include <sys/systm.h> 51 #include <sys/lock.h> 52 #include <sys/mutex.h> 53 #include <sys/sysproto.h> 54 #include <sys/filedesc.h> 55 #include <sys/priv.h> 56 #include <sys/proc.h> 57 #include <sys/racct.h> 58 #include <sys/resource.h> 59 #include <sys/resourcevar.h> 60 #include <sys/vnode.h> 61 #include <sys/fcntl.h> 62 #include <sys/file.h> 63 #include <sys/mman.h> 64 #include <sys/mount.h> 65 #include <sys/conf.h> 66 #include <sys/stat.h> 67 #include <sys/sysent.h> 68 #include <sys/vmmeter.h> 69 70 #include <security/mac/mac_framework.h> 71 72 #include <vm/vm.h> 73 #include <vm/vm_param.h> 74 #include <vm/pmap.h> 75 #include <vm/vm_map.h> 76 #include <vm/vm_object.h> 77 #include <vm/vm_page.h> 78 #include <vm/vm_pager.h> 79 #include <vm/vm_pageout.h> 80 #include <vm/vm_extern.h> 81 #include <vm/vm_page.h> 82 83 #ifdef HWPMC_HOOKS 84 #include <sys/pmckern.h> 85 #endif 86 87 #ifndef _SYS_SYSPROTO_H_ 88 struct sbrk_args { 89 int incr; 90 }; 91 #endif 92 93 static int vm_mmap_vnode(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *, 94 int *, struct vnode *, vm_ooffset_t *, vm_object_t *); 95 static int vm_mmap_cdev(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *, 96 int *, struct cdev *, vm_ooffset_t *, vm_object_t *); 97 static int vm_mmap_shm(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *, 98 int *, struct shmfd *, vm_ooffset_t, vm_object_t *); 99 100 /* 101 * MPSAFE 102 */ 103 /* ARGSUSED */ 104 int 105 sbrk(td, uap) 106 struct thread *td; 107 struct sbrk_args *uap; 108 { 109 /* Not yet implemented */ 110 return (EOPNOTSUPP); 111 } 112 113 #ifndef _SYS_SYSPROTO_H_ 114 struct sstk_args { 115 int incr; 116 }; 117 #endif 118 119 /* 120 * MPSAFE 121 */ 122 /* ARGSUSED */ 123 int 124 sstk(td, uap) 125 struct thread *td; 126 struct sstk_args *uap; 127 { 128 /* Not yet implemented */ 129 return (EOPNOTSUPP); 130 } 131 132 #if defined(COMPAT_43) 133 #ifndef _SYS_SYSPROTO_H_ 134 struct getpagesize_args { 135 int dummy; 136 }; 137 #endif 138 139 /* ARGSUSED */ 140 int 141 ogetpagesize(td, uap) 142 struct thread *td; 143 struct getpagesize_args *uap; 144 { 145 /* MP SAFE */ 146 td->td_retval[0] = PAGE_SIZE; 147 return (0); 148 } 149 #endif /* COMPAT_43 */ 150 151 152 /* 153 * Memory Map (mmap) system call. Note that the file offset 154 * and address are allowed to be NOT page aligned, though if 155 * the MAP_FIXED flag it set, both must have the same remainder 156 * modulo the PAGE_SIZE (POSIX 1003.1b). If the address is not 157 * page-aligned, the actual mapping starts at trunc_page(addr) 158 * and the return value is adjusted up by the page offset. 159 * 160 * Generally speaking, only character devices which are themselves 161 * memory-based, such as a video framebuffer, can be mmap'd. Otherwise 162 * there would be no cache coherency between a descriptor and a VM mapping 163 * both to the same character device. 164 */ 165 #ifndef _SYS_SYSPROTO_H_ 166 struct mmap_args { 167 void *addr; 168 size_t len; 169 int prot; 170 int flags; 171 int fd; 172 long pad; 173 off_t pos; 174 }; 175 #endif 176 177 /* 178 * MPSAFE 179 */ 180 int 181 mmap(td, uap) 182 struct thread *td; 183 struct mmap_args *uap; 184 { 185 #ifdef HWPMC_HOOKS 186 struct pmckern_map_in pkm; 187 #endif 188 struct file *fp; 189 struct vnode *vp; 190 vm_offset_t addr; 191 vm_size_t size, pageoff; 192 vm_prot_t prot, maxprot; 193 void *handle; 194 objtype_t handle_type; 195 int flags, error; 196 off_t pos; 197 struct vmspace *vms = td->td_proc->p_vmspace; 198 199 addr = (vm_offset_t) uap->addr; 200 size = uap->len; 201 prot = uap->prot & VM_PROT_ALL; 202 flags = uap->flags; 203 pos = uap->pos; 204 205 fp = NULL; 206 207 /* Make sure mapping fits into numeric range, etc. */ 208 if ((uap->len == 0 && !SV_CURPROC_FLAG(SV_AOUT) && 209 curproc->p_osrel >= P_OSREL_MAP_ANON) || 210 ((flags & MAP_ANON) && (uap->fd != -1 || pos != 0))) 211 return (EINVAL); 212 213 if (flags & MAP_STACK) { 214 if ((uap->fd != -1) || 215 ((prot & (PROT_READ | PROT_WRITE)) != (PROT_READ | PROT_WRITE))) 216 return (EINVAL); 217 flags |= MAP_ANON; 218 pos = 0; 219 } 220 221 /* 222 * Align the file position to a page boundary, 223 * and save its page offset component. 224 */ 225 pageoff = (pos & PAGE_MASK); 226 pos -= pageoff; 227 228 /* Adjust size for rounding (on both ends). */ 229 size += pageoff; /* low end... */ 230 size = (vm_size_t) round_page(size); /* hi end */ 231 232 /* 233 * Check for illegal addresses. Watch out for address wrap... Note 234 * that VM_*_ADDRESS are not constants due to casts (argh). 235 */ 236 if (flags & MAP_FIXED) { 237 /* 238 * The specified address must have the same remainder 239 * as the file offset taken modulo PAGE_SIZE, so it 240 * should be aligned after adjustment by pageoff. 241 */ 242 addr -= pageoff; 243 if (addr & PAGE_MASK) 244 return (EINVAL); 245 246 /* Address range must be all in user VM space. */ 247 if (addr < vm_map_min(&vms->vm_map) || 248 addr + size > vm_map_max(&vms->vm_map)) 249 return (EINVAL); 250 if (addr + size < addr) 251 return (EINVAL); 252 } else { 253 /* 254 * XXX for non-fixed mappings where no hint is provided or 255 * the hint would fall in the potential heap space, 256 * place it after the end of the largest possible heap. 257 * 258 * There should really be a pmap call to determine a reasonable 259 * location. 260 */ 261 PROC_LOCK(td->td_proc); 262 if (addr == 0 || 263 (addr >= round_page((vm_offset_t)vms->vm_taddr) && 264 addr < round_page((vm_offset_t)vms->vm_daddr + 265 lim_max(td->td_proc, RLIMIT_DATA)))) 266 addr = round_page((vm_offset_t)vms->vm_daddr + 267 lim_max(td->td_proc, RLIMIT_DATA)); 268 PROC_UNLOCK(td->td_proc); 269 } 270 if (flags & MAP_ANON) { 271 /* 272 * Mapping blank space is trivial. 273 */ 274 handle = NULL; 275 handle_type = OBJT_DEFAULT; 276 maxprot = VM_PROT_ALL; 277 } else { 278 /* 279 * Mapping file, get fp for validation and 280 * don't let the descriptor disappear on us if we block. 281 */ 282 if ((error = fget(td, uap->fd, &fp)) != 0) 283 goto done; 284 if (fp->f_type == DTYPE_SHM) { 285 handle = fp->f_data; 286 handle_type = OBJT_SWAP; 287 maxprot = VM_PROT_NONE; 288 289 /* FREAD should always be set. */ 290 if (fp->f_flag & FREAD) 291 maxprot |= VM_PROT_EXECUTE | VM_PROT_READ; 292 if (fp->f_flag & FWRITE) 293 maxprot |= VM_PROT_WRITE; 294 goto map; 295 } 296 if (fp->f_type != DTYPE_VNODE) { 297 error = ENODEV; 298 goto done; 299 } 300 #if defined(COMPAT_FREEBSD7) || defined(COMPAT_FREEBSD6) || \ 301 defined(COMPAT_FREEBSD5) || defined(COMPAT_FREEBSD4) 302 /* 303 * POSIX shared-memory objects are defined to have 304 * kernel persistence, and are not defined to support 305 * read(2)/write(2) -- or even open(2). Thus, we can 306 * use MAP_ASYNC to trade on-disk coherence for speed. 307 * The shm_open(3) library routine turns on the FPOSIXSHM 308 * flag to request this behavior. 309 */ 310 if (fp->f_flag & FPOSIXSHM) 311 flags |= MAP_NOSYNC; 312 #endif 313 vp = fp->f_vnode; 314 /* 315 * Ensure that file and memory protections are 316 * compatible. Note that we only worry about 317 * writability if mapping is shared; in this case, 318 * current and max prot are dictated by the open file. 319 * XXX use the vnode instead? Problem is: what 320 * credentials do we use for determination? What if 321 * proc does a setuid? 322 */ 323 if (vp->v_mount != NULL && vp->v_mount->mnt_flag & MNT_NOEXEC) 324 maxprot = VM_PROT_NONE; 325 else 326 maxprot = VM_PROT_EXECUTE; 327 if (fp->f_flag & FREAD) { 328 maxprot |= VM_PROT_READ; 329 } else if (prot & PROT_READ) { 330 error = EACCES; 331 goto done; 332 } 333 /* 334 * If we are sharing potential changes (either via 335 * MAP_SHARED or via the implicit sharing of character 336 * device mappings), and we are trying to get write 337 * permission although we opened it without asking 338 * for it, bail out. 339 */ 340 if ((flags & MAP_SHARED) != 0) { 341 if ((fp->f_flag & FWRITE) != 0) { 342 maxprot |= VM_PROT_WRITE; 343 } else if ((prot & PROT_WRITE) != 0) { 344 error = EACCES; 345 goto done; 346 } 347 } else if (vp->v_type != VCHR || (fp->f_flag & FWRITE) != 0) { 348 maxprot |= VM_PROT_WRITE; 349 } 350 handle = (void *)vp; 351 handle_type = OBJT_VNODE; 352 } 353 map: 354 td->td_fpop = fp; 355 error = vm_mmap(&vms->vm_map, &addr, size, prot, maxprot, 356 flags, handle_type, handle, pos); 357 td->td_fpop = NULL; 358 #ifdef HWPMC_HOOKS 359 /* inform hwpmc(4) if an executable is being mapped */ 360 if (error == 0 && handle_type == OBJT_VNODE && 361 (prot & PROT_EXEC)) { 362 pkm.pm_file = handle; 363 pkm.pm_address = (uintptr_t) addr; 364 PMC_CALL_HOOK(td, PMC_FN_MMAP, (void *) &pkm); 365 } 366 #endif 367 if (error == 0) 368 td->td_retval[0] = (register_t) (addr + pageoff); 369 done: 370 if (fp) 371 fdrop(fp, td); 372 373 return (error); 374 } 375 376 int 377 freebsd6_mmap(struct thread *td, struct freebsd6_mmap_args *uap) 378 { 379 struct mmap_args oargs; 380 381 oargs.addr = uap->addr; 382 oargs.len = uap->len; 383 oargs.prot = uap->prot; 384 oargs.flags = uap->flags; 385 oargs.fd = uap->fd; 386 oargs.pos = uap->pos; 387 return (mmap(td, &oargs)); 388 } 389 390 #ifdef COMPAT_43 391 #ifndef _SYS_SYSPROTO_H_ 392 struct ommap_args { 393 caddr_t addr; 394 int len; 395 int prot; 396 int flags; 397 int fd; 398 long pos; 399 }; 400 #endif 401 int 402 ommap(td, uap) 403 struct thread *td; 404 struct ommap_args *uap; 405 { 406 struct mmap_args nargs; 407 static const char cvtbsdprot[8] = { 408 0, 409 PROT_EXEC, 410 PROT_WRITE, 411 PROT_EXEC | PROT_WRITE, 412 PROT_READ, 413 PROT_EXEC | PROT_READ, 414 PROT_WRITE | PROT_READ, 415 PROT_EXEC | PROT_WRITE | PROT_READ, 416 }; 417 418 #define OMAP_ANON 0x0002 419 #define OMAP_COPY 0x0020 420 #define OMAP_SHARED 0x0010 421 #define OMAP_FIXED 0x0100 422 423 nargs.addr = uap->addr; 424 nargs.len = uap->len; 425 nargs.prot = cvtbsdprot[uap->prot & 0x7]; 426 nargs.flags = 0; 427 if (uap->flags & OMAP_ANON) 428 nargs.flags |= MAP_ANON; 429 if (uap->flags & OMAP_COPY) 430 nargs.flags |= MAP_COPY; 431 if (uap->flags & OMAP_SHARED) 432 nargs.flags |= MAP_SHARED; 433 else 434 nargs.flags |= MAP_PRIVATE; 435 if (uap->flags & OMAP_FIXED) 436 nargs.flags |= MAP_FIXED; 437 nargs.fd = uap->fd; 438 nargs.pos = uap->pos; 439 return (mmap(td, &nargs)); 440 } 441 #endif /* COMPAT_43 */ 442 443 444 #ifndef _SYS_SYSPROTO_H_ 445 struct msync_args { 446 void *addr; 447 size_t len; 448 int flags; 449 }; 450 #endif 451 /* 452 * MPSAFE 453 */ 454 int 455 msync(td, uap) 456 struct thread *td; 457 struct msync_args *uap; 458 { 459 vm_offset_t addr; 460 vm_size_t size, pageoff; 461 int flags; 462 vm_map_t map; 463 int rv; 464 465 addr = (vm_offset_t) uap->addr; 466 size = uap->len; 467 flags = uap->flags; 468 469 pageoff = (addr & PAGE_MASK); 470 addr -= pageoff; 471 size += pageoff; 472 size = (vm_size_t) round_page(size); 473 if (addr + size < addr) 474 return (EINVAL); 475 476 if ((flags & (MS_ASYNC|MS_INVALIDATE)) == (MS_ASYNC|MS_INVALIDATE)) 477 return (EINVAL); 478 479 map = &td->td_proc->p_vmspace->vm_map; 480 481 /* 482 * Clean the pages and interpret the return value. 483 */ 484 rv = vm_map_sync(map, addr, addr + size, (flags & MS_ASYNC) == 0, 485 (flags & MS_INVALIDATE) != 0); 486 switch (rv) { 487 case KERN_SUCCESS: 488 return (0); 489 case KERN_INVALID_ADDRESS: 490 return (EINVAL); /* Sun returns ENOMEM? */ 491 case KERN_INVALID_ARGUMENT: 492 return (EBUSY); 493 default: 494 return (EINVAL); 495 } 496 } 497 498 #ifndef _SYS_SYSPROTO_H_ 499 struct munmap_args { 500 void *addr; 501 size_t len; 502 }; 503 #endif 504 /* 505 * MPSAFE 506 */ 507 int 508 munmap(td, uap) 509 struct thread *td; 510 struct munmap_args *uap; 511 { 512 #ifdef HWPMC_HOOKS 513 struct pmckern_map_out pkm; 514 vm_map_entry_t entry; 515 #endif 516 vm_offset_t addr; 517 vm_size_t size, pageoff; 518 vm_map_t map; 519 520 addr = (vm_offset_t) uap->addr; 521 size = uap->len; 522 if (size == 0) 523 return (EINVAL); 524 525 pageoff = (addr & PAGE_MASK); 526 addr -= pageoff; 527 size += pageoff; 528 size = (vm_size_t) round_page(size); 529 if (addr + size < addr) 530 return (EINVAL); 531 532 /* 533 * Check for illegal addresses. Watch out for address wrap... 534 */ 535 map = &td->td_proc->p_vmspace->vm_map; 536 if (addr < vm_map_min(map) || addr + size > vm_map_max(map)) 537 return (EINVAL); 538 vm_map_lock(map); 539 #ifdef HWPMC_HOOKS 540 /* 541 * Inform hwpmc if the address range being unmapped contains 542 * an executable region. 543 */ 544 pkm.pm_address = (uintptr_t) NULL; 545 if (vm_map_lookup_entry(map, addr, &entry)) { 546 for (; 547 entry != &map->header && entry->start < addr + size; 548 entry = entry->next) { 549 if (vm_map_check_protection(map, entry->start, 550 entry->end, VM_PROT_EXECUTE) == TRUE) { 551 pkm.pm_address = (uintptr_t) addr; 552 pkm.pm_size = (size_t) size; 553 break; 554 } 555 } 556 } 557 #endif 558 vm_map_delete(map, addr, addr + size); 559 560 #ifdef HWPMC_HOOKS 561 /* downgrade the lock to prevent a LOR with the pmc-sx lock */ 562 vm_map_lock_downgrade(map); 563 if (pkm.pm_address != (uintptr_t) NULL) 564 PMC_CALL_HOOK(td, PMC_FN_MUNMAP, (void *) &pkm); 565 vm_map_unlock_read(map); 566 #else 567 vm_map_unlock(map); 568 #endif 569 /* vm_map_delete returns nothing but KERN_SUCCESS anyway */ 570 return (0); 571 } 572 573 #ifndef _SYS_SYSPROTO_H_ 574 struct mprotect_args { 575 const void *addr; 576 size_t len; 577 int prot; 578 }; 579 #endif 580 /* 581 * MPSAFE 582 */ 583 int 584 mprotect(td, uap) 585 struct thread *td; 586 struct mprotect_args *uap; 587 { 588 vm_offset_t addr; 589 vm_size_t size, pageoff; 590 vm_prot_t prot; 591 592 addr = (vm_offset_t) uap->addr; 593 size = uap->len; 594 prot = uap->prot & VM_PROT_ALL; 595 596 pageoff = (addr & PAGE_MASK); 597 addr -= pageoff; 598 size += pageoff; 599 size = (vm_size_t) round_page(size); 600 if (addr + size < addr) 601 return (EINVAL); 602 603 switch (vm_map_protect(&td->td_proc->p_vmspace->vm_map, addr, 604 addr + size, prot, FALSE)) { 605 case KERN_SUCCESS: 606 return (0); 607 case KERN_PROTECTION_FAILURE: 608 return (EACCES); 609 case KERN_RESOURCE_SHORTAGE: 610 return (ENOMEM); 611 } 612 return (EINVAL); 613 } 614 615 #ifndef _SYS_SYSPROTO_H_ 616 struct minherit_args { 617 void *addr; 618 size_t len; 619 int inherit; 620 }; 621 #endif 622 /* 623 * MPSAFE 624 */ 625 int 626 minherit(td, uap) 627 struct thread *td; 628 struct minherit_args *uap; 629 { 630 vm_offset_t addr; 631 vm_size_t size, pageoff; 632 vm_inherit_t inherit; 633 634 addr = (vm_offset_t)uap->addr; 635 size = uap->len; 636 inherit = uap->inherit; 637 638 pageoff = (addr & PAGE_MASK); 639 addr -= pageoff; 640 size += pageoff; 641 size = (vm_size_t) round_page(size); 642 if (addr + size < addr) 643 return (EINVAL); 644 645 switch (vm_map_inherit(&td->td_proc->p_vmspace->vm_map, addr, 646 addr + size, inherit)) { 647 case KERN_SUCCESS: 648 return (0); 649 case KERN_PROTECTION_FAILURE: 650 return (EACCES); 651 } 652 return (EINVAL); 653 } 654 655 #ifndef _SYS_SYSPROTO_H_ 656 struct madvise_args { 657 void *addr; 658 size_t len; 659 int behav; 660 }; 661 #endif 662 663 /* 664 * MPSAFE 665 */ 666 /* ARGSUSED */ 667 int 668 madvise(td, uap) 669 struct thread *td; 670 struct madvise_args *uap; 671 { 672 vm_offset_t start, end; 673 vm_map_t map; 674 struct proc *p; 675 int error; 676 677 /* 678 * Check for our special case, advising the swap pager we are 679 * "immortal." 680 */ 681 if (uap->behav == MADV_PROTECT) { 682 error = priv_check(td, PRIV_VM_MADV_PROTECT); 683 if (error == 0) { 684 p = td->td_proc; 685 PROC_LOCK(p); 686 p->p_flag |= P_PROTECTED; 687 PROC_UNLOCK(p); 688 } 689 return (error); 690 } 691 /* 692 * Check for illegal behavior 693 */ 694 if (uap->behav < 0 || uap->behav > MADV_CORE) 695 return (EINVAL); 696 /* 697 * Check for illegal addresses. Watch out for address wrap... Note 698 * that VM_*_ADDRESS are not constants due to casts (argh). 699 */ 700 map = &td->td_proc->p_vmspace->vm_map; 701 if ((vm_offset_t)uap->addr < vm_map_min(map) || 702 (vm_offset_t)uap->addr + uap->len > vm_map_max(map)) 703 return (EINVAL); 704 if (((vm_offset_t) uap->addr + uap->len) < (vm_offset_t) uap->addr) 705 return (EINVAL); 706 707 /* 708 * Since this routine is only advisory, we default to conservative 709 * behavior. 710 */ 711 start = trunc_page((vm_offset_t) uap->addr); 712 end = round_page((vm_offset_t) uap->addr + uap->len); 713 714 if (vm_map_madvise(map, start, end, uap->behav)) 715 return (EINVAL); 716 return (0); 717 } 718 719 #ifndef _SYS_SYSPROTO_H_ 720 struct mincore_args { 721 const void *addr; 722 size_t len; 723 char *vec; 724 }; 725 #endif 726 727 /* 728 * MPSAFE 729 */ 730 /* ARGSUSED */ 731 int 732 mincore(td, uap) 733 struct thread *td; 734 struct mincore_args *uap; 735 { 736 vm_offset_t addr, first_addr; 737 vm_offset_t end, cend; 738 pmap_t pmap; 739 vm_map_t map; 740 char *vec; 741 int error = 0; 742 int vecindex, lastvecindex; 743 vm_map_entry_t current; 744 vm_map_entry_t entry; 745 vm_object_t object; 746 vm_paddr_t locked_pa; 747 vm_page_t m; 748 vm_pindex_t pindex; 749 int mincoreinfo; 750 unsigned int timestamp; 751 boolean_t locked; 752 753 /* 754 * Make sure that the addresses presented are valid for user 755 * mode. 756 */ 757 first_addr = addr = trunc_page((vm_offset_t) uap->addr); 758 end = addr + (vm_size_t)round_page(uap->len); 759 map = &td->td_proc->p_vmspace->vm_map; 760 if (end > vm_map_max(map) || end < addr) 761 return (ENOMEM); 762 763 /* 764 * Address of byte vector 765 */ 766 vec = uap->vec; 767 768 pmap = vmspace_pmap(td->td_proc->p_vmspace); 769 770 vm_map_lock_read(map); 771 RestartScan: 772 timestamp = map->timestamp; 773 774 if (!vm_map_lookup_entry(map, addr, &entry)) { 775 vm_map_unlock_read(map); 776 return (ENOMEM); 777 } 778 779 /* 780 * Do this on a map entry basis so that if the pages are not 781 * in the current processes address space, we can easily look 782 * up the pages elsewhere. 783 */ 784 lastvecindex = -1; 785 for (current = entry; 786 (current != &map->header) && (current->start < end); 787 current = current->next) { 788 789 /* 790 * check for contiguity 791 */ 792 if (current->end < end && 793 (entry->next == &map->header || 794 current->next->start > current->end)) { 795 vm_map_unlock_read(map); 796 return (ENOMEM); 797 } 798 799 /* 800 * ignore submaps (for now) or null objects 801 */ 802 if ((current->eflags & MAP_ENTRY_IS_SUB_MAP) || 803 current->object.vm_object == NULL) 804 continue; 805 806 /* 807 * limit this scan to the current map entry and the 808 * limits for the mincore call 809 */ 810 if (addr < current->start) 811 addr = current->start; 812 cend = current->end; 813 if (cend > end) 814 cend = end; 815 816 /* 817 * scan this entry one page at a time 818 */ 819 while (addr < cend) { 820 /* 821 * Check pmap first, it is likely faster, also 822 * it can provide info as to whether we are the 823 * one referencing or modifying the page. 824 */ 825 object = NULL; 826 locked_pa = 0; 827 retry: 828 m = NULL; 829 mincoreinfo = pmap_mincore(pmap, addr, &locked_pa); 830 if (locked_pa != 0) { 831 /* 832 * The page is mapped by this process but not 833 * both accessed and modified. It is also 834 * managed. Acquire the object lock so that 835 * other mappings might be examined. 836 */ 837 m = PHYS_TO_VM_PAGE(locked_pa); 838 if (m->object != object) { 839 if (object != NULL) 840 VM_OBJECT_UNLOCK(object); 841 object = m->object; 842 locked = VM_OBJECT_TRYLOCK(object); 843 vm_page_unlock(m); 844 if (!locked) { 845 VM_OBJECT_LOCK(object); 846 vm_page_lock(m); 847 goto retry; 848 } 849 } else 850 vm_page_unlock(m); 851 KASSERT(m->valid == VM_PAGE_BITS_ALL, 852 ("mincore: page %p is mapped but invalid", 853 m)); 854 } else if (mincoreinfo == 0) { 855 /* 856 * The page is not mapped by this process. If 857 * the object implements managed pages, then 858 * determine if the page is resident so that 859 * the mappings might be examined. 860 */ 861 if (current->object.vm_object != object) { 862 if (object != NULL) 863 VM_OBJECT_UNLOCK(object); 864 object = current->object.vm_object; 865 VM_OBJECT_LOCK(object); 866 } 867 if (object->type == OBJT_DEFAULT || 868 object->type == OBJT_SWAP || 869 object->type == OBJT_VNODE) { 870 pindex = OFF_TO_IDX(current->offset + 871 (addr - current->start)); 872 m = vm_page_lookup(object, pindex); 873 if (m != NULL && m->valid == 0) 874 m = NULL; 875 if (m != NULL) 876 mincoreinfo = MINCORE_INCORE; 877 } 878 } 879 if (m != NULL) { 880 /* Examine other mappings to the page. */ 881 if (m->dirty == 0 && pmap_is_modified(m)) 882 vm_page_dirty(m); 883 if (m->dirty != 0) 884 mincoreinfo |= MINCORE_MODIFIED_OTHER; 885 /* 886 * The first test for PG_REFERENCED is an 887 * optimization. The second test is 888 * required because a concurrent pmap 889 * operation could clear the last reference 890 * and set PG_REFERENCED before the call to 891 * pmap_is_referenced(). 892 */ 893 if ((m->flags & PG_REFERENCED) != 0 || 894 pmap_is_referenced(m) || 895 (m->flags & PG_REFERENCED) != 0) 896 mincoreinfo |= MINCORE_REFERENCED_OTHER; 897 } 898 if (object != NULL) 899 VM_OBJECT_UNLOCK(object); 900 901 /* 902 * subyte may page fault. In case it needs to modify 903 * the map, we release the lock. 904 */ 905 vm_map_unlock_read(map); 906 907 /* 908 * calculate index into user supplied byte vector 909 */ 910 vecindex = OFF_TO_IDX(addr - first_addr); 911 912 /* 913 * If we have skipped map entries, we need to make sure that 914 * the byte vector is zeroed for those skipped entries. 915 */ 916 while ((lastvecindex + 1) < vecindex) { 917 error = subyte(vec + lastvecindex, 0); 918 if (error) { 919 error = EFAULT; 920 goto done2; 921 } 922 ++lastvecindex; 923 } 924 925 /* 926 * Pass the page information to the user 927 */ 928 error = subyte(vec + vecindex, mincoreinfo); 929 if (error) { 930 error = EFAULT; 931 goto done2; 932 } 933 934 /* 935 * If the map has changed, due to the subyte, the previous 936 * output may be invalid. 937 */ 938 vm_map_lock_read(map); 939 if (timestamp != map->timestamp) 940 goto RestartScan; 941 942 lastvecindex = vecindex; 943 addr += PAGE_SIZE; 944 } 945 } 946 947 /* 948 * subyte may page fault. In case it needs to modify 949 * the map, we release the lock. 950 */ 951 vm_map_unlock_read(map); 952 953 /* 954 * Zero the last entries in the byte vector. 955 */ 956 vecindex = OFF_TO_IDX(end - first_addr); 957 while ((lastvecindex + 1) < vecindex) { 958 error = subyte(vec + lastvecindex, 0); 959 if (error) { 960 error = EFAULT; 961 goto done2; 962 } 963 ++lastvecindex; 964 } 965 966 /* 967 * If the map has changed, due to the subyte, the previous 968 * output may be invalid. 969 */ 970 vm_map_lock_read(map); 971 if (timestamp != map->timestamp) 972 goto RestartScan; 973 vm_map_unlock_read(map); 974 done2: 975 return (error); 976 } 977 978 #ifndef _SYS_SYSPROTO_H_ 979 struct mlock_args { 980 const void *addr; 981 size_t len; 982 }; 983 #endif 984 /* 985 * MPSAFE 986 */ 987 int 988 mlock(td, uap) 989 struct thread *td; 990 struct mlock_args *uap; 991 { 992 struct proc *proc; 993 vm_offset_t addr, end, last, start; 994 vm_size_t npages, size; 995 unsigned long nsize; 996 int error; 997 998 error = priv_check(td, PRIV_VM_MLOCK); 999 if (error) 1000 return (error); 1001 addr = (vm_offset_t)uap->addr; 1002 size = uap->len; 1003 last = addr + size; 1004 start = trunc_page(addr); 1005 end = round_page(last); 1006 if (last < addr || end < addr) 1007 return (EINVAL); 1008 npages = atop(end - start); 1009 if (npages > vm_page_max_wired) 1010 return (ENOMEM); 1011 proc = td->td_proc; 1012 PROC_LOCK(proc); 1013 nsize = ptoa(npages + 1014 pmap_wired_count(vm_map_pmap(&proc->p_vmspace->vm_map))); 1015 if (nsize > lim_cur(proc, RLIMIT_MEMLOCK)) { 1016 PROC_UNLOCK(proc); 1017 return (ENOMEM); 1018 } 1019 PROC_UNLOCK(proc); 1020 if (npages + cnt.v_wire_count > vm_page_max_wired) 1021 return (EAGAIN); 1022 PROC_LOCK(proc); 1023 error = racct_set(proc, RACCT_MEMLOCK, nsize); 1024 PROC_UNLOCK(proc); 1025 if (error != 0) 1026 return (ENOMEM); 1027 error = vm_map_wire(&proc->p_vmspace->vm_map, start, end, 1028 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 1029 if (error != KERN_SUCCESS) { 1030 PROC_LOCK(proc); 1031 racct_set(proc, RACCT_MEMLOCK, 1032 ptoa(pmap_wired_count(vm_map_pmap(&proc->p_vmspace->vm_map)))); 1033 PROC_UNLOCK(proc); 1034 } 1035 return (error == KERN_SUCCESS ? 0 : ENOMEM); 1036 } 1037 1038 #ifndef _SYS_SYSPROTO_H_ 1039 struct mlockall_args { 1040 int how; 1041 }; 1042 #endif 1043 1044 /* 1045 * MPSAFE 1046 */ 1047 int 1048 mlockall(td, uap) 1049 struct thread *td; 1050 struct mlockall_args *uap; 1051 { 1052 vm_map_t map; 1053 int error; 1054 1055 map = &td->td_proc->p_vmspace->vm_map; 1056 error = 0; 1057 1058 if ((uap->how == 0) || ((uap->how & ~(MCL_CURRENT|MCL_FUTURE)) != 0)) 1059 return (EINVAL); 1060 1061 #if 0 1062 /* 1063 * If wiring all pages in the process would cause it to exceed 1064 * a hard resource limit, return ENOMEM. 1065 */ 1066 PROC_LOCK(td->td_proc); 1067 if (map->size > lim_cur(td->td_proc, RLIMIT_MEMLOCK)) { 1068 PROC_UNLOCK(td->td_proc); 1069 return (ENOMEM); 1070 } 1071 PROC_UNLOCK(td->td_proc); 1072 #else 1073 error = priv_check(td, PRIV_VM_MLOCK); 1074 if (error) 1075 return (error); 1076 #endif 1077 PROC_LOCK(td->td_proc); 1078 error = racct_set(td->td_proc, RACCT_MEMLOCK, map->size); 1079 PROC_UNLOCK(td->td_proc); 1080 if (error != 0) 1081 return (ENOMEM); 1082 1083 if (uap->how & MCL_FUTURE) { 1084 vm_map_lock(map); 1085 vm_map_modflags(map, MAP_WIREFUTURE, 0); 1086 vm_map_unlock(map); 1087 error = 0; 1088 } 1089 1090 if (uap->how & MCL_CURRENT) { 1091 /* 1092 * P1003.1-2001 mandates that all currently mapped pages 1093 * will be memory resident and locked (wired) upon return 1094 * from mlockall(). vm_map_wire() will wire pages, by 1095 * calling vm_fault_wire() for each page in the region. 1096 */ 1097 error = vm_map_wire(map, vm_map_min(map), vm_map_max(map), 1098 VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK); 1099 error = (error == KERN_SUCCESS ? 0 : EAGAIN); 1100 } 1101 if (error != KERN_SUCCESS) { 1102 PROC_LOCK(td->td_proc); 1103 racct_set(td->td_proc, RACCT_MEMLOCK, 1104 ptoa(pmap_wired_count(vm_map_pmap(&td->td_proc->p_vmspace->vm_map)))); 1105 PROC_UNLOCK(td->td_proc); 1106 } 1107 1108 return (error); 1109 } 1110 1111 #ifndef _SYS_SYSPROTO_H_ 1112 struct munlockall_args { 1113 register_t dummy; 1114 }; 1115 #endif 1116 1117 /* 1118 * MPSAFE 1119 */ 1120 int 1121 munlockall(td, uap) 1122 struct thread *td; 1123 struct munlockall_args *uap; 1124 { 1125 vm_map_t map; 1126 int error; 1127 1128 map = &td->td_proc->p_vmspace->vm_map; 1129 error = priv_check(td, PRIV_VM_MUNLOCK); 1130 if (error) 1131 return (error); 1132 1133 /* Clear the MAP_WIREFUTURE flag from this vm_map. */ 1134 vm_map_lock(map); 1135 vm_map_modflags(map, 0, MAP_WIREFUTURE); 1136 vm_map_unlock(map); 1137 1138 /* Forcibly unwire all pages. */ 1139 error = vm_map_unwire(map, vm_map_min(map), vm_map_max(map), 1140 VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK); 1141 if (error == KERN_SUCCESS) { 1142 PROC_LOCK(td->td_proc); 1143 racct_set(td->td_proc, RACCT_MEMLOCK, 0); 1144 PROC_UNLOCK(td->td_proc); 1145 } 1146 1147 return (error); 1148 } 1149 1150 #ifndef _SYS_SYSPROTO_H_ 1151 struct munlock_args { 1152 const void *addr; 1153 size_t len; 1154 }; 1155 #endif 1156 /* 1157 * MPSAFE 1158 */ 1159 int 1160 munlock(td, uap) 1161 struct thread *td; 1162 struct munlock_args *uap; 1163 { 1164 vm_offset_t addr, end, last, start; 1165 vm_size_t size; 1166 int error; 1167 1168 error = priv_check(td, PRIV_VM_MUNLOCK); 1169 if (error) 1170 return (error); 1171 addr = (vm_offset_t)uap->addr; 1172 size = uap->len; 1173 last = addr + size; 1174 start = trunc_page(addr); 1175 end = round_page(last); 1176 if (last < addr || end < addr) 1177 return (EINVAL); 1178 error = vm_map_unwire(&td->td_proc->p_vmspace->vm_map, start, end, 1179 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 1180 if (error == KERN_SUCCESS) { 1181 PROC_LOCK(td->td_proc); 1182 racct_sub(td->td_proc, RACCT_MEMLOCK, ptoa(end - start)); 1183 PROC_UNLOCK(td->td_proc); 1184 } 1185 return (error == KERN_SUCCESS ? 0 : ENOMEM); 1186 } 1187 1188 /* 1189 * vm_mmap_vnode() 1190 * 1191 * MPSAFE 1192 * 1193 * Helper function for vm_mmap. Perform sanity check specific for mmap 1194 * operations on vnodes. 1195 */ 1196 int 1197 vm_mmap_vnode(struct thread *td, vm_size_t objsize, 1198 vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp, 1199 struct vnode *vp, vm_ooffset_t *foffp, vm_object_t *objp) 1200 { 1201 struct vattr va; 1202 vm_object_t obj; 1203 vm_offset_t foff; 1204 struct mount *mp; 1205 struct ucred *cred; 1206 int error, flags; 1207 int vfslocked; 1208 1209 mp = vp->v_mount; 1210 cred = td->td_ucred; 1211 vfslocked = VFS_LOCK_GIANT(mp); 1212 if ((error = vget(vp, LK_SHARED, td)) != 0) { 1213 VFS_UNLOCK_GIANT(vfslocked); 1214 return (error); 1215 } 1216 foff = *foffp; 1217 flags = *flagsp; 1218 obj = vp->v_object; 1219 if (vp->v_type == VREG) { 1220 /* 1221 * Get the proper underlying object 1222 */ 1223 if (obj == NULL) { 1224 error = EINVAL; 1225 goto done; 1226 } 1227 if (obj->handle != vp) { 1228 vput(vp); 1229 vp = (struct vnode*)obj->handle; 1230 vget(vp, LK_SHARED, td); 1231 } 1232 } else if (vp->v_type == VCHR) { 1233 error = vm_mmap_cdev(td, objsize, prot, maxprotp, flagsp, 1234 vp->v_rdev, foffp, objp); 1235 if (error == 0) 1236 goto mark_atime; 1237 goto done; 1238 } else { 1239 error = EINVAL; 1240 goto done; 1241 } 1242 if ((error = VOP_GETATTR(vp, &va, cred))) 1243 goto done; 1244 #ifdef MAC 1245 error = mac_vnode_check_mmap(cred, vp, prot, flags); 1246 if (error != 0) 1247 goto done; 1248 #endif 1249 if ((flags & MAP_SHARED) != 0) { 1250 if ((va.va_flags & (SF_SNAPSHOT|IMMUTABLE|APPEND)) != 0) { 1251 if (prot & PROT_WRITE) { 1252 error = EPERM; 1253 goto done; 1254 } 1255 *maxprotp &= ~VM_PROT_WRITE; 1256 } 1257 } 1258 /* 1259 * If it is a regular file without any references 1260 * we do not need to sync it. 1261 * Adjust object size to be the size of actual file. 1262 */ 1263 objsize = round_page(va.va_size); 1264 if (va.va_nlink == 0) 1265 flags |= MAP_NOSYNC; 1266 obj = vm_pager_allocate(OBJT_VNODE, vp, objsize, prot, foff, td->td_ucred); 1267 if (obj == NULL) { 1268 error = ENOMEM; 1269 goto done; 1270 } 1271 *objp = obj; 1272 *flagsp = flags; 1273 1274 mark_atime: 1275 vfs_mark_atime(vp, cred); 1276 1277 done: 1278 vput(vp); 1279 VFS_UNLOCK_GIANT(vfslocked); 1280 return (error); 1281 } 1282 1283 /* 1284 * vm_mmap_cdev() 1285 * 1286 * MPSAFE 1287 * 1288 * Helper function for vm_mmap. Perform sanity check specific for mmap 1289 * operations on cdevs. 1290 */ 1291 int 1292 vm_mmap_cdev(struct thread *td, vm_size_t objsize, 1293 vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp, 1294 struct cdev *cdev, vm_ooffset_t *foff, vm_object_t *objp) 1295 { 1296 vm_object_t obj; 1297 struct cdevsw *dsw; 1298 int error, flags, ref; 1299 1300 flags = *flagsp; 1301 1302 dsw = dev_refthread(cdev, &ref); 1303 if (dsw == NULL) 1304 return (ENXIO); 1305 if (dsw->d_flags & D_MMAP_ANON) { 1306 dev_relthread(cdev, ref); 1307 *maxprotp = VM_PROT_ALL; 1308 *flagsp |= MAP_ANON; 1309 return (0); 1310 } 1311 /* 1312 * cdevs do not provide private mappings of any kind. 1313 */ 1314 if ((*maxprotp & VM_PROT_WRITE) == 0 && 1315 (prot & PROT_WRITE) != 0) { 1316 dev_relthread(cdev, ref); 1317 return (EACCES); 1318 } 1319 if (flags & (MAP_PRIVATE|MAP_COPY)) { 1320 dev_relthread(cdev, ref); 1321 return (EINVAL); 1322 } 1323 /* 1324 * Force device mappings to be shared. 1325 */ 1326 flags |= MAP_SHARED; 1327 #ifdef MAC_XXX 1328 error = mac_cdev_check_mmap(td->td_ucred, cdev, prot); 1329 if (error != 0) { 1330 dev_relthread(cdev, ref); 1331 return (error); 1332 } 1333 #endif 1334 /* 1335 * First, try d_mmap_single(). If that is not implemented 1336 * (returns ENODEV), fall back to using the device pager. 1337 * Note that d_mmap_single() must return a reference to the 1338 * object (it needs to bump the reference count of the object 1339 * it returns somehow). 1340 * 1341 * XXX assumes VM_PROT_* == PROT_* 1342 */ 1343 error = dsw->d_mmap_single(cdev, foff, objsize, objp, (int)prot); 1344 dev_relthread(cdev, ref); 1345 if (error != ENODEV) 1346 return (error); 1347 obj = vm_pager_allocate(OBJT_DEVICE, cdev, objsize, prot, *foff, 1348 td->td_ucred); 1349 if (obj == NULL) 1350 return (EINVAL); 1351 *objp = obj; 1352 *flagsp = flags; 1353 return (0); 1354 } 1355 1356 /* 1357 * vm_mmap_shm() 1358 * 1359 * MPSAFE 1360 * 1361 * Helper function for vm_mmap. Perform sanity check specific for mmap 1362 * operations on shm file descriptors. 1363 */ 1364 int 1365 vm_mmap_shm(struct thread *td, vm_size_t objsize, 1366 vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp, 1367 struct shmfd *shmfd, vm_ooffset_t foff, vm_object_t *objp) 1368 { 1369 int error; 1370 1371 if ((*flagsp & MAP_SHARED) != 0 && 1372 (*maxprotp & VM_PROT_WRITE) == 0 && 1373 (prot & PROT_WRITE) != 0) 1374 return (EACCES); 1375 #ifdef MAC 1376 error = mac_posixshm_check_mmap(td->td_ucred, shmfd, prot, *flagsp); 1377 if (error != 0) 1378 return (error); 1379 #endif 1380 error = shm_mmap(shmfd, objsize, foff, objp); 1381 if (error) 1382 return (error); 1383 return (0); 1384 } 1385 1386 /* 1387 * vm_mmap() 1388 * 1389 * MPSAFE 1390 * 1391 * Internal version of mmap. Currently used by mmap, exec, and sys5 1392 * shared memory. Handle is either a vnode pointer or NULL for MAP_ANON. 1393 */ 1394 int 1395 vm_mmap(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot, 1396 vm_prot_t maxprot, int flags, 1397 objtype_t handle_type, void *handle, 1398 vm_ooffset_t foff) 1399 { 1400 boolean_t fitit; 1401 vm_object_t object = NULL; 1402 int rv = KERN_SUCCESS; 1403 int docow, error; 1404 struct thread *td = curthread; 1405 1406 if (size == 0) 1407 return (0); 1408 1409 size = round_page(size); 1410 1411 PROC_LOCK(td->td_proc); 1412 if (td->td_proc->p_vmspace->vm_map.size + size > 1413 lim_cur(td->td_proc, RLIMIT_VMEM)) { 1414 PROC_UNLOCK(td->td_proc); 1415 return(ENOMEM); 1416 } 1417 if (racct_set(td->td_proc, RACCT_VMEM, 1418 td->td_proc->p_vmspace->vm_map.size + size)) { 1419 PROC_UNLOCK(td->td_proc); 1420 return (ENOMEM); 1421 } 1422 PROC_UNLOCK(td->td_proc); 1423 1424 /* 1425 * We currently can only deal with page aligned file offsets. 1426 * The check is here rather than in the syscall because the 1427 * kernel calls this function internally for other mmaping 1428 * operations (such as in exec) and non-aligned offsets will 1429 * cause pmap inconsistencies...so we want to be sure to 1430 * disallow this in all cases. 1431 */ 1432 if (foff & PAGE_MASK) 1433 return (EINVAL); 1434 1435 if ((flags & MAP_FIXED) == 0) { 1436 fitit = TRUE; 1437 *addr = round_page(*addr); 1438 } else { 1439 if (*addr != trunc_page(*addr)) 1440 return (EINVAL); 1441 fitit = FALSE; 1442 } 1443 /* 1444 * Lookup/allocate object. 1445 */ 1446 switch (handle_type) { 1447 case OBJT_DEVICE: 1448 error = vm_mmap_cdev(td, size, prot, &maxprot, &flags, 1449 handle, &foff, &object); 1450 break; 1451 case OBJT_VNODE: 1452 error = vm_mmap_vnode(td, size, prot, &maxprot, &flags, 1453 handle, &foff, &object); 1454 break; 1455 case OBJT_SWAP: 1456 error = vm_mmap_shm(td, size, prot, &maxprot, &flags, 1457 handle, foff, &object); 1458 break; 1459 case OBJT_DEFAULT: 1460 if (handle == NULL) { 1461 error = 0; 1462 break; 1463 } 1464 /* FALLTHROUGH */ 1465 default: 1466 error = EINVAL; 1467 break; 1468 } 1469 if (error) 1470 return (error); 1471 if (flags & MAP_ANON) { 1472 object = NULL; 1473 docow = 0; 1474 /* 1475 * Unnamed anonymous regions always start at 0. 1476 */ 1477 if (handle == 0) 1478 foff = 0; 1479 } else if (flags & MAP_PREFAULT_READ) 1480 docow = MAP_PREFAULT; 1481 else 1482 docow = MAP_PREFAULT_PARTIAL; 1483 1484 if ((flags & (MAP_ANON|MAP_SHARED)) == 0) 1485 docow |= MAP_COPY_ON_WRITE; 1486 if (flags & MAP_NOSYNC) 1487 docow |= MAP_DISABLE_SYNCER; 1488 if (flags & MAP_NOCORE) 1489 docow |= MAP_DISABLE_COREDUMP; 1490 1491 if (flags & MAP_STACK) 1492 rv = vm_map_stack(map, *addr, size, prot, maxprot, 1493 docow | MAP_STACK_GROWS_DOWN); 1494 else if (fitit) 1495 rv = vm_map_find(map, object, foff, addr, size, 1496 object != NULL && object->type == OBJT_DEVICE ? 1497 VMFS_ALIGNED_SPACE : VMFS_ANY_SPACE, prot, maxprot, docow); 1498 else 1499 rv = vm_map_fixed(map, object, foff, *addr, size, 1500 prot, maxprot, docow); 1501 1502 if (rv != KERN_SUCCESS) { 1503 /* 1504 * Lose the object reference. Will destroy the 1505 * object if it's an unnamed anonymous mapping 1506 * or named anonymous without other references. 1507 */ 1508 vm_object_deallocate(object); 1509 } else if (flags & MAP_SHARED) { 1510 /* 1511 * Shared memory is also shared with children. 1512 */ 1513 rv = vm_map_inherit(map, *addr, *addr + size, VM_INHERIT_SHARE); 1514 if (rv != KERN_SUCCESS) 1515 (void) vm_map_remove(map, *addr, *addr + size); 1516 } 1517 1518 /* 1519 * If the process has requested that all future mappings 1520 * be wired, then heed this. 1521 */ 1522 if ((rv == KERN_SUCCESS) && (map->flags & MAP_WIREFUTURE)) 1523 vm_map_wire(map, *addr, *addr + size, 1524 VM_MAP_WIRE_USER|VM_MAP_WIRE_NOHOLES); 1525 1526 switch (rv) { 1527 case KERN_SUCCESS: 1528 return (0); 1529 case KERN_INVALID_ADDRESS: 1530 case KERN_NO_SPACE: 1531 return (ENOMEM); 1532 case KERN_PROTECTION_FAILURE: 1533 return (EACCES); 1534 default: 1535 return (EINVAL); 1536 } 1537 } 1538