1 /*- 2 * Copyright (c) 1988 University of Utah. 3 * Copyright (c) 1991, 1993 4 * The Regents of the University of California. All rights reserved. 5 * 6 * This code is derived from software contributed to Berkeley by 7 * the Systems Programming Group of the University of Utah Computer 8 * Science Department. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 4. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * from: Utah $Hdr: vm_mmap.c 1.6 91/10/21$ 35 * 36 * @(#)vm_mmap.c 8.4 (Berkeley) 1/12/94 37 */ 38 39 /* 40 * Mapped file (mmap) interface to VM 41 */ 42 43 #include <sys/cdefs.h> 44 __FBSDID("$FreeBSD$"); 45 46 #include "opt_compat.h" 47 #include "opt_hwpmc_hooks.h" 48 #include "opt_mac.h" 49 50 #include <sys/param.h> 51 #include <sys/systm.h> 52 #include <sys/kernel.h> 53 #include <sys/lock.h> 54 #include <sys/mutex.h> 55 #include <sys/sysproto.h> 56 #include <sys/filedesc.h> 57 #include <sys/priv.h> 58 #include <sys/proc.h> 59 #include <sys/resource.h> 60 #include <sys/resourcevar.h> 61 #include <sys/vnode.h> 62 #include <sys/fcntl.h> 63 #include <sys/file.h> 64 #include <sys/mman.h> 65 #include <sys/mount.h> 66 #include <sys/conf.h> 67 #include <sys/stat.h> 68 #include <sys/vmmeter.h> 69 #include <sys/sysctl.h> 70 71 #include <security/mac/mac_framework.h> 72 73 #include <vm/vm.h> 74 #include <vm/vm_param.h> 75 #include <vm/pmap.h> 76 #include <vm/vm_map.h> 77 #include <vm/vm_object.h> 78 #include <vm/vm_page.h> 79 #include <vm/vm_pager.h> 80 #include <vm/vm_pageout.h> 81 #include <vm/vm_extern.h> 82 #include <vm/vm_page.h> 83 #include <vm/vm_kern.h> 84 85 #ifdef HWPMC_HOOKS 86 #include <sys/pmckern.h> 87 #endif 88 89 #ifndef _SYS_SYSPROTO_H_ 90 struct sbrk_args { 91 int incr; 92 }; 93 #endif 94 95 static int max_proc_mmap; 96 SYSCTL_INT(_vm, OID_AUTO, max_proc_mmap, CTLFLAG_RW, &max_proc_mmap, 0, ""); 97 98 /* 99 * Set the maximum number of vm_map_entry structures per process. Roughly 100 * speaking vm_map_entry structures are tiny, so allowing them to eat 1/100 101 * of our KVM malloc space still results in generous limits. We want a 102 * default that is good enough to prevent the kernel running out of resources 103 * if attacked from compromised user account but generous enough such that 104 * multi-threaded processes are not unduly inconvenienced. 105 */ 106 static void vmmapentry_rsrc_init(void *); 107 SYSINIT(vmmersrc, SI_SUB_KVM_RSRC, SI_ORDER_FIRST, vmmapentry_rsrc_init, NULL) 108 109 static void 110 vmmapentry_rsrc_init(dummy) 111 void *dummy; 112 { 113 max_proc_mmap = vm_kmem_size / sizeof(struct vm_map_entry); 114 max_proc_mmap /= 100; 115 } 116 117 static int vm_mmap_vnode(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *, 118 int *, struct vnode *, vm_ooffset_t, vm_object_t *); 119 static int vm_mmap_cdev(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *, 120 int *, struct cdev *, vm_ooffset_t, vm_object_t *); 121 static int vm_mmap_shm(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *, 122 int *, struct shmfd *, vm_ooffset_t, vm_object_t *); 123 124 /* 125 * MPSAFE 126 */ 127 /* ARGSUSED */ 128 int 129 sbrk(td, uap) 130 struct thread *td; 131 struct sbrk_args *uap; 132 { 133 /* Not yet implemented */ 134 return (EOPNOTSUPP); 135 } 136 137 #ifndef _SYS_SYSPROTO_H_ 138 struct sstk_args { 139 int incr; 140 }; 141 #endif 142 143 /* 144 * MPSAFE 145 */ 146 /* ARGSUSED */ 147 int 148 sstk(td, uap) 149 struct thread *td; 150 struct sstk_args *uap; 151 { 152 /* Not yet implemented */ 153 return (EOPNOTSUPP); 154 } 155 156 #if defined(COMPAT_43) 157 #ifndef _SYS_SYSPROTO_H_ 158 struct getpagesize_args { 159 int dummy; 160 }; 161 #endif 162 163 /* ARGSUSED */ 164 int 165 ogetpagesize(td, uap) 166 struct thread *td; 167 struct getpagesize_args *uap; 168 { 169 /* MP SAFE */ 170 td->td_retval[0] = PAGE_SIZE; 171 return (0); 172 } 173 #endif /* COMPAT_43 */ 174 175 176 /* 177 * Memory Map (mmap) system call. Note that the file offset 178 * and address are allowed to be NOT page aligned, though if 179 * the MAP_FIXED flag it set, both must have the same remainder 180 * modulo the PAGE_SIZE (POSIX 1003.1b). If the address is not 181 * page-aligned, the actual mapping starts at trunc_page(addr) 182 * and the return value is adjusted up by the page offset. 183 * 184 * Generally speaking, only character devices which are themselves 185 * memory-based, such as a video framebuffer, can be mmap'd. Otherwise 186 * there would be no cache coherency between a descriptor and a VM mapping 187 * both to the same character device. 188 * 189 * Block devices can be mmap'd no matter what they represent. Cache coherency 190 * is maintained as long as you do not write directly to the underlying 191 * character device. 192 */ 193 #ifndef _SYS_SYSPROTO_H_ 194 struct mmap_args { 195 void *addr; 196 size_t len; 197 int prot; 198 int flags; 199 int fd; 200 long pad; 201 off_t pos; 202 }; 203 #endif 204 205 /* 206 * MPSAFE 207 */ 208 int 209 mmap(td, uap) 210 struct thread *td; 211 struct mmap_args *uap; 212 { 213 #ifdef HWPMC_HOOKS 214 struct pmckern_map_in pkm; 215 #endif 216 struct file *fp; 217 struct vnode *vp; 218 vm_offset_t addr; 219 vm_size_t size, pageoff; 220 vm_prot_t prot, maxprot; 221 void *handle; 222 objtype_t handle_type; 223 int flags, error; 224 off_t pos; 225 struct vmspace *vms = td->td_proc->p_vmspace; 226 227 addr = (vm_offset_t) uap->addr; 228 size = uap->len; 229 prot = uap->prot & VM_PROT_ALL; 230 flags = uap->flags; 231 pos = uap->pos; 232 233 fp = NULL; 234 /* make sure mapping fits into numeric range etc */ 235 if ((ssize_t) uap->len < 0 || 236 ((flags & MAP_ANON) && uap->fd != -1)) 237 return (EINVAL); 238 239 if (flags & MAP_STACK) { 240 if ((uap->fd != -1) || 241 ((prot & (PROT_READ | PROT_WRITE)) != (PROT_READ | PROT_WRITE))) 242 return (EINVAL); 243 flags |= MAP_ANON; 244 pos = 0; 245 } 246 247 /* 248 * Align the file position to a page boundary, 249 * and save its page offset component. 250 */ 251 pageoff = (pos & PAGE_MASK); 252 pos -= pageoff; 253 254 /* Adjust size for rounding (on both ends). */ 255 size += pageoff; /* low end... */ 256 size = (vm_size_t) round_page(size); /* hi end */ 257 258 /* 259 * Check for illegal addresses. Watch out for address wrap... Note 260 * that VM_*_ADDRESS are not constants due to casts (argh). 261 */ 262 if (flags & MAP_FIXED) { 263 /* 264 * The specified address must have the same remainder 265 * as the file offset taken modulo PAGE_SIZE, so it 266 * should be aligned after adjustment by pageoff. 267 */ 268 addr -= pageoff; 269 if (addr & PAGE_MASK) 270 return (EINVAL); 271 /* Address range must be all in user VM space. */ 272 if (addr < vm_map_min(&vms->vm_map) || 273 addr + size > vm_map_max(&vms->vm_map)) 274 return (EINVAL); 275 if (addr + size < addr) 276 return (EINVAL); 277 } else { 278 /* 279 * XXX for non-fixed mappings where no hint is provided or 280 * the hint would fall in the potential heap space, 281 * place it after the end of the largest possible heap. 282 * 283 * There should really be a pmap call to determine a reasonable 284 * location. 285 */ 286 PROC_LOCK(td->td_proc); 287 if (addr == 0 || 288 (addr >= round_page((vm_offset_t)vms->vm_taddr) && 289 addr < round_page((vm_offset_t)vms->vm_daddr + 290 lim_max(td->td_proc, RLIMIT_DATA)))) 291 addr = round_page((vm_offset_t)vms->vm_daddr + 292 lim_max(td->td_proc, RLIMIT_DATA)); 293 PROC_UNLOCK(td->td_proc); 294 } 295 if (flags & MAP_ANON) { 296 /* 297 * Mapping blank space is trivial. 298 */ 299 handle = NULL; 300 handle_type = OBJT_DEFAULT; 301 maxprot = VM_PROT_ALL; 302 pos = 0; 303 } else { 304 /* 305 * Mapping file, get fp for validation and 306 * don't let the descriptor disappear on us if we block. 307 */ 308 if ((error = fget(td, uap->fd, &fp)) != 0) 309 goto done; 310 if (fp->f_type == DTYPE_SHM) { 311 handle = fp->f_data; 312 handle_type = OBJT_SWAP; 313 maxprot = VM_PROT_NONE; 314 315 /* FREAD should always be set. */ 316 if (fp->f_flag & FREAD) 317 maxprot |= VM_PROT_EXECUTE | VM_PROT_READ; 318 if (fp->f_flag & FWRITE) 319 maxprot |= VM_PROT_WRITE; 320 goto map; 321 } 322 if (fp->f_type != DTYPE_VNODE) { 323 error = ENODEV; 324 goto done; 325 } 326 #if defined(COMPAT_FREEBSD7) || defined(COMPAT_FREEBSD6) || \ 327 defined(COMPAT_FREEBSD5) || defined(COMPAT_FREEBSD4) 328 /* 329 * POSIX shared-memory objects are defined to have 330 * kernel persistence, and are not defined to support 331 * read(2)/write(2) -- or even open(2). Thus, we can 332 * use MAP_ASYNC to trade on-disk coherence for speed. 333 * The shm_open(3) library routine turns on the FPOSIXSHM 334 * flag to request this behavior. 335 */ 336 if (fp->f_flag & FPOSIXSHM) 337 flags |= MAP_NOSYNC; 338 #endif 339 vp = fp->f_vnode; 340 /* 341 * Ensure that file and memory protections are 342 * compatible. Note that we only worry about 343 * writability if mapping is shared; in this case, 344 * current and max prot are dictated by the open file. 345 * XXX use the vnode instead? Problem is: what 346 * credentials do we use for determination? What if 347 * proc does a setuid? 348 */ 349 if (vp->v_mount != NULL && vp->v_mount->mnt_flag & MNT_NOEXEC) 350 maxprot = VM_PROT_NONE; 351 else 352 maxprot = VM_PROT_EXECUTE; 353 if (fp->f_flag & FREAD) { 354 maxprot |= VM_PROT_READ; 355 } else if (prot & PROT_READ) { 356 error = EACCES; 357 goto done; 358 } 359 /* 360 * If we are sharing potential changes (either via 361 * MAP_SHARED or via the implicit sharing of character 362 * device mappings), and we are trying to get write 363 * permission although we opened it without asking 364 * for it, bail out. 365 */ 366 if ((flags & MAP_SHARED) != 0) { 367 if ((fp->f_flag & FWRITE) != 0) { 368 maxprot |= VM_PROT_WRITE; 369 } else if ((prot & PROT_WRITE) != 0) { 370 error = EACCES; 371 goto done; 372 } 373 } else if (vp->v_type != VCHR || (fp->f_flag & FWRITE) != 0) { 374 maxprot |= VM_PROT_WRITE; 375 } 376 handle = (void *)vp; 377 handle_type = OBJT_VNODE; 378 } 379 map: 380 381 /* 382 * Do not allow more then a certain number of vm_map_entry structures 383 * per process. Scale with the number of rforks sharing the map 384 * to make the limit reasonable for threads. 385 */ 386 if (max_proc_mmap && 387 vms->vm_map.nentries >= max_proc_mmap * vms->vm_refcnt) { 388 error = ENOMEM; 389 goto done; 390 } 391 392 error = vm_mmap(&vms->vm_map, &addr, size, prot, maxprot, 393 flags, handle_type, handle, pos); 394 #ifdef HWPMC_HOOKS 395 /* inform hwpmc(4) if an executable is being mapped */ 396 if (error == 0 && handle_type == OBJT_VNODE && 397 (prot & PROT_EXEC)) { 398 pkm.pm_file = handle; 399 pkm.pm_address = (uintptr_t) addr; 400 PMC_CALL_HOOK(td, PMC_FN_MMAP, (void *) &pkm); 401 } 402 #endif 403 if (error == 0) 404 td->td_retval[0] = (register_t) (addr + pageoff); 405 done: 406 if (fp) 407 fdrop(fp, td); 408 409 return (error); 410 } 411 412 int 413 freebsd6_mmap(struct thread *td, struct freebsd6_mmap_args *uap) 414 { 415 struct mmap_args oargs; 416 417 oargs.addr = uap->addr; 418 oargs.len = uap->len; 419 oargs.prot = uap->prot; 420 oargs.flags = uap->flags; 421 oargs.fd = uap->fd; 422 oargs.pos = uap->pos; 423 return (mmap(td, &oargs)); 424 } 425 426 #ifdef COMPAT_43 427 #ifndef _SYS_SYSPROTO_H_ 428 struct ommap_args { 429 caddr_t addr; 430 int len; 431 int prot; 432 int flags; 433 int fd; 434 long pos; 435 }; 436 #endif 437 int 438 ommap(td, uap) 439 struct thread *td; 440 struct ommap_args *uap; 441 { 442 struct mmap_args nargs; 443 static const char cvtbsdprot[8] = { 444 0, 445 PROT_EXEC, 446 PROT_WRITE, 447 PROT_EXEC | PROT_WRITE, 448 PROT_READ, 449 PROT_EXEC | PROT_READ, 450 PROT_WRITE | PROT_READ, 451 PROT_EXEC | PROT_WRITE | PROT_READ, 452 }; 453 454 #define OMAP_ANON 0x0002 455 #define OMAP_COPY 0x0020 456 #define OMAP_SHARED 0x0010 457 #define OMAP_FIXED 0x0100 458 459 nargs.addr = uap->addr; 460 nargs.len = uap->len; 461 nargs.prot = cvtbsdprot[uap->prot & 0x7]; 462 nargs.flags = 0; 463 if (uap->flags & OMAP_ANON) 464 nargs.flags |= MAP_ANON; 465 if (uap->flags & OMAP_COPY) 466 nargs.flags |= MAP_COPY; 467 if (uap->flags & OMAP_SHARED) 468 nargs.flags |= MAP_SHARED; 469 else 470 nargs.flags |= MAP_PRIVATE; 471 if (uap->flags & OMAP_FIXED) 472 nargs.flags |= MAP_FIXED; 473 nargs.fd = uap->fd; 474 nargs.pos = uap->pos; 475 return (mmap(td, &nargs)); 476 } 477 #endif /* COMPAT_43 */ 478 479 480 #ifndef _SYS_SYSPROTO_H_ 481 struct msync_args { 482 void *addr; 483 size_t len; 484 int flags; 485 }; 486 #endif 487 /* 488 * MPSAFE 489 */ 490 int 491 msync(td, uap) 492 struct thread *td; 493 struct msync_args *uap; 494 { 495 vm_offset_t addr; 496 vm_size_t size, pageoff; 497 int flags; 498 vm_map_t map; 499 int rv; 500 501 addr = (vm_offset_t) uap->addr; 502 size = uap->len; 503 flags = uap->flags; 504 505 pageoff = (addr & PAGE_MASK); 506 addr -= pageoff; 507 size += pageoff; 508 size = (vm_size_t) round_page(size); 509 if (addr + size < addr) 510 return (EINVAL); 511 512 if ((flags & (MS_ASYNC|MS_INVALIDATE)) == (MS_ASYNC|MS_INVALIDATE)) 513 return (EINVAL); 514 515 map = &td->td_proc->p_vmspace->vm_map; 516 517 /* 518 * Clean the pages and interpret the return value. 519 */ 520 rv = vm_map_sync(map, addr, addr + size, (flags & MS_ASYNC) == 0, 521 (flags & MS_INVALIDATE) != 0); 522 switch (rv) { 523 case KERN_SUCCESS: 524 return (0); 525 case KERN_INVALID_ADDRESS: 526 return (EINVAL); /* Sun returns ENOMEM? */ 527 case KERN_INVALID_ARGUMENT: 528 return (EBUSY); 529 default: 530 return (EINVAL); 531 } 532 } 533 534 #ifndef _SYS_SYSPROTO_H_ 535 struct munmap_args { 536 void *addr; 537 size_t len; 538 }; 539 #endif 540 /* 541 * MPSAFE 542 */ 543 int 544 munmap(td, uap) 545 struct thread *td; 546 struct munmap_args *uap; 547 { 548 #ifdef HWPMC_HOOKS 549 struct pmckern_map_out pkm; 550 vm_map_entry_t entry; 551 #endif 552 vm_offset_t addr; 553 vm_size_t size, pageoff; 554 vm_map_t map; 555 556 addr = (vm_offset_t) uap->addr; 557 size = uap->len; 558 if (size == 0) 559 return (EINVAL); 560 561 pageoff = (addr & PAGE_MASK); 562 addr -= pageoff; 563 size += pageoff; 564 size = (vm_size_t) round_page(size); 565 if (addr + size < addr) 566 return (EINVAL); 567 568 /* 569 * Check for illegal addresses. Watch out for address wrap... 570 */ 571 map = &td->td_proc->p_vmspace->vm_map; 572 if (addr < vm_map_min(map) || addr + size > vm_map_max(map)) 573 return (EINVAL); 574 vm_map_lock(map); 575 /* 576 * Make sure entire range is allocated. 577 */ 578 if (!vm_map_check_protection(map, addr, addr + size, VM_PROT_NONE)) { 579 vm_map_unlock(map); 580 return (EINVAL); 581 } 582 #ifdef HWPMC_HOOKS 583 /* 584 * Inform hwpmc if the address range being unmapped contains 585 * an executable region. 586 */ 587 if (vm_map_lookup_entry(map, addr, &entry)) { 588 for (; 589 entry != &map->header && entry->start < addr + size; 590 entry = entry->next) { 591 if (vm_map_check_protection(map, entry->start, 592 entry->end, VM_PROT_EXECUTE) == TRUE) { 593 pkm.pm_address = (uintptr_t) addr; 594 pkm.pm_size = (size_t) size; 595 PMC_CALL_HOOK(td, PMC_FN_MUNMAP, 596 (void *) &pkm); 597 break; 598 } 599 } 600 } 601 #endif 602 /* returns nothing but KERN_SUCCESS anyway */ 603 vm_map_delete(map, addr, addr + size); 604 vm_map_unlock(map); 605 return (0); 606 } 607 608 #ifndef _SYS_SYSPROTO_H_ 609 struct mprotect_args { 610 const void *addr; 611 size_t len; 612 int prot; 613 }; 614 #endif 615 /* 616 * MPSAFE 617 */ 618 int 619 mprotect(td, uap) 620 struct thread *td; 621 struct mprotect_args *uap; 622 { 623 vm_offset_t addr; 624 vm_size_t size, pageoff; 625 vm_prot_t prot; 626 627 addr = (vm_offset_t) uap->addr; 628 size = uap->len; 629 prot = uap->prot & VM_PROT_ALL; 630 #if defined(VM_PROT_READ_IS_EXEC) 631 if (prot & VM_PROT_READ) 632 prot |= VM_PROT_EXECUTE; 633 #endif 634 635 pageoff = (addr & PAGE_MASK); 636 addr -= pageoff; 637 size += pageoff; 638 size = (vm_size_t) round_page(size); 639 if (addr + size < addr) 640 return (EINVAL); 641 642 switch (vm_map_protect(&td->td_proc->p_vmspace->vm_map, addr, 643 addr + size, prot, FALSE)) { 644 case KERN_SUCCESS: 645 return (0); 646 case KERN_PROTECTION_FAILURE: 647 return (EACCES); 648 } 649 return (EINVAL); 650 } 651 652 #ifndef _SYS_SYSPROTO_H_ 653 struct minherit_args { 654 void *addr; 655 size_t len; 656 int inherit; 657 }; 658 #endif 659 /* 660 * MPSAFE 661 */ 662 int 663 minherit(td, uap) 664 struct thread *td; 665 struct minherit_args *uap; 666 { 667 vm_offset_t addr; 668 vm_size_t size, pageoff; 669 vm_inherit_t inherit; 670 671 addr = (vm_offset_t)uap->addr; 672 size = uap->len; 673 inherit = uap->inherit; 674 675 pageoff = (addr & PAGE_MASK); 676 addr -= pageoff; 677 size += pageoff; 678 size = (vm_size_t) round_page(size); 679 if (addr + size < addr) 680 return (EINVAL); 681 682 switch (vm_map_inherit(&td->td_proc->p_vmspace->vm_map, addr, 683 addr + size, inherit)) { 684 case KERN_SUCCESS: 685 return (0); 686 case KERN_PROTECTION_FAILURE: 687 return (EACCES); 688 } 689 return (EINVAL); 690 } 691 692 #ifndef _SYS_SYSPROTO_H_ 693 struct madvise_args { 694 void *addr; 695 size_t len; 696 int behav; 697 }; 698 #endif 699 700 /* 701 * MPSAFE 702 */ 703 /* ARGSUSED */ 704 int 705 madvise(td, uap) 706 struct thread *td; 707 struct madvise_args *uap; 708 { 709 vm_offset_t start, end; 710 vm_map_t map; 711 struct proc *p; 712 int error; 713 714 /* 715 * Check for our special case, advising the swap pager we are 716 * "immortal." 717 */ 718 if (uap->behav == MADV_PROTECT) { 719 error = priv_check(td, PRIV_VM_MADV_PROTECT); 720 if (error == 0) { 721 p = td->td_proc; 722 PROC_LOCK(p); 723 p->p_flag |= P_PROTECTED; 724 PROC_UNLOCK(p); 725 } 726 return (error); 727 } 728 /* 729 * Check for illegal behavior 730 */ 731 if (uap->behav < 0 || uap->behav > MADV_CORE) 732 return (EINVAL); 733 /* 734 * Check for illegal addresses. Watch out for address wrap... Note 735 * that VM_*_ADDRESS are not constants due to casts (argh). 736 */ 737 map = &td->td_proc->p_vmspace->vm_map; 738 if ((vm_offset_t)uap->addr < vm_map_min(map) || 739 (vm_offset_t)uap->addr + uap->len > vm_map_max(map)) 740 return (EINVAL); 741 if (((vm_offset_t) uap->addr + uap->len) < (vm_offset_t) uap->addr) 742 return (EINVAL); 743 744 /* 745 * Since this routine is only advisory, we default to conservative 746 * behavior. 747 */ 748 start = trunc_page((vm_offset_t) uap->addr); 749 end = round_page((vm_offset_t) uap->addr + uap->len); 750 751 if (vm_map_madvise(map, start, end, uap->behav)) 752 return (EINVAL); 753 return (0); 754 } 755 756 #ifndef _SYS_SYSPROTO_H_ 757 struct mincore_args { 758 const void *addr; 759 size_t len; 760 char *vec; 761 }; 762 #endif 763 764 /* 765 * MPSAFE 766 */ 767 /* ARGSUSED */ 768 int 769 mincore(td, uap) 770 struct thread *td; 771 struct mincore_args *uap; 772 { 773 vm_offset_t addr, first_addr; 774 vm_offset_t end, cend; 775 pmap_t pmap; 776 vm_map_t map; 777 char *vec; 778 int error = 0; 779 int vecindex, lastvecindex; 780 vm_map_entry_t current; 781 vm_map_entry_t entry; 782 int mincoreinfo; 783 unsigned int timestamp; 784 785 /* 786 * Make sure that the addresses presented are valid for user 787 * mode. 788 */ 789 first_addr = addr = trunc_page((vm_offset_t) uap->addr); 790 end = addr + (vm_size_t)round_page(uap->len); 791 map = &td->td_proc->p_vmspace->vm_map; 792 if (end > vm_map_max(map) || end < addr) 793 return (ENOMEM); 794 795 /* 796 * Address of byte vector 797 */ 798 vec = uap->vec; 799 800 pmap = vmspace_pmap(td->td_proc->p_vmspace); 801 802 vm_map_lock_read(map); 803 RestartScan: 804 timestamp = map->timestamp; 805 806 if (!vm_map_lookup_entry(map, addr, &entry)) { 807 vm_map_unlock_read(map); 808 return (ENOMEM); 809 } 810 811 /* 812 * Do this on a map entry basis so that if the pages are not 813 * in the current processes address space, we can easily look 814 * up the pages elsewhere. 815 */ 816 lastvecindex = -1; 817 for (current = entry; 818 (current != &map->header) && (current->start < end); 819 current = current->next) { 820 821 /* 822 * check for contiguity 823 */ 824 if (current->end < end && 825 (entry->next == &map->header || 826 current->next->start > current->end)) { 827 vm_map_unlock_read(map); 828 return (ENOMEM); 829 } 830 831 /* 832 * ignore submaps (for now) or null objects 833 */ 834 if ((current->eflags & MAP_ENTRY_IS_SUB_MAP) || 835 current->object.vm_object == NULL) 836 continue; 837 838 /* 839 * limit this scan to the current map entry and the 840 * limits for the mincore call 841 */ 842 if (addr < current->start) 843 addr = current->start; 844 cend = current->end; 845 if (cend > end) 846 cend = end; 847 848 /* 849 * scan this entry one page at a time 850 */ 851 while (addr < cend) { 852 /* 853 * Check pmap first, it is likely faster, also 854 * it can provide info as to whether we are the 855 * one referencing or modifying the page. 856 */ 857 mincoreinfo = pmap_mincore(pmap, addr); 858 if (!mincoreinfo) { 859 vm_pindex_t pindex; 860 vm_ooffset_t offset; 861 vm_page_t m; 862 /* 863 * calculate the page index into the object 864 */ 865 offset = current->offset + (addr - current->start); 866 pindex = OFF_TO_IDX(offset); 867 VM_OBJECT_LOCK(current->object.vm_object); 868 m = vm_page_lookup(current->object.vm_object, 869 pindex); 870 /* 871 * if the page is resident, then gather information about 872 * it. 873 */ 874 if (m != NULL && m->valid != 0) { 875 mincoreinfo = MINCORE_INCORE; 876 vm_page_lock_queues(); 877 if (m->dirty || 878 pmap_is_modified(m)) 879 mincoreinfo |= MINCORE_MODIFIED_OTHER; 880 if ((m->flags & PG_REFERENCED) || 881 pmap_ts_referenced(m)) { 882 vm_page_flag_set(m, PG_REFERENCED); 883 mincoreinfo |= MINCORE_REFERENCED_OTHER; 884 } 885 vm_page_unlock_queues(); 886 } 887 VM_OBJECT_UNLOCK(current->object.vm_object); 888 } 889 890 /* 891 * subyte may page fault. In case it needs to modify 892 * the map, we release the lock. 893 */ 894 vm_map_unlock_read(map); 895 896 /* 897 * calculate index into user supplied byte vector 898 */ 899 vecindex = OFF_TO_IDX(addr - first_addr); 900 901 /* 902 * If we have skipped map entries, we need to make sure that 903 * the byte vector is zeroed for those skipped entries. 904 */ 905 while ((lastvecindex + 1) < vecindex) { 906 error = subyte(vec + lastvecindex, 0); 907 if (error) { 908 error = EFAULT; 909 goto done2; 910 } 911 ++lastvecindex; 912 } 913 914 /* 915 * Pass the page information to the user 916 */ 917 error = subyte(vec + vecindex, mincoreinfo); 918 if (error) { 919 error = EFAULT; 920 goto done2; 921 } 922 923 /* 924 * If the map has changed, due to the subyte, the previous 925 * output may be invalid. 926 */ 927 vm_map_lock_read(map); 928 if (timestamp != map->timestamp) 929 goto RestartScan; 930 931 lastvecindex = vecindex; 932 addr += PAGE_SIZE; 933 } 934 } 935 936 /* 937 * subyte may page fault. In case it needs to modify 938 * the map, we release the lock. 939 */ 940 vm_map_unlock_read(map); 941 942 /* 943 * Zero the last entries in the byte vector. 944 */ 945 vecindex = OFF_TO_IDX(end - first_addr); 946 while ((lastvecindex + 1) < vecindex) { 947 error = subyte(vec + lastvecindex, 0); 948 if (error) { 949 error = EFAULT; 950 goto done2; 951 } 952 ++lastvecindex; 953 } 954 955 /* 956 * If the map has changed, due to the subyte, the previous 957 * output may be invalid. 958 */ 959 vm_map_lock_read(map); 960 if (timestamp != map->timestamp) 961 goto RestartScan; 962 vm_map_unlock_read(map); 963 done2: 964 return (error); 965 } 966 967 #ifndef _SYS_SYSPROTO_H_ 968 struct mlock_args { 969 const void *addr; 970 size_t len; 971 }; 972 #endif 973 /* 974 * MPSAFE 975 */ 976 int 977 mlock(td, uap) 978 struct thread *td; 979 struct mlock_args *uap; 980 { 981 struct proc *proc; 982 vm_offset_t addr, end, last, start; 983 vm_size_t npages, size; 984 int error; 985 986 error = priv_check(td, PRIV_VM_MLOCK); 987 if (error) 988 return (error); 989 addr = (vm_offset_t)uap->addr; 990 size = uap->len; 991 last = addr + size; 992 start = trunc_page(addr); 993 end = round_page(last); 994 if (last < addr || end < addr) 995 return (EINVAL); 996 npages = atop(end - start); 997 if (npages > vm_page_max_wired) 998 return (ENOMEM); 999 proc = td->td_proc; 1000 PROC_LOCK(proc); 1001 if (ptoa(npages + 1002 pmap_wired_count(vm_map_pmap(&proc->p_vmspace->vm_map))) > 1003 lim_cur(proc, RLIMIT_MEMLOCK)) { 1004 PROC_UNLOCK(proc); 1005 return (ENOMEM); 1006 } 1007 PROC_UNLOCK(proc); 1008 if (npages + cnt.v_wire_count > vm_page_max_wired) 1009 return (EAGAIN); 1010 error = vm_map_wire(&proc->p_vmspace->vm_map, start, end, 1011 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 1012 return (error == KERN_SUCCESS ? 0 : ENOMEM); 1013 } 1014 1015 #ifndef _SYS_SYSPROTO_H_ 1016 struct mlockall_args { 1017 int how; 1018 }; 1019 #endif 1020 1021 /* 1022 * MPSAFE 1023 */ 1024 int 1025 mlockall(td, uap) 1026 struct thread *td; 1027 struct mlockall_args *uap; 1028 { 1029 vm_map_t map; 1030 int error; 1031 1032 map = &td->td_proc->p_vmspace->vm_map; 1033 error = 0; 1034 1035 if ((uap->how == 0) || ((uap->how & ~(MCL_CURRENT|MCL_FUTURE)) != 0)) 1036 return (EINVAL); 1037 1038 #if 0 1039 /* 1040 * If wiring all pages in the process would cause it to exceed 1041 * a hard resource limit, return ENOMEM. 1042 */ 1043 PROC_LOCK(td->td_proc); 1044 if (map->size - ptoa(pmap_wired_count(vm_map_pmap(map)) > 1045 lim_cur(td->td_proc, RLIMIT_MEMLOCK))) { 1046 PROC_UNLOCK(td->td_proc); 1047 return (ENOMEM); 1048 } 1049 PROC_UNLOCK(td->td_proc); 1050 #else 1051 error = priv_check(td, PRIV_VM_MLOCK); 1052 if (error) 1053 return (error); 1054 #endif 1055 1056 if (uap->how & MCL_FUTURE) { 1057 vm_map_lock(map); 1058 vm_map_modflags(map, MAP_WIREFUTURE, 0); 1059 vm_map_unlock(map); 1060 error = 0; 1061 } 1062 1063 if (uap->how & MCL_CURRENT) { 1064 /* 1065 * P1003.1-2001 mandates that all currently mapped pages 1066 * will be memory resident and locked (wired) upon return 1067 * from mlockall(). vm_map_wire() will wire pages, by 1068 * calling vm_fault_wire() for each page in the region. 1069 */ 1070 error = vm_map_wire(map, vm_map_min(map), vm_map_max(map), 1071 VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK); 1072 error = (error == KERN_SUCCESS ? 0 : EAGAIN); 1073 } 1074 1075 return (error); 1076 } 1077 1078 #ifndef _SYS_SYSPROTO_H_ 1079 struct munlockall_args { 1080 register_t dummy; 1081 }; 1082 #endif 1083 1084 /* 1085 * MPSAFE 1086 */ 1087 int 1088 munlockall(td, uap) 1089 struct thread *td; 1090 struct munlockall_args *uap; 1091 { 1092 vm_map_t map; 1093 int error; 1094 1095 map = &td->td_proc->p_vmspace->vm_map; 1096 error = priv_check(td, PRIV_VM_MUNLOCK); 1097 if (error) 1098 return (error); 1099 1100 /* Clear the MAP_WIREFUTURE flag from this vm_map. */ 1101 vm_map_lock(map); 1102 vm_map_modflags(map, 0, MAP_WIREFUTURE); 1103 vm_map_unlock(map); 1104 1105 /* Forcibly unwire all pages. */ 1106 error = vm_map_unwire(map, vm_map_min(map), vm_map_max(map), 1107 VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK); 1108 1109 return (error); 1110 } 1111 1112 #ifndef _SYS_SYSPROTO_H_ 1113 struct munlock_args { 1114 const void *addr; 1115 size_t len; 1116 }; 1117 #endif 1118 /* 1119 * MPSAFE 1120 */ 1121 int 1122 munlock(td, uap) 1123 struct thread *td; 1124 struct munlock_args *uap; 1125 { 1126 vm_offset_t addr, end, last, start; 1127 vm_size_t size; 1128 int error; 1129 1130 error = priv_check(td, PRIV_VM_MUNLOCK); 1131 if (error) 1132 return (error); 1133 addr = (vm_offset_t)uap->addr; 1134 size = uap->len; 1135 last = addr + size; 1136 start = trunc_page(addr); 1137 end = round_page(last); 1138 if (last < addr || end < addr) 1139 return (EINVAL); 1140 error = vm_map_unwire(&td->td_proc->p_vmspace->vm_map, start, end, 1141 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 1142 return (error == KERN_SUCCESS ? 0 : ENOMEM); 1143 } 1144 1145 /* 1146 * vm_mmap_vnode() 1147 * 1148 * MPSAFE 1149 * 1150 * Helper function for vm_mmap. Perform sanity check specific for mmap 1151 * operations on vnodes. 1152 */ 1153 int 1154 vm_mmap_vnode(struct thread *td, vm_size_t objsize, 1155 vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp, 1156 struct vnode *vp, vm_ooffset_t foff, vm_object_t *objp) 1157 { 1158 struct vattr va; 1159 void *handle; 1160 vm_object_t obj; 1161 struct mount *mp; 1162 int error, flags, type; 1163 int vfslocked; 1164 1165 mp = vp->v_mount; 1166 vfslocked = VFS_LOCK_GIANT(mp); 1167 if ((error = vget(vp, LK_EXCLUSIVE, td)) != 0) { 1168 VFS_UNLOCK_GIANT(vfslocked); 1169 return (error); 1170 } 1171 flags = *flagsp; 1172 obj = vp->v_object; 1173 if (vp->v_type == VREG) { 1174 /* 1175 * Get the proper underlying object 1176 */ 1177 if (obj == NULL) { 1178 error = EINVAL; 1179 goto done; 1180 } 1181 if (obj->handle != vp) { 1182 vput(vp); 1183 vp = (struct vnode*)obj->handle; 1184 vget(vp, LK_EXCLUSIVE, td); 1185 } 1186 type = OBJT_VNODE; 1187 handle = vp; 1188 } else if (vp->v_type == VCHR) { 1189 type = OBJT_DEVICE; 1190 handle = vp->v_rdev; 1191 1192 /* XXX: lack thredref on device */ 1193 if(vp->v_rdev->si_devsw->d_flags & D_MMAP_ANON) { 1194 *maxprotp = VM_PROT_ALL; 1195 *flagsp |= MAP_ANON; 1196 error = 0; 1197 goto done; 1198 } 1199 /* 1200 * cdevs does not provide private mappings of any kind. 1201 */ 1202 if ((*maxprotp & VM_PROT_WRITE) == 0 && 1203 (prot & PROT_WRITE) != 0) { 1204 error = EACCES; 1205 goto done; 1206 } 1207 if (flags & (MAP_PRIVATE|MAP_COPY)) { 1208 error = EINVAL; 1209 goto done; 1210 } 1211 /* 1212 * Force device mappings to be shared. 1213 */ 1214 flags |= MAP_SHARED; 1215 } else { 1216 error = EINVAL; 1217 goto done; 1218 } 1219 if ((error = VOP_GETATTR(vp, &va, td->td_ucred, td))) { 1220 goto done; 1221 } 1222 #ifdef MAC 1223 error = mac_vnode_check_mmap(td->td_ucred, vp, prot, flags); 1224 if (error != 0) 1225 goto done; 1226 #endif 1227 if ((flags & MAP_SHARED) != 0) { 1228 if ((va.va_flags & (SF_SNAPSHOT|IMMUTABLE|APPEND)) != 0) { 1229 if (prot & PROT_WRITE) { 1230 error = EPERM; 1231 goto done; 1232 } 1233 *maxprotp &= ~VM_PROT_WRITE; 1234 } 1235 } 1236 /* 1237 * If it is a regular file without any references 1238 * we do not need to sync it. 1239 * Adjust object size to be the size of actual file. 1240 */ 1241 if (vp->v_type == VREG) { 1242 objsize = round_page(va.va_size); 1243 if (va.va_nlink == 0) 1244 flags |= MAP_NOSYNC; 1245 } 1246 obj = vm_pager_allocate(type, handle, objsize, prot, foff); 1247 if (obj == NULL) { 1248 error = (type == OBJT_DEVICE ? EINVAL : ENOMEM); 1249 goto done; 1250 } 1251 *objp = obj; 1252 *flagsp = flags; 1253 vfs_mark_atime(vp, td); 1254 1255 done: 1256 vput(vp); 1257 VFS_UNLOCK_GIANT(vfslocked); 1258 return (error); 1259 } 1260 1261 /* 1262 * vm_mmap_cdev() 1263 * 1264 * MPSAFE 1265 * 1266 * Helper function for vm_mmap. Perform sanity check specific for mmap 1267 * operations on cdevs. 1268 */ 1269 int 1270 vm_mmap_cdev(struct thread *td, vm_size_t objsize, 1271 vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp, 1272 struct cdev *cdev, vm_ooffset_t foff, vm_object_t *objp) 1273 { 1274 vm_object_t obj; 1275 int flags; 1276 1277 flags = *flagsp; 1278 1279 /* XXX: lack thredref on device */ 1280 if (cdev->si_devsw->d_flags & D_MMAP_ANON) { 1281 *maxprotp = VM_PROT_ALL; 1282 *flagsp |= MAP_ANON; 1283 return (0); 1284 } 1285 /* 1286 * cdevs does not provide private mappings of any kind. 1287 */ 1288 if ((*maxprotp & VM_PROT_WRITE) == 0 && 1289 (prot & PROT_WRITE) != 0) 1290 return (EACCES); 1291 if (flags & (MAP_PRIVATE|MAP_COPY)) 1292 return (EINVAL); 1293 /* 1294 * Force device mappings to be shared. 1295 */ 1296 flags |= MAP_SHARED; 1297 #ifdef MAC_XXX 1298 error = mac_check_cdev_mmap(td->td_ucred, cdev, prot); 1299 if (error != 0) 1300 return (error); 1301 #endif 1302 obj = vm_pager_allocate(OBJT_DEVICE, cdev, objsize, prot, foff); 1303 if (obj == NULL) 1304 return (EINVAL); 1305 *objp = obj; 1306 *flagsp = flags; 1307 return (0); 1308 } 1309 1310 /* 1311 * vm_mmap_shm() 1312 * 1313 * MPSAFE 1314 * 1315 * Helper function for vm_mmap. Perform sanity check specific for mmap 1316 * operations on shm file descriptors. 1317 */ 1318 int 1319 vm_mmap_shm(struct thread *td, vm_size_t objsize, 1320 vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp, 1321 struct shmfd *shmfd, vm_ooffset_t foff, vm_object_t *objp) 1322 { 1323 int error; 1324 1325 if ((*maxprotp & VM_PROT_WRITE) == 0 && 1326 (prot & PROT_WRITE) != 0) 1327 return (EACCES); 1328 #ifdef MAC 1329 error = mac_posixshm_check_mmap(td->td_ucred, shmfd, prot, *flagsp); 1330 if (error != 0) 1331 return (error); 1332 #endif 1333 error = shm_mmap(shmfd, objsize, foff, objp); 1334 if (error) 1335 return (error); 1336 return (0); 1337 } 1338 1339 /* 1340 * vm_mmap() 1341 * 1342 * MPSAFE 1343 * 1344 * Internal version of mmap. Currently used by mmap, exec, and sys5 1345 * shared memory. Handle is either a vnode pointer or NULL for MAP_ANON. 1346 */ 1347 int 1348 vm_mmap(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot, 1349 vm_prot_t maxprot, int flags, 1350 objtype_t handle_type, void *handle, 1351 vm_ooffset_t foff) 1352 { 1353 boolean_t fitit; 1354 vm_object_t object = NULL; 1355 int rv = KERN_SUCCESS; 1356 int docow, error; 1357 struct thread *td = curthread; 1358 1359 if (size == 0) 1360 return (0); 1361 1362 size = round_page(size); 1363 1364 PROC_LOCK(td->td_proc); 1365 if (td->td_proc->p_vmspace->vm_map.size + size > 1366 lim_cur(td->td_proc, RLIMIT_VMEM)) { 1367 PROC_UNLOCK(td->td_proc); 1368 return(ENOMEM); 1369 } 1370 PROC_UNLOCK(td->td_proc); 1371 1372 /* 1373 * We currently can only deal with page aligned file offsets. 1374 * The check is here rather than in the syscall because the 1375 * kernel calls this function internally for other mmaping 1376 * operations (such as in exec) and non-aligned offsets will 1377 * cause pmap inconsistencies...so we want to be sure to 1378 * disallow this in all cases. 1379 */ 1380 if (foff & PAGE_MASK) 1381 return (EINVAL); 1382 1383 if ((flags & MAP_FIXED) == 0) { 1384 fitit = TRUE; 1385 *addr = round_page(*addr); 1386 } else { 1387 if (*addr != trunc_page(*addr)) 1388 return (EINVAL); 1389 fitit = FALSE; 1390 } 1391 /* 1392 * Lookup/allocate object. 1393 */ 1394 switch (handle_type) { 1395 case OBJT_DEVICE: 1396 error = vm_mmap_cdev(td, size, prot, &maxprot, &flags, 1397 handle, foff, &object); 1398 break; 1399 case OBJT_VNODE: 1400 error = vm_mmap_vnode(td, size, prot, &maxprot, &flags, 1401 handle, foff, &object); 1402 break; 1403 case OBJT_SWAP: 1404 error = vm_mmap_shm(td, size, prot, &maxprot, &flags, 1405 handle, foff, &object); 1406 break; 1407 case OBJT_DEFAULT: 1408 if (handle == NULL) { 1409 error = 0; 1410 break; 1411 } 1412 /* FALLTHROUGH */ 1413 default: 1414 error = EINVAL; 1415 break; 1416 } 1417 if (error) 1418 return (error); 1419 if (flags & MAP_ANON) { 1420 object = NULL; 1421 docow = 0; 1422 /* 1423 * Unnamed anonymous regions always start at 0. 1424 */ 1425 if (handle == 0) 1426 foff = 0; 1427 } else { 1428 docow = MAP_PREFAULT_PARTIAL; 1429 } 1430 1431 if ((flags & (MAP_ANON|MAP_SHARED)) == 0) 1432 docow |= MAP_COPY_ON_WRITE; 1433 if (flags & MAP_NOSYNC) 1434 docow |= MAP_DISABLE_SYNCER; 1435 if (flags & MAP_NOCORE) 1436 docow |= MAP_DISABLE_COREDUMP; 1437 1438 #if defined(VM_PROT_READ_IS_EXEC) 1439 if (prot & VM_PROT_READ) 1440 prot |= VM_PROT_EXECUTE; 1441 1442 if (maxprot & VM_PROT_READ) 1443 maxprot |= VM_PROT_EXECUTE; 1444 #endif 1445 1446 if (fitit) 1447 *addr = pmap_addr_hint(object, *addr, size); 1448 1449 if (flags & MAP_STACK) 1450 rv = vm_map_stack(map, *addr, size, prot, maxprot, 1451 docow | MAP_STACK_GROWS_DOWN); 1452 else if (fitit) 1453 rv = vm_map_find(map, object, foff, addr, size, TRUE, 1454 prot, maxprot, docow); 1455 else 1456 rv = vm_map_fixed(map, object, foff, addr, size, 1457 prot, maxprot, docow); 1458 1459 if (rv != KERN_SUCCESS) { 1460 /* 1461 * Lose the object reference. Will destroy the 1462 * object if it's an unnamed anonymous mapping 1463 * or named anonymous without other references. 1464 */ 1465 vm_object_deallocate(object); 1466 } else if (flags & MAP_SHARED) { 1467 /* 1468 * Shared memory is also shared with children. 1469 */ 1470 rv = vm_map_inherit(map, *addr, *addr + size, VM_INHERIT_SHARE); 1471 if (rv != KERN_SUCCESS) 1472 (void) vm_map_remove(map, *addr, *addr + size); 1473 } 1474 1475 /* 1476 * If the process has requested that all future mappings 1477 * be wired, then heed this. 1478 */ 1479 if ((rv == KERN_SUCCESS) && (map->flags & MAP_WIREFUTURE)) 1480 vm_map_wire(map, *addr, *addr + size, 1481 VM_MAP_WIRE_USER|VM_MAP_WIRE_NOHOLES); 1482 1483 switch (rv) { 1484 case KERN_SUCCESS: 1485 return (0); 1486 case KERN_INVALID_ADDRESS: 1487 case KERN_NO_SPACE: 1488 return (ENOMEM); 1489 case KERN_PROTECTION_FAILURE: 1490 return (EACCES); 1491 default: 1492 return (EINVAL); 1493 } 1494 } 1495