1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * mm/mmap.c 4 * 5 * Written by obz. 6 * 7 * Address space accounting code <alan@lxorguk.ukuu.org.uk> 8 */ 9 10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 11 12 #include <linux/kernel.h> 13 #include <linux/slab.h> 14 #include <linux/backing-dev.h> 15 #include <linux/mm.h> 16 #include <linux/mm_inline.h> 17 #include <linux/shm.h> 18 #include <linux/mman.h> 19 #include <linux/pagemap.h> 20 #include <linux/swap.h> 21 #include <linux/syscalls.h> 22 #include <linux/capability.h> 23 #include <linux/init.h> 24 #include <linux/file.h> 25 #include <linux/fs.h> 26 #include <linux/personality.h> 27 #include <linux/security.h> 28 #include <linux/hugetlb.h> 29 #include <linux/shmem_fs.h> 30 #include <linux/profile.h> 31 #include <linux/export.h> 32 #include <linux/mount.h> 33 #include <linux/mempolicy.h> 34 #include <linux/rmap.h> 35 #include <linux/mmu_notifier.h> 36 #include <linux/mmdebug.h> 37 #include <linux/perf_event.h> 38 #include <linux/audit.h> 39 #include <linux/khugepaged.h> 40 #include <linux/uprobes.h> 41 #include <linux/notifier.h> 42 #include <linux/memory.h> 43 #include <linux/printk.h> 44 #include <linux/userfaultfd_k.h> 45 #include <linux/moduleparam.h> 46 #include <linux/pkeys.h> 47 #include <linux/oom.h> 48 #include <linux/sched/mm.h> 49 #include <linux/ksm.h> 50 #include <linux/memfd.h> 51 52 #include <linux/uaccess.h> 53 #include <asm/cacheflush.h> 54 #include <asm/tlb.h> 55 #include <asm/mmu_context.h> 56 57 #define CREATE_TRACE_POINTS 58 #include <trace/events/mmap.h> 59 60 #include "internal.h" 61 62 #ifndef arch_mmap_check 63 #define arch_mmap_check(addr, len, flags) (0) 64 #endif 65 66 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS 67 const int mmap_rnd_bits_min = CONFIG_ARCH_MMAP_RND_BITS_MIN; 68 int mmap_rnd_bits_max __ro_after_init = CONFIG_ARCH_MMAP_RND_BITS_MAX; 69 int mmap_rnd_bits __read_mostly = CONFIG_ARCH_MMAP_RND_BITS; 70 #endif 71 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS 72 const int mmap_rnd_compat_bits_min = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MIN; 73 const int mmap_rnd_compat_bits_max = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MAX; 74 int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS; 75 #endif 76 77 static bool ignore_rlimit_data; 78 core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644); 79 80 /* Update vma->vm_page_prot to reflect vma->vm_flags. */ 81 void vma_set_page_prot(struct vm_area_struct *vma) 82 { 83 vm_flags_t vm_flags = vma->vm_flags; 84 pgprot_t vm_page_prot; 85 86 vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags); 87 if (vma_wants_writenotify(vma, vm_page_prot)) { 88 vm_flags &= ~VM_SHARED; 89 vm_page_prot = vm_pgprot_modify(vm_page_prot, vm_flags); 90 } 91 /* remove_protection_ptes reads vma->vm_page_prot without mmap_lock */ 92 WRITE_ONCE(vma->vm_page_prot, vm_page_prot); 93 } 94 95 /* 96 * check_brk_limits() - Use platform specific check of range & verify mlock 97 * limits. 98 * @addr: The address to check 99 * @len: The size of increase. 100 * 101 * Return: 0 on success. 102 */ 103 static int check_brk_limits(unsigned long addr, unsigned long len) 104 { 105 unsigned long mapped_addr; 106 107 mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED); 108 if (IS_ERR_VALUE(mapped_addr)) 109 return mapped_addr; 110 111 return mlock_future_ok(current->mm, current->mm->def_flags, len) 112 ? 0 : -EAGAIN; 113 } 114 115 SYSCALL_DEFINE1(brk, unsigned long, brk) 116 { 117 unsigned long newbrk, oldbrk, origbrk; 118 struct mm_struct *mm = current->mm; 119 struct vm_area_struct *brkvma, *next = NULL; 120 unsigned long min_brk; 121 bool populate = false; 122 LIST_HEAD(uf); 123 struct vma_iterator vmi; 124 125 if (mmap_write_lock_killable(mm)) 126 return -EINTR; 127 128 origbrk = mm->brk; 129 130 min_brk = mm->start_brk; 131 #ifdef CONFIG_COMPAT_BRK 132 /* 133 * CONFIG_COMPAT_BRK can still be overridden by setting 134 * randomize_va_space to 2, which will still cause mm->start_brk 135 * to be arbitrarily shifted 136 */ 137 if (!current->brk_randomized) 138 min_brk = mm->end_data; 139 #endif 140 if (brk < min_brk) 141 goto out; 142 143 /* 144 * Check against rlimit here. If this check is done later after the test 145 * of oldbrk with newbrk then it can escape the test and let the data 146 * segment grow beyond its set limit the in case where the limit is 147 * not page aligned -Ram Gupta 148 */ 149 if (check_data_rlimit(rlimit(RLIMIT_DATA), brk, mm->start_brk, 150 mm->end_data, mm->start_data)) 151 goto out; 152 153 newbrk = PAGE_ALIGN(brk); 154 oldbrk = PAGE_ALIGN(mm->brk); 155 if (oldbrk == newbrk) { 156 mm->brk = brk; 157 goto success; 158 } 159 160 /* Always allow shrinking brk. */ 161 if (brk <= mm->brk) { 162 /* Search one past newbrk */ 163 vma_iter_init(&vmi, mm, newbrk); 164 brkvma = vma_find(&vmi, oldbrk); 165 if (!brkvma || brkvma->vm_start >= oldbrk) 166 goto out; /* mapping intersects with an existing non-brk vma. */ 167 /* 168 * mm->brk must be protected by write mmap_lock. 169 * do_vmi_align_munmap() will drop the lock on success, so 170 * update it before calling do_vma_munmap(). 171 */ 172 mm->brk = brk; 173 if (do_vmi_align_munmap(&vmi, brkvma, mm, newbrk, oldbrk, &uf, 174 /* unlock = */ true)) 175 goto out; 176 177 goto success_unlocked; 178 } 179 180 if (check_brk_limits(oldbrk, newbrk - oldbrk)) 181 goto out; 182 183 /* 184 * Only check if the next VMA is within the stack_guard_gap of the 185 * expansion area 186 */ 187 vma_iter_init(&vmi, mm, oldbrk); 188 next = vma_find(&vmi, newbrk + PAGE_SIZE + stack_guard_gap); 189 if (next && newbrk + PAGE_SIZE > vm_start_gap(next)) 190 goto out; 191 192 brkvma = vma_prev_limit(&vmi, mm->start_brk); 193 /* Ok, looks good - let it rip. */ 194 if (do_brk_flags(&vmi, brkvma, oldbrk, newbrk - oldbrk, 0) < 0) 195 goto out; 196 197 mm->brk = brk; 198 if (mm->def_flags & VM_LOCKED) 199 populate = true; 200 201 success: 202 mmap_write_unlock(mm); 203 success_unlocked: 204 userfaultfd_unmap_complete(mm, &uf); 205 if (populate) 206 mm_populate(oldbrk, newbrk - oldbrk); 207 return brk; 208 209 out: 210 mm->brk = origbrk; 211 mmap_write_unlock(mm); 212 return origbrk; 213 } 214 215 /* 216 * If a hint addr is less than mmap_min_addr change hint to be as 217 * low as possible but still greater than mmap_min_addr 218 */ 219 static inline unsigned long round_hint_to_min(unsigned long hint) 220 { 221 hint &= PAGE_MASK; 222 if (((void *)hint != NULL) && 223 (hint < mmap_min_addr)) 224 return PAGE_ALIGN(mmap_min_addr); 225 return hint; 226 } 227 228 bool mlock_future_ok(const struct mm_struct *mm, vm_flags_t vm_flags, 229 unsigned long bytes) 230 { 231 unsigned long locked_pages, limit_pages; 232 233 if (!(vm_flags & VM_LOCKED) || capable(CAP_IPC_LOCK)) 234 return true; 235 236 locked_pages = bytes >> PAGE_SHIFT; 237 locked_pages += mm->locked_vm; 238 239 limit_pages = rlimit(RLIMIT_MEMLOCK); 240 limit_pages >>= PAGE_SHIFT; 241 242 return locked_pages <= limit_pages; 243 } 244 245 static inline u64 file_mmap_size_max(struct file *file, struct inode *inode) 246 { 247 if (S_ISREG(inode->i_mode)) 248 return MAX_LFS_FILESIZE; 249 250 if (S_ISBLK(inode->i_mode)) 251 return MAX_LFS_FILESIZE; 252 253 if (S_ISSOCK(inode->i_mode)) 254 return MAX_LFS_FILESIZE; 255 256 /* Special "we do even unsigned file positions" case */ 257 if (file->f_op->fop_flags & FOP_UNSIGNED_OFFSET) 258 return 0; 259 260 /* Yes, random drivers might want more. But I'm tired of buggy drivers */ 261 return ULONG_MAX; 262 } 263 264 static inline bool file_mmap_ok(struct file *file, struct inode *inode, 265 unsigned long pgoff, unsigned long len) 266 { 267 u64 maxsize = file_mmap_size_max(file, inode); 268 269 if (maxsize && len > maxsize) 270 return false; 271 maxsize -= len; 272 if (pgoff > maxsize >> PAGE_SHIFT) 273 return false; 274 return true; 275 } 276 277 /** 278 * do_mmap() - Perform a userland memory mapping into the current process 279 * address space of length @len with protection bits @prot, mmap flags @flags 280 * (from which VMA flags will be inferred), and any additional VMA flags to 281 * apply @vm_flags. If this is a file-backed mapping then the file is specified 282 * in @file and page offset into the file via @pgoff. 283 * 284 * This function does not perform security checks on the file and assumes, if 285 * @uf is non-NULL, the caller has provided a list head to track unmap events 286 * for userfaultfd @uf. 287 * 288 * It also simply indicates whether memory population is required by setting 289 * @populate, which must be non-NULL, expecting the caller to actually perform 290 * this task itself if appropriate. 291 * 292 * This function will invoke architecture-specific (and if provided and 293 * relevant, file system-specific) logic to determine the most appropriate 294 * unmapped area in which to place the mapping if not MAP_FIXED. 295 * 296 * Callers which require userland mmap() behaviour should invoke vm_mmap(), 297 * which is also exported for module use. 298 * 299 * Those which require this behaviour less security checks, userfaultfd and 300 * populate behaviour, and who handle the mmap write lock themselves, should 301 * call this function. 302 * 303 * Note that the returned address may reside within a merged VMA if an 304 * appropriate merge were to take place, so it doesn't necessarily specify the 305 * start of a VMA, rather only the start of a valid mapped range of length 306 * @len bytes, rounded down to the nearest page size. 307 * 308 * The caller must write-lock current->mm->mmap_lock. 309 * 310 * @file: An optional struct file pointer describing the file which is to be 311 * mapped, if a file-backed mapping. 312 * @addr: If non-zero, hints at (or if @flags has MAP_FIXED set, specifies) the 313 * address at which to perform this mapping. See mmap (2) for details. Must be 314 * page-aligned. 315 * @len: The length of the mapping. Will be page-aligned and must be at least 1 316 * page in size. 317 * @prot: Protection bits describing access required to the mapping. See mmap 318 * (2) for details. 319 * @flags: Flags specifying how the mapping should be performed, see mmap (2) 320 * for details. 321 * @vm_flags: VMA flags which should be set by default, or 0 otherwise. 322 * @pgoff: Page offset into the @file if file-backed, should be 0 otherwise. 323 * @populate: A pointer to a value which will be set to 0 if no population of 324 * the range is required, or the number of bytes to populate if it is. Must be 325 * non-NULL. See mmap (2) for details as to under what circumstances population 326 * of the range occurs. 327 * @uf: An optional pointer to a list head to track userfaultfd unmap events 328 * should unmapping events arise. If provided, it is up to the caller to manage 329 * this. 330 * 331 * Returns: Either an error, or the address at which the requested mapping has 332 * been performed. 333 */ 334 unsigned long do_mmap(struct file *file, unsigned long addr, 335 unsigned long len, unsigned long prot, 336 unsigned long flags, vm_flags_t vm_flags, 337 unsigned long pgoff, unsigned long *populate, 338 struct list_head *uf) 339 { 340 struct mm_struct *mm = current->mm; 341 int pkey = 0; 342 343 *populate = 0; 344 345 mmap_assert_write_locked(mm); 346 347 if (!len) 348 return -EINVAL; 349 350 /* 351 * Does the application expect PROT_READ to imply PROT_EXEC? 352 * 353 * (the exception is when the underlying filesystem is noexec 354 * mounted, in which case we don't add PROT_EXEC.) 355 */ 356 if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC)) 357 if (!(file && path_noexec(&file->f_path))) 358 prot |= PROT_EXEC; 359 360 /* force arch specific MAP_FIXED handling in get_unmapped_area */ 361 if (flags & MAP_FIXED_NOREPLACE) 362 flags |= MAP_FIXED; 363 364 if (!(flags & MAP_FIXED)) 365 addr = round_hint_to_min(addr); 366 367 /* Careful about overflows.. */ 368 len = PAGE_ALIGN(len); 369 if (!len) 370 return -ENOMEM; 371 372 /* offset overflow? */ 373 if ((pgoff + (len >> PAGE_SHIFT)) < pgoff) 374 return -EOVERFLOW; 375 376 /* Too many mappings? */ 377 if (mm->map_count > sysctl_max_map_count) 378 return -ENOMEM; 379 380 /* 381 * addr is returned from get_unmapped_area, 382 * There are two cases: 383 * 1> MAP_FIXED == false 384 * unallocated memory, no need to check sealing. 385 * 1> MAP_FIXED == true 386 * sealing is checked inside mmap_region when 387 * do_vmi_munmap is called. 388 */ 389 390 if (prot == PROT_EXEC) { 391 pkey = execute_only_pkey(mm); 392 if (pkey < 0) 393 pkey = 0; 394 } 395 396 /* Do simple checking here so the lower-level routines won't have 397 * to. we assume access permissions have been handled by the open 398 * of the memory object, so we don't do any here. 399 */ 400 vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(file, flags) | 401 mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; 402 403 /* Obtain the address to map to. we verify (or select) it and ensure 404 * that it represents a valid section of the address space. 405 */ 406 addr = __get_unmapped_area(file, addr, len, pgoff, flags, vm_flags); 407 if (IS_ERR_VALUE(addr)) 408 return addr; 409 410 if (flags & MAP_FIXED_NOREPLACE) { 411 if (find_vma_intersection(mm, addr, addr + len)) 412 return -EEXIST; 413 } 414 415 if (flags & MAP_LOCKED) 416 if (!can_do_mlock()) 417 return -EPERM; 418 419 if (!mlock_future_ok(mm, vm_flags, len)) 420 return -EAGAIN; 421 422 if (file) { 423 struct inode *inode = file_inode(file); 424 unsigned long flags_mask; 425 int err; 426 427 if (!file_mmap_ok(file, inode, pgoff, len)) 428 return -EOVERFLOW; 429 430 flags_mask = LEGACY_MAP_MASK; 431 if (file->f_op->fop_flags & FOP_MMAP_SYNC) 432 flags_mask |= MAP_SYNC; 433 434 switch (flags & MAP_TYPE) { 435 case MAP_SHARED: 436 /* 437 * Force use of MAP_SHARED_VALIDATE with non-legacy 438 * flags. E.g. MAP_SYNC is dangerous to use with 439 * MAP_SHARED as you don't know which consistency model 440 * you will get. We silently ignore unsupported flags 441 * with MAP_SHARED to preserve backward compatibility. 442 */ 443 flags &= LEGACY_MAP_MASK; 444 fallthrough; 445 case MAP_SHARED_VALIDATE: 446 if (flags & ~flags_mask) 447 return -EOPNOTSUPP; 448 if (prot & PROT_WRITE) { 449 if (!(file->f_mode & FMODE_WRITE)) 450 return -EACCES; 451 if (IS_SWAPFILE(file->f_mapping->host)) 452 return -ETXTBSY; 453 } 454 455 /* 456 * Make sure we don't allow writing to an append-only 457 * file.. 458 */ 459 if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE)) 460 return -EACCES; 461 462 vm_flags |= VM_SHARED | VM_MAYSHARE; 463 if (!(file->f_mode & FMODE_WRITE)) 464 vm_flags &= ~(VM_MAYWRITE | VM_SHARED); 465 fallthrough; 466 case MAP_PRIVATE: 467 if (!(file->f_mode & FMODE_READ)) 468 return -EACCES; 469 if (path_noexec(&file->f_path)) { 470 if (vm_flags & VM_EXEC) 471 return -EPERM; 472 vm_flags &= ~VM_MAYEXEC; 473 } 474 475 if (!can_mmap_file(file)) 476 return -ENODEV; 477 if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) 478 return -EINVAL; 479 break; 480 481 default: 482 return -EINVAL; 483 } 484 485 /* 486 * Check to see if we are violating any seals and update VMA 487 * flags if necessary to avoid future seal violations. 488 */ 489 err = memfd_check_seals_mmap(file, &vm_flags); 490 if (err) 491 return (unsigned long)err; 492 } else { 493 switch (flags & MAP_TYPE) { 494 case MAP_SHARED: 495 if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) 496 return -EINVAL; 497 /* 498 * Ignore pgoff. 499 */ 500 pgoff = 0; 501 vm_flags |= VM_SHARED | VM_MAYSHARE; 502 break; 503 case MAP_DROPPABLE: 504 if (VM_DROPPABLE == VM_NONE) 505 return -ENOTSUPP; 506 /* 507 * A locked or stack area makes no sense to be droppable. 508 * 509 * Also, since droppable pages can just go away at any time 510 * it makes no sense to copy them on fork or dump them. 511 * 512 * And don't attempt to combine with hugetlb for now. 513 */ 514 if (flags & (MAP_LOCKED | MAP_HUGETLB)) 515 return -EINVAL; 516 if (vm_flags & (VM_GROWSDOWN | VM_GROWSUP)) 517 return -EINVAL; 518 519 vm_flags |= VM_DROPPABLE; 520 521 /* 522 * If the pages can be dropped, then it doesn't make 523 * sense to reserve them. 524 */ 525 vm_flags |= VM_NORESERVE; 526 527 /* 528 * Likewise, they're volatile enough that they 529 * shouldn't survive forks or coredumps. 530 */ 531 vm_flags |= VM_WIPEONFORK | VM_DONTDUMP; 532 fallthrough; 533 case MAP_PRIVATE: 534 /* 535 * Set pgoff according to addr for anon_vma. 536 */ 537 pgoff = addr >> PAGE_SHIFT; 538 break; 539 default: 540 return -EINVAL; 541 } 542 } 543 544 /* 545 * Set 'VM_NORESERVE' if we should not account for the 546 * memory use of this mapping. 547 */ 548 if (flags & MAP_NORESERVE) { 549 /* We honor MAP_NORESERVE if allowed to overcommit */ 550 if (sysctl_overcommit_memory != OVERCOMMIT_NEVER) 551 vm_flags |= VM_NORESERVE; 552 553 /* hugetlb applies strict overcommit unless MAP_NORESERVE */ 554 if (file && is_file_hugepages(file)) 555 vm_flags |= VM_NORESERVE; 556 } 557 558 addr = mmap_region(file, addr, len, vm_flags, pgoff, uf); 559 if (!IS_ERR_VALUE(addr) && 560 ((vm_flags & VM_LOCKED) || 561 (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE)) 562 *populate = len; 563 return addr; 564 } 565 566 unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, 567 unsigned long prot, unsigned long flags, 568 unsigned long fd, unsigned long pgoff) 569 { 570 struct file *file = NULL; 571 unsigned long retval; 572 573 if (!(flags & MAP_ANONYMOUS)) { 574 audit_mmap_fd(fd, flags); 575 file = fget(fd); 576 if (!file) 577 return -EBADF; 578 if (is_file_hugepages(file)) { 579 len = ALIGN(len, huge_page_size(hstate_file(file))); 580 } else if (unlikely(flags & MAP_HUGETLB)) { 581 retval = -EINVAL; 582 goto out_fput; 583 } 584 } else if (flags & MAP_HUGETLB) { 585 struct hstate *hs; 586 587 hs = hstate_sizelog((flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK); 588 if (!hs) 589 return -EINVAL; 590 591 len = ALIGN(len, huge_page_size(hs)); 592 /* 593 * VM_NORESERVE is used because the reservations will be 594 * taken when vm_ops->mmap() is called 595 */ 596 file = hugetlb_file_setup(HUGETLB_ANON_FILE, len, 597 VM_NORESERVE, 598 HUGETLB_ANONHUGE_INODE, 599 (flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK); 600 if (IS_ERR(file)) 601 return PTR_ERR(file); 602 } 603 604 retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff); 605 out_fput: 606 if (file) 607 fput(file); 608 return retval; 609 } 610 611 SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len, 612 unsigned long, prot, unsigned long, flags, 613 unsigned long, fd, unsigned long, pgoff) 614 { 615 return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff); 616 } 617 618 #ifdef __ARCH_WANT_SYS_OLD_MMAP 619 struct mmap_arg_struct { 620 unsigned long addr; 621 unsigned long len; 622 unsigned long prot; 623 unsigned long flags; 624 unsigned long fd; 625 unsigned long offset; 626 }; 627 628 SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg) 629 { 630 struct mmap_arg_struct a; 631 632 if (copy_from_user(&a, arg, sizeof(a))) 633 return -EFAULT; 634 if (offset_in_page(a.offset)) 635 return -EINVAL; 636 637 return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd, 638 a.offset >> PAGE_SHIFT); 639 } 640 #endif /* __ARCH_WANT_SYS_OLD_MMAP */ 641 642 /* 643 * Determine if the allocation needs to ensure that there is no 644 * existing mapping within it's guard gaps, for use as start_gap. 645 */ 646 static inline unsigned long stack_guard_placement(vm_flags_t vm_flags) 647 { 648 if (vm_flags & VM_SHADOW_STACK) 649 return PAGE_SIZE; 650 651 return 0; 652 } 653 654 /* 655 * Search for an unmapped address range. 656 * 657 * We are looking for a range that: 658 * - does not intersect with any VMA; 659 * - is contained within the [low_limit, high_limit) interval; 660 * - is at least the desired size. 661 * - satisfies (begin_addr & align_mask) == (align_offset & align_mask) 662 */ 663 unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info) 664 { 665 unsigned long addr; 666 667 if (info->flags & VM_UNMAPPED_AREA_TOPDOWN) 668 addr = unmapped_area_topdown(info); 669 else 670 addr = unmapped_area(info); 671 672 trace_vm_unmapped_area(addr, info); 673 return addr; 674 } 675 676 /* Get an address range which is currently unmapped. 677 * For shmat() with addr=0. 678 * 679 * Ugly calling convention alert: 680 * Return value with the low bits set means error value, 681 * ie 682 * if (ret & ~PAGE_MASK) 683 * error = ret; 684 * 685 * This function "knows" that -ENOMEM has the bits set. 686 */ 687 unsigned long 688 generic_get_unmapped_area(struct file *filp, unsigned long addr, 689 unsigned long len, unsigned long pgoff, 690 unsigned long flags, vm_flags_t vm_flags) 691 { 692 struct mm_struct *mm = current->mm; 693 struct vm_area_struct *vma, *prev; 694 struct vm_unmapped_area_info info = {}; 695 const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags); 696 697 if (len > mmap_end - mmap_min_addr) 698 return -ENOMEM; 699 700 if (flags & MAP_FIXED) 701 return addr; 702 703 if (addr) { 704 addr = PAGE_ALIGN(addr); 705 vma = find_vma_prev(mm, addr, &prev); 706 if (mmap_end - len >= addr && addr >= mmap_min_addr && 707 (!vma || addr + len <= vm_start_gap(vma)) && 708 (!prev || addr >= vm_end_gap(prev))) 709 return addr; 710 } 711 712 info.length = len; 713 info.low_limit = mm->mmap_base; 714 info.high_limit = mmap_end; 715 info.start_gap = stack_guard_placement(vm_flags); 716 if (filp && is_file_hugepages(filp)) 717 info.align_mask = huge_page_mask_align(filp); 718 return vm_unmapped_area(&info); 719 } 720 721 #ifndef HAVE_ARCH_UNMAPPED_AREA 722 unsigned long 723 arch_get_unmapped_area(struct file *filp, unsigned long addr, 724 unsigned long len, unsigned long pgoff, 725 unsigned long flags, vm_flags_t vm_flags) 726 { 727 return generic_get_unmapped_area(filp, addr, len, pgoff, flags, 728 vm_flags); 729 } 730 #endif 731 732 /* 733 * This mmap-allocator allocates new areas top-down from below the 734 * stack's low limit (the base): 735 */ 736 unsigned long 737 generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, 738 unsigned long len, unsigned long pgoff, 739 unsigned long flags, vm_flags_t vm_flags) 740 { 741 struct vm_area_struct *vma, *prev; 742 struct mm_struct *mm = current->mm; 743 struct vm_unmapped_area_info info = {}; 744 const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags); 745 746 /* requested length too big for entire address space */ 747 if (len > mmap_end - mmap_min_addr) 748 return -ENOMEM; 749 750 if (flags & MAP_FIXED) 751 return addr; 752 753 /* requesting a specific address */ 754 if (addr) { 755 addr = PAGE_ALIGN(addr); 756 vma = find_vma_prev(mm, addr, &prev); 757 if (mmap_end - len >= addr && addr >= mmap_min_addr && 758 (!vma || addr + len <= vm_start_gap(vma)) && 759 (!prev || addr >= vm_end_gap(prev))) 760 return addr; 761 } 762 763 info.flags = VM_UNMAPPED_AREA_TOPDOWN; 764 info.length = len; 765 info.low_limit = PAGE_SIZE; 766 info.high_limit = arch_get_mmap_base(addr, mm->mmap_base); 767 info.start_gap = stack_guard_placement(vm_flags); 768 if (filp && is_file_hugepages(filp)) 769 info.align_mask = huge_page_mask_align(filp); 770 addr = vm_unmapped_area(&info); 771 772 /* 773 * A failed mmap() very likely causes application failure, 774 * so fall back to the bottom-up function here. This scenario 775 * can happen with large stack limits and large mmap() 776 * allocations. 777 */ 778 if (offset_in_page(addr)) { 779 VM_BUG_ON(addr != -ENOMEM); 780 info.flags = 0; 781 info.low_limit = TASK_UNMAPPED_BASE; 782 info.high_limit = mmap_end; 783 addr = vm_unmapped_area(&info); 784 } 785 786 return addr; 787 } 788 789 #ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN 790 unsigned long 791 arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, 792 unsigned long len, unsigned long pgoff, 793 unsigned long flags, vm_flags_t vm_flags) 794 { 795 return generic_get_unmapped_area_topdown(filp, addr, len, pgoff, flags, 796 vm_flags); 797 } 798 #endif 799 800 unsigned long mm_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, 801 unsigned long len, unsigned long pgoff, 802 unsigned long flags, vm_flags_t vm_flags) 803 { 804 if (mm_flags_test(MMF_TOPDOWN, current->mm)) 805 return arch_get_unmapped_area_topdown(filp, addr, len, pgoff, 806 flags, vm_flags); 807 return arch_get_unmapped_area(filp, addr, len, pgoff, flags, vm_flags); 808 } 809 810 unsigned long 811 __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, 812 unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags) 813 { 814 unsigned long (*get_area)(struct file *, unsigned long, 815 unsigned long, unsigned long, unsigned long) 816 = NULL; 817 818 unsigned long error = arch_mmap_check(addr, len, flags); 819 if (error) 820 return error; 821 822 /* Careful about overflows.. */ 823 if (len > TASK_SIZE) 824 return -ENOMEM; 825 826 if (file) { 827 if (file->f_op->get_unmapped_area) 828 get_area = file->f_op->get_unmapped_area; 829 } else if (flags & MAP_SHARED) { 830 /* 831 * mmap_region() will call shmem_zero_setup() to create a file, 832 * so use shmem's get_unmapped_area in case it can be huge. 833 */ 834 get_area = shmem_get_unmapped_area; 835 } 836 837 /* Always treat pgoff as zero for anonymous memory. */ 838 if (!file) 839 pgoff = 0; 840 841 if (get_area) { 842 addr = get_area(file, addr, len, pgoff, flags); 843 } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && !file 844 && !addr /* no hint */ 845 && IS_ALIGNED(len, PMD_SIZE)) { 846 /* Ensures that larger anonymous mappings are THP aligned. */ 847 addr = thp_get_unmapped_area_vmflags(file, addr, len, 848 pgoff, flags, vm_flags); 849 } else { 850 addr = mm_get_unmapped_area_vmflags(file, addr, len, 851 pgoff, flags, vm_flags); 852 } 853 if (IS_ERR_VALUE(addr)) 854 return addr; 855 856 if (addr > TASK_SIZE - len) 857 return -ENOMEM; 858 if (offset_in_page(addr)) 859 return -EINVAL; 860 861 error = security_mmap_addr(addr); 862 return error ? error : addr; 863 } 864 865 unsigned long 866 mm_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, 867 unsigned long pgoff, unsigned long flags) 868 { 869 return mm_get_unmapped_area_vmflags(file, addr, len, pgoff, flags, 0); 870 } 871 EXPORT_SYMBOL(mm_get_unmapped_area); 872 873 /** 874 * find_vma_intersection() - Look up the first VMA which intersects the interval 875 * @mm: The process address space. 876 * @start_addr: The inclusive start user address. 877 * @end_addr: The exclusive end user address. 878 * 879 * Returns: The first VMA within the provided range, %NULL otherwise. Assumes 880 * start_addr < end_addr. 881 */ 882 struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, 883 unsigned long start_addr, 884 unsigned long end_addr) 885 { 886 unsigned long index = start_addr; 887 888 mmap_assert_locked(mm); 889 return mt_find(&mm->mm_mt, &index, end_addr - 1); 890 } 891 EXPORT_SYMBOL(find_vma_intersection); 892 893 /** 894 * find_vma() - Find the VMA for a given address, or the next VMA. 895 * @mm: The mm_struct to check 896 * @addr: The address 897 * 898 * Returns: The VMA associated with addr, or the next VMA. 899 * May return %NULL in the case of no VMA at addr or above. 900 */ 901 struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) 902 { 903 unsigned long index = addr; 904 905 mmap_assert_locked(mm); 906 return mt_find(&mm->mm_mt, &index, ULONG_MAX); 907 } 908 EXPORT_SYMBOL(find_vma); 909 910 /** 911 * find_vma_prev() - Find the VMA for a given address, or the next vma and 912 * set %pprev to the previous VMA, if any. 913 * @mm: The mm_struct to check 914 * @addr: The address 915 * @pprev: The pointer to set to the previous VMA 916 * 917 * Note that RCU lock is missing here since the external mmap_lock() is used 918 * instead. 919 * 920 * Returns: The VMA associated with @addr, or the next vma. 921 * May return %NULL in the case of no vma at addr or above. 922 */ 923 struct vm_area_struct * 924 find_vma_prev(struct mm_struct *mm, unsigned long addr, 925 struct vm_area_struct **pprev) 926 { 927 struct vm_area_struct *vma; 928 VMA_ITERATOR(vmi, mm, addr); 929 930 vma = vma_iter_load(&vmi); 931 *pprev = vma_prev(&vmi); 932 if (!vma) 933 vma = vma_next(&vmi); 934 return vma; 935 } 936 937 /* enforced gap between the expanding stack and other mappings. */ 938 unsigned long stack_guard_gap = 256UL<<PAGE_SHIFT; 939 940 static int __init cmdline_parse_stack_guard_gap(char *p) 941 { 942 unsigned long val; 943 char *endptr; 944 945 val = simple_strtoul(p, &endptr, 10); 946 if (!*endptr) 947 stack_guard_gap = val << PAGE_SHIFT; 948 949 return 1; 950 } 951 __setup("stack_guard_gap=", cmdline_parse_stack_guard_gap); 952 953 #ifdef CONFIG_STACK_GROWSUP 954 int expand_stack_locked(struct vm_area_struct *vma, unsigned long address) 955 { 956 return expand_upwards(vma, address); 957 } 958 959 struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr) 960 { 961 struct vm_area_struct *vma, *prev; 962 963 addr &= PAGE_MASK; 964 vma = find_vma_prev(mm, addr, &prev); 965 if (vma && (vma->vm_start <= addr)) 966 return vma; 967 if (!prev) 968 return NULL; 969 if (expand_stack_locked(prev, addr)) 970 return NULL; 971 if (prev->vm_flags & VM_LOCKED) 972 populate_vma_page_range(prev, addr, prev->vm_end, NULL); 973 return prev; 974 } 975 #else 976 int expand_stack_locked(struct vm_area_struct *vma, unsigned long address) 977 { 978 return expand_downwards(vma, address); 979 } 980 981 struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr) 982 { 983 struct vm_area_struct *vma; 984 unsigned long start; 985 986 addr &= PAGE_MASK; 987 vma = find_vma(mm, addr); 988 if (!vma) 989 return NULL; 990 if (vma->vm_start <= addr) 991 return vma; 992 start = vma->vm_start; 993 if (expand_stack_locked(vma, addr)) 994 return NULL; 995 if (vma->vm_flags & VM_LOCKED) 996 populate_vma_page_range(vma, addr, start, NULL); 997 return vma; 998 } 999 #endif 1000 1001 #if defined(CONFIG_STACK_GROWSUP) 1002 1003 #define vma_expand_up(vma,addr) expand_upwards(vma, addr) 1004 #define vma_expand_down(vma, addr) (-EFAULT) 1005 1006 #else 1007 1008 #define vma_expand_up(vma,addr) (-EFAULT) 1009 #define vma_expand_down(vma, addr) expand_downwards(vma, addr) 1010 1011 #endif 1012 1013 /* 1014 * expand_stack(): legacy interface for page faulting. Don't use unless 1015 * you have to. 1016 * 1017 * This is called with the mm locked for reading, drops the lock, takes 1018 * the lock for writing, tries to look up a vma again, expands it if 1019 * necessary, and downgrades the lock to reading again. 1020 * 1021 * If no vma is found or it can't be expanded, it returns NULL and has 1022 * dropped the lock. 1023 */ 1024 struct vm_area_struct *expand_stack(struct mm_struct *mm, unsigned long addr) 1025 { 1026 struct vm_area_struct *vma, *prev; 1027 1028 mmap_read_unlock(mm); 1029 if (mmap_write_lock_killable(mm)) 1030 return NULL; 1031 1032 vma = find_vma_prev(mm, addr, &prev); 1033 if (vma && vma->vm_start <= addr) 1034 goto success; 1035 1036 if (prev && !vma_expand_up(prev, addr)) { 1037 vma = prev; 1038 goto success; 1039 } 1040 1041 if (vma && !vma_expand_down(vma, addr)) 1042 goto success; 1043 1044 mmap_write_unlock(mm); 1045 return NULL; 1046 1047 success: 1048 mmap_write_downgrade(mm); 1049 return vma; 1050 } 1051 1052 /* do_munmap() - Wrapper function for non-maple tree aware do_munmap() calls. 1053 * @mm: The mm_struct 1054 * @start: The start address to munmap 1055 * @len: The length to be munmapped. 1056 * @uf: The userfaultfd list_head 1057 * 1058 * Return: 0 on success, error otherwise. 1059 */ 1060 int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, 1061 struct list_head *uf) 1062 { 1063 VMA_ITERATOR(vmi, mm, start); 1064 1065 return do_vmi_munmap(&vmi, mm, start, len, uf, false); 1066 } 1067 1068 int vm_munmap(unsigned long start, size_t len) 1069 { 1070 return __vm_munmap(start, len, false); 1071 } 1072 EXPORT_SYMBOL(vm_munmap); 1073 1074 SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len) 1075 { 1076 addr = untagged_addr(addr); 1077 return __vm_munmap(addr, len, true); 1078 } 1079 1080 1081 /* 1082 * Emulation of deprecated remap_file_pages() syscall. 1083 */ 1084 SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, 1085 unsigned long, prot, unsigned long, pgoff, unsigned long, flags) 1086 { 1087 1088 struct mm_struct *mm = current->mm; 1089 struct vm_area_struct *vma; 1090 unsigned long populate = 0; 1091 unsigned long ret = -EINVAL; 1092 struct file *file; 1093 vm_flags_t vm_flags; 1094 1095 pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/mm/remap_file_pages.rst.\n", 1096 current->comm, current->pid); 1097 1098 if (prot) 1099 return ret; 1100 start = start & PAGE_MASK; 1101 size = size & PAGE_MASK; 1102 1103 if (start + size <= start) 1104 return ret; 1105 1106 /* Does pgoff wrap? */ 1107 if (pgoff + (size >> PAGE_SHIFT) < pgoff) 1108 return ret; 1109 1110 if (mmap_read_lock_killable(mm)) 1111 return -EINTR; 1112 1113 /* 1114 * Look up VMA under read lock first so we can perform the security 1115 * without holding locks (which can be problematic). We reacquire a 1116 * write lock later and check nothing changed underneath us. 1117 */ 1118 vma = vma_lookup(mm, start); 1119 1120 if (!vma || !(vma->vm_flags & VM_SHARED)) { 1121 mmap_read_unlock(mm); 1122 return -EINVAL; 1123 } 1124 1125 prot |= vma->vm_flags & VM_READ ? PROT_READ : 0; 1126 prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0; 1127 prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0; 1128 1129 flags &= MAP_NONBLOCK; 1130 flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE; 1131 if (vma->vm_flags & VM_LOCKED) 1132 flags |= MAP_LOCKED; 1133 1134 /* Save vm_flags used to calculate prot and flags, and recheck later. */ 1135 vm_flags = vma->vm_flags; 1136 file = get_file(vma->vm_file); 1137 1138 mmap_read_unlock(mm); 1139 1140 /* Call outside mmap_lock to be consistent with other callers. */ 1141 ret = security_mmap_file(file, prot, flags); 1142 if (ret) { 1143 fput(file); 1144 return ret; 1145 } 1146 1147 ret = -EINVAL; 1148 1149 /* OK security check passed, take write lock + let it rip. */ 1150 if (mmap_write_lock_killable(mm)) { 1151 fput(file); 1152 return -EINTR; 1153 } 1154 1155 vma = vma_lookup(mm, start); 1156 1157 if (!vma) 1158 goto out; 1159 1160 /* Make sure things didn't change under us. */ 1161 if (vma->vm_flags != vm_flags) 1162 goto out; 1163 if (vma->vm_file != file) 1164 goto out; 1165 1166 if (start + size > vma->vm_end) { 1167 VMA_ITERATOR(vmi, mm, vma->vm_end); 1168 struct vm_area_struct *next, *prev = vma; 1169 1170 for_each_vma_range(vmi, next, start + size) { 1171 /* hole between vmas ? */ 1172 if (next->vm_start != prev->vm_end) 1173 goto out; 1174 1175 if (next->vm_file != vma->vm_file) 1176 goto out; 1177 1178 if (next->vm_flags != vma->vm_flags) 1179 goto out; 1180 1181 if (start + size <= next->vm_end) 1182 break; 1183 1184 prev = next; 1185 } 1186 1187 if (!next) 1188 goto out; 1189 } 1190 1191 ret = do_mmap(vma->vm_file, start, size, 1192 prot, flags, 0, pgoff, &populate, NULL); 1193 out: 1194 mmap_write_unlock(mm); 1195 fput(file); 1196 if (populate) 1197 mm_populate(ret, populate); 1198 if (!IS_ERR_VALUE(ret)) 1199 ret = 0; 1200 return ret; 1201 } 1202 1203 int vm_brk_flags(unsigned long addr, unsigned long request, vm_flags_t vm_flags) 1204 { 1205 struct mm_struct *mm = current->mm; 1206 struct vm_area_struct *vma = NULL; 1207 unsigned long len; 1208 int ret; 1209 bool populate; 1210 LIST_HEAD(uf); 1211 VMA_ITERATOR(vmi, mm, addr); 1212 1213 len = PAGE_ALIGN(request); 1214 if (len < request) 1215 return -ENOMEM; 1216 if (!len) 1217 return 0; 1218 1219 /* Until we need other flags, refuse anything except VM_EXEC. */ 1220 if ((vm_flags & (~VM_EXEC)) != 0) 1221 return -EINVAL; 1222 1223 if (mmap_write_lock_killable(mm)) 1224 return -EINTR; 1225 1226 ret = check_brk_limits(addr, len); 1227 if (ret) 1228 goto limits_failed; 1229 1230 ret = do_vmi_munmap(&vmi, mm, addr, len, &uf, 0); 1231 if (ret) 1232 goto munmap_failed; 1233 1234 vma = vma_prev(&vmi); 1235 ret = do_brk_flags(&vmi, vma, addr, len, vm_flags); 1236 populate = ((mm->def_flags & VM_LOCKED) != 0); 1237 mmap_write_unlock(mm); 1238 userfaultfd_unmap_complete(mm, &uf); 1239 if (populate && !ret) 1240 mm_populate(addr, len); 1241 return ret; 1242 1243 munmap_failed: 1244 limits_failed: 1245 mmap_write_unlock(mm); 1246 return ret; 1247 } 1248 EXPORT_SYMBOL(vm_brk_flags); 1249 1250 /* Release all mmaps. */ 1251 void exit_mmap(struct mm_struct *mm) 1252 { 1253 struct mmu_gather tlb; 1254 struct vm_area_struct *vma; 1255 unsigned long nr_accounted = 0; 1256 VMA_ITERATOR(vmi, mm, 0); 1257 int count = 0; 1258 1259 /* mm's last user has gone, and its about to be pulled down */ 1260 mmu_notifier_release(mm); 1261 1262 mmap_read_lock(mm); 1263 arch_exit_mmap(mm); 1264 1265 vma = vma_next(&vmi); 1266 if (!vma || unlikely(xa_is_zero(vma))) { 1267 /* Can happen if dup_mmap() received an OOM */ 1268 mmap_read_unlock(mm); 1269 mmap_write_lock(mm); 1270 goto destroy; 1271 } 1272 1273 flush_cache_mm(mm); 1274 tlb_gather_mmu_fullmm(&tlb, mm); 1275 /* update_hiwater_rss(mm) here? but nobody should be looking */ 1276 /* Use ULONG_MAX here to ensure all VMAs in the mm are unmapped */ 1277 unmap_vmas(&tlb, &vmi.mas, vma, 0, ULONG_MAX, ULONG_MAX); 1278 mmap_read_unlock(mm); 1279 1280 /* 1281 * Set MMF_OOM_SKIP to hide this task from the oom killer/reaper 1282 * because the memory has been already freed. 1283 */ 1284 mm_flags_set(MMF_OOM_SKIP, mm); 1285 mmap_write_lock(mm); 1286 mt_clear_in_rcu(&mm->mm_mt); 1287 vma_iter_set(&vmi, vma->vm_end); 1288 free_pgtables(&tlb, &vmi.mas, vma, FIRST_USER_ADDRESS, 1289 USER_PGTABLES_CEILING, true); 1290 tlb_finish_mmu(&tlb); 1291 1292 /* 1293 * Walk the list again, actually closing and freeing it, with preemption 1294 * enabled, without holding any MM locks besides the unreachable 1295 * mmap_write_lock. 1296 */ 1297 vma_iter_set(&vmi, vma->vm_end); 1298 do { 1299 if (vma->vm_flags & VM_ACCOUNT) 1300 nr_accounted += vma_pages(vma); 1301 vma_mark_detached(vma); 1302 remove_vma(vma); 1303 count++; 1304 cond_resched(); 1305 vma = vma_next(&vmi); 1306 } while (vma && likely(!xa_is_zero(vma))); 1307 1308 BUG_ON(count != mm->map_count); 1309 1310 trace_exit_mmap(mm); 1311 destroy: 1312 __mt_destroy(&mm->mm_mt); 1313 mmap_write_unlock(mm); 1314 vm_unacct_memory(nr_accounted); 1315 } 1316 1317 /* 1318 * Return true if the calling process may expand its vm space by the passed 1319 * number of pages 1320 */ 1321 bool may_expand_vm(struct mm_struct *mm, vm_flags_t flags, unsigned long npages) 1322 { 1323 if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> PAGE_SHIFT) 1324 return false; 1325 1326 if (is_data_mapping(flags) && 1327 mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) { 1328 /* Workaround for Valgrind */ 1329 if (rlimit(RLIMIT_DATA) == 0 && 1330 mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> PAGE_SHIFT) 1331 return true; 1332 1333 pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Update limits%s.\n", 1334 current->comm, current->pid, 1335 (mm->data_vm + npages) << PAGE_SHIFT, 1336 rlimit(RLIMIT_DATA), 1337 ignore_rlimit_data ? "" : " or use boot option ignore_rlimit_data"); 1338 1339 if (!ignore_rlimit_data) 1340 return false; 1341 } 1342 1343 return true; 1344 } 1345 1346 void vm_stat_account(struct mm_struct *mm, vm_flags_t flags, long npages) 1347 { 1348 WRITE_ONCE(mm->total_vm, READ_ONCE(mm->total_vm)+npages); 1349 1350 if (is_exec_mapping(flags)) 1351 mm->exec_vm += npages; 1352 else if (is_stack_mapping(flags)) 1353 mm->stack_vm += npages; 1354 else if (is_data_mapping(flags)) 1355 mm->data_vm += npages; 1356 } 1357 1358 static vm_fault_t special_mapping_fault(struct vm_fault *vmf); 1359 1360 /* 1361 * Close hook, called for unmap() and on the old vma for mremap(). 1362 * 1363 * Having a close hook prevents vma merging regardless of flags. 1364 */ 1365 static void special_mapping_close(struct vm_area_struct *vma) 1366 { 1367 const struct vm_special_mapping *sm = vma->vm_private_data; 1368 1369 if (sm->close) 1370 sm->close(sm, vma); 1371 } 1372 1373 static const char *special_mapping_name(struct vm_area_struct *vma) 1374 { 1375 return ((struct vm_special_mapping *)vma->vm_private_data)->name; 1376 } 1377 1378 static int special_mapping_mremap(struct vm_area_struct *new_vma) 1379 { 1380 struct vm_special_mapping *sm = new_vma->vm_private_data; 1381 1382 if (WARN_ON_ONCE(current->mm != new_vma->vm_mm)) 1383 return -EFAULT; 1384 1385 if (sm->mremap) 1386 return sm->mremap(sm, new_vma); 1387 1388 return 0; 1389 } 1390 1391 static int special_mapping_split(struct vm_area_struct *vma, unsigned long addr) 1392 { 1393 /* 1394 * Forbid splitting special mappings - kernel has expectations over 1395 * the number of pages in mapping. Together with VM_DONTEXPAND 1396 * the size of vma should stay the same over the special mapping's 1397 * lifetime. 1398 */ 1399 return -EINVAL; 1400 } 1401 1402 static const struct vm_operations_struct special_mapping_vmops = { 1403 .close = special_mapping_close, 1404 .fault = special_mapping_fault, 1405 .mremap = special_mapping_mremap, 1406 .name = special_mapping_name, 1407 /* vDSO code relies that VVAR can't be accessed remotely */ 1408 .access = NULL, 1409 .may_split = special_mapping_split, 1410 }; 1411 1412 static vm_fault_t special_mapping_fault(struct vm_fault *vmf) 1413 { 1414 struct vm_area_struct *vma = vmf->vma; 1415 pgoff_t pgoff; 1416 struct page **pages; 1417 struct vm_special_mapping *sm = vma->vm_private_data; 1418 1419 if (sm->fault) 1420 return sm->fault(sm, vmf->vma, vmf); 1421 1422 pages = sm->pages; 1423 1424 for (pgoff = vmf->pgoff; pgoff && *pages; ++pages) 1425 pgoff--; 1426 1427 if (*pages) { 1428 struct page *page = *pages; 1429 get_page(page); 1430 vmf->page = page; 1431 return 0; 1432 } 1433 1434 return VM_FAULT_SIGBUS; 1435 } 1436 1437 static struct vm_area_struct *__install_special_mapping( 1438 struct mm_struct *mm, 1439 unsigned long addr, unsigned long len, 1440 vm_flags_t vm_flags, void *priv, 1441 const struct vm_operations_struct *ops) 1442 { 1443 int ret; 1444 struct vm_area_struct *vma; 1445 1446 vma = vm_area_alloc(mm); 1447 if (unlikely(vma == NULL)) 1448 return ERR_PTR(-ENOMEM); 1449 1450 vma_set_range(vma, addr, addr + len, 0); 1451 vm_flags |= mm->def_flags | VM_DONTEXPAND; 1452 if (pgtable_supports_soft_dirty()) 1453 vm_flags |= VM_SOFTDIRTY; 1454 vm_flags_init(vma, vm_flags & ~VM_LOCKED_MASK); 1455 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); 1456 1457 vma->vm_ops = ops; 1458 vma->vm_private_data = priv; 1459 1460 ret = insert_vm_struct(mm, vma); 1461 if (ret) 1462 goto out; 1463 1464 vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT); 1465 1466 perf_event_mmap(vma); 1467 1468 return vma; 1469 1470 out: 1471 vm_area_free(vma); 1472 return ERR_PTR(ret); 1473 } 1474 1475 bool vma_is_special_mapping(const struct vm_area_struct *vma, 1476 const struct vm_special_mapping *sm) 1477 { 1478 return vma->vm_private_data == sm && 1479 vma->vm_ops == &special_mapping_vmops; 1480 } 1481 1482 /* 1483 * Called with mm->mmap_lock held for writing. 1484 * Insert a new vma covering the given region, with the given flags. 1485 * Its pages are supplied by the given array of struct page *. 1486 * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated. 1487 * The region past the last page supplied will always produce SIGBUS. 1488 * The array pointer and the pages it points to are assumed to stay alive 1489 * for as long as this mapping might exist. 1490 */ 1491 struct vm_area_struct *_install_special_mapping( 1492 struct mm_struct *mm, 1493 unsigned long addr, unsigned long len, 1494 vm_flags_t vm_flags, const struct vm_special_mapping *spec) 1495 { 1496 return __install_special_mapping(mm, addr, len, vm_flags, (void *)spec, 1497 &special_mapping_vmops); 1498 } 1499 1500 #ifdef CONFIG_SYSCTL 1501 #if defined(HAVE_ARCH_PICK_MMAP_LAYOUT) || \ 1502 defined(CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT) 1503 int sysctl_legacy_va_layout; 1504 #endif 1505 1506 static const struct ctl_table mmap_table[] = { 1507 { 1508 .procname = "max_map_count", 1509 .data = &sysctl_max_map_count, 1510 .maxlen = sizeof(sysctl_max_map_count), 1511 .mode = 0644, 1512 .proc_handler = proc_dointvec_minmax, 1513 .extra1 = SYSCTL_ZERO, 1514 }, 1515 #if defined(HAVE_ARCH_PICK_MMAP_LAYOUT) || \ 1516 defined(CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT) 1517 { 1518 .procname = "legacy_va_layout", 1519 .data = &sysctl_legacy_va_layout, 1520 .maxlen = sizeof(sysctl_legacy_va_layout), 1521 .mode = 0644, 1522 .proc_handler = proc_dointvec_minmax, 1523 .extra1 = SYSCTL_ZERO, 1524 }, 1525 #endif 1526 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS 1527 { 1528 .procname = "mmap_rnd_bits", 1529 .data = &mmap_rnd_bits, 1530 .maxlen = sizeof(mmap_rnd_bits), 1531 .mode = 0600, 1532 .proc_handler = proc_dointvec_minmax, 1533 .extra1 = (void *)&mmap_rnd_bits_min, 1534 .extra2 = (void *)&mmap_rnd_bits_max, 1535 }, 1536 #endif 1537 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS 1538 { 1539 .procname = "mmap_rnd_compat_bits", 1540 .data = &mmap_rnd_compat_bits, 1541 .maxlen = sizeof(mmap_rnd_compat_bits), 1542 .mode = 0600, 1543 .proc_handler = proc_dointvec_minmax, 1544 .extra1 = (void *)&mmap_rnd_compat_bits_min, 1545 .extra2 = (void *)&mmap_rnd_compat_bits_max, 1546 }, 1547 #endif 1548 }; 1549 #endif /* CONFIG_SYSCTL */ 1550 1551 /* 1552 * initialise the percpu counter for VM, initialise VMA state. 1553 */ 1554 void __init mmap_init(void) 1555 { 1556 int ret; 1557 1558 ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL); 1559 VM_BUG_ON(ret); 1560 #ifdef CONFIG_SYSCTL 1561 register_sysctl_init("vm", mmap_table); 1562 #endif 1563 vma_state_init(); 1564 } 1565 1566 /* 1567 * Initialise sysctl_user_reserve_kbytes. 1568 * 1569 * This is intended to prevent a user from starting a single memory hogging 1570 * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER 1571 * mode. 1572 * 1573 * The default value is min(3% of free memory, 128MB) 1574 * 128MB is enough to recover with sshd/login, bash, and top/kill. 1575 */ 1576 static int init_user_reserve(void) 1577 { 1578 unsigned long free_kbytes; 1579 1580 free_kbytes = K(global_zone_page_state(NR_FREE_PAGES)); 1581 1582 sysctl_user_reserve_kbytes = min(free_kbytes / 32, SZ_128K); 1583 return 0; 1584 } 1585 subsys_initcall(init_user_reserve); 1586 1587 /* 1588 * Initialise sysctl_admin_reserve_kbytes. 1589 * 1590 * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin 1591 * to log in and kill a memory hogging process. 1592 * 1593 * Systems with more than 256MB will reserve 8MB, enough to recover 1594 * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will 1595 * only reserve 3% of free pages by default. 1596 */ 1597 static int init_admin_reserve(void) 1598 { 1599 unsigned long free_kbytes; 1600 1601 free_kbytes = K(global_zone_page_state(NR_FREE_PAGES)); 1602 1603 sysctl_admin_reserve_kbytes = min(free_kbytes / 32, SZ_8K); 1604 return 0; 1605 } 1606 subsys_initcall(init_admin_reserve); 1607 1608 /* 1609 * Reinititalise user and admin reserves if memory is added or removed. 1610 * 1611 * The default user reserve max is 128MB, and the default max for the 1612 * admin reserve is 8MB. These are usually, but not always, enough to 1613 * enable recovery from a memory hogging process using login/sshd, a shell, 1614 * and tools like top. It may make sense to increase or even disable the 1615 * reserve depending on the existence of swap or variations in the recovery 1616 * tools. So, the admin may have changed them. 1617 * 1618 * If memory is added and the reserves have been eliminated or increased above 1619 * the default max, then we'll trust the admin. 1620 * 1621 * If memory is removed and there isn't enough free memory, then we 1622 * need to reset the reserves. 1623 * 1624 * Otherwise keep the reserve set by the admin. 1625 */ 1626 static int reserve_mem_notifier(struct notifier_block *nb, 1627 unsigned long action, void *data) 1628 { 1629 unsigned long tmp, free_kbytes; 1630 1631 switch (action) { 1632 case MEM_ONLINE: 1633 /* Default max is 128MB. Leave alone if modified by operator. */ 1634 tmp = sysctl_user_reserve_kbytes; 1635 if (tmp > 0 && tmp < SZ_128K) 1636 init_user_reserve(); 1637 1638 /* Default max is 8MB. Leave alone if modified by operator. */ 1639 tmp = sysctl_admin_reserve_kbytes; 1640 if (tmp > 0 && tmp < SZ_8K) 1641 init_admin_reserve(); 1642 1643 break; 1644 case MEM_OFFLINE: 1645 free_kbytes = K(global_zone_page_state(NR_FREE_PAGES)); 1646 1647 if (sysctl_user_reserve_kbytes > free_kbytes) { 1648 init_user_reserve(); 1649 pr_info("vm.user_reserve_kbytes reset to %lu\n", 1650 sysctl_user_reserve_kbytes); 1651 } 1652 1653 if (sysctl_admin_reserve_kbytes > free_kbytes) { 1654 init_admin_reserve(); 1655 pr_info("vm.admin_reserve_kbytes reset to %lu\n", 1656 sysctl_admin_reserve_kbytes); 1657 } 1658 break; 1659 default: 1660 break; 1661 } 1662 return NOTIFY_OK; 1663 } 1664 1665 static int __meminit init_reserve_notifier(void) 1666 { 1667 if (hotplug_memory_notifier(reserve_mem_notifier, DEFAULT_CALLBACK_PRI)) 1668 pr_err("Failed registering memory add/remove notifier for admin reserve\n"); 1669 1670 return 0; 1671 } 1672 subsys_initcall(init_reserve_notifier); 1673 1674 /* 1675 * Obtain a read lock on mm->mmap_lock, if the specified address is below the 1676 * start of the VMA, the intent is to perform a write, and it is a 1677 * downward-growing stack, then attempt to expand the stack to contain it. 1678 * 1679 * This function is intended only for obtaining an argument page from an ELF 1680 * image, and is almost certainly NOT what you want to use for any other 1681 * purpose. 1682 * 1683 * IMPORTANT - VMA fields are accessed without an mmap lock being held, so the 1684 * VMA referenced must not be linked in any user-visible tree, i.e. it must be a 1685 * new VMA being mapped. 1686 * 1687 * The function assumes that addr is either contained within the VMA or below 1688 * it, and makes no attempt to validate this value beyond that. 1689 * 1690 * Returns true if the read lock was obtained and a stack was perhaps expanded, 1691 * false if the stack expansion failed. 1692 * 1693 * On stack expansion the function temporarily acquires an mmap write lock 1694 * before downgrading it. 1695 */ 1696 bool mmap_read_lock_maybe_expand(struct mm_struct *mm, 1697 struct vm_area_struct *new_vma, 1698 unsigned long addr, bool write) 1699 { 1700 if (!write || addr >= new_vma->vm_start) { 1701 mmap_read_lock(mm); 1702 return true; 1703 } 1704 1705 if (!(new_vma->vm_flags & VM_GROWSDOWN)) 1706 return false; 1707 1708 mmap_write_lock(mm); 1709 if (expand_downwards(new_vma, addr)) { 1710 mmap_write_unlock(mm); 1711 return false; 1712 } 1713 1714 mmap_write_downgrade(mm); 1715 return true; 1716 } 1717 1718 __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) 1719 { 1720 struct vm_area_struct *mpnt, *tmp; 1721 int retval; 1722 unsigned long charge = 0; 1723 LIST_HEAD(uf); 1724 VMA_ITERATOR(vmi, mm, 0); 1725 1726 if (mmap_write_lock_killable(oldmm)) 1727 return -EINTR; 1728 flush_cache_dup_mm(oldmm); 1729 uprobe_dup_mmap(oldmm, mm); 1730 /* 1731 * Not linked in yet - no deadlock potential: 1732 */ 1733 mmap_write_lock_nested(mm, SINGLE_DEPTH_NESTING); 1734 1735 /* No ordering required: file already has been exposed. */ 1736 dup_mm_exe_file(mm, oldmm); 1737 1738 mm->total_vm = oldmm->total_vm; 1739 mm->data_vm = oldmm->data_vm; 1740 mm->exec_vm = oldmm->exec_vm; 1741 mm->stack_vm = oldmm->stack_vm; 1742 1743 /* Use __mt_dup() to efficiently build an identical maple tree. */ 1744 retval = __mt_dup(&oldmm->mm_mt, &mm->mm_mt, GFP_KERNEL); 1745 if (unlikely(retval)) 1746 goto out; 1747 1748 mt_clear_in_rcu(vmi.mas.tree); 1749 for_each_vma(vmi, mpnt) { 1750 struct file *file; 1751 1752 retval = vma_start_write_killable(mpnt); 1753 if (retval < 0) 1754 goto loop_out; 1755 if (mpnt->vm_flags & VM_DONTCOPY) { 1756 retval = vma_iter_clear_gfp(&vmi, mpnt->vm_start, 1757 mpnt->vm_end, GFP_KERNEL); 1758 if (retval) 1759 goto loop_out; 1760 1761 vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt)); 1762 continue; 1763 } 1764 charge = 0; 1765 if (mpnt->vm_flags & VM_ACCOUNT) { 1766 unsigned long len = vma_pages(mpnt); 1767 1768 if (security_vm_enough_memory_mm(oldmm, len)) /* sic */ 1769 goto fail_nomem; 1770 charge = len; 1771 } 1772 1773 tmp = vm_area_dup(mpnt); 1774 if (!tmp) 1775 goto fail_nomem; 1776 retval = vma_dup_policy(mpnt, tmp); 1777 if (retval) 1778 goto fail_nomem_policy; 1779 tmp->vm_mm = mm; 1780 retval = dup_userfaultfd(tmp, &uf); 1781 if (retval) 1782 goto fail_nomem_anon_vma_fork; 1783 if (tmp->vm_flags & VM_WIPEONFORK) { 1784 /* 1785 * VM_WIPEONFORK gets a clean slate in the child. 1786 * Don't prepare anon_vma until fault since we don't 1787 * copy page for current vma. 1788 */ 1789 tmp->anon_vma = NULL; 1790 } else if (anon_vma_fork(tmp, mpnt)) 1791 goto fail_nomem_anon_vma_fork; 1792 vm_flags_clear(tmp, VM_LOCKED_MASK); 1793 /* 1794 * Copy/update hugetlb private vma information. 1795 */ 1796 if (is_vm_hugetlb_page(tmp)) 1797 hugetlb_dup_vma_private(tmp); 1798 1799 /* 1800 * Link the vma into the MT. After using __mt_dup(), memory 1801 * allocation is not necessary here, so it cannot fail. 1802 */ 1803 vma_iter_bulk_store(&vmi, tmp); 1804 1805 mm->map_count++; 1806 1807 if (tmp->vm_ops && tmp->vm_ops->open) 1808 tmp->vm_ops->open(tmp); 1809 1810 file = tmp->vm_file; 1811 if (file) { 1812 struct address_space *mapping = file->f_mapping; 1813 1814 get_file(file); 1815 i_mmap_lock_write(mapping); 1816 if (vma_is_shared_maywrite(tmp)) 1817 mapping_allow_writable(mapping); 1818 flush_dcache_mmap_lock(mapping); 1819 /* insert tmp into the share list, just after mpnt */ 1820 vma_interval_tree_insert_after(tmp, mpnt, 1821 &mapping->i_mmap); 1822 flush_dcache_mmap_unlock(mapping); 1823 i_mmap_unlock_write(mapping); 1824 } 1825 1826 if (!(tmp->vm_flags & VM_WIPEONFORK)) 1827 retval = copy_page_range(tmp, mpnt); 1828 1829 if (retval) { 1830 mpnt = vma_next(&vmi); 1831 goto loop_out; 1832 } 1833 } 1834 /* a new mm has just been created */ 1835 retval = arch_dup_mmap(oldmm, mm); 1836 loop_out: 1837 vma_iter_free(&vmi); 1838 if (!retval) { 1839 mt_set_in_rcu(vmi.mas.tree); 1840 ksm_fork(mm, oldmm); 1841 khugepaged_fork(mm, oldmm); 1842 } else { 1843 1844 /* 1845 * The entire maple tree has already been duplicated. If the 1846 * mmap duplication fails, mark the failure point with 1847 * XA_ZERO_ENTRY. In exit_mmap(), if this marker is encountered, 1848 * stop releasing VMAs that have not been duplicated after this 1849 * point. 1850 */ 1851 if (mpnt) { 1852 mas_set_range(&vmi.mas, mpnt->vm_start, mpnt->vm_end - 1); 1853 mas_store(&vmi.mas, XA_ZERO_ENTRY); 1854 /* Avoid OOM iterating a broken tree */ 1855 mm_flags_set(MMF_OOM_SKIP, mm); 1856 } 1857 /* 1858 * The mm_struct is going to exit, but the locks will be dropped 1859 * first. Set the mm_struct as unstable is advisable as it is 1860 * not fully initialised. 1861 */ 1862 mm_flags_set(MMF_UNSTABLE, mm); 1863 } 1864 out: 1865 mmap_write_unlock(mm); 1866 flush_tlb_mm(oldmm); 1867 mmap_write_unlock(oldmm); 1868 if (!retval) 1869 dup_userfaultfd_complete(&uf); 1870 else 1871 dup_userfaultfd_fail(&uf); 1872 return retval; 1873 1874 fail_nomem_anon_vma_fork: 1875 mpol_put(vma_policy(tmp)); 1876 fail_nomem_policy: 1877 vm_area_free(tmp); 1878 fail_nomem: 1879 retval = -ENOMEM; 1880 vm_unacct_memory(charge); 1881 goto loop_out; 1882 } 1883