1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * mm/mmap.c 4 * 5 * Written by obz. 6 * 7 * Address space accounting code <alan@lxorguk.ukuu.org.uk> 8 */ 9 10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 11 12 #include <linux/kernel.h> 13 #include <linux/slab.h> 14 #include <linux/backing-dev.h> 15 #include <linux/mm.h> 16 #include <linux/mm_inline.h> 17 #include <linux/shm.h> 18 #include <linux/mman.h> 19 #include <linux/pagemap.h> 20 #include <linux/swap.h> 21 #include <linux/syscalls.h> 22 #include <linux/capability.h> 23 #include <linux/init.h> 24 #include <linux/file.h> 25 #include <linux/fs.h> 26 #include <linux/personality.h> 27 #include <linux/security.h> 28 #include <linux/hugetlb.h> 29 #include <linux/shmem_fs.h> 30 #include <linux/profile.h> 31 #include <linux/export.h> 32 #include <linux/mount.h> 33 #include <linux/mempolicy.h> 34 #include <linux/rmap.h> 35 #include <linux/mmu_notifier.h> 36 #include <linux/mmdebug.h> 37 #include <linux/perf_event.h> 38 #include <linux/audit.h> 39 #include <linux/khugepaged.h> 40 #include <linux/uprobes.h> 41 #include <linux/notifier.h> 42 #include <linux/memory.h> 43 #include <linux/printk.h> 44 #include <linux/userfaultfd_k.h> 45 #include <linux/moduleparam.h> 46 #include <linux/pkeys.h> 47 #include <linux/oom.h> 48 #include <linux/sched/mm.h> 49 #include <linux/ksm.h> 50 51 #include <linux/uaccess.h> 52 #include <asm/cacheflush.h> 53 #include <asm/tlb.h> 54 #include <asm/mmu_context.h> 55 56 #define CREATE_TRACE_POINTS 57 #include <trace/events/mmap.h> 58 59 #include "internal.h" 60 61 #ifndef arch_mmap_check 62 #define arch_mmap_check(addr, len, flags) (0) 63 #endif 64 65 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS 66 const int mmap_rnd_bits_min = CONFIG_ARCH_MMAP_RND_BITS_MIN; 67 int mmap_rnd_bits_max __ro_after_init = CONFIG_ARCH_MMAP_RND_BITS_MAX; 68 int mmap_rnd_bits __read_mostly = CONFIG_ARCH_MMAP_RND_BITS; 69 #endif 70 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS 71 const int mmap_rnd_compat_bits_min = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MIN; 72 const int mmap_rnd_compat_bits_max = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MAX; 73 int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS; 74 #endif 75 76 static bool ignore_rlimit_data; 77 core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644); 78 79 static void unmap_region(struct mm_struct *mm, struct ma_state *mas, 80 struct vm_area_struct *vma, struct vm_area_struct *prev, 81 struct vm_area_struct *next, unsigned long start, 82 unsigned long end, unsigned long tree_end, bool mm_wr_locked); 83 84 static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags) 85 { 86 return pgprot_modify(oldprot, vm_get_page_prot(vm_flags)); 87 } 88 89 /* Update vma->vm_page_prot to reflect vma->vm_flags. */ 90 void vma_set_page_prot(struct vm_area_struct *vma) 91 { 92 unsigned long vm_flags = vma->vm_flags; 93 pgprot_t vm_page_prot; 94 95 vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags); 96 if (vma_wants_writenotify(vma, vm_page_prot)) { 97 vm_flags &= ~VM_SHARED; 98 vm_page_prot = vm_pgprot_modify(vm_page_prot, vm_flags); 99 } 100 /* remove_protection_ptes reads vma->vm_page_prot without mmap_lock */ 101 WRITE_ONCE(vma->vm_page_prot, vm_page_prot); 102 } 103 104 /* 105 * Requires inode->i_mapping->i_mmap_rwsem 106 */ 107 static void __remove_shared_vm_struct(struct vm_area_struct *vma, 108 struct address_space *mapping) 109 { 110 if (vma_is_shared_maywrite(vma)) 111 mapping_unmap_writable(mapping); 112 113 flush_dcache_mmap_lock(mapping); 114 vma_interval_tree_remove(vma, &mapping->i_mmap); 115 flush_dcache_mmap_unlock(mapping); 116 } 117 118 /* 119 * Unlink a file-based vm structure from its interval tree, to hide 120 * vma from rmap and vmtruncate before freeing its page tables. 121 */ 122 void unlink_file_vma(struct vm_area_struct *vma) 123 { 124 struct file *file = vma->vm_file; 125 126 if (file) { 127 struct address_space *mapping = file->f_mapping; 128 i_mmap_lock_write(mapping); 129 __remove_shared_vm_struct(vma, mapping); 130 i_mmap_unlock_write(mapping); 131 } 132 } 133 134 /* 135 * Close a vm structure and free it. 136 */ 137 static void remove_vma(struct vm_area_struct *vma, bool unreachable) 138 { 139 might_sleep(); 140 if (vma->vm_ops && vma->vm_ops->close) 141 vma->vm_ops->close(vma); 142 if (vma->vm_file) 143 fput(vma->vm_file); 144 mpol_put(vma_policy(vma)); 145 if (unreachable) 146 __vm_area_free(vma); 147 else 148 vm_area_free(vma); 149 } 150 151 static inline struct vm_area_struct *vma_prev_limit(struct vma_iterator *vmi, 152 unsigned long min) 153 { 154 return mas_prev(&vmi->mas, min); 155 } 156 157 /* 158 * check_brk_limits() - Use platform specific check of range & verify mlock 159 * limits. 160 * @addr: The address to check 161 * @len: The size of increase. 162 * 163 * Return: 0 on success. 164 */ 165 static int check_brk_limits(unsigned long addr, unsigned long len) 166 { 167 unsigned long mapped_addr; 168 169 mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED); 170 if (IS_ERR_VALUE(mapped_addr)) 171 return mapped_addr; 172 173 return mlock_future_ok(current->mm, current->mm->def_flags, len) 174 ? 0 : -EAGAIN; 175 } 176 static int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma, 177 unsigned long addr, unsigned long request, unsigned long flags); 178 SYSCALL_DEFINE1(brk, unsigned long, brk) 179 { 180 unsigned long newbrk, oldbrk, origbrk; 181 struct mm_struct *mm = current->mm; 182 struct vm_area_struct *brkvma, *next = NULL; 183 unsigned long min_brk; 184 bool populate = false; 185 LIST_HEAD(uf); 186 struct vma_iterator vmi; 187 188 if (mmap_write_lock_killable(mm)) 189 return -EINTR; 190 191 origbrk = mm->brk; 192 193 #ifdef CONFIG_COMPAT_BRK 194 /* 195 * CONFIG_COMPAT_BRK can still be overridden by setting 196 * randomize_va_space to 2, which will still cause mm->start_brk 197 * to be arbitrarily shifted 198 */ 199 if (current->brk_randomized) 200 min_brk = mm->start_brk; 201 else 202 min_brk = mm->end_data; 203 #else 204 min_brk = mm->start_brk; 205 #endif 206 if (brk < min_brk) 207 goto out; 208 209 /* 210 * Check against rlimit here. If this check is done later after the test 211 * of oldbrk with newbrk then it can escape the test and let the data 212 * segment grow beyond its set limit the in case where the limit is 213 * not page aligned -Ram Gupta 214 */ 215 if (check_data_rlimit(rlimit(RLIMIT_DATA), brk, mm->start_brk, 216 mm->end_data, mm->start_data)) 217 goto out; 218 219 newbrk = PAGE_ALIGN(brk); 220 oldbrk = PAGE_ALIGN(mm->brk); 221 if (oldbrk == newbrk) { 222 mm->brk = brk; 223 goto success; 224 } 225 226 /* Always allow shrinking brk. */ 227 if (brk <= mm->brk) { 228 /* Search one past newbrk */ 229 vma_iter_init(&vmi, mm, newbrk); 230 brkvma = vma_find(&vmi, oldbrk); 231 if (!brkvma || brkvma->vm_start >= oldbrk) 232 goto out; /* mapping intersects with an existing non-brk vma. */ 233 /* 234 * mm->brk must be protected by write mmap_lock. 235 * do_vma_munmap() will drop the lock on success, so update it 236 * before calling do_vma_munmap(). 237 */ 238 mm->brk = brk; 239 if (do_vma_munmap(&vmi, brkvma, newbrk, oldbrk, &uf, true)) 240 goto out; 241 242 goto success_unlocked; 243 } 244 245 if (check_brk_limits(oldbrk, newbrk - oldbrk)) 246 goto out; 247 248 /* 249 * Only check if the next VMA is within the stack_guard_gap of the 250 * expansion area 251 */ 252 vma_iter_init(&vmi, mm, oldbrk); 253 next = vma_find(&vmi, newbrk + PAGE_SIZE + stack_guard_gap); 254 if (next && newbrk + PAGE_SIZE > vm_start_gap(next)) 255 goto out; 256 257 brkvma = vma_prev_limit(&vmi, mm->start_brk); 258 /* Ok, looks good - let it rip. */ 259 if (do_brk_flags(&vmi, brkvma, oldbrk, newbrk - oldbrk, 0) < 0) 260 goto out; 261 262 mm->brk = brk; 263 if (mm->def_flags & VM_LOCKED) 264 populate = true; 265 266 success: 267 mmap_write_unlock(mm); 268 success_unlocked: 269 userfaultfd_unmap_complete(mm, &uf); 270 if (populate) 271 mm_populate(oldbrk, newbrk - oldbrk); 272 return brk; 273 274 out: 275 mm->brk = origbrk; 276 mmap_write_unlock(mm); 277 return origbrk; 278 } 279 280 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE) 281 static void validate_mm(struct mm_struct *mm) 282 { 283 int bug = 0; 284 int i = 0; 285 struct vm_area_struct *vma; 286 VMA_ITERATOR(vmi, mm, 0); 287 288 mt_validate(&mm->mm_mt); 289 for_each_vma(vmi, vma) { 290 #ifdef CONFIG_DEBUG_VM_RB 291 struct anon_vma *anon_vma = vma->anon_vma; 292 struct anon_vma_chain *avc; 293 #endif 294 unsigned long vmi_start, vmi_end; 295 bool warn = 0; 296 297 vmi_start = vma_iter_addr(&vmi); 298 vmi_end = vma_iter_end(&vmi); 299 if (VM_WARN_ON_ONCE_MM(vma->vm_end != vmi_end, mm)) 300 warn = 1; 301 302 if (VM_WARN_ON_ONCE_MM(vma->vm_start != vmi_start, mm)) 303 warn = 1; 304 305 if (warn) { 306 pr_emerg("issue in %s\n", current->comm); 307 dump_stack(); 308 dump_vma(vma); 309 pr_emerg("tree range: %px start %lx end %lx\n", vma, 310 vmi_start, vmi_end - 1); 311 vma_iter_dump_tree(&vmi); 312 } 313 314 #ifdef CONFIG_DEBUG_VM_RB 315 if (anon_vma) { 316 anon_vma_lock_read(anon_vma); 317 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 318 anon_vma_interval_tree_verify(avc); 319 anon_vma_unlock_read(anon_vma); 320 } 321 #endif 322 i++; 323 } 324 if (i != mm->map_count) { 325 pr_emerg("map_count %d vma iterator %d\n", mm->map_count, i); 326 bug = 1; 327 } 328 VM_BUG_ON_MM(bug, mm); 329 } 330 331 #else /* !CONFIG_DEBUG_VM_MAPLE_TREE */ 332 #define validate_mm(mm) do { } while (0) 333 #endif /* CONFIG_DEBUG_VM_MAPLE_TREE */ 334 335 /* 336 * vma has some anon_vma assigned, and is already inserted on that 337 * anon_vma's interval trees. 338 * 339 * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the 340 * vma must be removed from the anon_vma's interval trees using 341 * anon_vma_interval_tree_pre_update_vma(). 342 * 343 * After the update, the vma will be reinserted using 344 * anon_vma_interval_tree_post_update_vma(). 345 * 346 * The entire update must be protected by exclusive mmap_lock and by 347 * the root anon_vma's mutex. 348 */ 349 static inline void 350 anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma) 351 { 352 struct anon_vma_chain *avc; 353 354 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 355 anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root); 356 } 357 358 static inline void 359 anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma) 360 { 361 struct anon_vma_chain *avc; 362 363 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 364 anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root); 365 } 366 367 static unsigned long count_vma_pages_range(struct mm_struct *mm, 368 unsigned long addr, unsigned long end) 369 { 370 VMA_ITERATOR(vmi, mm, addr); 371 struct vm_area_struct *vma; 372 unsigned long nr_pages = 0; 373 374 for_each_vma_range(vmi, vma, end) { 375 unsigned long vm_start = max(addr, vma->vm_start); 376 unsigned long vm_end = min(end, vma->vm_end); 377 378 nr_pages += PHYS_PFN(vm_end - vm_start); 379 } 380 381 return nr_pages; 382 } 383 384 static void __vma_link_file(struct vm_area_struct *vma, 385 struct address_space *mapping) 386 { 387 if (vma_is_shared_maywrite(vma)) 388 mapping_allow_writable(mapping); 389 390 flush_dcache_mmap_lock(mapping); 391 vma_interval_tree_insert(vma, &mapping->i_mmap); 392 flush_dcache_mmap_unlock(mapping); 393 } 394 395 static void vma_link_file(struct vm_area_struct *vma) 396 { 397 struct file *file = vma->vm_file; 398 struct address_space *mapping; 399 400 if (file) { 401 mapping = file->f_mapping; 402 i_mmap_lock_write(mapping); 403 __vma_link_file(vma, mapping); 404 i_mmap_unlock_write(mapping); 405 } 406 } 407 408 static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) 409 { 410 VMA_ITERATOR(vmi, mm, 0); 411 412 vma_iter_config(&vmi, vma->vm_start, vma->vm_end); 413 if (vma_iter_prealloc(&vmi, vma)) 414 return -ENOMEM; 415 416 vma_start_write(vma); 417 vma_iter_store(&vmi, vma); 418 vma_link_file(vma); 419 mm->map_count++; 420 validate_mm(mm); 421 return 0; 422 } 423 424 /* 425 * init_multi_vma_prep() - Initializer for struct vma_prepare 426 * @vp: The vma_prepare struct 427 * @vma: The vma that will be altered once locked 428 * @next: The next vma if it is to be adjusted 429 * @remove: The first vma to be removed 430 * @remove2: The second vma to be removed 431 */ 432 static inline void init_multi_vma_prep(struct vma_prepare *vp, 433 struct vm_area_struct *vma, struct vm_area_struct *next, 434 struct vm_area_struct *remove, struct vm_area_struct *remove2) 435 { 436 memset(vp, 0, sizeof(struct vma_prepare)); 437 vp->vma = vma; 438 vp->anon_vma = vma->anon_vma; 439 vp->remove = remove; 440 vp->remove2 = remove2; 441 vp->adj_next = next; 442 if (!vp->anon_vma && next) 443 vp->anon_vma = next->anon_vma; 444 445 vp->file = vma->vm_file; 446 if (vp->file) 447 vp->mapping = vma->vm_file->f_mapping; 448 449 } 450 451 /* 452 * init_vma_prep() - Initializer wrapper for vma_prepare struct 453 * @vp: The vma_prepare struct 454 * @vma: The vma that will be altered once locked 455 */ 456 static inline void init_vma_prep(struct vma_prepare *vp, 457 struct vm_area_struct *vma) 458 { 459 init_multi_vma_prep(vp, vma, NULL, NULL, NULL); 460 } 461 462 463 /* 464 * vma_prepare() - Helper function for handling locking VMAs prior to altering 465 * @vp: The initialized vma_prepare struct 466 */ 467 static inline void vma_prepare(struct vma_prepare *vp) 468 { 469 if (vp->file) { 470 uprobe_munmap(vp->vma, vp->vma->vm_start, vp->vma->vm_end); 471 472 if (vp->adj_next) 473 uprobe_munmap(vp->adj_next, vp->adj_next->vm_start, 474 vp->adj_next->vm_end); 475 476 i_mmap_lock_write(vp->mapping); 477 if (vp->insert && vp->insert->vm_file) { 478 /* 479 * Put into interval tree now, so instantiated pages 480 * are visible to arm/parisc __flush_dcache_page 481 * throughout; but we cannot insert into address 482 * space until vma start or end is updated. 483 */ 484 __vma_link_file(vp->insert, 485 vp->insert->vm_file->f_mapping); 486 } 487 } 488 489 if (vp->anon_vma) { 490 anon_vma_lock_write(vp->anon_vma); 491 anon_vma_interval_tree_pre_update_vma(vp->vma); 492 if (vp->adj_next) 493 anon_vma_interval_tree_pre_update_vma(vp->adj_next); 494 } 495 496 if (vp->file) { 497 flush_dcache_mmap_lock(vp->mapping); 498 vma_interval_tree_remove(vp->vma, &vp->mapping->i_mmap); 499 if (vp->adj_next) 500 vma_interval_tree_remove(vp->adj_next, 501 &vp->mapping->i_mmap); 502 } 503 504 } 505 506 /* 507 * vma_complete- Helper function for handling the unlocking after altering VMAs, 508 * or for inserting a VMA. 509 * 510 * @vp: The vma_prepare struct 511 * @vmi: The vma iterator 512 * @mm: The mm_struct 513 */ 514 static inline void vma_complete(struct vma_prepare *vp, 515 struct vma_iterator *vmi, struct mm_struct *mm) 516 { 517 if (vp->file) { 518 if (vp->adj_next) 519 vma_interval_tree_insert(vp->adj_next, 520 &vp->mapping->i_mmap); 521 vma_interval_tree_insert(vp->vma, &vp->mapping->i_mmap); 522 flush_dcache_mmap_unlock(vp->mapping); 523 } 524 525 if (vp->remove && vp->file) { 526 __remove_shared_vm_struct(vp->remove, vp->mapping); 527 if (vp->remove2) 528 __remove_shared_vm_struct(vp->remove2, vp->mapping); 529 } else if (vp->insert) { 530 /* 531 * split_vma has split insert from vma, and needs 532 * us to insert it before dropping the locks 533 * (it may either follow vma or precede it). 534 */ 535 vma_iter_store(vmi, vp->insert); 536 mm->map_count++; 537 } 538 539 if (vp->anon_vma) { 540 anon_vma_interval_tree_post_update_vma(vp->vma); 541 if (vp->adj_next) 542 anon_vma_interval_tree_post_update_vma(vp->adj_next); 543 anon_vma_unlock_write(vp->anon_vma); 544 } 545 546 if (vp->file) { 547 i_mmap_unlock_write(vp->mapping); 548 uprobe_mmap(vp->vma); 549 550 if (vp->adj_next) 551 uprobe_mmap(vp->adj_next); 552 } 553 554 if (vp->remove) { 555 again: 556 vma_mark_detached(vp->remove, true); 557 if (vp->file) { 558 uprobe_munmap(vp->remove, vp->remove->vm_start, 559 vp->remove->vm_end); 560 fput(vp->file); 561 } 562 if (vp->remove->anon_vma) 563 anon_vma_merge(vp->vma, vp->remove); 564 mm->map_count--; 565 mpol_put(vma_policy(vp->remove)); 566 if (!vp->remove2) 567 WARN_ON_ONCE(vp->vma->vm_end < vp->remove->vm_end); 568 vm_area_free(vp->remove); 569 570 /* 571 * In mprotect's case 6 (see comments on vma_merge), 572 * we are removing both mid and next vmas 573 */ 574 if (vp->remove2) { 575 vp->remove = vp->remove2; 576 vp->remove2 = NULL; 577 goto again; 578 } 579 } 580 if (vp->insert && vp->file) 581 uprobe_mmap(vp->insert); 582 validate_mm(mm); 583 } 584 585 /* 586 * dup_anon_vma() - Helper function to duplicate anon_vma 587 * @dst: The destination VMA 588 * @src: The source VMA 589 * @dup: Pointer to the destination VMA when successful. 590 * 591 * Returns: 0 on success. 592 */ 593 static inline int dup_anon_vma(struct vm_area_struct *dst, 594 struct vm_area_struct *src, struct vm_area_struct **dup) 595 { 596 /* 597 * Easily overlooked: when mprotect shifts the boundary, make sure the 598 * expanding vma has anon_vma set if the shrinking vma had, to cover any 599 * anon pages imported. 600 */ 601 if (src->anon_vma && !dst->anon_vma) { 602 int ret; 603 604 vma_assert_write_locked(dst); 605 dst->anon_vma = src->anon_vma; 606 ret = anon_vma_clone(dst, src); 607 if (ret) 608 return ret; 609 610 *dup = dst; 611 } 612 613 return 0; 614 } 615 616 /* 617 * vma_expand - Expand an existing VMA 618 * 619 * @vmi: The vma iterator 620 * @vma: The vma to expand 621 * @start: The start of the vma 622 * @end: The exclusive end of the vma 623 * @pgoff: The page offset of vma 624 * @next: The current of next vma. 625 * 626 * Expand @vma to @start and @end. Can expand off the start and end. Will 627 * expand over @next if it's different from @vma and @end == @next->vm_end. 628 * Checking if the @vma can expand and merge with @next needs to be handled by 629 * the caller. 630 * 631 * Returns: 0 on success 632 */ 633 int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma, 634 unsigned long start, unsigned long end, pgoff_t pgoff, 635 struct vm_area_struct *next) 636 { 637 struct vm_area_struct *anon_dup = NULL; 638 bool remove_next = false; 639 struct vma_prepare vp; 640 641 vma_start_write(vma); 642 if (next && (vma != next) && (end == next->vm_end)) { 643 int ret; 644 645 remove_next = true; 646 vma_start_write(next); 647 ret = dup_anon_vma(vma, next, &anon_dup); 648 if (ret) 649 return ret; 650 } 651 652 init_multi_vma_prep(&vp, vma, NULL, remove_next ? next : NULL, NULL); 653 /* Not merging but overwriting any part of next is not handled. */ 654 VM_WARN_ON(next && !vp.remove && 655 next != vma && end > next->vm_start); 656 /* Only handles expanding */ 657 VM_WARN_ON(vma->vm_start < start || vma->vm_end > end); 658 659 /* Note: vma iterator must be pointing to 'start' */ 660 vma_iter_config(vmi, start, end); 661 if (vma_iter_prealloc(vmi, vma)) 662 goto nomem; 663 664 vma_prepare(&vp); 665 vma_adjust_trans_huge(vma, start, end, 0); 666 vma_set_range(vma, start, end, pgoff); 667 vma_iter_store(vmi, vma); 668 669 vma_complete(&vp, vmi, vma->vm_mm); 670 return 0; 671 672 nomem: 673 if (anon_dup) 674 unlink_anon_vmas(anon_dup); 675 return -ENOMEM; 676 } 677 678 /* 679 * vma_shrink() - Reduce an existing VMAs memory area 680 * @vmi: The vma iterator 681 * @vma: The VMA to modify 682 * @start: The new start 683 * @end: The new end 684 * 685 * Returns: 0 on success, -ENOMEM otherwise 686 */ 687 int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, 688 unsigned long start, unsigned long end, pgoff_t pgoff) 689 { 690 struct vma_prepare vp; 691 692 WARN_ON((vma->vm_start != start) && (vma->vm_end != end)); 693 694 if (vma->vm_start < start) 695 vma_iter_config(vmi, vma->vm_start, start); 696 else 697 vma_iter_config(vmi, end, vma->vm_end); 698 699 if (vma_iter_prealloc(vmi, NULL)) 700 return -ENOMEM; 701 702 vma_start_write(vma); 703 704 init_vma_prep(&vp, vma); 705 vma_prepare(&vp); 706 vma_adjust_trans_huge(vma, start, end, 0); 707 708 vma_iter_clear(vmi); 709 vma_set_range(vma, start, end, pgoff); 710 vma_complete(&vp, vmi, vma->vm_mm); 711 return 0; 712 } 713 714 /* 715 * If the vma has a ->close operation then the driver probably needs to release 716 * per-vma resources, so we don't attempt to merge those if the caller indicates 717 * the current vma may be removed as part of the merge. 718 */ 719 static inline bool is_mergeable_vma(struct vm_area_struct *vma, 720 struct file *file, unsigned long vm_flags, 721 struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 722 struct anon_vma_name *anon_name, bool may_remove_vma) 723 { 724 /* 725 * VM_SOFTDIRTY should not prevent from VMA merging, if we 726 * match the flags but dirty bit -- the caller should mark 727 * merged VMA as dirty. If dirty bit won't be excluded from 728 * comparison, we increase pressure on the memory system forcing 729 * the kernel to generate new VMAs when old one could be 730 * extended instead. 731 */ 732 if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY) 733 return false; 734 if (vma->vm_file != file) 735 return false; 736 if (may_remove_vma && vma->vm_ops && vma->vm_ops->close) 737 return false; 738 if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx)) 739 return false; 740 if (!anon_vma_name_eq(anon_vma_name(vma), anon_name)) 741 return false; 742 return true; 743 } 744 745 static inline bool is_mergeable_anon_vma(struct anon_vma *anon_vma1, 746 struct anon_vma *anon_vma2, struct vm_area_struct *vma) 747 { 748 /* 749 * The list_is_singular() test is to avoid merging VMA cloned from 750 * parents. This can improve scalability caused by anon_vma lock. 751 */ 752 if ((!anon_vma1 || !anon_vma2) && (!vma || 753 list_is_singular(&vma->anon_vma_chain))) 754 return true; 755 return anon_vma1 == anon_vma2; 756 } 757 758 /* 759 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff) 760 * in front of (at a lower virtual address and file offset than) the vma. 761 * 762 * We cannot merge two vmas if they have differently assigned (non-NULL) 763 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible. 764 * 765 * We don't check here for the merged mmap wrapping around the end of pagecache 766 * indices (16TB on ia32) because do_mmap() does not permit mmap's which 767 * wrap, nor mmaps which cover the final page at index -1UL. 768 * 769 * We assume the vma may be removed as part of the merge. 770 */ 771 static bool 772 can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags, 773 struct anon_vma *anon_vma, struct file *file, 774 pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 775 struct anon_vma_name *anon_name) 776 { 777 if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, true) && 778 is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { 779 if (vma->vm_pgoff == vm_pgoff) 780 return true; 781 } 782 return false; 783 } 784 785 /* 786 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff) 787 * beyond (at a higher virtual address and file offset than) the vma. 788 * 789 * We cannot merge two vmas if they have differently assigned (non-NULL) 790 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible. 791 * 792 * We assume that vma is not removed as part of the merge. 793 */ 794 static bool 795 can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags, 796 struct anon_vma *anon_vma, struct file *file, 797 pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 798 struct anon_vma_name *anon_name) 799 { 800 if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, false) && 801 is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { 802 pgoff_t vm_pglen; 803 vm_pglen = vma_pages(vma); 804 if (vma->vm_pgoff + vm_pglen == vm_pgoff) 805 return true; 806 } 807 return false; 808 } 809 810 /* 811 * Given a mapping request (addr,end,vm_flags,file,pgoff,anon_name), 812 * figure out whether that can be merged with its predecessor or its 813 * successor. Or both (it neatly fills a hole). 814 * 815 * In most cases - when called for mmap, brk or mremap - [addr,end) is 816 * certain not to be mapped by the time vma_merge is called; but when 817 * called for mprotect, it is certain to be already mapped (either at 818 * an offset within prev, or at the start of next), and the flags of 819 * this area are about to be changed to vm_flags - and the no-change 820 * case has already been eliminated. 821 * 822 * The following mprotect cases have to be considered, where **** is 823 * the area passed down from mprotect_fixup, never extending beyond one 824 * vma, PPPP is the previous vma, CCCC is a concurrent vma that starts 825 * at the same address as **** and is of the same or larger span, and 826 * NNNN the next vma after ****: 827 * 828 * **** **** **** 829 * PPPPPPNNNNNN PPPPPPNNNNNN PPPPPPCCCCCC 830 * cannot merge might become might become 831 * PPNNNNNNNNNN PPPPPPPPPPCC 832 * mmap, brk or case 4 below case 5 below 833 * mremap move: 834 * **** **** 835 * PPPP NNNN PPPPCCCCNNNN 836 * might become might become 837 * PPPPPPPPPPPP 1 or PPPPPPPPPPPP 6 or 838 * PPPPPPPPNNNN 2 or PPPPPPPPNNNN 7 or 839 * PPPPNNNNNNNN 3 PPPPNNNNNNNN 8 840 * 841 * It is important for case 8 that the vma CCCC overlapping the 842 * region **** is never going to extended over NNNN. Instead NNNN must 843 * be extended in region **** and CCCC must be removed. This way in 844 * all cases where vma_merge succeeds, the moment vma_merge drops the 845 * rmap_locks, the properties of the merged vma will be already 846 * correct for the whole merged range. Some of those properties like 847 * vm_page_prot/vm_flags may be accessed by rmap_walks and they must 848 * be correct for the whole merged range immediately after the 849 * rmap_locks are released. Otherwise if NNNN would be removed and 850 * CCCC would be extended over the NNNN range, remove_migration_ptes 851 * or other rmap walkers (if working on addresses beyond the "end" 852 * parameter) may establish ptes with the wrong permissions of CCCC 853 * instead of the right permissions of NNNN. 854 * 855 * In the code below: 856 * PPPP is represented by *prev 857 * CCCC is represented by *curr or not represented at all (NULL) 858 * NNNN is represented by *next or not represented at all (NULL) 859 * **** is not represented - it will be merged and the vma containing the 860 * area is returned, or the function will return NULL 861 */ 862 static struct vm_area_struct 863 *vma_merge(struct vma_iterator *vmi, struct vm_area_struct *prev, 864 struct vm_area_struct *src, unsigned long addr, unsigned long end, 865 unsigned long vm_flags, pgoff_t pgoff, struct mempolicy *policy, 866 struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 867 struct anon_vma_name *anon_name) 868 { 869 struct mm_struct *mm = src->vm_mm; 870 struct anon_vma *anon_vma = src->anon_vma; 871 struct file *file = src->vm_file; 872 struct vm_area_struct *curr, *next, *res; 873 struct vm_area_struct *vma, *adjust, *remove, *remove2; 874 struct vm_area_struct *anon_dup = NULL; 875 struct vma_prepare vp; 876 pgoff_t vma_pgoff; 877 int err = 0; 878 bool merge_prev = false; 879 bool merge_next = false; 880 bool vma_expanded = false; 881 unsigned long vma_start = addr; 882 unsigned long vma_end = end; 883 pgoff_t pglen = (end - addr) >> PAGE_SHIFT; 884 long adj_start = 0; 885 886 /* 887 * We later require that vma->vm_flags == vm_flags, 888 * so this tests vma->vm_flags & VM_SPECIAL, too. 889 */ 890 if (vm_flags & VM_SPECIAL) 891 return NULL; 892 893 /* Does the input range span an existing VMA? (cases 5 - 8) */ 894 curr = find_vma_intersection(mm, prev ? prev->vm_end : 0, end); 895 896 if (!curr || /* cases 1 - 4 */ 897 end == curr->vm_end) /* cases 6 - 8, adjacent VMA */ 898 next = vma_lookup(mm, end); 899 else 900 next = NULL; /* case 5 */ 901 902 if (prev) { 903 vma_start = prev->vm_start; 904 vma_pgoff = prev->vm_pgoff; 905 906 /* Can we merge the predecessor? */ 907 if (addr == prev->vm_end && mpol_equal(vma_policy(prev), policy) 908 && can_vma_merge_after(prev, vm_flags, anon_vma, file, 909 pgoff, vm_userfaultfd_ctx, anon_name)) { 910 merge_prev = true; 911 vma_prev(vmi); 912 } 913 } 914 915 /* Can we merge the successor? */ 916 if (next && mpol_equal(policy, vma_policy(next)) && 917 can_vma_merge_before(next, vm_flags, anon_vma, file, pgoff+pglen, 918 vm_userfaultfd_ctx, anon_name)) { 919 merge_next = true; 920 } 921 922 /* Verify some invariant that must be enforced by the caller. */ 923 VM_WARN_ON(prev && addr <= prev->vm_start); 924 VM_WARN_ON(curr && (addr != curr->vm_start || end > curr->vm_end)); 925 VM_WARN_ON(addr >= end); 926 927 if (!merge_prev && !merge_next) 928 return NULL; /* Not mergeable. */ 929 930 if (merge_prev) 931 vma_start_write(prev); 932 933 res = vma = prev; 934 remove = remove2 = adjust = NULL; 935 936 /* Can we merge both the predecessor and the successor? */ 937 if (merge_prev && merge_next && 938 is_mergeable_anon_vma(prev->anon_vma, next->anon_vma, NULL)) { 939 vma_start_write(next); 940 remove = next; /* case 1 */ 941 vma_end = next->vm_end; 942 err = dup_anon_vma(prev, next, &anon_dup); 943 if (curr) { /* case 6 */ 944 vma_start_write(curr); 945 remove = curr; 946 remove2 = next; 947 /* 948 * Note that the dup_anon_vma below cannot overwrite err 949 * since the first caller would do nothing unless next 950 * has an anon_vma. 951 */ 952 if (!next->anon_vma) 953 err = dup_anon_vma(prev, curr, &anon_dup); 954 } 955 } else if (merge_prev) { /* case 2 */ 956 if (curr) { 957 vma_start_write(curr); 958 if (end == curr->vm_end) { /* case 7 */ 959 /* 960 * can_vma_merge_after() assumed we would not be 961 * removing prev vma, so it skipped the check 962 * for vm_ops->close, but we are removing curr 963 */ 964 if (curr->vm_ops && curr->vm_ops->close) 965 err = -EINVAL; 966 remove = curr; 967 } else { /* case 5 */ 968 adjust = curr; 969 adj_start = (end - curr->vm_start); 970 } 971 if (!err) 972 err = dup_anon_vma(prev, curr, &anon_dup); 973 } 974 } else { /* merge_next */ 975 vma_start_write(next); 976 res = next; 977 if (prev && addr < prev->vm_end) { /* case 4 */ 978 vma_start_write(prev); 979 vma_end = addr; 980 adjust = next; 981 adj_start = -(prev->vm_end - addr); 982 err = dup_anon_vma(next, prev, &anon_dup); 983 } else { 984 /* 985 * Note that cases 3 and 8 are the ONLY ones where prev 986 * is permitted to be (but is not necessarily) NULL. 987 */ 988 vma = next; /* case 3 */ 989 vma_start = addr; 990 vma_end = next->vm_end; 991 vma_pgoff = next->vm_pgoff - pglen; 992 if (curr) { /* case 8 */ 993 vma_pgoff = curr->vm_pgoff; 994 vma_start_write(curr); 995 remove = curr; 996 err = dup_anon_vma(next, curr, &anon_dup); 997 } 998 } 999 } 1000 1001 /* Error in anon_vma clone. */ 1002 if (err) 1003 goto anon_vma_fail; 1004 1005 if (vma_start < vma->vm_start || vma_end > vma->vm_end) 1006 vma_expanded = true; 1007 1008 if (vma_expanded) { 1009 vma_iter_config(vmi, vma_start, vma_end); 1010 } else { 1011 vma_iter_config(vmi, adjust->vm_start + adj_start, 1012 adjust->vm_end); 1013 } 1014 1015 if (vma_iter_prealloc(vmi, vma)) 1016 goto prealloc_fail; 1017 1018 init_multi_vma_prep(&vp, vma, adjust, remove, remove2); 1019 VM_WARN_ON(vp.anon_vma && adjust && adjust->anon_vma && 1020 vp.anon_vma != adjust->anon_vma); 1021 1022 vma_prepare(&vp); 1023 vma_adjust_trans_huge(vma, vma_start, vma_end, adj_start); 1024 vma_set_range(vma, vma_start, vma_end, vma_pgoff); 1025 1026 if (vma_expanded) 1027 vma_iter_store(vmi, vma); 1028 1029 if (adj_start) { 1030 adjust->vm_start += adj_start; 1031 adjust->vm_pgoff += adj_start >> PAGE_SHIFT; 1032 if (adj_start < 0) { 1033 WARN_ON(vma_expanded); 1034 vma_iter_store(vmi, next); 1035 } 1036 } 1037 1038 vma_complete(&vp, vmi, mm); 1039 khugepaged_enter_vma(res, vm_flags); 1040 return res; 1041 1042 prealloc_fail: 1043 if (anon_dup) 1044 unlink_anon_vmas(anon_dup); 1045 1046 anon_vma_fail: 1047 vma_iter_set(vmi, addr); 1048 vma_iter_load(vmi); 1049 return NULL; 1050 } 1051 1052 /* 1053 * Rough compatibility check to quickly see if it's even worth looking 1054 * at sharing an anon_vma. 1055 * 1056 * They need to have the same vm_file, and the flags can only differ 1057 * in things that mprotect may change. 1058 * 1059 * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that 1060 * we can merge the two vma's. For example, we refuse to merge a vma if 1061 * there is a vm_ops->close() function, because that indicates that the 1062 * driver is doing some kind of reference counting. But that doesn't 1063 * really matter for the anon_vma sharing case. 1064 */ 1065 static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b) 1066 { 1067 return a->vm_end == b->vm_start && 1068 mpol_equal(vma_policy(a), vma_policy(b)) && 1069 a->vm_file == b->vm_file && 1070 !((a->vm_flags ^ b->vm_flags) & ~(VM_ACCESS_FLAGS | VM_SOFTDIRTY)) && 1071 b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT); 1072 } 1073 1074 /* 1075 * Do some basic sanity checking to see if we can re-use the anon_vma 1076 * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be 1077 * the same as 'old', the other will be the new one that is trying 1078 * to share the anon_vma. 1079 * 1080 * NOTE! This runs with mmap_lock held for reading, so it is possible that 1081 * the anon_vma of 'old' is concurrently in the process of being set up 1082 * by another page fault trying to merge _that_. But that's ok: if it 1083 * is being set up, that automatically means that it will be a singleton 1084 * acceptable for merging, so we can do all of this optimistically. But 1085 * we do that READ_ONCE() to make sure that we never re-load the pointer. 1086 * 1087 * IOW: that the "list_is_singular()" test on the anon_vma_chain only 1088 * matters for the 'stable anon_vma' case (ie the thing we want to avoid 1089 * is to return an anon_vma that is "complex" due to having gone through 1090 * a fork). 1091 * 1092 * We also make sure that the two vma's are compatible (adjacent, 1093 * and with the same memory policies). That's all stable, even with just 1094 * a read lock on the mmap_lock. 1095 */ 1096 static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b) 1097 { 1098 if (anon_vma_compatible(a, b)) { 1099 struct anon_vma *anon_vma = READ_ONCE(old->anon_vma); 1100 1101 if (anon_vma && list_is_singular(&old->anon_vma_chain)) 1102 return anon_vma; 1103 } 1104 return NULL; 1105 } 1106 1107 /* 1108 * find_mergeable_anon_vma is used by anon_vma_prepare, to check 1109 * neighbouring vmas for a suitable anon_vma, before it goes off 1110 * to allocate a new anon_vma. It checks because a repetitive 1111 * sequence of mprotects and faults may otherwise lead to distinct 1112 * anon_vmas being allocated, preventing vma merge in subsequent 1113 * mprotect. 1114 */ 1115 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma) 1116 { 1117 MA_STATE(mas, &vma->vm_mm->mm_mt, vma->vm_end, vma->vm_end); 1118 struct anon_vma *anon_vma = NULL; 1119 struct vm_area_struct *prev, *next; 1120 1121 /* Try next first. */ 1122 next = mas_walk(&mas); 1123 if (next) { 1124 anon_vma = reusable_anon_vma(next, vma, next); 1125 if (anon_vma) 1126 return anon_vma; 1127 } 1128 1129 prev = mas_prev(&mas, 0); 1130 VM_BUG_ON_VMA(prev != vma, vma); 1131 prev = mas_prev(&mas, 0); 1132 /* Try prev next. */ 1133 if (prev) 1134 anon_vma = reusable_anon_vma(prev, prev, vma); 1135 1136 /* 1137 * We might reach here with anon_vma == NULL if we can't find 1138 * any reusable anon_vma. 1139 * There's no absolute need to look only at touching neighbours: 1140 * we could search further afield for "compatible" anon_vmas. 1141 * But it would probably just be a waste of time searching, 1142 * or lead to too many vmas hanging off the same anon_vma. 1143 * We're trying to allow mprotect remerging later on, 1144 * not trying to minimize memory used for anon_vmas. 1145 */ 1146 return anon_vma; 1147 } 1148 1149 /* 1150 * If a hint addr is less than mmap_min_addr change hint to be as 1151 * low as possible but still greater than mmap_min_addr 1152 */ 1153 static inline unsigned long round_hint_to_min(unsigned long hint) 1154 { 1155 hint &= PAGE_MASK; 1156 if (((void *)hint != NULL) && 1157 (hint < mmap_min_addr)) 1158 return PAGE_ALIGN(mmap_min_addr); 1159 return hint; 1160 } 1161 1162 bool mlock_future_ok(struct mm_struct *mm, unsigned long flags, 1163 unsigned long bytes) 1164 { 1165 unsigned long locked_pages, limit_pages; 1166 1167 if (!(flags & VM_LOCKED) || capable(CAP_IPC_LOCK)) 1168 return true; 1169 1170 locked_pages = bytes >> PAGE_SHIFT; 1171 locked_pages += mm->locked_vm; 1172 1173 limit_pages = rlimit(RLIMIT_MEMLOCK); 1174 limit_pages >>= PAGE_SHIFT; 1175 1176 return locked_pages <= limit_pages; 1177 } 1178 1179 static inline u64 file_mmap_size_max(struct file *file, struct inode *inode) 1180 { 1181 if (S_ISREG(inode->i_mode)) 1182 return MAX_LFS_FILESIZE; 1183 1184 if (S_ISBLK(inode->i_mode)) 1185 return MAX_LFS_FILESIZE; 1186 1187 if (S_ISSOCK(inode->i_mode)) 1188 return MAX_LFS_FILESIZE; 1189 1190 /* Special "we do even unsigned file positions" case */ 1191 if (file->f_mode & FMODE_UNSIGNED_OFFSET) 1192 return 0; 1193 1194 /* Yes, random drivers might want more. But I'm tired of buggy drivers */ 1195 return ULONG_MAX; 1196 } 1197 1198 static inline bool file_mmap_ok(struct file *file, struct inode *inode, 1199 unsigned long pgoff, unsigned long len) 1200 { 1201 u64 maxsize = file_mmap_size_max(file, inode); 1202 1203 if (maxsize && len > maxsize) 1204 return false; 1205 maxsize -= len; 1206 if (pgoff > maxsize >> PAGE_SHIFT) 1207 return false; 1208 return true; 1209 } 1210 1211 /* 1212 * The caller must write-lock current->mm->mmap_lock. 1213 */ 1214 unsigned long do_mmap(struct file *file, unsigned long addr, 1215 unsigned long len, unsigned long prot, 1216 unsigned long flags, vm_flags_t vm_flags, 1217 unsigned long pgoff, unsigned long *populate, 1218 struct list_head *uf) 1219 { 1220 struct mm_struct *mm = current->mm; 1221 int pkey = 0; 1222 1223 *populate = 0; 1224 1225 if (!len) 1226 return -EINVAL; 1227 1228 /* 1229 * Does the application expect PROT_READ to imply PROT_EXEC? 1230 * 1231 * (the exception is when the underlying filesystem is noexec 1232 * mounted, in which case we don't add PROT_EXEC.) 1233 */ 1234 if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC)) 1235 if (!(file && path_noexec(&file->f_path))) 1236 prot |= PROT_EXEC; 1237 1238 /* force arch specific MAP_FIXED handling in get_unmapped_area */ 1239 if (flags & MAP_FIXED_NOREPLACE) 1240 flags |= MAP_FIXED; 1241 1242 if (!(flags & MAP_FIXED)) 1243 addr = round_hint_to_min(addr); 1244 1245 /* Careful about overflows.. */ 1246 len = PAGE_ALIGN(len); 1247 if (!len) 1248 return -ENOMEM; 1249 1250 /* offset overflow? */ 1251 if ((pgoff + (len >> PAGE_SHIFT)) < pgoff) 1252 return -EOVERFLOW; 1253 1254 /* Too many mappings? */ 1255 if (mm->map_count > sysctl_max_map_count) 1256 return -ENOMEM; 1257 1258 /* Obtain the address to map to. we verify (or select) it and ensure 1259 * that it represents a valid section of the address space. 1260 */ 1261 addr = get_unmapped_area(file, addr, len, pgoff, flags); 1262 if (IS_ERR_VALUE(addr)) 1263 return addr; 1264 1265 if (flags & MAP_FIXED_NOREPLACE) { 1266 if (find_vma_intersection(mm, addr, addr + len)) 1267 return -EEXIST; 1268 } 1269 1270 if (prot == PROT_EXEC) { 1271 pkey = execute_only_pkey(mm); 1272 if (pkey < 0) 1273 pkey = 0; 1274 } 1275 1276 /* Do simple checking here so the lower-level routines won't have 1277 * to. we assume access permissions have been handled by the open 1278 * of the memory object, so we don't do any here. 1279 */ 1280 vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) | 1281 mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; 1282 1283 if (flags & MAP_LOCKED) 1284 if (!can_do_mlock()) 1285 return -EPERM; 1286 1287 if (!mlock_future_ok(mm, vm_flags, len)) 1288 return -EAGAIN; 1289 1290 if (file) { 1291 struct inode *inode = file_inode(file); 1292 unsigned long flags_mask; 1293 1294 if (!file_mmap_ok(file, inode, pgoff, len)) 1295 return -EOVERFLOW; 1296 1297 flags_mask = LEGACY_MAP_MASK | file->f_op->mmap_supported_flags; 1298 1299 switch (flags & MAP_TYPE) { 1300 case MAP_SHARED: 1301 /* 1302 * Force use of MAP_SHARED_VALIDATE with non-legacy 1303 * flags. E.g. MAP_SYNC is dangerous to use with 1304 * MAP_SHARED as you don't know which consistency model 1305 * you will get. We silently ignore unsupported flags 1306 * with MAP_SHARED to preserve backward compatibility. 1307 */ 1308 flags &= LEGACY_MAP_MASK; 1309 fallthrough; 1310 case MAP_SHARED_VALIDATE: 1311 if (flags & ~flags_mask) 1312 return -EOPNOTSUPP; 1313 if (prot & PROT_WRITE) { 1314 if (!(file->f_mode & FMODE_WRITE)) 1315 return -EACCES; 1316 if (IS_SWAPFILE(file->f_mapping->host)) 1317 return -ETXTBSY; 1318 } 1319 1320 /* 1321 * Make sure we don't allow writing to an append-only 1322 * file.. 1323 */ 1324 if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE)) 1325 return -EACCES; 1326 1327 vm_flags |= VM_SHARED | VM_MAYSHARE; 1328 if (!(file->f_mode & FMODE_WRITE)) 1329 vm_flags &= ~(VM_MAYWRITE | VM_SHARED); 1330 fallthrough; 1331 case MAP_PRIVATE: 1332 if (!(file->f_mode & FMODE_READ)) 1333 return -EACCES; 1334 if (path_noexec(&file->f_path)) { 1335 if (vm_flags & VM_EXEC) 1336 return -EPERM; 1337 vm_flags &= ~VM_MAYEXEC; 1338 } 1339 1340 if (!file->f_op->mmap) 1341 return -ENODEV; 1342 if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) 1343 return -EINVAL; 1344 break; 1345 1346 default: 1347 return -EINVAL; 1348 } 1349 } else { 1350 switch (flags & MAP_TYPE) { 1351 case MAP_SHARED: 1352 if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) 1353 return -EINVAL; 1354 /* 1355 * Ignore pgoff. 1356 */ 1357 pgoff = 0; 1358 vm_flags |= VM_SHARED | VM_MAYSHARE; 1359 break; 1360 case MAP_PRIVATE: 1361 /* 1362 * Set pgoff according to addr for anon_vma. 1363 */ 1364 pgoff = addr >> PAGE_SHIFT; 1365 break; 1366 default: 1367 return -EINVAL; 1368 } 1369 } 1370 1371 /* 1372 * Set 'VM_NORESERVE' if we should not account for the 1373 * memory use of this mapping. 1374 */ 1375 if (flags & MAP_NORESERVE) { 1376 /* We honor MAP_NORESERVE if allowed to overcommit */ 1377 if (sysctl_overcommit_memory != OVERCOMMIT_NEVER) 1378 vm_flags |= VM_NORESERVE; 1379 1380 /* hugetlb applies strict overcommit unless MAP_NORESERVE */ 1381 if (file && is_file_hugepages(file)) 1382 vm_flags |= VM_NORESERVE; 1383 } 1384 1385 addr = mmap_region(file, addr, len, vm_flags, pgoff, uf); 1386 if (!IS_ERR_VALUE(addr) && 1387 ((vm_flags & VM_LOCKED) || 1388 (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE)) 1389 *populate = len; 1390 return addr; 1391 } 1392 1393 unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, 1394 unsigned long prot, unsigned long flags, 1395 unsigned long fd, unsigned long pgoff) 1396 { 1397 struct file *file = NULL; 1398 unsigned long retval; 1399 1400 if (!(flags & MAP_ANONYMOUS)) { 1401 audit_mmap_fd(fd, flags); 1402 file = fget(fd); 1403 if (!file) 1404 return -EBADF; 1405 if (is_file_hugepages(file)) { 1406 len = ALIGN(len, huge_page_size(hstate_file(file))); 1407 } else if (unlikely(flags & MAP_HUGETLB)) { 1408 retval = -EINVAL; 1409 goto out_fput; 1410 } 1411 } else if (flags & MAP_HUGETLB) { 1412 struct hstate *hs; 1413 1414 hs = hstate_sizelog((flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK); 1415 if (!hs) 1416 return -EINVAL; 1417 1418 len = ALIGN(len, huge_page_size(hs)); 1419 /* 1420 * VM_NORESERVE is used because the reservations will be 1421 * taken when vm_ops->mmap() is called 1422 */ 1423 file = hugetlb_file_setup(HUGETLB_ANON_FILE, len, 1424 VM_NORESERVE, 1425 HUGETLB_ANONHUGE_INODE, 1426 (flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK); 1427 if (IS_ERR(file)) 1428 return PTR_ERR(file); 1429 } 1430 1431 retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff); 1432 out_fput: 1433 if (file) 1434 fput(file); 1435 return retval; 1436 } 1437 1438 SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len, 1439 unsigned long, prot, unsigned long, flags, 1440 unsigned long, fd, unsigned long, pgoff) 1441 { 1442 return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff); 1443 } 1444 1445 #ifdef __ARCH_WANT_SYS_OLD_MMAP 1446 struct mmap_arg_struct { 1447 unsigned long addr; 1448 unsigned long len; 1449 unsigned long prot; 1450 unsigned long flags; 1451 unsigned long fd; 1452 unsigned long offset; 1453 }; 1454 1455 SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg) 1456 { 1457 struct mmap_arg_struct a; 1458 1459 if (copy_from_user(&a, arg, sizeof(a))) 1460 return -EFAULT; 1461 if (offset_in_page(a.offset)) 1462 return -EINVAL; 1463 1464 return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd, 1465 a.offset >> PAGE_SHIFT); 1466 } 1467 #endif /* __ARCH_WANT_SYS_OLD_MMAP */ 1468 1469 static bool vm_ops_needs_writenotify(const struct vm_operations_struct *vm_ops) 1470 { 1471 return vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite); 1472 } 1473 1474 static bool vma_is_shared_writable(struct vm_area_struct *vma) 1475 { 1476 return (vma->vm_flags & (VM_WRITE | VM_SHARED)) == 1477 (VM_WRITE | VM_SHARED); 1478 } 1479 1480 static bool vma_fs_can_writeback(struct vm_area_struct *vma) 1481 { 1482 /* No managed pages to writeback. */ 1483 if (vma->vm_flags & VM_PFNMAP) 1484 return false; 1485 1486 return vma->vm_file && vma->vm_file->f_mapping && 1487 mapping_can_writeback(vma->vm_file->f_mapping); 1488 } 1489 1490 /* 1491 * Does this VMA require the underlying folios to have their dirty state 1492 * tracked? 1493 */ 1494 bool vma_needs_dirty_tracking(struct vm_area_struct *vma) 1495 { 1496 /* Only shared, writable VMAs require dirty tracking. */ 1497 if (!vma_is_shared_writable(vma)) 1498 return false; 1499 1500 /* Does the filesystem need to be notified? */ 1501 if (vm_ops_needs_writenotify(vma->vm_ops)) 1502 return true; 1503 1504 /* 1505 * Even if the filesystem doesn't indicate a need for writenotify, if it 1506 * can writeback, dirty tracking is still required. 1507 */ 1508 return vma_fs_can_writeback(vma); 1509 } 1510 1511 /* 1512 * Some shared mappings will want the pages marked read-only 1513 * to track write events. If so, we'll downgrade vm_page_prot 1514 * to the private version (using protection_map[] without the 1515 * VM_SHARED bit). 1516 */ 1517 int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot) 1518 { 1519 /* If it was private or non-writable, the write bit is already clear */ 1520 if (!vma_is_shared_writable(vma)) 1521 return 0; 1522 1523 /* The backer wishes to know when pages are first written to? */ 1524 if (vm_ops_needs_writenotify(vma->vm_ops)) 1525 return 1; 1526 1527 /* The open routine did something to the protections that pgprot_modify 1528 * won't preserve? */ 1529 if (pgprot_val(vm_page_prot) != 1530 pgprot_val(vm_pgprot_modify(vm_page_prot, vma->vm_flags))) 1531 return 0; 1532 1533 /* 1534 * Do we need to track softdirty? hugetlb does not support softdirty 1535 * tracking yet. 1536 */ 1537 if (vma_soft_dirty_enabled(vma) && !is_vm_hugetlb_page(vma)) 1538 return 1; 1539 1540 /* Do we need write faults for uffd-wp tracking? */ 1541 if (userfaultfd_wp(vma)) 1542 return 1; 1543 1544 /* Can the mapping track the dirty pages? */ 1545 return vma_fs_can_writeback(vma); 1546 } 1547 1548 /* 1549 * We account for memory if it's a private writeable mapping, 1550 * not hugepages and VM_NORESERVE wasn't set. 1551 */ 1552 static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags) 1553 { 1554 /* 1555 * hugetlb has its own accounting separate from the core VM 1556 * VM_HUGETLB may not be set yet so we cannot check for that flag. 1557 */ 1558 if (file && is_file_hugepages(file)) 1559 return 0; 1560 1561 return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE; 1562 } 1563 1564 /** 1565 * unmapped_area() - Find an area between the low_limit and the high_limit with 1566 * the correct alignment and offset, all from @info. Note: current->mm is used 1567 * for the search. 1568 * 1569 * @info: The unmapped area information including the range [low_limit - 1570 * high_limit), the alignment offset and mask. 1571 * 1572 * Return: A memory address or -ENOMEM. 1573 */ 1574 static unsigned long unmapped_area(struct vm_unmapped_area_info *info) 1575 { 1576 unsigned long length, gap; 1577 unsigned long low_limit, high_limit; 1578 struct vm_area_struct *tmp; 1579 1580 MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); 1581 1582 /* Adjust search length to account for worst case alignment overhead */ 1583 length = info->length + info->align_mask; 1584 if (length < info->length) 1585 return -ENOMEM; 1586 1587 low_limit = info->low_limit; 1588 if (low_limit < mmap_min_addr) 1589 low_limit = mmap_min_addr; 1590 high_limit = info->high_limit; 1591 retry: 1592 if (mas_empty_area(&mas, low_limit, high_limit - 1, length)) 1593 return -ENOMEM; 1594 1595 gap = mas.index; 1596 gap += (info->align_offset - gap) & info->align_mask; 1597 tmp = mas_next(&mas, ULONG_MAX); 1598 if (tmp && (tmp->vm_flags & VM_STARTGAP_FLAGS)) { /* Avoid prev check if possible */ 1599 if (vm_start_gap(tmp) < gap + length - 1) { 1600 low_limit = tmp->vm_end; 1601 mas_reset(&mas); 1602 goto retry; 1603 } 1604 } else { 1605 tmp = mas_prev(&mas, 0); 1606 if (tmp && vm_end_gap(tmp) > gap) { 1607 low_limit = vm_end_gap(tmp); 1608 mas_reset(&mas); 1609 goto retry; 1610 } 1611 } 1612 1613 return gap; 1614 } 1615 1616 /** 1617 * unmapped_area_topdown() - Find an area between the low_limit and the 1618 * high_limit with the correct alignment and offset at the highest available 1619 * address, all from @info. Note: current->mm is used for the search. 1620 * 1621 * @info: The unmapped area information including the range [low_limit - 1622 * high_limit), the alignment offset and mask. 1623 * 1624 * Return: A memory address or -ENOMEM. 1625 */ 1626 static unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info) 1627 { 1628 unsigned long length, gap, gap_end; 1629 unsigned long low_limit, high_limit; 1630 struct vm_area_struct *tmp; 1631 1632 MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); 1633 /* Adjust search length to account for worst case alignment overhead */ 1634 length = info->length + info->align_mask; 1635 if (length < info->length) 1636 return -ENOMEM; 1637 1638 low_limit = info->low_limit; 1639 if (low_limit < mmap_min_addr) 1640 low_limit = mmap_min_addr; 1641 high_limit = info->high_limit; 1642 retry: 1643 if (mas_empty_area_rev(&mas, low_limit, high_limit - 1, length)) 1644 return -ENOMEM; 1645 1646 gap = mas.last + 1 - info->length; 1647 gap -= (gap - info->align_offset) & info->align_mask; 1648 gap_end = mas.last; 1649 tmp = mas_next(&mas, ULONG_MAX); 1650 if (tmp && (tmp->vm_flags & VM_STARTGAP_FLAGS)) { /* Avoid prev check if possible */ 1651 if (vm_start_gap(tmp) <= gap_end) { 1652 high_limit = vm_start_gap(tmp); 1653 mas_reset(&mas); 1654 goto retry; 1655 } 1656 } else { 1657 tmp = mas_prev(&mas, 0); 1658 if (tmp && vm_end_gap(tmp) > gap) { 1659 high_limit = tmp->vm_start; 1660 mas_reset(&mas); 1661 goto retry; 1662 } 1663 } 1664 1665 return gap; 1666 } 1667 1668 /* 1669 * Search for an unmapped address range. 1670 * 1671 * We are looking for a range that: 1672 * - does not intersect with any VMA; 1673 * - is contained within the [low_limit, high_limit) interval; 1674 * - is at least the desired size. 1675 * - satisfies (begin_addr & align_mask) == (align_offset & align_mask) 1676 */ 1677 unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info) 1678 { 1679 unsigned long addr; 1680 1681 if (info->flags & VM_UNMAPPED_AREA_TOPDOWN) 1682 addr = unmapped_area_topdown(info); 1683 else 1684 addr = unmapped_area(info); 1685 1686 trace_vm_unmapped_area(addr, info); 1687 return addr; 1688 } 1689 1690 /* Get an address range which is currently unmapped. 1691 * For shmat() with addr=0. 1692 * 1693 * Ugly calling convention alert: 1694 * Return value with the low bits set means error value, 1695 * ie 1696 * if (ret & ~PAGE_MASK) 1697 * error = ret; 1698 * 1699 * This function "knows" that -ENOMEM has the bits set. 1700 */ 1701 unsigned long 1702 generic_get_unmapped_area(struct file *filp, unsigned long addr, 1703 unsigned long len, unsigned long pgoff, 1704 unsigned long flags) 1705 { 1706 struct mm_struct *mm = current->mm; 1707 struct vm_area_struct *vma, *prev; 1708 struct vm_unmapped_area_info info; 1709 const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags); 1710 1711 if (len > mmap_end - mmap_min_addr) 1712 return -ENOMEM; 1713 1714 if (flags & MAP_FIXED) 1715 return addr; 1716 1717 if (addr) { 1718 addr = PAGE_ALIGN(addr); 1719 vma = find_vma_prev(mm, addr, &prev); 1720 if (mmap_end - len >= addr && addr >= mmap_min_addr && 1721 (!vma || addr + len <= vm_start_gap(vma)) && 1722 (!prev || addr >= vm_end_gap(prev))) 1723 return addr; 1724 } 1725 1726 info.flags = 0; 1727 info.length = len; 1728 info.low_limit = mm->mmap_base; 1729 info.high_limit = mmap_end; 1730 info.align_mask = 0; 1731 info.align_offset = 0; 1732 return vm_unmapped_area(&info); 1733 } 1734 1735 #ifndef HAVE_ARCH_UNMAPPED_AREA 1736 unsigned long 1737 arch_get_unmapped_area(struct file *filp, unsigned long addr, 1738 unsigned long len, unsigned long pgoff, 1739 unsigned long flags) 1740 { 1741 return generic_get_unmapped_area(filp, addr, len, pgoff, flags); 1742 } 1743 #endif 1744 1745 /* 1746 * This mmap-allocator allocates new areas top-down from below the 1747 * stack's low limit (the base): 1748 */ 1749 unsigned long 1750 generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, 1751 unsigned long len, unsigned long pgoff, 1752 unsigned long flags) 1753 { 1754 struct vm_area_struct *vma, *prev; 1755 struct mm_struct *mm = current->mm; 1756 struct vm_unmapped_area_info info; 1757 const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags); 1758 1759 /* requested length too big for entire address space */ 1760 if (len > mmap_end - mmap_min_addr) 1761 return -ENOMEM; 1762 1763 if (flags & MAP_FIXED) 1764 return addr; 1765 1766 /* requesting a specific address */ 1767 if (addr) { 1768 addr = PAGE_ALIGN(addr); 1769 vma = find_vma_prev(mm, addr, &prev); 1770 if (mmap_end - len >= addr && addr >= mmap_min_addr && 1771 (!vma || addr + len <= vm_start_gap(vma)) && 1772 (!prev || addr >= vm_end_gap(prev))) 1773 return addr; 1774 } 1775 1776 info.flags = VM_UNMAPPED_AREA_TOPDOWN; 1777 info.length = len; 1778 info.low_limit = PAGE_SIZE; 1779 info.high_limit = arch_get_mmap_base(addr, mm->mmap_base); 1780 info.align_mask = 0; 1781 info.align_offset = 0; 1782 addr = vm_unmapped_area(&info); 1783 1784 /* 1785 * A failed mmap() very likely causes application failure, 1786 * so fall back to the bottom-up function here. This scenario 1787 * can happen with large stack limits and large mmap() 1788 * allocations. 1789 */ 1790 if (offset_in_page(addr)) { 1791 VM_BUG_ON(addr != -ENOMEM); 1792 info.flags = 0; 1793 info.low_limit = TASK_UNMAPPED_BASE; 1794 info.high_limit = mmap_end; 1795 addr = vm_unmapped_area(&info); 1796 } 1797 1798 return addr; 1799 } 1800 1801 #ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN 1802 unsigned long 1803 arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, 1804 unsigned long len, unsigned long pgoff, 1805 unsigned long flags) 1806 { 1807 return generic_get_unmapped_area_topdown(filp, addr, len, pgoff, flags); 1808 } 1809 #endif 1810 1811 unsigned long 1812 get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, 1813 unsigned long pgoff, unsigned long flags) 1814 { 1815 unsigned long (*get_area)(struct file *, unsigned long, 1816 unsigned long, unsigned long, unsigned long); 1817 1818 unsigned long error = arch_mmap_check(addr, len, flags); 1819 if (error) 1820 return error; 1821 1822 /* Careful about overflows.. */ 1823 if (len > TASK_SIZE) 1824 return -ENOMEM; 1825 1826 get_area = current->mm->get_unmapped_area; 1827 if (file) { 1828 if (file->f_op->get_unmapped_area) 1829 get_area = file->f_op->get_unmapped_area; 1830 } else if (flags & MAP_SHARED) { 1831 /* 1832 * mmap_region() will call shmem_zero_setup() to create a file, 1833 * so use shmem's get_unmapped_area in case it can be huge. 1834 */ 1835 get_area = shmem_get_unmapped_area; 1836 } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { 1837 /* Ensures that larger anonymous mappings are THP aligned. */ 1838 get_area = thp_get_unmapped_area; 1839 } 1840 1841 /* Always treat pgoff as zero for anonymous memory. */ 1842 if (!file) 1843 pgoff = 0; 1844 1845 addr = get_area(file, addr, len, pgoff, flags); 1846 if (IS_ERR_VALUE(addr)) 1847 return addr; 1848 1849 if (addr > TASK_SIZE - len) 1850 return -ENOMEM; 1851 if (offset_in_page(addr)) 1852 return -EINVAL; 1853 1854 error = security_mmap_addr(addr); 1855 return error ? error : addr; 1856 } 1857 1858 EXPORT_SYMBOL(get_unmapped_area); 1859 1860 /** 1861 * find_vma_intersection() - Look up the first VMA which intersects the interval 1862 * @mm: The process address space. 1863 * @start_addr: The inclusive start user address. 1864 * @end_addr: The exclusive end user address. 1865 * 1866 * Returns: The first VMA within the provided range, %NULL otherwise. Assumes 1867 * start_addr < end_addr. 1868 */ 1869 struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, 1870 unsigned long start_addr, 1871 unsigned long end_addr) 1872 { 1873 unsigned long index = start_addr; 1874 1875 mmap_assert_locked(mm); 1876 return mt_find(&mm->mm_mt, &index, end_addr - 1); 1877 } 1878 EXPORT_SYMBOL(find_vma_intersection); 1879 1880 /** 1881 * find_vma() - Find the VMA for a given address, or the next VMA. 1882 * @mm: The mm_struct to check 1883 * @addr: The address 1884 * 1885 * Returns: The VMA associated with addr, or the next VMA. 1886 * May return %NULL in the case of no VMA at addr or above. 1887 */ 1888 struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) 1889 { 1890 unsigned long index = addr; 1891 1892 mmap_assert_locked(mm); 1893 return mt_find(&mm->mm_mt, &index, ULONG_MAX); 1894 } 1895 EXPORT_SYMBOL(find_vma); 1896 1897 /** 1898 * find_vma_prev() - Find the VMA for a given address, or the next vma and 1899 * set %pprev to the previous VMA, if any. 1900 * @mm: The mm_struct to check 1901 * @addr: The address 1902 * @pprev: The pointer to set to the previous VMA 1903 * 1904 * Note that RCU lock is missing here since the external mmap_lock() is used 1905 * instead. 1906 * 1907 * Returns: The VMA associated with @addr, or the next vma. 1908 * May return %NULL in the case of no vma at addr or above. 1909 */ 1910 struct vm_area_struct * 1911 find_vma_prev(struct mm_struct *mm, unsigned long addr, 1912 struct vm_area_struct **pprev) 1913 { 1914 struct vm_area_struct *vma; 1915 MA_STATE(mas, &mm->mm_mt, addr, addr); 1916 1917 vma = mas_walk(&mas); 1918 *pprev = mas_prev(&mas, 0); 1919 if (!vma) 1920 vma = mas_next(&mas, ULONG_MAX); 1921 return vma; 1922 } 1923 1924 /* 1925 * Verify that the stack growth is acceptable and 1926 * update accounting. This is shared with both the 1927 * grow-up and grow-down cases. 1928 */ 1929 static int acct_stack_growth(struct vm_area_struct *vma, 1930 unsigned long size, unsigned long grow) 1931 { 1932 struct mm_struct *mm = vma->vm_mm; 1933 unsigned long new_start; 1934 1935 /* address space limit tests */ 1936 if (!may_expand_vm(mm, vma->vm_flags, grow)) 1937 return -ENOMEM; 1938 1939 /* Stack limit test */ 1940 if (size > rlimit(RLIMIT_STACK)) 1941 return -ENOMEM; 1942 1943 /* mlock limit tests */ 1944 if (!mlock_future_ok(mm, vma->vm_flags, grow << PAGE_SHIFT)) 1945 return -ENOMEM; 1946 1947 /* Check to ensure the stack will not grow into a hugetlb-only region */ 1948 new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start : 1949 vma->vm_end - size; 1950 if (is_hugepage_only_range(vma->vm_mm, new_start, size)) 1951 return -EFAULT; 1952 1953 /* 1954 * Overcommit.. This must be the final test, as it will 1955 * update security statistics. 1956 */ 1957 if (security_vm_enough_memory_mm(mm, grow)) 1958 return -ENOMEM; 1959 1960 return 0; 1961 } 1962 1963 #if defined(CONFIG_STACK_GROWSUP) 1964 /* 1965 * PA-RISC uses this for its stack. 1966 * vma is the last one with address > vma->vm_end. Have to extend vma. 1967 */ 1968 static int expand_upwards(struct vm_area_struct *vma, unsigned long address) 1969 { 1970 struct mm_struct *mm = vma->vm_mm; 1971 struct vm_area_struct *next; 1972 unsigned long gap_addr; 1973 int error = 0; 1974 MA_STATE(mas, &mm->mm_mt, vma->vm_start, address); 1975 1976 if (!(vma->vm_flags & VM_GROWSUP)) 1977 return -EFAULT; 1978 1979 /* Guard against exceeding limits of the address space. */ 1980 address &= PAGE_MASK; 1981 if (address >= (TASK_SIZE & PAGE_MASK)) 1982 return -ENOMEM; 1983 address += PAGE_SIZE; 1984 1985 /* Enforce stack_guard_gap */ 1986 gap_addr = address + stack_guard_gap; 1987 1988 /* Guard against overflow */ 1989 if (gap_addr < address || gap_addr > TASK_SIZE) 1990 gap_addr = TASK_SIZE; 1991 1992 next = find_vma_intersection(mm, vma->vm_end, gap_addr); 1993 if (next && vma_is_accessible(next)) { 1994 if (!(next->vm_flags & VM_GROWSUP)) 1995 return -ENOMEM; 1996 /* Check that both stack segments have the same anon_vma? */ 1997 } 1998 1999 if (next) 2000 mas_prev_range(&mas, address); 2001 2002 __mas_set_range(&mas, vma->vm_start, address - 1); 2003 if (mas_preallocate(&mas, vma, GFP_KERNEL)) 2004 return -ENOMEM; 2005 2006 /* We must make sure the anon_vma is allocated. */ 2007 if (unlikely(anon_vma_prepare(vma))) { 2008 mas_destroy(&mas); 2009 return -ENOMEM; 2010 } 2011 2012 /* Lock the VMA before expanding to prevent concurrent page faults */ 2013 vma_start_write(vma); 2014 /* 2015 * vma->vm_start/vm_end cannot change under us because the caller 2016 * is required to hold the mmap_lock in read mode. We need the 2017 * anon_vma lock to serialize against concurrent expand_stacks. 2018 */ 2019 anon_vma_lock_write(vma->anon_vma); 2020 2021 /* Somebody else might have raced and expanded it already */ 2022 if (address > vma->vm_end) { 2023 unsigned long size, grow; 2024 2025 size = address - vma->vm_start; 2026 grow = (address - vma->vm_end) >> PAGE_SHIFT; 2027 2028 error = -ENOMEM; 2029 if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) { 2030 error = acct_stack_growth(vma, size, grow); 2031 if (!error) { 2032 /* 2033 * We only hold a shared mmap_lock lock here, so 2034 * we need to protect against concurrent vma 2035 * expansions. anon_vma_lock_write() doesn't 2036 * help here, as we don't guarantee that all 2037 * growable vmas in a mm share the same root 2038 * anon vma. So, we reuse mm->page_table_lock 2039 * to guard against concurrent vma expansions. 2040 */ 2041 spin_lock(&mm->page_table_lock); 2042 if (vma->vm_flags & VM_LOCKED) 2043 mm->locked_vm += grow; 2044 vm_stat_account(mm, vma->vm_flags, grow); 2045 anon_vma_interval_tree_pre_update_vma(vma); 2046 vma->vm_end = address; 2047 /* Overwrite old entry in mtree. */ 2048 mas_store_prealloc(&mas, vma); 2049 anon_vma_interval_tree_post_update_vma(vma); 2050 spin_unlock(&mm->page_table_lock); 2051 2052 perf_event_mmap(vma); 2053 } 2054 } 2055 } 2056 anon_vma_unlock_write(vma->anon_vma); 2057 mas_destroy(&mas); 2058 validate_mm(mm); 2059 return error; 2060 } 2061 #endif /* CONFIG_STACK_GROWSUP */ 2062 2063 /* 2064 * vma is the first one with address < vma->vm_start. Have to extend vma. 2065 * mmap_lock held for writing. 2066 */ 2067 int expand_downwards(struct vm_area_struct *vma, unsigned long address) 2068 { 2069 struct mm_struct *mm = vma->vm_mm; 2070 MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_start); 2071 struct vm_area_struct *prev; 2072 int error = 0; 2073 2074 if (!(vma->vm_flags & VM_GROWSDOWN)) 2075 return -EFAULT; 2076 2077 address &= PAGE_MASK; 2078 if (address < mmap_min_addr || address < FIRST_USER_ADDRESS) 2079 return -EPERM; 2080 2081 /* Enforce stack_guard_gap */ 2082 prev = mas_prev(&mas, 0); 2083 /* Check that both stack segments have the same anon_vma? */ 2084 if (prev) { 2085 if (!(prev->vm_flags & VM_GROWSDOWN) && 2086 vma_is_accessible(prev) && 2087 (address - prev->vm_end < stack_guard_gap)) 2088 return -ENOMEM; 2089 } 2090 2091 if (prev) 2092 mas_next_range(&mas, vma->vm_start); 2093 2094 __mas_set_range(&mas, address, vma->vm_end - 1); 2095 if (mas_preallocate(&mas, vma, GFP_KERNEL)) 2096 return -ENOMEM; 2097 2098 /* We must make sure the anon_vma is allocated. */ 2099 if (unlikely(anon_vma_prepare(vma))) { 2100 mas_destroy(&mas); 2101 return -ENOMEM; 2102 } 2103 2104 /* Lock the VMA before expanding to prevent concurrent page faults */ 2105 vma_start_write(vma); 2106 /* 2107 * vma->vm_start/vm_end cannot change under us because the caller 2108 * is required to hold the mmap_lock in read mode. We need the 2109 * anon_vma lock to serialize against concurrent expand_stacks. 2110 */ 2111 anon_vma_lock_write(vma->anon_vma); 2112 2113 /* Somebody else might have raced and expanded it already */ 2114 if (address < vma->vm_start) { 2115 unsigned long size, grow; 2116 2117 size = vma->vm_end - address; 2118 grow = (vma->vm_start - address) >> PAGE_SHIFT; 2119 2120 error = -ENOMEM; 2121 if (grow <= vma->vm_pgoff) { 2122 error = acct_stack_growth(vma, size, grow); 2123 if (!error) { 2124 /* 2125 * We only hold a shared mmap_lock lock here, so 2126 * we need to protect against concurrent vma 2127 * expansions. anon_vma_lock_write() doesn't 2128 * help here, as we don't guarantee that all 2129 * growable vmas in a mm share the same root 2130 * anon vma. So, we reuse mm->page_table_lock 2131 * to guard against concurrent vma expansions. 2132 */ 2133 spin_lock(&mm->page_table_lock); 2134 if (vma->vm_flags & VM_LOCKED) 2135 mm->locked_vm += grow; 2136 vm_stat_account(mm, vma->vm_flags, grow); 2137 anon_vma_interval_tree_pre_update_vma(vma); 2138 vma->vm_start = address; 2139 vma->vm_pgoff -= grow; 2140 /* Overwrite old entry in mtree. */ 2141 mas_store_prealloc(&mas, vma); 2142 anon_vma_interval_tree_post_update_vma(vma); 2143 spin_unlock(&mm->page_table_lock); 2144 2145 perf_event_mmap(vma); 2146 } 2147 } 2148 } 2149 anon_vma_unlock_write(vma->anon_vma); 2150 mas_destroy(&mas); 2151 validate_mm(mm); 2152 return error; 2153 } 2154 2155 /* enforced gap between the expanding stack and other mappings. */ 2156 unsigned long stack_guard_gap = 256UL<<PAGE_SHIFT; 2157 2158 static int __init cmdline_parse_stack_guard_gap(char *p) 2159 { 2160 unsigned long val; 2161 char *endptr; 2162 2163 val = simple_strtoul(p, &endptr, 10); 2164 if (!*endptr) 2165 stack_guard_gap = val << PAGE_SHIFT; 2166 2167 return 1; 2168 } 2169 __setup("stack_guard_gap=", cmdline_parse_stack_guard_gap); 2170 2171 #ifdef CONFIG_STACK_GROWSUP 2172 int expand_stack_locked(struct vm_area_struct *vma, unsigned long address) 2173 { 2174 return expand_upwards(vma, address); 2175 } 2176 2177 struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr) 2178 { 2179 struct vm_area_struct *vma, *prev; 2180 2181 addr &= PAGE_MASK; 2182 vma = find_vma_prev(mm, addr, &prev); 2183 if (vma && (vma->vm_start <= addr)) 2184 return vma; 2185 if (!prev) 2186 return NULL; 2187 if (expand_stack_locked(prev, addr)) 2188 return NULL; 2189 if (prev->vm_flags & VM_LOCKED) 2190 populate_vma_page_range(prev, addr, prev->vm_end, NULL); 2191 return prev; 2192 } 2193 #else 2194 int expand_stack_locked(struct vm_area_struct *vma, unsigned long address) 2195 { 2196 return expand_downwards(vma, address); 2197 } 2198 2199 struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr) 2200 { 2201 struct vm_area_struct *vma; 2202 unsigned long start; 2203 2204 addr &= PAGE_MASK; 2205 vma = find_vma(mm, addr); 2206 if (!vma) 2207 return NULL; 2208 if (vma->vm_start <= addr) 2209 return vma; 2210 start = vma->vm_start; 2211 if (expand_stack_locked(vma, addr)) 2212 return NULL; 2213 if (vma->vm_flags & VM_LOCKED) 2214 populate_vma_page_range(vma, addr, start, NULL); 2215 return vma; 2216 } 2217 #endif 2218 2219 #if defined(CONFIG_STACK_GROWSUP) 2220 2221 #define vma_expand_up(vma,addr) expand_upwards(vma, addr) 2222 #define vma_expand_down(vma, addr) (-EFAULT) 2223 2224 #else 2225 2226 #define vma_expand_up(vma,addr) (-EFAULT) 2227 #define vma_expand_down(vma, addr) expand_downwards(vma, addr) 2228 2229 #endif 2230 2231 /* 2232 * expand_stack(): legacy interface for page faulting. Don't use unless 2233 * you have to. 2234 * 2235 * This is called with the mm locked for reading, drops the lock, takes 2236 * the lock for writing, tries to look up a vma again, expands it if 2237 * necessary, and downgrades the lock to reading again. 2238 * 2239 * If no vma is found or it can't be expanded, it returns NULL and has 2240 * dropped the lock. 2241 */ 2242 struct vm_area_struct *expand_stack(struct mm_struct *mm, unsigned long addr) 2243 { 2244 struct vm_area_struct *vma, *prev; 2245 2246 mmap_read_unlock(mm); 2247 if (mmap_write_lock_killable(mm)) 2248 return NULL; 2249 2250 vma = find_vma_prev(mm, addr, &prev); 2251 if (vma && vma->vm_start <= addr) 2252 goto success; 2253 2254 if (prev && !vma_expand_up(prev, addr)) { 2255 vma = prev; 2256 goto success; 2257 } 2258 2259 if (vma && !vma_expand_down(vma, addr)) 2260 goto success; 2261 2262 mmap_write_unlock(mm); 2263 return NULL; 2264 2265 success: 2266 mmap_write_downgrade(mm); 2267 return vma; 2268 } 2269 2270 /* 2271 * Ok - we have the memory areas we should free on a maple tree so release them, 2272 * and do the vma updates. 2273 * 2274 * Called with the mm semaphore held. 2275 */ 2276 static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas) 2277 { 2278 unsigned long nr_accounted = 0; 2279 struct vm_area_struct *vma; 2280 2281 /* Update high watermark before we lower total_vm */ 2282 update_hiwater_vm(mm); 2283 mas_for_each(mas, vma, ULONG_MAX) { 2284 long nrpages = vma_pages(vma); 2285 2286 if (vma->vm_flags & VM_ACCOUNT) 2287 nr_accounted += nrpages; 2288 vm_stat_account(mm, vma->vm_flags, -nrpages); 2289 remove_vma(vma, false); 2290 } 2291 vm_unacct_memory(nr_accounted); 2292 } 2293 2294 /* 2295 * Get rid of page table information in the indicated region. 2296 * 2297 * Called with the mm semaphore held. 2298 */ 2299 static void unmap_region(struct mm_struct *mm, struct ma_state *mas, 2300 struct vm_area_struct *vma, struct vm_area_struct *prev, 2301 struct vm_area_struct *next, unsigned long start, 2302 unsigned long end, unsigned long tree_end, bool mm_wr_locked) 2303 { 2304 struct mmu_gather tlb; 2305 unsigned long mt_start = mas->index; 2306 2307 lru_add_drain(); 2308 tlb_gather_mmu(&tlb, mm); 2309 update_hiwater_rss(mm); 2310 unmap_vmas(&tlb, mas, vma, start, end, tree_end, mm_wr_locked); 2311 mas_set(mas, mt_start); 2312 free_pgtables(&tlb, mas, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, 2313 next ? next->vm_start : USER_PGTABLES_CEILING, 2314 mm_wr_locked); 2315 tlb_finish_mmu(&tlb); 2316 } 2317 2318 /* 2319 * __split_vma() bypasses sysctl_max_map_count checking. We use this where it 2320 * has already been checked or doesn't make sense to fail. 2321 * VMA Iterator will point to the end VMA. 2322 */ 2323 static int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, 2324 unsigned long addr, int new_below) 2325 { 2326 struct vma_prepare vp; 2327 struct vm_area_struct *new; 2328 int err; 2329 2330 WARN_ON(vma->vm_start >= addr); 2331 WARN_ON(vma->vm_end <= addr); 2332 2333 if (vma->vm_ops && vma->vm_ops->may_split) { 2334 err = vma->vm_ops->may_split(vma, addr); 2335 if (err) 2336 return err; 2337 } 2338 2339 new = vm_area_dup(vma); 2340 if (!new) 2341 return -ENOMEM; 2342 2343 if (new_below) { 2344 new->vm_end = addr; 2345 } else { 2346 new->vm_start = addr; 2347 new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT); 2348 } 2349 2350 err = -ENOMEM; 2351 vma_iter_config(vmi, new->vm_start, new->vm_end); 2352 if (vma_iter_prealloc(vmi, new)) 2353 goto out_free_vma; 2354 2355 err = vma_dup_policy(vma, new); 2356 if (err) 2357 goto out_free_vmi; 2358 2359 err = anon_vma_clone(new, vma); 2360 if (err) 2361 goto out_free_mpol; 2362 2363 if (new->vm_file) 2364 get_file(new->vm_file); 2365 2366 if (new->vm_ops && new->vm_ops->open) 2367 new->vm_ops->open(new); 2368 2369 vma_start_write(vma); 2370 vma_start_write(new); 2371 2372 init_vma_prep(&vp, vma); 2373 vp.insert = new; 2374 vma_prepare(&vp); 2375 vma_adjust_trans_huge(vma, vma->vm_start, addr, 0); 2376 2377 if (new_below) { 2378 vma->vm_start = addr; 2379 vma->vm_pgoff += (addr - new->vm_start) >> PAGE_SHIFT; 2380 } else { 2381 vma->vm_end = addr; 2382 } 2383 2384 /* vma_complete stores the new vma */ 2385 vma_complete(&vp, vmi, vma->vm_mm); 2386 2387 /* Success. */ 2388 if (new_below) 2389 vma_next(vmi); 2390 return 0; 2391 2392 out_free_mpol: 2393 mpol_put(vma_policy(new)); 2394 out_free_vmi: 2395 vma_iter_free(vmi); 2396 out_free_vma: 2397 vm_area_free(new); 2398 return err; 2399 } 2400 2401 /* 2402 * Split a vma into two pieces at address 'addr', a new vma is allocated 2403 * either for the first part or the tail. 2404 */ 2405 static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, 2406 unsigned long addr, int new_below) 2407 { 2408 if (vma->vm_mm->map_count >= sysctl_max_map_count) 2409 return -ENOMEM; 2410 2411 return __split_vma(vmi, vma, addr, new_below); 2412 } 2413 2414 /* 2415 * We are about to modify one or multiple of a VMA's flags, policy, userfaultfd 2416 * context and anonymous VMA name within the range [start, end). 2417 * 2418 * As a result, we might be able to merge the newly modified VMA range with an 2419 * adjacent VMA with identical properties. 2420 * 2421 * If no merge is possible and the range does not span the entirety of the VMA, 2422 * we then need to split the VMA to accommodate the change. 2423 * 2424 * The function returns either the merged VMA, the original VMA if a split was 2425 * required instead, or an error if the split failed. 2426 */ 2427 struct vm_area_struct *vma_modify(struct vma_iterator *vmi, 2428 struct vm_area_struct *prev, 2429 struct vm_area_struct *vma, 2430 unsigned long start, unsigned long end, 2431 unsigned long vm_flags, 2432 struct mempolicy *policy, 2433 struct vm_userfaultfd_ctx uffd_ctx, 2434 struct anon_vma_name *anon_name) 2435 { 2436 pgoff_t pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); 2437 struct vm_area_struct *merged; 2438 2439 merged = vma_merge(vmi, prev, vma, start, end, vm_flags, 2440 pgoff, policy, uffd_ctx, anon_name); 2441 if (merged) 2442 return merged; 2443 2444 if (vma->vm_start < start) { 2445 int err = split_vma(vmi, vma, start, 1); 2446 2447 if (err) 2448 return ERR_PTR(err); 2449 } 2450 2451 if (vma->vm_end > end) { 2452 int err = split_vma(vmi, vma, end, 0); 2453 2454 if (err) 2455 return ERR_PTR(err); 2456 } 2457 2458 return vma; 2459 } 2460 2461 /* 2462 * Attempt to merge a newly mapped VMA with those adjacent to it. The caller 2463 * must ensure that [start, end) does not overlap any existing VMA. 2464 */ 2465 static struct vm_area_struct 2466 *vma_merge_new_vma(struct vma_iterator *vmi, struct vm_area_struct *prev, 2467 struct vm_area_struct *vma, unsigned long start, 2468 unsigned long end, pgoff_t pgoff) 2469 { 2470 return vma_merge(vmi, prev, vma, start, end, vma->vm_flags, pgoff, 2471 vma_policy(vma), vma->vm_userfaultfd_ctx, anon_vma_name(vma)); 2472 } 2473 2474 /* 2475 * Expand vma by delta bytes, potentially merging with an immediately adjacent 2476 * VMA with identical properties. 2477 */ 2478 struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi, 2479 struct vm_area_struct *vma, 2480 unsigned long delta) 2481 { 2482 pgoff_t pgoff = vma->vm_pgoff + vma_pages(vma); 2483 2484 /* vma is specified as prev, so case 1 or 2 will apply. */ 2485 return vma_merge(vmi, vma, vma, vma->vm_end, vma->vm_end + delta, 2486 vma->vm_flags, pgoff, vma_policy(vma), 2487 vma->vm_userfaultfd_ctx, anon_vma_name(vma)); 2488 } 2489 2490 /* 2491 * do_vmi_align_munmap() - munmap the aligned region from @start to @end. 2492 * @vmi: The vma iterator 2493 * @vma: The starting vm_area_struct 2494 * @mm: The mm_struct 2495 * @start: The aligned start address to munmap. 2496 * @end: The aligned end address to munmap. 2497 * @uf: The userfaultfd list_head 2498 * @unlock: Set to true to drop the mmap_lock. unlocking only happens on 2499 * success. 2500 * 2501 * Return: 0 on success and drops the lock if so directed, error and leaves the 2502 * lock held otherwise. 2503 */ 2504 static int 2505 do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, 2506 struct mm_struct *mm, unsigned long start, 2507 unsigned long end, struct list_head *uf, bool unlock) 2508 { 2509 struct vm_area_struct *prev, *next = NULL; 2510 struct maple_tree mt_detach; 2511 int count = 0; 2512 int error = -ENOMEM; 2513 unsigned long locked_vm = 0; 2514 MA_STATE(mas_detach, &mt_detach, 0, 0); 2515 mt_init_flags(&mt_detach, vmi->mas.tree->ma_flags & MT_FLAGS_LOCK_MASK); 2516 mt_on_stack(mt_detach); 2517 2518 /* 2519 * If we need to split any vma, do it now to save pain later. 2520 * 2521 * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially 2522 * unmapped vm_area_struct will remain in use: so lower split_vma 2523 * places tmp vma above, and higher split_vma places tmp vma below. 2524 */ 2525 2526 /* Does it split the first one? */ 2527 if (start > vma->vm_start) { 2528 2529 /* 2530 * Make sure that map_count on return from munmap() will 2531 * not exceed its limit; but let map_count go just above 2532 * its limit temporarily, to help free resources as expected. 2533 */ 2534 if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count) 2535 goto map_count_exceeded; 2536 2537 error = __split_vma(vmi, vma, start, 1); 2538 if (error) 2539 goto start_split_failed; 2540 } 2541 2542 /* 2543 * Detach a range of VMAs from the mm. Using next as a temp variable as 2544 * it is always overwritten. 2545 */ 2546 next = vma; 2547 do { 2548 /* Does it split the end? */ 2549 if (next->vm_end > end) { 2550 error = __split_vma(vmi, next, end, 0); 2551 if (error) 2552 goto end_split_failed; 2553 } 2554 vma_start_write(next); 2555 mas_set(&mas_detach, count); 2556 error = mas_store_gfp(&mas_detach, next, GFP_KERNEL); 2557 if (error) 2558 goto munmap_gather_failed; 2559 vma_mark_detached(next, true); 2560 if (next->vm_flags & VM_LOCKED) 2561 locked_vm += vma_pages(next); 2562 2563 count++; 2564 if (unlikely(uf)) { 2565 /* 2566 * If userfaultfd_unmap_prep returns an error the vmas 2567 * will remain split, but userland will get a 2568 * highly unexpected error anyway. This is no 2569 * different than the case where the first of the two 2570 * __split_vma fails, but we don't undo the first 2571 * split, despite we could. This is unlikely enough 2572 * failure that it's not worth optimizing it for. 2573 */ 2574 error = userfaultfd_unmap_prep(next, start, end, uf); 2575 2576 if (error) 2577 goto userfaultfd_error; 2578 } 2579 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE 2580 BUG_ON(next->vm_start < start); 2581 BUG_ON(next->vm_start > end); 2582 #endif 2583 } for_each_vma_range(*vmi, next, end); 2584 2585 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE) 2586 /* Make sure no VMAs are about to be lost. */ 2587 { 2588 MA_STATE(test, &mt_detach, 0, 0); 2589 struct vm_area_struct *vma_mas, *vma_test; 2590 int test_count = 0; 2591 2592 vma_iter_set(vmi, start); 2593 rcu_read_lock(); 2594 vma_test = mas_find(&test, count - 1); 2595 for_each_vma_range(*vmi, vma_mas, end) { 2596 BUG_ON(vma_mas != vma_test); 2597 test_count++; 2598 vma_test = mas_next(&test, count - 1); 2599 } 2600 rcu_read_unlock(); 2601 BUG_ON(count != test_count); 2602 } 2603 #endif 2604 2605 while (vma_iter_addr(vmi) > start) 2606 vma_iter_prev_range(vmi); 2607 2608 error = vma_iter_clear_gfp(vmi, start, end, GFP_KERNEL); 2609 if (error) 2610 goto clear_tree_failed; 2611 2612 /* Point of no return */ 2613 mm->locked_vm -= locked_vm; 2614 mm->map_count -= count; 2615 if (unlock) 2616 mmap_write_downgrade(mm); 2617 2618 prev = vma_iter_prev_range(vmi); 2619 next = vma_next(vmi); 2620 if (next) 2621 vma_iter_prev_range(vmi); 2622 2623 /* 2624 * We can free page tables without write-locking mmap_lock because VMAs 2625 * were isolated before we downgraded mmap_lock. 2626 */ 2627 mas_set(&mas_detach, 1); 2628 unmap_region(mm, &mas_detach, vma, prev, next, start, end, count, 2629 !unlock); 2630 /* Statistics and freeing VMAs */ 2631 mas_set(&mas_detach, 0); 2632 remove_mt(mm, &mas_detach); 2633 validate_mm(mm); 2634 if (unlock) 2635 mmap_read_unlock(mm); 2636 2637 __mt_destroy(&mt_detach); 2638 return 0; 2639 2640 clear_tree_failed: 2641 userfaultfd_error: 2642 munmap_gather_failed: 2643 end_split_failed: 2644 mas_set(&mas_detach, 0); 2645 mas_for_each(&mas_detach, next, end) 2646 vma_mark_detached(next, false); 2647 2648 __mt_destroy(&mt_detach); 2649 start_split_failed: 2650 map_count_exceeded: 2651 validate_mm(mm); 2652 return error; 2653 } 2654 2655 /* 2656 * do_vmi_munmap() - munmap a given range. 2657 * @vmi: The vma iterator 2658 * @mm: The mm_struct 2659 * @start: The start address to munmap 2660 * @len: The length of the range to munmap 2661 * @uf: The userfaultfd list_head 2662 * @unlock: set to true if the user wants to drop the mmap_lock on success 2663 * 2664 * This function takes a @mas that is either pointing to the previous VMA or set 2665 * to MA_START and sets it up to remove the mapping(s). The @len will be 2666 * aligned and any arch_unmap work will be preformed. 2667 * 2668 * Return: 0 on success and drops the lock if so directed, error and leaves the 2669 * lock held otherwise. 2670 */ 2671 int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, 2672 unsigned long start, size_t len, struct list_head *uf, 2673 bool unlock) 2674 { 2675 unsigned long end; 2676 struct vm_area_struct *vma; 2677 2678 if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) 2679 return -EINVAL; 2680 2681 end = start + PAGE_ALIGN(len); 2682 if (end == start) 2683 return -EINVAL; 2684 2685 /* arch_unmap() might do unmaps itself. */ 2686 arch_unmap(mm, start, end); 2687 2688 /* Find the first overlapping VMA */ 2689 vma = vma_find(vmi, end); 2690 if (!vma) { 2691 if (unlock) 2692 mmap_write_unlock(mm); 2693 return 0; 2694 } 2695 2696 return do_vmi_align_munmap(vmi, vma, mm, start, end, uf, unlock); 2697 } 2698 2699 /* do_munmap() - Wrapper function for non-maple tree aware do_munmap() calls. 2700 * @mm: The mm_struct 2701 * @start: The start address to munmap 2702 * @len: The length to be munmapped. 2703 * @uf: The userfaultfd list_head 2704 * 2705 * Return: 0 on success, error otherwise. 2706 */ 2707 int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, 2708 struct list_head *uf) 2709 { 2710 VMA_ITERATOR(vmi, mm, start); 2711 2712 return do_vmi_munmap(&vmi, mm, start, len, uf, false); 2713 } 2714 2715 unsigned long mmap_region(struct file *file, unsigned long addr, 2716 unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, 2717 struct list_head *uf) 2718 { 2719 struct mm_struct *mm = current->mm; 2720 struct vm_area_struct *vma = NULL; 2721 struct vm_area_struct *next, *prev, *merge; 2722 pgoff_t pglen = len >> PAGE_SHIFT; 2723 unsigned long charged = 0; 2724 unsigned long end = addr + len; 2725 unsigned long merge_start = addr, merge_end = end; 2726 bool writable_file_mapping = false; 2727 pgoff_t vm_pgoff; 2728 int error; 2729 VMA_ITERATOR(vmi, mm, addr); 2730 2731 /* Check against address space limit. */ 2732 if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) { 2733 unsigned long nr_pages; 2734 2735 /* 2736 * MAP_FIXED may remove pages of mappings that intersects with 2737 * requested mapping. Account for the pages it would unmap. 2738 */ 2739 nr_pages = count_vma_pages_range(mm, addr, end); 2740 2741 if (!may_expand_vm(mm, vm_flags, 2742 (len >> PAGE_SHIFT) - nr_pages)) 2743 return -ENOMEM; 2744 } 2745 2746 /* Unmap any existing mapping in the area */ 2747 if (do_vmi_munmap(&vmi, mm, addr, len, uf, false)) 2748 return -ENOMEM; 2749 2750 /* 2751 * Private writable mapping: check memory availability 2752 */ 2753 if (accountable_mapping(file, vm_flags)) { 2754 charged = len >> PAGE_SHIFT; 2755 if (security_vm_enough_memory_mm(mm, charged)) 2756 return -ENOMEM; 2757 vm_flags |= VM_ACCOUNT; 2758 } 2759 2760 next = vma_next(&vmi); 2761 prev = vma_prev(&vmi); 2762 if (vm_flags & VM_SPECIAL) { 2763 if (prev) 2764 vma_iter_next_range(&vmi); 2765 goto cannot_expand; 2766 } 2767 2768 /* Attempt to expand an old mapping */ 2769 /* Check next */ 2770 if (next && next->vm_start == end && !vma_policy(next) && 2771 can_vma_merge_before(next, vm_flags, NULL, file, pgoff+pglen, 2772 NULL_VM_UFFD_CTX, NULL)) { 2773 merge_end = next->vm_end; 2774 vma = next; 2775 vm_pgoff = next->vm_pgoff - pglen; 2776 } 2777 2778 /* Check prev */ 2779 if (prev && prev->vm_end == addr && !vma_policy(prev) && 2780 (vma ? can_vma_merge_after(prev, vm_flags, vma->anon_vma, file, 2781 pgoff, vma->vm_userfaultfd_ctx, NULL) : 2782 can_vma_merge_after(prev, vm_flags, NULL, file, pgoff, 2783 NULL_VM_UFFD_CTX, NULL))) { 2784 merge_start = prev->vm_start; 2785 vma = prev; 2786 vm_pgoff = prev->vm_pgoff; 2787 } else if (prev) { 2788 vma_iter_next_range(&vmi); 2789 } 2790 2791 /* Actually expand, if possible */ 2792 if (vma && 2793 !vma_expand(&vmi, vma, merge_start, merge_end, vm_pgoff, next)) { 2794 khugepaged_enter_vma(vma, vm_flags); 2795 goto expanded; 2796 } 2797 2798 if (vma == prev) 2799 vma_iter_set(&vmi, addr); 2800 cannot_expand: 2801 2802 /* 2803 * Determine the object being mapped and call the appropriate 2804 * specific mapper. the address has already been validated, but 2805 * not unmapped, but the maps are removed from the list. 2806 */ 2807 vma = vm_area_alloc(mm); 2808 if (!vma) { 2809 error = -ENOMEM; 2810 goto unacct_error; 2811 } 2812 2813 vma_iter_config(&vmi, addr, end); 2814 vma_set_range(vma, addr, end, pgoff); 2815 vm_flags_init(vma, vm_flags); 2816 vma->vm_page_prot = vm_get_page_prot(vm_flags); 2817 2818 if (file) { 2819 vma->vm_file = get_file(file); 2820 error = call_mmap(file, vma); 2821 if (error) 2822 goto unmap_and_free_vma; 2823 2824 if (vma_is_shared_maywrite(vma)) { 2825 error = mapping_map_writable(file->f_mapping); 2826 if (error) 2827 goto close_and_free_vma; 2828 2829 writable_file_mapping = true; 2830 } 2831 2832 /* 2833 * Expansion is handled above, merging is handled below. 2834 * Drivers should not alter the address of the VMA. 2835 */ 2836 error = -EINVAL; 2837 if (WARN_ON((addr != vma->vm_start))) 2838 goto close_and_free_vma; 2839 2840 vma_iter_config(&vmi, addr, end); 2841 /* 2842 * If vm_flags changed after call_mmap(), we should try merge 2843 * vma again as we may succeed this time. 2844 */ 2845 if (unlikely(vm_flags != vma->vm_flags && prev)) { 2846 merge = vma_merge_new_vma(&vmi, prev, vma, 2847 vma->vm_start, vma->vm_end, 2848 vma->vm_pgoff); 2849 if (merge) { 2850 /* 2851 * ->mmap() can change vma->vm_file and fput 2852 * the original file. So fput the vma->vm_file 2853 * here or we would add an extra fput for file 2854 * and cause general protection fault 2855 * ultimately. 2856 */ 2857 fput(vma->vm_file); 2858 vm_area_free(vma); 2859 vma = merge; 2860 /* Update vm_flags to pick up the change. */ 2861 vm_flags = vma->vm_flags; 2862 goto unmap_writable; 2863 } 2864 } 2865 2866 vm_flags = vma->vm_flags; 2867 } else if (vm_flags & VM_SHARED) { 2868 error = shmem_zero_setup(vma); 2869 if (error) 2870 goto free_vma; 2871 } else { 2872 vma_set_anonymous(vma); 2873 } 2874 2875 if (map_deny_write_exec(vma, vma->vm_flags)) { 2876 error = -EACCES; 2877 goto close_and_free_vma; 2878 } 2879 2880 /* Allow architectures to sanity-check the vm_flags */ 2881 error = -EINVAL; 2882 if (!arch_validate_flags(vma->vm_flags)) 2883 goto close_and_free_vma; 2884 2885 error = -ENOMEM; 2886 if (vma_iter_prealloc(&vmi, vma)) 2887 goto close_and_free_vma; 2888 2889 /* Lock the VMA since it is modified after insertion into VMA tree */ 2890 vma_start_write(vma); 2891 vma_iter_store(&vmi, vma); 2892 mm->map_count++; 2893 vma_link_file(vma); 2894 2895 /* 2896 * vma_merge() calls khugepaged_enter_vma() either, the below 2897 * call covers the non-merge case. 2898 */ 2899 khugepaged_enter_vma(vma, vma->vm_flags); 2900 2901 /* Once vma denies write, undo our temporary denial count */ 2902 unmap_writable: 2903 if (writable_file_mapping) 2904 mapping_unmap_writable(file->f_mapping); 2905 file = vma->vm_file; 2906 ksm_add_vma(vma); 2907 expanded: 2908 perf_event_mmap(vma); 2909 2910 vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT); 2911 if (vm_flags & VM_LOCKED) { 2912 if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) || 2913 is_vm_hugetlb_page(vma) || 2914 vma == get_gate_vma(current->mm)) 2915 vm_flags_clear(vma, VM_LOCKED_MASK); 2916 else 2917 mm->locked_vm += (len >> PAGE_SHIFT); 2918 } 2919 2920 if (file) 2921 uprobe_mmap(vma); 2922 2923 /* 2924 * New (or expanded) vma always get soft dirty status. 2925 * Otherwise user-space soft-dirty page tracker won't 2926 * be able to distinguish situation when vma area unmapped, 2927 * then new mapped in-place (which must be aimed as 2928 * a completely new data area). 2929 */ 2930 vm_flags_set(vma, VM_SOFTDIRTY); 2931 2932 vma_set_page_prot(vma); 2933 2934 validate_mm(mm); 2935 return addr; 2936 2937 close_and_free_vma: 2938 if (file && vma->vm_ops && vma->vm_ops->close) 2939 vma->vm_ops->close(vma); 2940 2941 if (file || vma->vm_file) { 2942 unmap_and_free_vma: 2943 fput(vma->vm_file); 2944 vma->vm_file = NULL; 2945 2946 vma_iter_set(&vmi, vma->vm_end); 2947 /* Undo any partial mapping done by a device driver. */ 2948 unmap_region(mm, &vmi.mas, vma, prev, next, vma->vm_start, 2949 vma->vm_end, vma->vm_end, true); 2950 } 2951 if (writable_file_mapping) 2952 mapping_unmap_writable(file->f_mapping); 2953 free_vma: 2954 vm_area_free(vma); 2955 unacct_error: 2956 if (charged) 2957 vm_unacct_memory(charged); 2958 validate_mm(mm); 2959 return error; 2960 } 2961 2962 static int __vm_munmap(unsigned long start, size_t len, bool unlock) 2963 { 2964 int ret; 2965 struct mm_struct *mm = current->mm; 2966 LIST_HEAD(uf); 2967 VMA_ITERATOR(vmi, mm, start); 2968 2969 if (mmap_write_lock_killable(mm)) 2970 return -EINTR; 2971 2972 ret = do_vmi_munmap(&vmi, mm, start, len, &uf, unlock); 2973 if (ret || !unlock) 2974 mmap_write_unlock(mm); 2975 2976 userfaultfd_unmap_complete(mm, &uf); 2977 return ret; 2978 } 2979 2980 int vm_munmap(unsigned long start, size_t len) 2981 { 2982 return __vm_munmap(start, len, false); 2983 } 2984 EXPORT_SYMBOL(vm_munmap); 2985 2986 SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len) 2987 { 2988 addr = untagged_addr(addr); 2989 return __vm_munmap(addr, len, true); 2990 } 2991 2992 2993 /* 2994 * Emulation of deprecated remap_file_pages() syscall. 2995 */ 2996 SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, 2997 unsigned long, prot, unsigned long, pgoff, unsigned long, flags) 2998 { 2999 3000 struct mm_struct *mm = current->mm; 3001 struct vm_area_struct *vma; 3002 unsigned long populate = 0; 3003 unsigned long ret = -EINVAL; 3004 struct file *file; 3005 3006 pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/mm/remap_file_pages.rst.\n", 3007 current->comm, current->pid); 3008 3009 if (prot) 3010 return ret; 3011 start = start & PAGE_MASK; 3012 size = size & PAGE_MASK; 3013 3014 if (start + size <= start) 3015 return ret; 3016 3017 /* Does pgoff wrap? */ 3018 if (pgoff + (size >> PAGE_SHIFT) < pgoff) 3019 return ret; 3020 3021 if (mmap_write_lock_killable(mm)) 3022 return -EINTR; 3023 3024 vma = vma_lookup(mm, start); 3025 3026 if (!vma || !(vma->vm_flags & VM_SHARED)) 3027 goto out; 3028 3029 if (start + size > vma->vm_end) { 3030 VMA_ITERATOR(vmi, mm, vma->vm_end); 3031 struct vm_area_struct *next, *prev = vma; 3032 3033 for_each_vma_range(vmi, next, start + size) { 3034 /* hole between vmas ? */ 3035 if (next->vm_start != prev->vm_end) 3036 goto out; 3037 3038 if (next->vm_file != vma->vm_file) 3039 goto out; 3040 3041 if (next->vm_flags != vma->vm_flags) 3042 goto out; 3043 3044 if (start + size <= next->vm_end) 3045 break; 3046 3047 prev = next; 3048 } 3049 3050 if (!next) 3051 goto out; 3052 } 3053 3054 prot |= vma->vm_flags & VM_READ ? PROT_READ : 0; 3055 prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0; 3056 prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0; 3057 3058 flags &= MAP_NONBLOCK; 3059 flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE; 3060 if (vma->vm_flags & VM_LOCKED) 3061 flags |= MAP_LOCKED; 3062 3063 file = get_file(vma->vm_file); 3064 ret = do_mmap(vma->vm_file, start, size, 3065 prot, flags, 0, pgoff, &populate, NULL); 3066 fput(file); 3067 out: 3068 mmap_write_unlock(mm); 3069 if (populate) 3070 mm_populate(ret, populate); 3071 if (!IS_ERR_VALUE(ret)) 3072 ret = 0; 3073 return ret; 3074 } 3075 3076 /* 3077 * do_vma_munmap() - Unmap a full or partial vma. 3078 * @vmi: The vma iterator pointing at the vma 3079 * @vma: The first vma to be munmapped 3080 * @start: the start of the address to unmap 3081 * @end: The end of the address to unmap 3082 * @uf: The userfaultfd list_head 3083 * @unlock: Drop the lock on success 3084 * 3085 * unmaps a VMA mapping when the vma iterator is already in position. 3086 * Does not handle alignment. 3087 * 3088 * Return: 0 on success drops the lock of so directed, error on failure and will 3089 * still hold the lock. 3090 */ 3091 int do_vma_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, 3092 unsigned long start, unsigned long end, struct list_head *uf, 3093 bool unlock) 3094 { 3095 struct mm_struct *mm = vma->vm_mm; 3096 3097 arch_unmap(mm, start, end); 3098 return do_vmi_align_munmap(vmi, vma, mm, start, end, uf, unlock); 3099 } 3100 3101 /* 3102 * do_brk_flags() - Increase the brk vma if the flags match. 3103 * @vmi: The vma iterator 3104 * @addr: The start address 3105 * @len: The length of the increase 3106 * @vma: The vma, 3107 * @flags: The VMA Flags 3108 * 3109 * Extend the brk VMA from addr to addr + len. If the VMA is NULL or the flags 3110 * do not match then create a new anonymous VMA. Eventually we may be able to 3111 * do some brk-specific accounting here. 3112 */ 3113 static int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, 3114 unsigned long addr, unsigned long len, unsigned long flags) 3115 { 3116 struct mm_struct *mm = current->mm; 3117 struct vma_prepare vp; 3118 3119 /* 3120 * Check against address space limits by the changed size 3121 * Note: This happens *after* clearing old mappings in some code paths. 3122 */ 3123 flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags; 3124 if (!may_expand_vm(mm, flags, len >> PAGE_SHIFT)) 3125 return -ENOMEM; 3126 3127 if (mm->map_count > sysctl_max_map_count) 3128 return -ENOMEM; 3129 3130 if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT)) 3131 return -ENOMEM; 3132 3133 /* 3134 * Expand the existing vma if possible; Note that singular lists do not 3135 * occur after forking, so the expand will only happen on new VMAs. 3136 */ 3137 if (vma && vma->vm_end == addr && !vma_policy(vma) && 3138 can_vma_merge_after(vma, flags, NULL, NULL, 3139 addr >> PAGE_SHIFT, NULL_VM_UFFD_CTX, NULL)) { 3140 vma_iter_config(vmi, vma->vm_start, addr + len); 3141 if (vma_iter_prealloc(vmi, vma)) 3142 goto unacct_fail; 3143 3144 vma_start_write(vma); 3145 3146 init_vma_prep(&vp, vma); 3147 vma_prepare(&vp); 3148 vma_adjust_trans_huge(vma, vma->vm_start, addr + len, 0); 3149 vma->vm_end = addr + len; 3150 vm_flags_set(vma, VM_SOFTDIRTY); 3151 vma_iter_store(vmi, vma); 3152 3153 vma_complete(&vp, vmi, mm); 3154 khugepaged_enter_vma(vma, flags); 3155 goto out; 3156 } 3157 3158 if (vma) 3159 vma_iter_next_range(vmi); 3160 /* create a vma struct for an anonymous mapping */ 3161 vma = vm_area_alloc(mm); 3162 if (!vma) 3163 goto unacct_fail; 3164 3165 vma_set_anonymous(vma); 3166 vma_set_range(vma, addr, addr + len, addr >> PAGE_SHIFT); 3167 vm_flags_init(vma, flags); 3168 vma->vm_page_prot = vm_get_page_prot(flags); 3169 vma_start_write(vma); 3170 if (vma_iter_store_gfp(vmi, vma, GFP_KERNEL)) 3171 goto mas_store_fail; 3172 3173 mm->map_count++; 3174 validate_mm(mm); 3175 ksm_add_vma(vma); 3176 out: 3177 perf_event_mmap(vma); 3178 mm->total_vm += len >> PAGE_SHIFT; 3179 mm->data_vm += len >> PAGE_SHIFT; 3180 if (flags & VM_LOCKED) 3181 mm->locked_vm += (len >> PAGE_SHIFT); 3182 vm_flags_set(vma, VM_SOFTDIRTY); 3183 return 0; 3184 3185 mas_store_fail: 3186 vm_area_free(vma); 3187 unacct_fail: 3188 vm_unacct_memory(len >> PAGE_SHIFT); 3189 return -ENOMEM; 3190 } 3191 3192 int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags) 3193 { 3194 struct mm_struct *mm = current->mm; 3195 struct vm_area_struct *vma = NULL; 3196 unsigned long len; 3197 int ret; 3198 bool populate; 3199 LIST_HEAD(uf); 3200 VMA_ITERATOR(vmi, mm, addr); 3201 3202 len = PAGE_ALIGN(request); 3203 if (len < request) 3204 return -ENOMEM; 3205 if (!len) 3206 return 0; 3207 3208 /* Until we need other flags, refuse anything except VM_EXEC. */ 3209 if ((flags & (~VM_EXEC)) != 0) 3210 return -EINVAL; 3211 3212 if (mmap_write_lock_killable(mm)) 3213 return -EINTR; 3214 3215 ret = check_brk_limits(addr, len); 3216 if (ret) 3217 goto limits_failed; 3218 3219 ret = do_vmi_munmap(&vmi, mm, addr, len, &uf, 0); 3220 if (ret) 3221 goto munmap_failed; 3222 3223 vma = vma_prev(&vmi); 3224 ret = do_brk_flags(&vmi, vma, addr, len, flags); 3225 populate = ((mm->def_flags & VM_LOCKED) != 0); 3226 mmap_write_unlock(mm); 3227 userfaultfd_unmap_complete(mm, &uf); 3228 if (populate && !ret) 3229 mm_populate(addr, len); 3230 return ret; 3231 3232 munmap_failed: 3233 limits_failed: 3234 mmap_write_unlock(mm); 3235 return ret; 3236 } 3237 EXPORT_SYMBOL(vm_brk_flags); 3238 3239 /* Release all mmaps. */ 3240 void exit_mmap(struct mm_struct *mm) 3241 { 3242 struct mmu_gather tlb; 3243 struct vm_area_struct *vma; 3244 unsigned long nr_accounted = 0; 3245 MA_STATE(mas, &mm->mm_mt, 0, 0); 3246 int count = 0; 3247 3248 /* mm's last user has gone, and its about to be pulled down */ 3249 mmu_notifier_release(mm); 3250 3251 mmap_read_lock(mm); 3252 arch_exit_mmap(mm); 3253 3254 vma = mas_find(&mas, ULONG_MAX); 3255 if (!vma || unlikely(xa_is_zero(vma))) { 3256 /* Can happen if dup_mmap() received an OOM */ 3257 mmap_read_unlock(mm); 3258 mmap_write_lock(mm); 3259 goto destroy; 3260 } 3261 3262 lru_add_drain(); 3263 flush_cache_mm(mm); 3264 tlb_gather_mmu_fullmm(&tlb, mm); 3265 /* update_hiwater_rss(mm) here? but nobody should be looking */ 3266 /* Use ULONG_MAX here to ensure all VMAs in the mm are unmapped */ 3267 unmap_vmas(&tlb, &mas, vma, 0, ULONG_MAX, ULONG_MAX, false); 3268 mmap_read_unlock(mm); 3269 3270 /* 3271 * Set MMF_OOM_SKIP to hide this task from the oom killer/reaper 3272 * because the memory has been already freed. 3273 */ 3274 set_bit(MMF_OOM_SKIP, &mm->flags); 3275 mmap_write_lock(mm); 3276 mt_clear_in_rcu(&mm->mm_mt); 3277 mas_set(&mas, vma->vm_end); 3278 free_pgtables(&tlb, &mas, vma, FIRST_USER_ADDRESS, 3279 USER_PGTABLES_CEILING, true); 3280 tlb_finish_mmu(&tlb); 3281 3282 /* 3283 * Walk the list again, actually closing and freeing it, with preemption 3284 * enabled, without holding any MM locks besides the unreachable 3285 * mmap_write_lock. 3286 */ 3287 mas_set(&mas, vma->vm_end); 3288 do { 3289 if (vma->vm_flags & VM_ACCOUNT) 3290 nr_accounted += vma_pages(vma); 3291 remove_vma(vma, true); 3292 count++; 3293 cond_resched(); 3294 vma = mas_find(&mas, ULONG_MAX); 3295 } while (vma && likely(!xa_is_zero(vma))); 3296 3297 BUG_ON(count != mm->map_count); 3298 3299 trace_exit_mmap(mm); 3300 destroy: 3301 __mt_destroy(&mm->mm_mt); 3302 mmap_write_unlock(mm); 3303 vm_unacct_memory(nr_accounted); 3304 } 3305 3306 /* Insert vm structure into process list sorted by address 3307 * and into the inode's i_mmap tree. If vm_file is non-NULL 3308 * then i_mmap_rwsem is taken here. 3309 */ 3310 int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) 3311 { 3312 unsigned long charged = vma_pages(vma); 3313 3314 3315 if (find_vma_intersection(mm, vma->vm_start, vma->vm_end)) 3316 return -ENOMEM; 3317 3318 if ((vma->vm_flags & VM_ACCOUNT) && 3319 security_vm_enough_memory_mm(mm, charged)) 3320 return -ENOMEM; 3321 3322 /* 3323 * The vm_pgoff of a purely anonymous vma should be irrelevant 3324 * until its first write fault, when page's anon_vma and index 3325 * are set. But now set the vm_pgoff it will almost certainly 3326 * end up with (unless mremap moves it elsewhere before that 3327 * first wfault), so /proc/pid/maps tells a consistent story. 3328 * 3329 * By setting it to reflect the virtual start address of the 3330 * vma, merges and splits can happen in a seamless way, just 3331 * using the existing file pgoff checks and manipulations. 3332 * Similarly in do_mmap and in do_brk_flags. 3333 */ 3334 if (vma_is_anonymous(vma)) { 3335 BUG_ON(vma->anon_vma); 3336 vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT; 3337 } 3338 3339 if (vma_link(mm, vma)) { 3340 if (vma->vm_flags & VM_ACCOUNT) 3341 vm_unacct_memory(charged); 3342 return -ENOMEM; 3343 } 3344 3345 return 0; 3346 } 3347 3348 /* 3349 * Copy the vma structure to a new location in the same mm, 3350 * prior to moving page table entries, to effect an mremap move. 3351 */ 3352 struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, 3353 unsigned long addr, unsigned long len, pgoff_t pgoff, 3354 bool *need_rmap_locks) 3355 { 3356 struct vm_area_struct *vma = *vmap; 3357 unsigned long vma_start = vma->vm_start; 3358 struct mm_struct *mm = vma->vm_mm; 3359 struct vm_area_struct *new_vma, *prev; 3360 bool faulted_in_anon_vma = true; 3361 VMA_ITERATOR(vmi, mm, addr); 3362 3363 /* 3364 * If anonymous vma has not yet been faulted, update new pgoff 3365 * to match new location, to increase its chance of merging. 3366 */ 3367 if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) { 3368 pgoff = addr >> PAGE_SHIFT; 3369 faulted_in_anon_vma = false; 3370 } 3371 3372 new_vma = find_vma_prev(mm, addr, &prev); 3373 if (new_vma && new_vma->vm_start < addr + len) 3374 return NULL; /* should never get here */ 3375 3376 new_vma = vma_merge_new_vma(&vmi, prev, vma, addr, addr + len, pgoff); 3377 if (new_vma) { 3378 /* 3379 * Source vma may have been merged into new_vma 3380 */ 3381 if (unlikely(vma_start >= new_vma->vm_start && 3382 vma_start < new_vma->vm_end)) { 3383 /* 3384 * The only way we can get a vma_merge with 3385 * self during an mremap is if the vma hasn't 3386 * been faulted in yet and we were allowed to 3387 * reset the dst vma->vm_pgoff to the 3388 * destination address of the mremap to allow 3389 * the merge to happen. mremap must change the 3390 * vm_pgoff linearity between src and dst vmas 3391 * (in turn preventing a vma_merge) to be 3392 * safe. It is only safe to keep the vm_pgoff 3393 * linear if there are no pages mapped yet. 3394 */ 3395 VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma); 3396 *vmap = vma = new_vma; 3397 } 3398 *need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff); 3399 } else { 3400 new_vma = vm_area_dup(vma); 3401 if (!new_vma) 3402 goto out; 3403 vma_set_range(new_vma, addr, addr + len, pgoff); 3404 if (vma_dup_policy(vma, new_vma)) 3405 goto out_free_vma; 3406 if (anon_vma_clone(new_vma, vma)) 3407 goto out_free_mempol; 3408 if (new_vma->vm_file) 3409 get_file(new_vma->vm_file); 3410 if (new_vma->vm_ops && new_vma->vm_ops->open) 3411 new_vma->vm_ops->open(new_vma); 3412 if (vma_link(mm, new_vma)) 3413 goto out_vma_link; 3414 *need_rmap_locks = false; 3415 } 3416 return new_vma; 3417 3418 out_vma_link: 3419 if (new_vma->vm_ops && new_vma->vm_ops->close) 3420 new_vma->vm_ops->close(new_vma); 3421 3422 if (new_vma->vm_file) 3423 fput(new_vma->vm_file); 3424 3425 unlink_anon_vmas(new_vma); 3426 out_free_mempol: 3427 mpol_put(vma_policy(new_vma)); 3428 out_free_vma: 3429 vm_area_free(new_vma); 3430 out: 3431 return NULL; 3432 } 3433 3434 /* 3435 * Return true if the calling process may expand its vm space by the passed 3436 * number of pages 3437 */ 3438 bool may_expand_vm(struct mm_struct *mm, vm_flags_t flags, unsigned long npages) 3439 { 3440 if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> PAGE_SHIFT) 3441 return false; 3442 3443 if (is_data_mapping(flags) && 3444 mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) { 3445 /* Workaround for Valgrind */ 3446 if (rlimit(RLIMIT_DATA) == 0 && 3447 mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> PAGE_SHIFT) 3448 return true; 3449 3450 pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Update limits%s.\n", 3451 current->comm, current->pid, 3452 (mm->data_vm + npages) << PAGE_SHIFT, 3453 rlimit(RLIMIT_DATA), 3454 ignore_rlimit_data ? "" : " or use boot option ignore_rlimit_data"); 3455 3456 if (!ignore_rlimit_data) 3457 return false; 3458 } 3459 3460 return true; 3461 } 3462 3463 void vm_stat_account(struct mm_struct *mm, vm_flags_t flags, long npages) 3464 { 3465 WRITE_ONCE(mm->total_vm, READ_ONCE(mm->total_vm)+npages); 3466 3467 if (is_exec_mapping(flags)) 3468 mm->exec_vm += npages; 3469 else if (is_stack_mapping(flags)) 3470 mm->stack_vm += npages; 3471 else if (is_data_mapping(flags)) 3472 mm->data_vm += npages; 3473 } 3474 3475 static vm_fault_t special_mapping_fault(struct vm_fault *vmf); 3476 3477 /* 3478 * Having a close hook prevents vma merging regardless of flags. 3479 */ 3480 static void special_mapping_close(struct vm_area_struct *vma) 3481 { 3482 } 3483 3484 static const char *special_mapping_name(struct vm_area_struct *vma) 3485 { 3486 return ((struct vm_special_mapping *)vma->vm_private_data)->name; 3487 } 3488 3489 static int special_mapping_mremap(struct vm_area_struct *new_vma) 3490 { 3491 struct vm_special_mapping *sm = new_vma->vm_private_data; 3492 3493 if (WARN_ON_ONCE(current->mm != new_vma->vm_mm)) 3494 return -EFAULT; 3495 3496 if (sm->mremap) 3497 return sm->mremap(sm, new_vma); 3498 3499 return 0; 3500 } 3501 3502 static int special_mapping_split(struct vm_area_struct *vma, unsigned long addr) 3503 { 3504 /* 3505 * Forbid splitting special mappings - kernel has expectations over 3506 * the number of pages in mapping. Together with VM_DONTEXPAND 3507 * the size of vma should stay the same over the special mapping's 3508 * lifetime. 3509 */ 3510 return -EINVAL; 3511 } 3512 3513 static const struct vm_operations_struct special_mapping_vmops = { 3514 .close = special_mapping_close, 3515 .fault = special_mapping_fault, 3516 .mremap = special_mapping_mremap, 3517 .name = special_mapping_name, 3518 /* vDSO code relies that VVAR can't be accessed remotely */ 3519 .access = NULL, 3520 .may_split = special_mapping_split, 3521 }; 3522 3523 static const struct vm_operations_struct legacy_special_mapping_vmops = { 3524 .close = special_mapping_close, 3525 .fault = special_mapping_fault, 3526 }; 3527 3528 static vm_fault_t special_mapping_fault(struct vm_fault *vmf) 3529 { 3530 struct vm_area_struct *vma = vmf->vma; 3531 pgoff_t pgoff; 3532 struct page **pages; 3533 3534 if (vma->vm_ops == &legacy_special_mapping_vmops) { 3535 pages = vma->vm_private_data; 3536 } else { 3537 struct vm_special_mapping *sm = vma->vm_private_data; 3538 3539 if (sm->fault) 3540 return sm->fault(sm, vmf->vma, vmf); 3541 3542 pages = sm->pages; 3543 } 3544 3545 for (pgoff = vmf->pgoff; pgoff && *pages; ++pages) 3546 pgoff--; 3547 3548 if (*pages) { 3549 struct page *page = *pages; 3550 get_page(page); 3551 vmf->page = page; 3552 return 0; 3553 } 3554 3555 return VM_FAULT_SIGBUS; 3556 } 3557 3558 static struct vm_area_struct *__install_special_mapping( 3559 struct mm_struct *mm, 3560 unsigned long addr, unsigned long len, 3561 unsigned long vm_flags, void *priv, 3562 const struct vm_operations_struct *ops) 3563 { 3564 int ret; 3565 struct vm_area_struct *vma; 3566 3567 vma = vm_area_alloc(mm); 3568 if (unlikely(vma == NULL)) 3569 return ERR_PTR(-ENOMEM); 3570 3571 vma_set_range(vma, addr, addr + len, 0); 3572 vm_flags_init(vma, (vm_flags | mm->def_flags | 3573 VM_DONTEXPAND | VM_SOFTDIRTY) & ~VM_LOCKED_MASK); 3574 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); 3575 3576 vma->vm_ops = ops; 3577 vma->vm_private_data = priv; 3578 3579 ret = insert_vm_struct(mm, vma); 3580 if (ret) 3581 goto out; 3582 3583 vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT); 3584 3585 perf_event_mmap(vma); 3586 3587 return vma; 3588 3589 out: 3590 vm_area_free(vma); 3591 return ERR_PTR(ret); 3592 } 3593 3594 bool vma_is_special_mapping(const struct vm_area_struct *vma, 3595 const struct vm_special_mapping *sm) 3596 { 3597 return vma->vm_private_data == sm && 3598 (vma->vm_ops == &special_mapping_vmops || 3599 vma->vm_ops == &legacy_special_mapping_vmops); 3600 } 3601 3602 /* 3603 * Called with mm->mmap_lock held for writing. 3604 * Insert a new vma covering the given region, with the given flags. 3605 * Its pages are supplied by the given array of struct page *. 3606 * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated. 3607 * The region past the last page supplied will always produce SIGBUS. 3608 * The array pointer and the pages it points to are assumed to stay alive 3609 * for as long as this mapping might exist. 3610 */ 3611 struct vm_area_struct *_install_special_mapping( 3612 struct mm_struct *mm, 3613 unsigned long addr, unsigned long len, 3614 unsigned long vm_flags, const struct vm_special_mapping *spec) 3615 { 3616 return __install_special_mapping(mm, addr, len, vm_flags, (void *)spec, 3617 &special_mapping_vmops); 3618 } 3619 3620 int install_special_mapping(struct mm_struct *mm, 3621 unsigned long addr, unsigned long len, 3622 unsigned long vm_flags, struct page **pages) 3623 { 3624 struct vm_area_struct *vma = __install_special_mapping( 3625 mm, addr, len, vm_flags, (void *)pages, 3626 &legacy_special_mapping_vmops); 3627 3628 return PTR_ERR_OR_ZERO(vma); 3629 } 3630 3631 static DEFINE_MUTEX(mm_all_locks_mutex); 3632 3633 static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma) 3634 { 3635 if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) { 3636 /* 3637 * The LSB of head.next can't change from under us 3638 * because we hold the mm_all_locks_mutex. 3639 */ 3640 down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_lock); 3641 /* 3642 * We can safely modify head.next after taking the 3643 * anon_vma->root->rwsem. If some other vma in this mm shares 3644 * the same anon_vma we won't take it again. 3645 * 3646 * No need of atomic instructions here, head.next 3647 * can't change from under us thanks to the 3648 * anon_vma->root->rwsem. 3649 */ 3650 if (__test_and_set_bit(0, (unsigned long *) 3651 &anon_vma->root->rb_root.rb_root.rb_node)) 3652 BUG(); 3653 } 3654 } 3655 3656 static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping) 3657 { 3658 if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) { 3659 /* 3660 * AS_MM_ALL_LOCKS can't change from under us because 3661 * we hold the mm_all_locks_mutex. 3662 * 3663 * Operations on ->flags have to be atomic because 3664 * even if AS_MM_ALL_LOCKS is stable thanks to the 3665 * mm_all_locks_mutex, there may be other cpus 3666 * changing other bitflags in parallel to us. 3667 */ 3668 if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags)) 3669 BUG(); 3670 down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_lock); 3671 } 3672 } 3673 3674 /* 3675 * This operation locks against the VM for all pte/vma/mm related 3676 * operations that could ever happen on a certain mm. This includes 3677 * vmtruncate, try_to_unmap, and all page faults. 3678 * 3679 * The caller must take the mmap_lock in write mode before calling 3680 * mm_take_all_locks(). The caller isn't allowed to release the 3681 * mmap_lock until mm_drop_all_locks() returns. 3682 * 3683 * mmap_lock in write mode is required in order to block all operations 3684 * that could modify pagetables and free pages without need of 3685 * altering the vma layout. It's also needed in write mode to avoid new 3686 * anon_vmas to be associated with existing vmas. 3687 * 3688 * A single task can't take more than one mm_take_all_locks() in a row 3689 * or it would deadlock. 3690 * 3691 * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in 3692 * mapping->flags avoid to take the same lock twice, if more than one 3693 * vma in this mm is backed by the same anon_vma or address_space. 3694 * 3695 * We take locks in following order, accordingly to comment at beginning 3696 * of mm/rmap.c: 3697 * - all hugetlbfs_i_mmap_rwsem_key locks (aka mapping->i_mmap_rwsem for 3698 * hugetlb mapping); 3699 * - all vmas marked locked 3700 * - all i_mmap_rwsem locks; 3701 * - all anon_vma->rwseml 3702 * 3703 * We can take all locks within these types randomly because the VM code 3704 * doesn't nest them and we protected from parallel mm_take_all_locks() by 3705 * mm_all_locks_mutex. 3706 * 3707 * mm_take_all_locks() and mm_drop_all_locks are expensive operations 3708 * that may have to take thousand of locks. 3709 * 3710 * mm_take_all_locks() can fail if it's interrupted by signals. 3711 */ 3712 int mm_take_all_locks(struct mm_struct *mm) 3713 { 3714 struct vm_area_struct *vma; 3715 struct anon_vma_chain *avc; 3716 MA_STATE(mas, &mm->mm_mt, 0, 0); 3717 3718 mmap_assert_write_locked(mm); 3719 3720 mutex_lock(&mm_all_locks_mutex); 3721 3722 /* 3723 * vma_start_write() does not have a complement in mm_drop_all_locks() 3724 * because vma_start_write() is always asymmetrical; it marks a VMA as 3725 * being written to until mmap_write_unlock() or mmap_write_downgrade() 3726 * is reached. 3727 */ 3728 mas_for_each(&mas, vma, ULONG_MAX) { 3729 if (signal_pending(current)) 3730 goto out_unlock; 3731 vma_start_write(vma); 3732 } 3733 3734 mas_set(&mas, 0); 3735 mas_for_each(&mas, vma, ULONG_MAX) { 3736 if (signal_pending(current)) 3737 goto out_unlock; 3738 if (vma->vm_file && vma->vm_file->f_mapping && 3739 is_vm_hugetlb_page(vma)) 3740 vm_lock_mapping(mm, vma->vm_file->f_mapping); 3741 } 3742 3743 mas_set(&mas, 0); 3744 mas_for_each(&mas, vma, ULONG_MAX) { 3745 if (signal_pending(current)) 3746 goto out_unlock; 3747 if (vma->vm_file && vma->vm_file->f_mapping && 3748 !is_vm_hugetlb_page(vma)) 3749 vm_lock_mapping(mm, vma->vm_file->f_mapping); 3750 } 3751 3752 mas_set(&mas, 0); 3753 mas_for_each(&mas, vma, ULONG_MAX) { 3754 if (signal_pending(current)) 3755 goto out_unlock; 3756 if (vma->anon_vma) 3757 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 3758 vm_lock_anon_vma(mm, avc->anon_vma); 3759 } 3760 3761 return 0; 3762 3763 out_unlock: 3764 mm_drop_all_locks(mm); 3765 return -EINTR; 3766 } 3767 3768 static void vm_unlock_anon_vma(struct anon_vma *anon_vma) 3769 { 3770 if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) { 3771 /* 3772 * The LSB of head.next can't change to 0 from under 3773 * us because we hold the mm_all_locks_mutex. 3774 * 3775 * We must however clear the bitflag before unlocking 3776 * the vma so the users using the anon_vma->rb_root will 3777 * never see our bitflag. 3778 * 3779 * No need of atomic instructions here, head.next 3780 * can't change from under us until we release the 3781 * anon_vma->root->rwsem. 3782 */ 3783 if (!__test_and_clear_bit(0, (unsigned long *) 3784 &anon_vma->root->rb_root.rb_root.rb_node)) 3785 BUG(); 3786 anon_vma_unlock_write(anon_vma); 3787 } 3788 } 3789 3790 static void vm_unlock_mapping(struct address_space *mapping) 3791 { 3792 if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) { 3793 /* 3794 * AS_MM_ALL_LOCKS can't change to 0 from under us 3795 * because we hold the mm_all_locks_mutex. 3796 */ 3797 i_mmap_unlock_write(mapping); 3798 if (!test_and_clear_bit(AS_MM_ALL_LOCKS, 3799 &mapping->flags)) 3800 BUG(); 3801 } 3802 } 3803 3804 /* 3805 * The mmap_lock cannot be released by the caller until 3806 * mm_drop_all_locks() returns. 3807 */ 3808 void mm_drop_all_locks(struct mm_struct *mm) 3809 { 3810 struct vm_area_struct *vma; 3811 struct anon_vma_chain *avc; 3812 MA_STATE(mas, &mm->mm_mt, 0, 0); 3813 3814 mmap_assert_write_locked(mm); 3815 BUG_ON(!mutex_is_locked(&mm_all_locks_mutex)); 3816 3817 mas_for_each(&mas, vma, ULONG_MAX) { 3818 if (vma->anon_vma) 3819 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 3820 vm_unlock_anon_vma(avc->anon_vma); 3821 if (vma->vm_file && vma->vm_file->f_mapping) 3822 vm_unlock_mapping(vma->vm_file->f_mapping); 3823 } 3824 3825 mutex_unlock(&mm_all_locks_mutex); 3826 } 3827 3828 /* 3829 * initialise the percpu counter for VM 3830 */ 3831 void __init mmap_init(void) 3832 { 3833 int ret; 3834 3835 ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL); 3836 VM_BUG_ON(ret); 3837 } 3838 3839 /* 3840 * Initialise sysctl_user_reserve_kbytes. 3841 * 3842 * This is intended to prevent a user from starting a single memory hogging 3843 * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER 3844 * mode. 3845 * 3846 * The default value is min(3% of free memory, 128MB) 3847 * 128MB is enough to recover with sshd/login, bash, and top/kill. 3848 */ 3849 static int init_user_reserve(void) 3850 { 3851 unsigned long free_kbytes; 3852 3853 free_kbytes = K(global_zone_page_state(NR_FREE_PAGES)); 3854 3855 sysctl_user_reserve_kbytes = min(free_kbytes / 32, SZ_128K); 3856 return 0; 3857 } 3858 subsys_initcall(init_user_reserve); 3859 3860 /* 3861 * Initialise sysctl_admin_reserve_kbytes. 3862 * 3863 * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin 3864 * to log in and kill a memory hogging process. 3865 * 3866 * Systems with more than 256MB will reserve 8MB, enough to recover 3867 * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will 3868 * only reserve 3% of free pages by default. 3869 */ 3870 static int init_admin_reserve(void) 3871 { 3872 unsigned long free_kbytes; 3873 3874 free_kbytes = K(global_zone_page_state(NR_FREE_PAGES)); 3875 3876 sysctl_admin_reserve_kbytes = min(free_kbytes / 32, SZ_8K); 3877 return 0; 3878 } 3879 subsys_initcall(init_admin_reserve); 3880 3881 /* 3882 * Reinititalise user and admin reserves if memory is added or removed. 3883 * 3884 * The default user reserve max is 128MB, and the default max for the 3885 * admin reserve is 8MB. These are usually, but not always, enough to 3886 * enable recovery from a memory hogging process using login/sshd, a shell, 3887 * and tools like top. It may make sense to increase or even disable the 3888 * reserve depending on the existence of swap or variations in the recovery 3889 * tools. So, the admin may have changed them. 3890 * 3891 * If memory is added and the reserves have been eliminated or increased above 3892 * the default max, then we'll trust the admin. 3893 * 3894 * If memory is removed and there isn't enough free memory, then we 3895 * need to reset the reserves. 3896 * 3897 * Otherwise keep the reserve set by the admin. 3898 */ 3899 static int reserve_mem_notifier(struct notifier_block *nb, 3900 unsigned long action, void *data) 3901 { 3902 unsigned long tmp, free_kbytes; 3903 3904 switch (action) { 3905 case MEM_ONLINE: 3906 /* Default max is 128MB. Leave alone if modified by operator. */ 3907 tmp = sysctl_user_reserve_kbytes; 3908 if (tmp > 0 && tmp < SZ_128K) 3909 init_user_reserve(); 3910 3911 /* Default max is 8MB. Leave alone if modified by operator. */ 3912 tmp = sysctl_admin_reserve_kbytes; 3913 if (tmp > 0 && tmp < SZ_8K) 3914 init_admin_reserve(); 3915 3916 break; 3917 case MEM_OFFLINE: 3918 free_kbytes = K(global_zone_page_state(NR_FREE_PAGES)); 3919 3920 if (sysctl_user_reserve_kbytes > free_kbytes) { 3921 init_user_reserve(); 3922 pr_info("vm.user_reserve_kbytes reset to %lu\n", 3923 sysctl_user_reserve_kbytes); 3924 } 3925 3926 if (sysctl_admin_reserve_kbytes > free_kbytes) { 3927 init_admin_reserve(); 3928 pr_info("vm.admin_reserve_kbytes reset to %lu\n", 3929 sysctl_admin_reserve_kbytes); 3930 } 3931 break; 3932 default: 3933 break; 3934 } 3935 return NOTIFY_OK; 3936 } 3937 3938 static int __meminit init_reserve_notifier(void) 3939 { 3940 if (hotplug_memory_notifier(reserve_mem_notifier, DEFAULT_CALLBACK_PRI)) 3941 pr_err("Failed registering memory add/remove notifier for admin reserve\n"); 3942 3943 return 0; 3944 } 3945 subsys_initcall(init_reserve_notifier); 3946