1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * linux/mm/madvise.c 4 * 5 * Copyright (C) 1999 Linus Torvalds 6 * Copyright (C) 2002 Christoph Hellwig 7 */ 8 9 #include <linux/mman.h> 10 #include <linux/pagemap.h> 11 #include <linux/syscalls.h> 12 #include <linux/mempolicy.h> 13 #include <linux/page-isolation.h> 14 #include <linux/page_idle.h> 15 #include <linux/userfaultfd_k.h> 16 #include <linux/hugetlb.h> 17 #include <linux/falloc.h> 18 #include <linux/fadvise.h> 19 #include <linux/sched.h> 20 #include <linux/sched/mm.h> 21 #include <linux/mm_inline.h> 22 #include <linux/string.h> 23 #include <linux/uio.h> 24 #include <linux/ksm.h> 25 #include <linux/fs.h> 26 #include <linux/file.h> 27 #include <linux/blkdev.h> 28 #include <linux/backing-dev.h> 29 #include <linux/pagewalk.h> 30 #include <linux/swap.h> 31 #include <linux/swapops.h> 32 #include <linux/shmem_fs.h> 33 #include <linux/mmu_notifier.h> 34 35 #include <asm/tlb.h> 36 37 #include "internal.h" 38 #include "swap.h" 39 40 /* 41 * Maximum number of attempts we make to install guard pages before we give up 42 * and return -ERESTARTNOINTR to have userspace try again. 43 */ 44 #define MAX_MADVISE_GUARD_RETRIES 3 45 46 struct madvise_walk_private { 47 struct mmu_gather *tlb; 48 bool pageout; 49 }; 50 51 /* 52 * Any behaviour which results in changes to the vma->vm_flags needs to 53 * take mmap_lock for writing. Others, which simply traverse vmas, need 54 * to only take it for reading. 55 */ 56 static int madvise_need_mmap_write(int behavior) 57 { 58 switch (behavior) { 59 case MADV_REMOVE: 60 case MADV_WILLNEED: 61 case MADV_DONTNEED: 62 case MADV_DONTNEED_LOCKED: 63 case MADV_COLD: 64 case MADV_PAGEOUT: 65 case MADV_FREE: 66 case MADV_POPULATE_READ: 67 case MADV_POPULATE_WRITE: 68 case MADV_COLLAPSE: 69 case MADV_GUARD_INSTALL: 70 case MADV_GUARD_REMOVE: 71 return 0; 72 default: 73 /* be safe, default to 1. list exceptions explicitly */ 74 return 1; 75 } 76 } 77 78 #ifdef CONFIG_ANON_VMA_NAME 79 struct anon_vma_name *anon_vma_name_alloc(const char *name) 80 { 81 struct anon_vma_name *anon_name; 82 size_t count; 83 84 /* Add 1 for NUL terminator at the end of the anon_name->name */ 85 count = strlen(name) + 1; 86 anon_name = kmalloc(struct_size(anon_name, name, count), GFP_KERNEL); 87 if (anon_name) { 88 kref_init(&anon_name->kref); 89 memcpy(anon_name->name, name, count); 90 } 91 92 return anon_name; 93 } 94 95 void anon_vma_name_free(struct kref *kref) 96 { 97 struct anon_vma_name *anon_name = 98 container_of(kref, struct anon_vma_name, kref); 99 kfree(anon_name); 100 } 101 102 struct anon_vma_name *anon_vma_name(struct vm_area_struct *vma) 103 { 104 mmap_assert_locked(vma->vm_mm); 105 106 return vma->anon_name; 107 } 108 109 /* mmap_lock should be write-locked */ 110 static int replace_anon_vma_name(struct vm_area_struct *vma, 111 struct anon_vma_name *anon_name) 112 { 113 struct anon_vma_name *orig_name = anon_vma_name(vma); 114 115 if (!anon_name) { 116 vma->anon_name = NULL; 117 anon_vma_name_put(orig_name); 118 return 0; 119 } 120 121 if (anon_vma_name_eq(orig_name, anon_name)) 122 return 0; 123 124 vma->anon_name = anon_vma_name_reuse(anon_name); 125 anon_vma_name_put(orig_name); 126 127 return 0; 128 } 129 #else /* CONFIG_ANON_VMA_NAME */ 130 static int replace_anon_vma_name(struct vm_area_struct *vma, 131 struct anon_vma_name *anon_name) 132 { 133 if (anon_name) 134 return -EINVAL; 135 136 return 0; 137 } 138 #endif /* CONFIG_ANON_VMA_NAME */ 139 /* 140 * Update the vm_flags on region of a vma, splitting it or merging it as 141 * necessary. Must be called with mmap_lock held for writing; 142 * Caller should ensure anon_name stability by raising its refcount even when 143 * anon_name belongs to a valid vma because this function might free that vma. 144 */ 145 static int madvise_update_vma(struct vm_area_struct *vma, 146 struct vm_area_struct **prev, unsigned long start, 147 unsigned long end, unsigned long new_flags, 148 struct anon_vma_name *anon_name) 149 { 150 struct mm_struct *mm = vma->vm_mm; 151 int error; 152 VMA_ITERATOR(vmi, mm, start); 153 154 if (new_flags == vma->vm_flags && anon_vma_name_eq(anon_vma_name(vma), anon_name)) { 155 *prev = vma; 156 return 0; 157 } 158 159 vma = vma_modify_flags_name(&vmi, *prev, vma, start, end, new_flags, 160 anon_name); 161 if (IS_ERR(vma)) 162 return PTR_ERR(vma); 163 164 *prev = vma; 165 166 /* vm_flags is protected by the mmap_lock held in write mode. */ 167 vma_start_write(vma); 168 vm_flags_reset(vma, new_flags); 169 if (!vma->vm_file || vma_is_anon_shmem(vma)) { 170 error = replace_anon_vma_name(vma, anon_name); 171 if (error) 172 return error; 173 } 174 175 return 0; 176 } 177 178 #ifdef CONFIG_SWAP 179 static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start, 180 unsigned long end, struct mm_walk *walk) 181 { 182 struct vm_area_struct *vma = walk->private; 183 struct swap_iocb *splug = NULL; 184 pte_t *ptep = NULL; 185 spinlock_t *ptl; 186 unsigned long addr; 187 188 for (addr = start; addr < end; addr += PAGE_SIZE) { 189 pte_t pte; 190 swp_entry_t entry; 191 struct folio *folio; 192 193 if (!ptep++) { 194 ptep = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 195 if (!ptep) 196 break; 197 } 198 199 pte = ptep_get(ptep); 200 if (!is_swap_pte(pte)) 201 continue; 202 entry = pte_to_swp_entry(pte); 203 if (unlikely(non_swap_entry(entry))) 204 continue; 205 206 pte_unmap_unlock(ptep, ptl); 207 ptep = NULL; 208 209 folio = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE, 210 vma, addr, &splug); 211 if (folio) 212 folio_put(folio); 213 } 214 215 if (ptep) 216 pte_unmap_unlock(ptep, ptl); 217 swap_read_unplug(splug); 218 cond_resched(); 219 220 return 0; 221 } 222 223 static const struct mm_walk_ops swapin_walk_ops = { 224 .pmd_entry = swapin_walk_pmd_entry, 225 .walk_lock = PGWALK_RDLOCK, 226 }; 227 228 static void shmem_swapin_range(struct vm_area_struct *vma, 229 unsigned long start, unsigned long end, 230 struct address_space *mapping) 231 { 232 XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start)); 233 pgoff_t end_index = linear_page_index(vma, end) - 1; 234 struct folio *folio; 235 struct swap_iocb *splug = NULL; 236 237 rcu_read_lock(); 238 xas_for_each(&xas, folio, end_index) { 239 unsigned long addr; 240 swp_entry_t entry; 241 242 if (!xa_is_value(folio)) 243 continue; 244 entry = radix_to_swp_entry(folio); 245 /* There might be swapin error entries in shmem mapping. */ 246 if (non_swap_entry(entry)) 247 continue; 248 249 addr = vma->vm_start + 250 ((xas.xa_index - vma->vm_pgoff) << PAGE_SHIFT); 251 xas_pause(&xas); 252 rcu_read_unlock(); 253 254 folio = read_swap_cache_async(entry, mapping_gfp_mask(mapping), 255 vma, addr, &splug); 256 if (folio) 257 folio_put(folio); 258 259 rcu_read_lock(); 260 } 261 rcu_read_unlock(); 262 swap_read_unplug(splug); 263 } 264 #endif /* CONFIG_SWAP */ 265 266 /* 267 * Schedule all required I/O operations. Do not wait for completion. 268 */ 269 static long madvise_willneed(struct vm_area_struct *vma, 270 struct vm_area_struct **prev, 271 unsigned long start, unsigned long end) 272 { 273 struct mm_struct *mm = vma->vm_mm; 274 struct file *file = vma->vm_file; 275 loff_t offset; 276 277 *prev = vma; 278 #ifdef CONFIG_SWAP 279 if (!file) { 280 walk_page_range(vma->vm_mm, start, end, &swapin_walk_ops, vma); 281 lru_add_drain(); /* Push any new pages onto the LRU now */ 282 return 0; 283 } 284 285 if (shmem_mapping(file->f_mapping)) { 286 shmem_swapin_range(vma, start, end, file->f_mapping); 287 lru_add_drain(); /* Push any new pages onto the LRU now */ 288 return 0; 289 } 290 #else 291 if (!file) 292 return -EBADF; 293 #endif 294 295 if (IS_DAX(file_inode(file))) { 296 /* no bad return value, but ignore advice */ 297 return 0; 298 } 299 300 /* 301 * Filesystem's fadvise may need to take various locks. We need to 302 * explicitly grab a reference because the vma (and hence the 303 * vma's reference to the file) can go away as soon as we drop 304 * mmap_lock. 305 */ 306 *prev = NULL; /* tell sys_madvise we drop mmap_lock */ 307 get_file(file); 308 offset = (loff_t)(start - vma->vm_start) 309 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); 310 mmap_read_unlock(mm); 311 vfs_fadvise(file, offset, end - start, POSIX_FADV_WILLNEED); 312 fput(file); 313 mmap_read_lock(mm); 314 return 0; 315 } 316 317 static inline bool can_do_file_pageout(struct vm_area_struct *vma) 318 { 319 if (!vma->vm_file) 320 return false; 321 /* 322 * paging out pagecache only for non-anonymous mappings that correspond 323 * to the files the calling process could (if tried) open for writing; 324 * otherwise we'd be including shared non-exclusive mappings, which 325 * opens a side channel. 326 */ 327 return inode_owner_or_capable(&nop_mnt_idmap, 328 file_inode(vma->vm_file)) || 329 file_permission(vma->vm_file, MAY_WRITE) == 0; 330 } 331 332 static inline int madvise_folio_pte_batch(unsigned long addr, unsigned long end, 333 struct folio *folio, pte_t *ptep, 334 pte_t pte, bool *any_young, 335 bool *any_dirty) 336 { 337 const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY; 338 int max_nr = (end - addr) / PAGE_SIZE; 339 340 return folio_pte_batch(folio, addr, ptep, pte, max_nr, fpb_flags, NULL, 341 any_young, any_dirty); 342 } 343 344 static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, 345 unsigned long addr, unsigned long end, 346 struct mm_walk *walk) 347 { 348 struct madvise_walk_private *private = walk->private; 349 struct mmu_gather *tlb = private->tlb; 350 bool pageout = private->pageout; 351 struct mm_struct *mm = tlb->mm; 352 struct vm_area_struct *vma = walk->vma; 353 pte_t *start_pte, *pte, ptent; 354 spinlock_t *ptl; 355 struct folio *folio = NULL; 356 LIST_HEAD(folio_list); 357 bool pageout_anon_only_filter; 358 unsigned int batch_count = 0; 359 int nr; 360 361 if (fatal_signal_pending(current)) 362 return -EINTR; 363 364 pageout_anon_only_filter = pageout && !vma_is_anonymous(vma) && 365 !can_do_file_pageout(vma); 366 367 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 368 if (pmd_trans_huge(*pmd)) { 369 pmd_t orig_pmd; 370 unsigned long next = pmd_addr_end(addr, end); 371 372 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 373 ptl = pmd_trans_huge_lock(pmd, vma); 374 if (!ptl) 375 return 0; 376 377 orig_pmd = *pmd; 378 if (is_huge_zero_pmd(orig_pmd)) 379 goto huge_unlock; 380 381 if (unlikely(!pmd_present(orig_pmd))) { 382 VM_BUG_ON(thp_migration_supported() && 383 !is_pmd_migration_entry(orig_pmd)); 384 goto huge_unlock; 385 } 386 387 folio = pmd_folio(orig_pmd); 388 389 /* Do not interfere with other mappings of this folio */ 390 if (folio_likely_mapped_shared(folio)) 391 goto huge_unlock; 392 393 if (pageout_anon_only_filter && !folio_test_anon(folio)) 394 goto huge_unlock; 395 396 if (next - addr != HPAGE_PMD_SIZE) { 397 int err; 398 399 folio_get(folio); 400 spin_unlock(ptl); 401 folio_lock(folio); 402 err = split_folio(folio); 403 folio_unlock(folio); 404 folio_put(folio); 405 if (!err) 406 goto regular_folio; 407 return 0; 408 } 409 410 if (!pageout && pmd_young(orig_pmd)) { 411 pmdp_invalidate(vma, addr, pmd); 412 orig_pmd = pmd_mkold(orig_pmd); 413 414 set_pmd_at(mm, addr, pmd, orig_pmd); 415 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 416 } 417 418 folio_clear_referenced(folio); 419 folio_test_clear_young(folio); 420 if (folio_test_active(folio)) 421 folio_set_workingset(folio); 422 if (pageout) { 423 if (folio_isolate_lru(folio)) { 424 if (folio_test_unevictable(folio)) 425 folio_putback_lru(folio); 426 else 427 list_add(&folio->lru, &folio_list); 428 } 429 } else 430 folio_deactivate(folio); 431 huge_unlock: 432 spin_unlock(ptl); 433 if (pageout) 434 reclaim_pages(&folio_list); 435 return 0; 436 } 437 438 regular_folio: 439 #endif 440 tlb_change_page_size(tlb, PAGE_SIZE); 441 restart: 442 start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 443 if (!start_pte) 444 return 0; 445 flush_tlb_batched_pending(mm); 446 arch_enter_lazy_mmu_mode(); 447 for (; addr < end; pte += nr, addr += nr * PAGE_SIZE) { 448 nr = 1; 449 ptent = ptep_get(pte); 450 451 if (++batch_count == SWAP_CLUSTER_MAX) { 452 batch_count = 0; 453 if (need_resched()) { 454 arch_leave_lazy_mmu_mode(); 455 pte_unmap_unlock(start_pte, ptl); 456 cond_resched(); 457 goto restart; 458 } 459 } 460 461 if (pte_none(ptent)) 462 continue; 463 464 if (!pte_present(ptent)) 465 continue; 466 467 folio = vm_normal_folio(vma, addr, ptent); 468 if (!folio || folio_is_zone_device(folio)) 469 continue; 470 471 /* 472 * If we encounter a large folio, only split it if it is not 473 * fully mapped within the range we are operating on. Otherwise 474 * leave it as is so that it can be swapped out whole. If we 475 * fail to split a folio, leave it in place and advance to the 476 * next pte in the range. 477 */ 478 if (folio_test_large(folio)) { 479 bool any_young; 480 481 nr = madvise_folio_pte_batch(addr, end, folio, pte, 482 ptent, &any_young, NULL); 483 if (any_young) 484 ptent = pte_mkyoung(ptent); 485 486 if (nr < folio_nr_pages(folio)) { 487 int err; 488 489 if (folio_likely_mapped_shared(folio)) 490 continue; 491 if (pageout_anon_only_filter && !folio_test_anon(folio)) 492 continue; 493 if (!folio_trylock(folio)) 494 continue; 495 folio_get(folio); 496 arch_leave_lazy_mmu_mode(); 497 pte_unmap_unlock(start_pte, ptl); 498 start_pte = NULL; 499 err = split_folio(folio); 500 folio_unlock(folio); 501 folio_put(folio); 502 start_pte = pte = 503 pte_offset_map_lock(mm, pmd, addr, &ptl); 504 if (!start_pte) 505 break; 506 arch_enter_lazy_mmu_mode(); 507 if (!err) 508 nr = 0; 509 continue; 510 } 511 } 512 513 /* 514 * Do not interfere with other mappings of this folio and 515 * non-LRU folio. If we have a large folio at this point, we 516 * know it is fully mapped so if its mapcount is the same as its 517 * number of pages, it must be exclusive. 518 */ 519 if (!folio_test_lru(folio) || 520 folio_mapcount(folio) != folio_nr_pages(folio)) 521 continue; 522 523 if (pageout_anon_only_filter && !folio_test_anon(folio)) 524 continue; 525 526 if (!pageout && pte_young(ptent)) { 527 clear_young_dirty_ptes(vma, addr, pte, nr, 528 CYDP_CLEAR_YOUNG); 529 tlb_remove_tlb_entries(tlb, pte, nr, addr); 530 } 531 532 /* 533 * We are deactivating a folio for accelerating reclaiming. 534 * VM couldn't reclaim the folio unless we clear PG_young. 535 * As a side effect, it makes confuse idle-page tracking 536 * because they will miss recent referenced history. 537 */ 538 folio_clear_referenced(folio); 539 folio_test_clear_young(folio); 540 if (folio_test_active(folio)) 541 folio_set_workingset(folio); 542 if (pageout) { 543 if (folio_isolate_lru(folio)) { 544 if (folio_test_unevictable(folio)) 545 folio_putback_lru(folio); 546 else 547 list_add(&folio->lru, &folio_list); 548 } 549 } else 550 folio_deactivate(folio); 551 } 552 553 if (start_pte) { 554 arch_leave_lazy_mmu_mode(); 555 pte_unmap_unlock(start_pte, ptl); 556 } 557 if (pageout) 558 reclaim_pages(&folio_list); 559 cond_resched(); 560 561 return 0; 562 } 563 564 static const struct mm_walk_ops cold_walk_ops = { 565 .pmd_entry = madvise_cold_or_pageout_pte_range, 566 .walk_lock = PGWALK_RDLOCK, 567 }; 568 569 static void madvise_cold_page_range(struct mmu_gather *tlb, 570 struct vm_area_struct *vma, 571 unsigned long addr, unsigned long end) 572 { 573 struct madvise_walk_private walk_private = { 574 .pageout = false, 575 .tlb = tlb, 576 }; 577 578 tlb_start_vma(tlb, vma); 579 walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private); 580 tlb_end_vma(tlb, vma); 581 } 582 583 static inline bool can_madv_lru_vma(struct vm_area_struct *vma) 584 { 585 return !(vma->vm_flags & (VM_LOCKED|VM_PFNMAP|VM_HUGETLB)); 586 } 587 588 static long madvise_cold(struct vm_area_struct *vma, 589 struct vm_area_struct **prev, 590 unsigned long start_addr, unsigned long end_addr) 591 { 592 struct mm_struct *mm = vma->vm_mm; 593 struct mmu_gather tlb; 594 595 *prev = vma; 596 if (!can_madv_lru_vma(vma)) 597 return -EINVAL; 598 599 lru_add_drain(); 600 tlb_gather_mmu(&tlb, mm); 601 madvise_cold_page_range(&tlb, vma, start_addr, end_addr); 602 tlb_finish_mmu(&tlb); 603 604 return 0; 605 } 606 607 static void madvise_pageout_page_range(struct mmu_gather *tlb, 608 struct vm_area_struct *vma, 609 unsigned long addr, unsigned long end) 610 { 611 struct madvise_walk_private walk_private = { 612 .pageout = true, 613 .tlb = tlb, 614 }; 615 616 tlb_start_vma(tlb, vma); 617 walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private); 618 tlb_end_vma(tlb, vma); 619 } 620 621 static long madvise_pageout(struct vm_area_struct *vma, 622 struct vm_area_struct **prev, 623 unsigned long start_addr, unsigned long end_addr) 624 { 625 struct mm_struct *mm = vma->vm_mm; 626 struct mmu_gather tlb; 627 628 *prev = vma; 629 if (!can_madv_lru_vma(vma)) 630 return -EINVAL; 631 632 /* 633 * If the VMA belongs to a private file mapping, there can be private 634 * dirty pages which can be paged out if even this process is neither 635 * owner nor write capable of the file. We allow private file mappings 636 * further to pageout dirty anon pages. 637 */ 638 if (!vma_is_anonymous(vma) && (!can_do_file_pageout(vma) && 639 (vma->vm_flags & VM_MAYSHARE))) 640 return 0; 641 642 lru_add_drain(); 643 tlb_gather_mmu(&tlb, mm); 644 madvise_pageout_page_range(&tlb, vma, start_addr, end_addr); 645 tlb_finish_mmu(&tlb); 646 647 return 0; 648 } 649 650 static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, 651 unsigned long end, struct mm_walk *walk) 652 653 { 654 const cydp_t cydp_flags = CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY; 655 struct mmu_gather *tlb = walk->private; 656 struct mm_struct *mm = tlb->mm; 657 struct vm_area_struct *vma = walk->vma; 658 spinlock_t *ptl; 659 pte_t *start_pte, *pte, ptent; 660 struct folio *folio; 661 int nr_swap = 0; 662 unsigned long next; 663 int nr, max_nr; 664 665 next = pmd_addr_end(addr, end); 666 if (pmd_trans_huge(*pmd)) 667 if (madvise_free_huge_pmd(tlb, vma, pmd, addr, next)) 668 return 0; 669 670 tlb_change_page_size(tlb, PAGE_SIZE); 671 start_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl); 672 if (!start_pte) 673 return 0; 674 flush_tlb_batched_pending(mm); 675 arch_enter_lazy_mmu_mode(); 676 for (; addr != end; pte += nr, addr += PAGE_SIZE * nr) { 677 nr = 1; 678 ptent = ptep_get(pte); 679 680 if (pte_none(ptent)) 681 continue; 682 /* 683 * If the pte has swp_entry, just clear page table to 684 * prevent swap-in which is more expensive rather than 685 * (page allocation + zeroing). 686 */ 687 if (!pte_present(ptent)) { 688 swp_entry_t entry; 689 690 entry = pte_to_swp_entry(ptent); 691 if (!non_swap_entry(entry)) { 692 max_nr = (end - addr) / PAGE_SIZE; 693 nr = swap_pte_batch(pte, max_nr, ptent); 694 nr_swap -= nr; 695 free_swap_and_cache_nr(entry, nr); 696 clear_not_present_full_ptes(mm, addr, pte, nr, tlb->fullmm); 697 } else if (is_hwpoison_entry(entry) || 698 is_poisoned_swp_entry(entry)) { 699 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); 700 } 701 continue; 702 } 703 704 folio = vm_normal_folio(vma, addr, ptent); 705 if (!folio || folio_is_zone_device(folio)) 706 continue; 707 708 /* 709 * If we encounter a large folio, only split it if it is not 710 * fully mapped within the range we are operating on. Otherwise 711 * leave it as is so that it can be marked as lazyfree. If we 712 * fail to split a folio, leave it in place and advance to the 713 * next pte in the range. 714 */ 715 if (folio_test_large(folio)) { 716 bool any_young, any_dirty; 717 718 nr = madvise_folio_pte_batch(addr, end, folio, pte, 719 ptent, &any_young, &any_dirty); 720 721 if (nr < folio_nr_pages(folio)) { 722 int err; 723 724 if (folio_likely_mapped_shared(folio)) 725 continue; 726 if (!folio_trylock(folio)) 727 continue; 728 folio_get(folio); 729 arch_leave_lazy_mmu_mode(); 730 pte_unmap_unlock(start_pte, ptl); 731 start_pte = NULL; 732 err = split_folio(folio); 733 folio_unlock(folio); 734 folio_put(folio); 735 pte = pte_offset_map_lock(mm, pmd, addr, &ptl); 736 start_pte = pte; 737 if (!start_pte) 738 break; 739 arch_enter_lazy_mmu_mode(); 740 if (!err) 741 nr = 0; 742 continue; 743 } 744 745 if (any_young) 746 ptent = pte_mkyoung(ptent); 747 if (any_dirty) 748 ptent = pte_mkdirty(ptent); 749 } 750 751 if (folio_test_swapcache(folio) || folio_test_dirty(folio)) { 752 if (!folio_trylock(folio)) 753 continue; 754 /* 755 * If we have a large folio at this point, we know it is 756 * fully mapped so if its mapcount is the same as its 757 * number of pages, it must be exclusive. 758 */ 759 if (folio_mapcount(folio) != folio_nr_pages(folio)) { 760 folio_unlock(folio); 761 continue; 762 } 763 764 if (folio_test_swapcache(folio) && 765 !folio_free_swap(folio)) { 766 folio_unlock(folio); 767 continue; 768 } 769 770 folio_clear_dirty(folio); 771 folio_unlock(folio); 772 } 773 774 if (pte_young(ptent) || pte_dirty(ptent)) { 775 clear_young_dirty_ptes(vma, addr, pte, nr, cydp_flags); 776 tlb_remove_tlb_entries(tlb, pte, nr, addr); 777 } 778 folio_mark_lazyfree(folio); 779 } 780 781 if (nr_swap) 782 add_mm_counter(mm, MM_SWAPENTS, nr_swap); 783 if (start_pte) { 784 arch_leave_lazy_mmu_mode(); 785 pte_unmap_unlock(start_pte, ptl); 786 } 787 cond_resched(); 788 789 return 0; 790 } 791 792 static const struct mm_walk_ops madvise_free_walk_ops = { 793 .pmd_entry = madvise_free_pte_range, 794 .walk_lock = PGWALK_RDLOCK, 795 }; 796 797 static int madvise_free_single_vma(struct vm_area_struct *vma, 798 unsigned long start_addr, unsigned long end_addr) 799 { 800 struct mm_struct *mm = vma->vm_mm; 801 struct mmu_notifier_range range; 802 struct mmu_gather tlb; 803 804 /* MADV_FREE works for only anon vma at the moment */ 805 if (!vma_is_anonymous(vma)) 806 return -EINVAL; 807 808 range.start = max(vma->vm_start, start_addr); 809 if (range.start >= vma->vm_end) 810 return -EINVAL; 811 range.end = min(vma->vm_end, end_addr); 812 if (range.end <= vma->vm_start) 813 return -EINVAL; 814 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, 815 range.start, range.end); 816 817 lru_add_drain(); 818 tlb_gather_mmu(&tlb, mm); 819 update_hiwater_rss(mm); 820 821 mmu_notifier_invalidate_range_start(&range); 822 tlb_start_vma(&tlb, vma); 823 walk_page_range(vma->vm_mm, range.start, range.end, 824 &madvise_free_walk_ops, &tlb); 825 tlb_end_vma(&tlb, vma); 826 mmu_notifier_invalidate_range_end(&range); 827 tlb_finish_mmu(&tlb); 828 829 return 0; 830 } 831 832 /* 833 * Application no longer needs these pages. If the pages are dirty, 834 * it's OK to just throw them away. The app will be more careful about 835 * data it wants to keep. Be sure to free swap resources too. The 836 * zap_page_range_single call sets things up for shrink_active_list to actually 837 * free these pages later if no one else has touched them in the meantime, 838 * although we could add these pages to a global reuse list for 839 * shrink_active_list to pick up before reclaiming other pages. 840 * 841 * NB: This interface discards data rather than pushes it out to swap, 842 * as some implementations do. This has performance implications for 843 * applications like large transactional databases which want to discard 844 * pages in anonymous maps after committing to backing store the data 845 * that was kept in them. There is no reason to write this data out to 846 * the swap area if the application is discarding it. 847 * 848 * An interface that causes the system to free clean pages and flush 849 * dirty pages is already available as msync(MS_INVALIDATE). 850 */ 851 static long madvise_dontneed_single_vma(struct vm_area_struct *vma, 852 unsigned long start, unsigned long end) 853 { 854 zap_page_range_single(vma, start, end - start, NULL); 855 return 0; 856 } 857 858 static bool madvise_dontneed_free_valid_vma(struct vm_area_struct *vma, 859 unsigned long start, 860 unsigned long *end, 861 int behavior) 862 { 863 if (!is_vm_hugetlb_page(vma)) { 864 unsigned int forbidden = VM_PFNMAP; 865 866 if (behavior != MADV_DONTNEED_LOCKED) 867 forbidden |= VM_LOCKED; 868 869 return !(vma->vm_flags & forbidden); 870 } 871 872 if (behavior != MADV_DONTNEED && behavior != MADV_DONTNEED_LOCKED) 873 return false; 874 if (start & ~huge_page_mask(hstate_vma(vma))) 875 return false; 876 877 /* 878 * Madvise callers expect the length to be rounded up to PAGE_SIZE 879 * boundaries, and may be unaware that this VMA uses huge pages. 880 * Avoid unexpected data loss by rounding down the number of 881 * huge pages freed. 882 */ 883 *end = ALIGN_DOWN(*end, huge_page_size(hstate_vma(vma))); 884 885 return true; 886 } 887 888 static long madvise_dontneed_free(struct vm_area_struct *vma, 889 struct vm_area_struct **prev, 890 unsigned long start, unsigned long end, 891 int behavior) 892 { 893 struct mm_struct *mm = vma->vm_mm; 894 895 *prev = vma; 896 if (!madvise_dontneed_free_valid_vma(vma, start, &end, behavior)) 897 return -EINVAL; 898 899 if (start == end) 900 return 0; 901 902 if (!userfaultfd_remove(vma, start, end)) { 903 *prev = NULL; /* mmap_lock has been dropped, prev is stale */ 904 905 mmap_read_lock(mm); 906 vma = vma_lookup(mm, start); 907 if (!vma) 908 return -ENOMEM; 909 /* 910 * Potential end adjustment for hugetlb vma is OK as 911 * the check below keeps end within vma. 912 */ 913 if (!madvise_dontneed_free_valid_vma(vma, start, &end, 914 behavior)) 915 return -EINVAL; 916 if (end > vma->vm_end) { 917 /* 918 * Don't fail if end > vma->vm_end. If the old 919 * vma was split while the mmap_lock was 920 * released the effect of the concurrent 921 * operation may not cause madvise() to 922 * have an undefined result. There may be an 923 * adjacent next vma that we'll walk 924 * next. userfaultfd_remove() will generate an 925 * UFFD_EVENT_REMOVE repetition on the 926 * end-vma->vm_end range, but the manager can 927 * handle a repetition fine. 928 */ 929 end = vma->vm_end; 930 } 931 VM_WARN_ON(start >= end); 932 } 933 934 if (behavior == MADV_DONTNEED || behavior == MADV_DONTNEED_LOCKED) 935 return madvise_dontneed_single_vma(vma, start, end); 936 else if (behavior == MADV_FREE) 937 return madvise_free_single_vma(vma, start, end); 938 else 939 return -EINVAL; 940 } 941 942 static long madvise_populate(struct mm_struct *mm, unsigned long start, 943 unsigned long end, int behavior) 944 { 945 const bool write = behavior == MADV_POPULATE_WRITE; 946 int locked = 1; 947 long pages; 948 949 while (start < end) { 950 /* Populate (prefault) page tables readable/writable. */ 951 pages = faultin_page_range(mm, start, end, write, &locked); 952 if (!locked) { 953 mmap_read_lock(mm); 954 locked = 1; 955 } 956 if (pages < 0) { 957 switch (pages) { 958 case -EINTR: 959 return -EINTR; 960 case -EINVAL: /* Incompatible mappings / permissions. */ 961 return -EINVAL; 962 case -EHWPOISON: 963 return -EHWPOISON; 964 case -EFAULT: /* VM_FAULT_SIGBUS or VM_FAULT_SIGSEGV */ 965 return -EFAULT; 966 default: 967 pr_warn_once("%s: unhandled return value: %ld\n", 968 __func__, pages); 969 fallthrough; 970 case -ENOMEM: /* No VMA or out of memory. */ 971 return -ENOMEM; 972 } 973 } 974 start += pages * PAGE_SIZE; 975 } 976 return 0; 977 } 978 979 /* 980 * Application wants to free up the pages and associated backing store. 981 * This is effectively punching a hole into the middle of a file. 982 */ 983 static long madvise_remove(struct vm_area_struct *vma, 984 struct vm_area_struct **prev, 985 unsigned long start, unsigned long end) 986 { 987 loff_t offset; 988 int error; 989 struct file *f; 990 struct mm_struct *mm = vma->vm_mm; 991 992 *prev = NULL; /* tell sys_madvise we drop mmap_lock */ 993 994 if (vma->vm_flags & VM_LOCKED) 995 return -EINVAL; 996 997 f = vma->vm_file; 998 999 if (!f || !f->f_mapping || !f->f_mapping->host) { 1000 return -EINVAL; 1001 } 1002 1003 if (!vma_is_shared_maywrite(vma)) 1004 return -EACCES; 1005 1006 offset = (loff_t)(start - vma->vm_start) 1007 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); 1008 1009 /* 1010 * Filesystem's fallocate may need to take i_rwsem. We need to 1011 * explicitly grab a reference because the vma (and hence the 1012 * vma's reference to the file) can go away as soon as we drop 1013 * mmap_lock. 1014 */ 1015 get_file(f); 1016 if (userfaultfd_remove(vma, start, end)) { 1017 /* mmap_lock was not released by userfaultfd_remove() */ 1018 mmap_read_unlock(mm); 1019 } 1020 error = vfs_fallocate(f, 1021 FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, 1022 offset, end - start); 1023 fput(f); 1024 mmap_read_lock(mm); 1025 return error; 1026 } 1027 1028 static bool is_valid_guard_vma(struct vm_area_struct *vma, bool allow_locked) 1029 { 1030 vm_flags_t disallowed = VM_SPECIAL | VM_HUGETLB; 1031 1032 /* 1033 * A user could lock after setting a guard range but that's fine, as 1034 * they'd not be able to fault in. The issue arises when we try to zap 1035 * existing locked VMAs. We don't want to do that. 1036 */ 1037 if (!allow_locked) 1038 disallowed |= VM_LOCKED; 1039 1040 if (!vma_is_anonymous(vma)) 1041 return false; 1042 1043 if ((vma->vm_flags & (VM_MAYWRITE | disallowed)) != VM_MAYWRITE) 1044 return false; 1045 1046 return true; 1047 } 1048 1049 static bool is_guard_pte_marker(pte_t ptent) 1050 { 1051 return is_pte_marker(ptent) && 1052 is_guard_swp_entry(pte_to_swp_entry(ptent)); 1053 } 1054 1055 static int guard_install_pud_entry(pud_t *pud, unsigned long addr, 1056 unsigned long next, struct mm_walk *walk) 1057 { 1058 pud_t pudval = pudp_get(pud); 1059 1060 /* If huge return >0 so we abort the operation + zap. */ 1061 return pud_trans_huge(pudval) || pud_devmap(pudval); 1062 } 1063 1064 static int guard_install_pmd_entry(pmd_t *pmd, unsigned long addr, 1065 unsigned long next, struct mm_walk *walk) 1066 { 1067 pmd_t pmdval = pmdp_get(pmd); 1068 1069 /* If huge return >0 so we abort the operation + zap. */ 1070 return pmd_trans_huge(pmdval) || pmd_devmap(pmdval); 1071 } 1072 1073 static int guard_install_pte_entry(pte_t *pte, unsigned long addr, 1074 unsigned long next, struct mm_walk *walk) 1075 { 1076 pte_t pteval = ptep_get(pte); 1077 unsigned long *nr_pages = (unsigned long *)walk->private; 1078 1079 /* If there is already a guard page marker, we have nothing to do. */ 1080 if (is_guard_pte_marker(pteval)) { 1081 (*nr_pages)++; 1082 1083 return 0; 1084 } 1085 1086 /* If populated return >0 so we abort the operation + zap. */ 1087 return 1; 1088 } 1089 1090 static int guard_install_set_pte(unsigned long addr, unsigned long next, 1091 pte_t *ptep, struct mm_walk *walk) 1092 { 1093 unsigned long *nr_pages = (unsigned long *)walk->private; 1094 1095 /* Simply install a PTE marker, this causes segfault on access. */ 1096 *ptep = make_pte_marker(PTE_MARKER_GUARD); 1097 (*nr_pages)++; 1098 1099 return 0; 1100 } 1101 1102 static const struct mm_walk_ops guard_install_walk_ops = { 1103 .pud_entry = guard_install_pud_entry, 1104 .pmd_entry = guard_install_pmd_entry, 1105 .pte_entry = guard_install_pte_entry, 1106 .install_pte = guard_install_set_pte, 1107 .walk_lock = PGWALK_RDLOCK, 1108 }; 1109 1110 static long madvise_guard_install(struct vm_area_struct *vma, 1111 struct vm_area_struct **prev, 1112 unsigned long start, unsigned long end) 1113 { 1114 long err; 1115 int i; 1116 1117 *prev = vma; 1118 if (!is_valid_guard_vma(vma, /* allow_locked = */false)) 1119 return -EINVAL; 1120 1121 /* 1122 * If we install guard markers, then the range is no longer 1123 * empty from a page table perspective and therefore it's 1124 * appropriate to have an anon_vma. 1125 * 1126 * This ensures that on fork, we copy page tables correctly. 1127 */ 1128 err = anon_vma_prepare(vma); 1129 if (err) 1130 return err; 1131 1132 /* 1133 * Optimistically try to install the guard marker pages first. If any 1134 * non-guard pages are encountered, give up and zap the range before 1135 * trying again. 1136 * 1137 * We try a few times before giving up and releasing back to userland to 1138 * loop around, releasing locks in the process to avoid contention. This 1139 * would only happen if there was a great many racing page faults. 1140 * 1141 * In most cases we should simply install the guard markers immediately 1142 * with no zap or looping. 1143 */ 1144 for (i = 0; i < MAX_MADVISE_GUARD_RETRIES; i++) { 1145 unsigned long nr_pages = 0; 1146 1147 /* Returns < 0 on error, == 0 if success, > 0 if zap needed. */ 1148 err = walk_page_range_mm(vma->vm_mm, start, end, 1149 &guard_install_walk_ops, &nr_pages); 1150 if (err < 0) 1151 return err; 1152 1153 if (err == 0) { 1154 unsigned long nr_expected_pages = PHYS_PFN(end - start); 1155 1156 VM_WARN_ON(nr_pages != nr_expected_pages); 1157 return 0; 1158 } 1159 1160 /* 1161 * OK some of the range have non-guard pages mapped, zap 1162 * them. This leaves existing guard pages in place. 1163 */ 1164 zap_page_range_single(vma, start, end - start, NULL); 1165 } 1166 1167 /* 1168 * We were unable to install the guard pages due to being raced by page 1169 * faults. This should not happen ordinarily. We return to userspace and 1170 * immediately retry, relieving lock contention. 1171 */ 1172 return restart_syscall(); 1173 } 1174 1175 static int guard_remove_pud_entry(pud_t *pud, unsigned long addr, 1176 unsigned long next, struct mm_walk *walk) 1177 { 1178 pud_t pudval = pudp_get(pud); 1179 1180 /* If huge, cannot have guard pages present, so no-op - skip. */ 1181 if (pud_trans_huge(pudval) || pud_devmap(pudval)) 1182 walk->action = ACTION_CONTINUE; 1183 1184 return 0; 1185 } 1186 1187 static int guard_remove_pmd_entry(pmd_t *pmd, unsigned long addr, 1188 unsigned long next, struct mm_walk *walk) 1189 { 1190 pmd_t pmdval = pmdp_get(pmd); 1191 1192 /* If huge, cannot have guard pages present, so no-op - skip. */ 1193 if (pmd_trans_huge(pmdval) || pmd_devmap(pmdval)) 1194 walk->action = ACTION_CONTINUE; 1195 1196 return 0; 1197 } 1198 1199 static int guard_remove_pte_entry(pte_t *pte, unsigned long addr, 1200 unsigned long next, struct mm_walk *walk) 1201 { 1202 pte_t ptent = ptep_get(pte); 1203 1204 if (is_guard_pte_marker(ptent)) { 1205 /* Simply clear the PTE marker. */ 1206 pte_clear_not_present_full(walk->mm, addr, pte, false); 1207 update_mmu_cache(walk->vma, addr, pte); 1208 } 1209 1210 return 0; 1211 } 1212 1213 static const struct mm_walk_ops guard_remove_walk_ops = { 1214 .pud_entry = guard_remove_pud_entry, 1215 .pmd_entry = guard_remove_pmd_entry, 1216 .pte_entry = guard_remove_pte_entry, 1217 .walk_lock = PGWALK_RDLOCK, 1218 }; 1219 1220 static long madvise_guard_remove(struct vm_area_struct *vma, 1221 struct vm_area_struct **prev, 1222 unsigned long start, unsigned long end) 1223 { 1224 *prev = vma; 1225 /* 1226 * We're ok with removing guards in mlock()'d ranges, as this is a 1227 * non-destructive action. 1228 */ 1229 if (!is_valid_guard_vma(vma, /* allow_locked = */true)) 1230 return -EINVAL; 1231 1232 return walk_page_range(vma->vm_mm, start, end, 1233 &guard_remove_walk_ops, NULL); 1234 } 1235 1236 /* 1237 * Apply an madvise behavior to a region of a vma. madvise_update_vma 1238 * will handle splitting a vm area into separate areas, each area with its own 1239 * behavior. 1240 */ 1241 static int madvise_vma_behavior(struct vm_area_struct *vma, 1242 struct vm_area_struct **prev, 1243 unsigned long start, unsigned long end, 1244 unsigned long behavior) 1245 { 1246 int error; 1247 struct anon_vma_name *anon_name; 1248 unsigned long new_flags = vma->vm_flags; 1249 1250 if (unlikely(!can_modify_vma_madv(vma, behavior))) 1251 return -EPERM; 1252 1253 switch (behavior) { 1254 case MADV_REMOVE: 1255 return madvise_remove(vma, prev, start, end); 1256 case MADV_WILLNEED: 1257 return madvise_willneed(vma, prev, start, end); 1258 case MADV_COLD: 1259 return madvise_cold(vma, prev, start, end); 1260 case MADV_PAGEOUT: 1261 return madvise_pageout(vma, prev, start, end); 1262 case MADV_FREE: 1263 case MADV_DONTNEED: 1264 case MADV_DONTNEED_LOCKED: 1265 return madvise_dontneed_free(vma, prev, start, end, behavior); 1266 case MADV_NORMAL: 1267 new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ; 1268 break; 1269 case MADV_SEQUENTIAL: 1270 new_flags = (new_flags & ~VM_RAND_READ) | VM_SEQ_READ; 1271 break; 1272 case MADV_RANDOM: 1273 new_flags = (new_flags & ~VM_SEQ_READ) | VM_RAND_READ; 1274 break; 1275 case MADV_DONTFORK: 1276 new_flags |= VM_DONTCOPY; 1277 break; 1278 case MADV_DOFORK: 1279 if (vma->vm_flags & VM_IO) 1280 return -EINVAL; 1281 new_flags &= ~VM_DONTCOPY; 1282 break; 1283 case MADV_WIPEONFORK: 1284 /* MADV_WIPEONFORK is only supported on anonymous memory. */ 1285 if (vma->vm_file || vma->vm_flags & VM_SHARED) 1286 return -EINVAL; 1287 new_flags |= VM_WIPEONFORK; 1288 break; 1289 case MADV_KEEPONFORK: 1290 if (vma->vm_flags & VM_DROPPABLE) 1291 return -EINVAL; 1292 new_flags &= ~VM_WIPEONFORK; 1293 break; 1294 case MADV_DONTDUMP: 1295 new_flags |= VM_DONTDUMP; 1296 break; 1297 case MADV_DODUMP: 1298 if ((!is_vm_hugetlb_page(vma) && new_flags & VM_SPECIAL) || 1299 (vma->vm_flags & VM_DROPPABLE)) 1300 return -EINVAL; 1301 new_flags &= ~VM_DONTDUMP; 1302 break; 1303 case MADV_MERGEABLE: 1304 case MADV_UNMERGEABLE: 1305 error = ksm_madvise(vma, start, end, behavior, &new_flags); 1306 if (error) 1307 goto out; 1308 break; 1309 case MADV_HUGEPAGE: 1310 case MADV_NOHUGEPAGE: 1311 error = hugepage_madvise(vma, &new_flags, behavior); 1312 if (error) 1313 goto out; 1314 break; 1315 case MADV_COLLAPSE: 1316 return madvise_collapse(vma, prev, start, end); 1317 case MADV_GUARD_INSTALL: 1318 return madvise_guard_install(vma, prev, start, end); 1319 case MADV_GUARD_REMOVE: 1320 return madvise_guard_remove(vma, prev, start, end); 1321 } 1322 1323 anon_name = anon_vma_name(vma); 1324 anon_vma_name_get(anon_name); 1325 error = madvise_update_vma(vma, prev, start, end, new_flags, 1326 anon_name); 1327 anon_vma_name_put(anon_name); 1328 1329 out: 1330 /* 1331 * madvise() returns EAGAIN if kernel resources, such as 1332 * slab, are temporarily unavailable. 1333 */ 1334 if (error == -ENOMEM) 1335 error = -EAGAIN; 1336 return error; 1337 } 1338 1339 #ifdef CONFIG_MEMORY_FAILURE 1340 /* 1341 * Error injection support for memory error handling. 1342 */ 1343 static int madvise_inject_error(int behavior, 1344 unsigned long start, unsigned long end) 1345 { 1346 unsigned long size; 1347 1348 if (!capable(CAP_SYS_ADMIN)) 1349 return -EPERM; 1350 1351 1352 for (; start < end; start += size) { 1353 unsigned long pfn; 1354 struct page *page; 1355 int ret; 1356 1357 ret = get_user_pages_fast(start, 1, 0, &page); 1358 if (ret != 1) 1359 return ret; 1360 pfn = page_to_pfn(page); 1361 1362 /* 1363 * When soft offlining hugepages, after migrating the page 1364 * we dissolve it, therefore in the second loop "page" will 1365 * no longer be a compound page. 1366 */ 1367 size = page_size(compound_head(page)); 1368 1369 if (behavior == MADV_SOFT_OFFLINE) { 1370 pr_info("Soft offlining pfn %#lx at process virtual address %#lx\n", 1371 pfn, start); 1372 ret = soft_offline_page(pfn, MF_COUNT_INCREASED); 1373 } else { 1374 pr_info("Injecting memory failure for pfn %#lx at process virtual address %#lx\n", 1375 pfn, start); 1376 ret = memory_failure(pfn, MF_ACTION_REQUIRED | MF_COUNT_INCREASED | MF_SW_SIMULATED); 1377 if (ret == -EOPNOTSUPP) 1378 ret = 0; 1379 } 1380 1381 if (ret) 1382 return ret; 1383 } 1384 1385 return 0; 1386 } 1387 #endif 1388 1389 static bool 1390 madvise_behavior_valid(int behavior) 1391 { 1392 switch (behavior) { 1393 case MADV_DOFORK: 1394 case MADV_DONTFORK: 1395 case MADV_NORMAL: 1396 case MADV_SEQUENTIAL: 1397 case MADV_RANDOM: 1398 case MADV_REMOVE: 1399 case MADV_WILLNEED: 1400 case MADV_DONTNEED: 1401 case MADV_DONTNEED_LOCKED: 1402 case MADV_FREE: 1403 case MADV_COLD: 1404 case MADV_PAGEOUT: 1405 case MADV_POPULATE_READ: 1406 case MADV_POPULATE_WRITE: 1407 #ifdef CONFIG_KSM 1408 case MADV_MERGEABLE: 1409 case MADV_UNMERGEABLE: 1410 #endif 1411 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1412 case MADV_HUGEPAGE: 1413 case MADV_NOHUGEPAGE: 1414 case MADV_COLLAPSE: 1415 #endif 1416 case MADV_DONTDUMP: 1417 case MADV_DODUMP: 1418 case MADV_WIPEONFORK: 1419 case MADV_KEEPONFORK: 1420 case MADV_GUARD_INSTALL: 1421 case MADV_GUARD_REMOVE: 1422 #ifdef CONFIG_MEMORY_FAILURE 1423 case MADV_SOFT_OFFLINE: 1424 case MADV_HWPOISON: 1425 #endif 1426 return true; 1427 1428 default: 1429 return false; 1430 } 1431 } 1432 1433 /* Can we invoke process_madvise() on a remote mm for the specified behavior? */ 1434 static bool process_madvise_remote_valid(int behavior) 1435 { 1436 switch (behavior) { 1437 case MADV_COLD: 1438 case MADV_PAGEOUT: 1439 case MADV_WILLNEED: 1440 case MADV_COLLAPSE: 1441 return true; 1442 default: 1443 return false; 1444 } 1445 } 1446 1447 /* 1448 * Walk the vmas in range [start,end), and call the visit function on each one. 1449 * The visit function will get start and end parameters that cover the overlap 1450 * between the current vma and the original range. Any unmapped regions in the 1451 * original range will result in this function returning -ENOMEM while still 1452 * calling the visit function on all of the existing vmas in the range. 1453 * Must be called with the mmap_lock held for reading or writing. 1454 */ 1455 static 1456 int madvise_walk_vmas(struct mm_struct *mm, unsigned long start, 1457 unsigned long end, unsigned long arg, 1458 int (*visit)(struct vm_area_struct *vma, 1459 struct vm_area_struct **prev, unsigned long start, 1460 unsigned long end, unsigned long arg)) 1461 { 1462 struct vm_area_struct *vma; 1463 struct vm_area_struct *prev; 1464 unsigned long tmp; 1465 int unmapped_error = 0; 1466 1467 /* 1468 * If the interval [start,end) covers some unmapped address 1469 * ranges, just ignore them, but return -ENOMEM at the end. 1470 * - different from the way of handling in mlock etc. 1471 */ 1472 vma = find_vma_prev(mm, start, &prev); 1473 if (vma && start > vma->vm_start) 1474 prev = vma; 1475 1476 for (;;) { 1477 int error; 1478 1479 /* Still start < end. */ 1480 if (!vma) 1481 return -ENOMEM; 1482 1483 /* Here start < (end|vma->vm_end). */ 1484 if (start < vma->vm_start) { 1485 unmapped_error = -ENOMEM; 1486 start = vma->vm_start; 1487 if (start >= end) 1488 break; 1489 } 1490 1491 /* Here vma->vm_start <= start < (end|vma->vm_end) */ 1492 tmp = vma->vm_end; 1493 if (end < tmp) 1494 tmp = end; 1495 1496 /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */ 1497 error = visit(vma, &prev, start, tmp, arg); 1498 if (error) 1499 return error; 1500 start = tmp; 1501 if (prev && start < prev->vm_end) 1502 start = prev->vm_end; 1503 if (start >= end) 1504 break; 1505 if (prev) 1506 vma = find_vma(mm, prev->vm_end); 1507 else /* madvise_remove dropped mmap_lock */ 1508 vma = find_vma(mm, start); 1509 } 1510 1511 return unmapped_error; 1512 } 1513 1514 #ifdef CONFIG_ANON_VMA_NAME 1515 static int madvise_vma_anon_name(struct vm_area_struct *vma, 1516 struct vm_area_struct **prev, 1517 unsigned long start, unsigned long end, 1518 unsigned long anon_name) 1519 { 1520 int error; 1521 1522 /* Only anonymous mappings can be named */ 1523 if (vma->vm_file && !vma_is_anon_shmem(vma)) 1524 return -EBADF; 1525 1526 error = madvise_update_vma(vma, prev, start, end, vma->vm_flags, 1527 (struct anon_vma_name *)anon_name); 1528 1529 /* 1530 * madvise() returns EAGAIN if kernel resources, such as 1531 * slab, are temporarily unavailable. 1532 */ 1533 if (error == -ENOMEM) 1534 error = -EAGAIN; 1535 return error; 1536 } 1537 1538 int madvise_set_anon_name(struct mm_struct *mm, unsigned long start, 1539 unsigned long len_in, struct anon_vma_name *anon_name) 1540 { 1541 unsigned long end; 1542 unsigned long len; 1543 1544 if (start & ~PAGE_MASK) 1545 return -EINVAL; 1546 len = (len_in + ~PAGE_MASK) & PAGE_MASK; 1547 1548 /* Check to see whether len was rounded up from small -ve to zero */ 1549 if (len_in && !len) 1550 return -EINVAL; 1551 1552 end = start + len; 1553 if (end < start) 1554 return -EINVAL; 1555 1556 if (end == start) 1557 return 0; 1558 1559 return madvise_walk_vmas(mm, start, end, (unsigned long)anon_name, 1560 madvise_vma_anon_name); 1561 } 1562 #endif /* CONFIG_ANON_VMA_NAME */ 1563 /* 1564 * The madvise(2) system call. 1565 * 1566 * Applications can use madvise() to advise the kernel how it should 1567 * handle paging I/O in this VM area. The idea is to help the kernel 1568 * use appropriate read-ahead and caching techniques. The information 1569 * provided is advisory only, and can be safely disregarded by the 1570 * kernel without affecting the correct operation of the application. 1571 * 1572 * behavior values: 1573 * MADV_NORMAL - the default behavior is to read clusters. This 1574 * results in some read-ahead and read-behind. 1575 * MADV_RANDOM - the system should read the minimum amount of data 1576 * on any access, since it is unlikely that the appli- 1577 * cation will need more than what it asks for. 1578 * MADV_SEQUENTIAL - pages in the given range will probably be accessed 1579 * once, so they can be aggressively read ahead, and 1580 * can be freed soon after they are accessed. 1581 * MADV_WILLNEED - the application is notifying the system to read 1582 * some pages ahead. 1583 * MADV_DONTNEED - the application is finished with the given range, 1584 * so the kernel can free resources associated with it. 1585 * MADV_FREE - the application marks pages in the given range as lazy free, 1586 * where actual purges are postponed until memory pressure happens. 1587 * MADV_REMOVE - the application wants to free up the given range of 1588 * pages and associated backing store. 1589 * MADV_DONTFORK - omit this area from child's address space when forking: 1590 * typically, to avoid COWing pages pinned by get_user_pages(). 1591 * MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking. 1592 * MADV_WIPEONFORK - present the child process with zero-filled memory in this 1593 * range after a fork. 1594 * MADV_KEEPONFORK - undo the effect of MADV_WIPEONFORK 1595 * MADV_HWPOISON - trigger memory error handler as if the given memory range 1596 * were corrupted by unrecoverable hardware memory failure. 1597 * MADV_SOFT_OFFLINE - try to soft-offline the given range of memory. 1598 * MADV_MERGEABLE - the application recommends that KSM try to merge pages in 1599 * this area with pages of identical content from other such areas. 1600 * MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others. 1601 * MADV_HUGEPAGE - the application wants to back the given range by transparent 1602 * huge pages in the future. Existing pages might be coalesced and 1603 * new pages might be allocated as THP. 1604 * MADV_NOHUGEPAGE - mark the given range as not worth being backed by 1605 * transparent huge pages so the existing pages will not be 1606 * coalesced into THP and new pages will not be allocated as THP. 1607 * MADV_COLLAPSE - synchronously coalesce pages into new THP. 1608 * MADV_DONTDUMP - the application wants to prevent pages in the given range 1609 * from being included in its core dump. 1610 * MADV_DODUMP - cancel MADV_DONTDUMP: no longer exclude from core dump. 1611 * MADV_COLD - the application is not expected to use this memory soon, 1612 * deactivate pages in this range so that they can be reclaimed 1613 * easily if memory pressure happens. 1614 * MADV_PAGEOUT - the application is not expected to use this memory soon, 1615 * page out the pages in this range immediately. 1616 * MADV_POPULATE_READ - populate (prefault) page tables readable by 1617 * triggering read faults if required 1618 * MADV_POPULATE_WRITE - populate (prefault) page tables writable by 1619 * triggering write faults if required 1620 * 1621 * return values: 1622 * zero - success 1623 * -EINVAL - start + len < 0, start is not page-aligned, 1624 * "behavior" is not a valid value, or application 1625 * is attempting to release locked or shared pages, 1626 * or the specified address range includes file, Huge TLB, 1627 * MAP_SHARED or VMPFNMAP range. 1628 * -ENOMEM - addresses in the specified range are not currently 1629 * mapped, or are outside the AS of the process. 1630 * -EIO - an I/O error occurred while paging in data. 1631 * -EBADF - map exists, but area maps something that isn't a file. 1632 * -EAGAIN - a kernel resource was temporarily unavailable. 1633 * -EPERM - memory is sealed. 1634 */ 1635 int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior) 1636 { 1637 unsigned long end; 1638 int error; 1639 int write; 1640 size_t len; 1641 struct blk_plug plug; 1642 1643 if (!madvise_behavior_valid(behavior)) 1644 return -EINVAL; 1645 1646 if (!PAGE_ALIGNED(start)) 1647 return -EINVAL; 1648 len = PAGE_ALIGN(len_in); 1649 1650 /* Check to see whether len was rounded up from small -ve to zero */ 1651 if (len_in && !len) 1652 return -EINVAL; 1653 1654 end = start + len; 1655 if (end < start) 1656 return -EINVAL; 1657 1658 if (end == start) 1659 return 0; 1660 1661 #ifdef CONFIG_MEMORY_FAILURE 1662 if (behavior == MADV_HWPOISON || behavior == MADV_SOFT_OFFLINE) 1663 return madvise_inject_error(behavior, start, start + len_in); 1664 #endif 1665 1666 write = madvise_need_mmap_write(behavior); 1667 if (write) { 1668 if (mmap_write_lock_killable(mm)) 1669 return -EINTR; 1670 } else { 1671 mmap_read_lock(mm); 1672 } 1673 1674 start = untagged_addr_remote(mm, start); 1675 end = start + len; 1676 1677 blk_start_plug(&plug); 1678 switch (behavior) { 1679 case MADV_POPULATE_READ: 1680 case MADV_POPULATE_WRITE: 1681 error = madvise_populate(mm, start, end, behavior); 1682 break; 1683 default: 1684 error = madvise_walk_vmas(mm, start, end, behavior, 1685 madvise_vma_behavior); 1686 break; 1687 } 1688 blk_finish_plug(&plug); 1689 1690 if (write) 1691 mmap_write_unlock(mm); 1692 else 1693 mmap_read_unlock(mm); 1694 1695 return error; 1696 } 1697 1698 SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior) 1699 { 1700 return do_madvise(current->mm, start, len_in, behavior); 1701 } 1702 1703 /* Perform an madvise operation over a vector of addresses and lengths. */ 1704 static ssize_t vector_madvise(struct mm_struct *mm, struct iov_iter *iter, 1705 int behavior) 1706 { 1707 ssize_t ret = 0; 1708 size_t total_len; 1709 1710 total_len = iov_iter_count(iter); 1711 1712 while (iov_iter_count(iter)) { 1713 ret = do_madvise(mm, (unsigned long)iter_iov_addr(iter), 1714 iter_iov_len(iter), behavior); 1715 /* 1716 * An madvise operation is attempting to restart the syscall, 1717 * but we cannot proceed as it would not be correct to repeat 1718 * the operation in aggregate, and would be surprising to the 1719 * user. 1720 * 1721 * As we have already dropped locks, it is safe to just loop and 1722 * try again. We check for fatal signals in case we need exit 1723 * early anyway. 1724 */ 1725 if (ret == -ERESTARTNOINTR) { 1726 if (fatal_signal_pending(current)) { 1727 ret = -EINTR; 1728 break; 1729 } 1730 continue; 1731 } 1732 if (ret < 0) 1733 break; 1734 iov_iter_advance(iter, iter_iov_len(iter)); 1735 } 1736 1737 ret = (total_len - iov_iter_count(iter)) ? : ret; 1738 1739 return ret; 1740 } 1741 1742 SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec, 1743 size_t, vlen, int, behavior, unsigned int, flags) 1744 { 1745 ssize_t ret; 1746 struct iovec iovstack[UIO_FASTIOV]; 1747 struct iovec *iov = iovstack; 1748 struct iov_iter iter; 1749 struct task_struct *task; 1750 struct mm_struct *mm; 1751 unsigned int f_flags; 1752 1753 if (flags != 0) { 1754 ret = -EINVAL; 1755 goto out; 1756 } 1757 1758 ret = import_iovec(ITER_DEST, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter); 1759 if (ret < 0) 1760 goto out; 1761 1762 task = pidfd_get_task(pidfd, &f_flags); 1763 if (IS_ERR(task)) { 1764 ret = PTR_ERR(task); 1765 goto free_iov; 1766 } 1767 1768 /* Require PTRACE_MODE_READ to avoid leaking ASLR metadata. */ 1769 mm = mm_access(task, PTRACE_MODE_READ_FSCREDS); 1770 if (IS_ERR(mm)) { 1771 ret = PTR_ERR(mm); 1772 goto release_task; 1773 } 1774 1775 /* 1776 * We need only perform this check if we are attempting to manipulate a 1777 * remote process's address space. 1778 */ 1779 if (mm != current->mm && !process_madvise_remote_valid(behavior)) { 1780 ret = -EINVAL; 1781 goto release_mm; 1782 } 1783 1784 /* 1785 * Require CAP_SYS_NICE for influencing process performance. Note that 1786 * only non-destructive hints are currently supported for remote 1787 * processes. 1788 */ 1789 if (mm != current->mm && !capable(CAP_SYS_NICE)) { 1790 ret = -EPERM; 1791 goto release_mm; 1792 } 1793 1794 ret = vector_madvise(mm, &iter, behavior); 1795 1796 release_mm: 1797 mmput(mm); 1798 release_task: 1799 put_task_struct(task); 1800 free_iov: 1801 kfree(iov); 1802 out: 1803 return ret; 1804 } 1805