1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * mm/rmap.c - physical to virtual reverse mappings 4 * 5 * Copyright 2001, Rik van Riel <riel@conectiva.com.br> 6 * 7 * Simple, low overhead reverse mapping scheme. 8 * Please try to keep this thing as modular as possible. 9 * 10 * Provides methods for unmapping each kind of mapped page: 11 * the anon methods track anonymous pages, and 12 * the file methods track pages belonging to an inode. 13 * 14 * Original design by Rik van Riel <riel@conectiva.com.br> 2001 15 * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004 16 * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004 17 * Contributions by Hugh Dickins 2003, 2004 18 */ 19 20 /* 21 * Lock ordering in mm: 22 * 23 * inode->i_rwsem (while writing or truncating, not reading or faulting) 24 * mm->mmap_lock 25 * mapping->invalidate_lock (in filemap_fault) 26 * folio_lock 27 * hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share, see hugetlbfs below) 28 * vma_start_write 29 * mapping->i_mmap_rwsem 30 * anon_vma->rwsem 31 * mm->page_table_lock or pte_lock 32 * swap_lock (in swap_duplicate, swap_info_get) 33 * mmlist_lock (in mmput, drain_mmlist and others) 34 * mapping->private_lock (in block_dirty_folio) 35 * i_pages lock (widely used) 36 * lruvec->lru_lock (in folio_lruvec_lock_irq) 37 * inode->i_lock (in set_page_dirty's __mark_inode_dirty) 38 * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty) 39 * sb_lock (within inode_lock in fs/fs-writeback.c) 40 * i_pages lock (widely used, in set_page_dirty, 41 * in arch-dependent flush_dcache_mmap_lock, 42 * within bdi.wb->list_lock in __sync_single_inode) 43 * 44 * anon_vma->rwsem,mapping->i_mmap_rwsem (memory_failure, collect_procs_anon) 45 * ->tasklist_lock 46 * pte map lock 47 * 48 * hugetlbfs PageHuge() take locks in this order: 49 * hugetlb_fault_mutex (hugetlbfs specific page fault mutex) 50 * vma_lock (hugetlb specific lock for pmd_sharing) 51 * mapping->i_mmap_rwsem (also used for hugetlb pmd sharing) 52 * folio_lock 53 */ 54 55 #include <linux/mm.h> 56 #include <linux/sched/mm.h> 57 #include <linux/sched/task.h> 58 #include <linux/pagemap.h> 59 #include <linux/swap.h> 60 #include <linux/leafops.h> 61 #include <linux/slab.h> 62 #include <linux/init.h> 63 #include <linux/ksm.h> 64 #include <linux/rmap.h> 65 #include <linux/rcupdate.h> 66 #include <linux/export.h> 67 #include <linux/memcontrol.h> 68 #include <linux/mmu_notifier.h> 69 #include <linux/migrate.h> 70 #include <linux/hugetlb.h> 71 #include <linux/huge_mm.h> 72 #include <linux/backing-dev.h> 73 #include <linux/page_idle.h> 74 #include <linux/memremap.h> 75 #include <linux/userfaultfd_k.h> 76 #include <linux/mm_inline.h> 77 #include <linux/oom.h> 78 79 #include <asm/tlb.h> 80 81 #define CREATE_TRACE_POINTS 82 #include <trace/events/migrate.h> 83 84 #include "internal.h" 85 #include "swap.h" 86 87 static struct kmem_cache *anon_vma_cachep; 88 static struct kmem_cache *anon_vma_chain_cachep; 89 90 static inline struct anon_vma *anon_vma_alloc(void) 91 { 92 struct anon_vma *anon_vma; 93 94 anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL); 95 if (anon_vma) { 96 atomic_set(&anon_vma->refcount, 1); 97 anon_vma->num_children = 0; 98 anon_vma->num_active_vmas = 0; 99 anon_vma->parent = anon_vma; 100 /* 101 * Initialise the anon_vma root to point to itself. If called 102 * from fork, the root will be reset to the parents anon_vma. 103 */ 104 anon_vma->root = anon_vma; 105 } 106 107 return anon_vma; 108 } 109 110 static inline void anon_vma_free(struct anon_vma *anon_vma) 111 { 112 VM_BUG_ON(atomic_read(&anon_vma->refcount)); 113 114 /* 115 * Synchronize against folio_lock_anon_vma_read() such that 116 * we can safely hold the lock without the anon_vma getting 117 * freed. 118 * 119 * Relies on the full mb implied by the atomic_dec_and_test() from 120 * put_anon_vma() against the acquire barrier implied by 121 * down_read_trylock() from folio_lock_anon_vma_read(). This orders: 122 * 123 * folio_lock_anon_vma_read() VS put_anon_vma() 124 * down_read_trylock() atomic_dec_and_test() 125 * LOCK MB 126 * atomic_read() rwsem_is_locked() 127 * 128 * LOCK should suffice since the actual taking of the lock must 129 * happen _before_ what follows. 130 */ 131 might_sleep(); 132 if (rwsem_is_locked(&anon_vma->root->rwsem)) { 133 anon_vma_lock_write(anon_vma); 134 anon_vma_unlock_write(anon_vma); 135 } 136 137 kmem_cache_free(anon_vma_cachep, anon_vma); 138 } 139 140 static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp) 141 { 142 return kmem_cache_alloc(anon_vma_chain_cachep, gfp); 143 } 144 145 static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain) 146 { 147 kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain); 148 } 149 150 static void anon_vma_chain_assign(struct vm_area_struct *vma, 151 struct anon_vma_chain *avc, 152 struct anon_vma *anon_vma) 153 { 154 avc->vma = vma; 155 avc->anon_vma = anon_vma; 156 list_add(&avc->same_vma, &vma->anon_vma_chain); 157 } 158 159 /** 160 * __anon_vma_prepare - attach an anon_vma to a memory region 161 * @vma: the memory region in question 162 * 163 * This makes sure the memory mapping described by 'vma' has 164 * an 'anon_vma' attached to it, so that we can associate the 165 * anonymous pages mapped into it with that anon_vma. 166 * 167 * The common case will be that we already have one, which 168 * is handled inline by anon_vma_prepare(). But if 169 * not we either need to find an adjacent mapping that we 170 * can re-use the anon_vma from (very common when the only 171 * reason for splitting a vma has been mprotect()), or we 172 * allocate a new one. 173 * 174 * Anon-vma allocations are very subtle, because we may have 175 * optimistically looked up an anon_vma in folio_lock_anon_vma_read() 176 * and that may actually touch the rwsem even in the newly 177 * allocated vma (it depends on RCU to make sure that the 178 * anon_vma isn't actually destroyed). 179 * 180 * As a result, we need to do proper anon_vma locking even 181 * for the new allocation. At the same time, we do not want 182 * to do any locking for the common case of already having 183 * an anon_vma. 184 */ 185 int __anon_vma_prepare(struct vm_area_struct *vma) 186 { 187 struct mm_struct *mm = vma->vm_mm; 188 struct anon_vma *anon_vma, *allocated; 189 struct anon_vma_chain *avc; 190 191 mmap_assert_locked(mm); 192 might_sleep(); 193 194 avc = anon_vma_chain_alloc(GFP_KERNEL); 195 if (!avc) 196 goto out_enomem; 197 198 anon_vma = find_mergeable_anon_vma(vma); 199 allocated = NULL; 200 if (!anon_vma) { 201 anon_vma = anon_vma_alloc(); 202 if (unlikely(!anon_vma)) 203 goto out_enomem_free_avc; 204 anon_vma->num_children++; /* self-parent link for new root */ 205 allocated = anon_vma; 206 } 207 208 anon_vma_lock_write(anon_vma); 209 /* page_table_lock to protect against threads */ 210 spin_lock(&mm->page_table_lock); 211 if (likely(!vma_has_anon_rmap(vma))) { 212 /* 213 * Make anon_vma fields visible before anon_vma is published. 214 * Paired with an address dependency in reusable_anon_vma(). 215 */ 216 smp_store_release(&vma->anon_vma, anon_vma); 217 anon_vma_chain_assign(vma, avc, anon_vma); 218 anon_rmap_tree_insert(avc, anon_vma); 219 anon_vma->num_active_vmas++; 220 allocated = NULL; 221 avc = NULL; 222 } 223 spin_unlock(&mm->page_table_lock); 224 anon_vma_unlock_write(anon_vma); 225 226 if (unlikely(allocated)) 227 put_anon_vma(allocated); 228 if (unlikely(avc)) 229 anon_vma_chain_free(avc); 230 231 return 0; 232 233 out_enomem_free_avc: 234 anon_vma_chain_free(avc); 235 out_enomem: 236 return -ENOMEM; 237 } 238 239 static void check_anon_vma_clone(struct vm_area_struct *dst, 240 struct vm_area_struct *src, 241 enum vma_operation operation) 242 { 243 /* The write lock must be held. */ 244 mmap_assert_write_locked(src->vm_mm); 245 /* If not a fork then must be on same mm. */ 246 VM_WARN_ON_ONCE(operation != VMA_OP_FORK && dst->vm_mm != src->vm_mm); 247 248 /* If we have anything to do src->anon_vma must be provided. */ 249 VM_WARN_ON_ONCE(!vma_has_anon_rmap(src) && !list_empty(&src->anon_vma_chain)); 250 VM_WARN_ON_ONCE(!vma_has_anon_rmap(src) && vma_has_anon_rmap(dst)); 251 /* We are establishing a new anon_vma_chain. */ 252 VM_WARN_ON_ONCE(!list_empty(&dst->anon_vma_chain)); 253 /* 254 * On fork, dst->anon_vma is set NULL (temporarily). Otherwise, anon_vma 255 * must be the same across dst and src. 256 */ 257 VM_WARN_ON_ONCE(vma_has_anon_rmap(dst) && dst->anon_vma != src->anon_vma); 258 /* 259 * Essentially equivalent to above - if not a no-op, we should expect 260 * dst->anon_vma to be set for everything except a fork. 261 */ 262 VM_WARN_ON_ONCE(operation != VMA_OP_FORK && vma_has_anon_rmap(src) && 263 !vma_has_anon_rmap(dst)); 264 /* For the anon_vma to be compatible, it can only be singular. */ 265 VM_WARN_ON_ONCE(operation == VMA_OP_MERGE_UNFAULTED && 266 !list_is_singular(&src->anon_vma_chain)); 267 /* Only merging an unfaulted VMA leaves the destination attached. */ 268 VM_WARN_ON_ONCE(operation != VMA_OP_MERGE_UNFAULTED && 269 vma_is_attached(dst)); 270 } 271 272 static void maybe_reuse_anon_vma(struct vm_area_struct *dst, 273 struct anon_vma *anon_vma) 274 { 275 /* If already populated, nothing to do.*/ 276 if (vma_has_anon_rmap(dst)) 277 return; 278 279 /* 280 * We reuse an anon_vma if any linking VMAs were unmapped and it has 281 * only a single child at most. 282 */ 283 if (anon_vma->num_active_vmas > 0) 284 return; 285 if (anon_vma->num_children > 1) 286 return; 287 288 dst->anon_vma = anon_vma; 289 anon_vma->num_active_vmas++; 290 } 291 292 static void cleanup_partial_anon_vmas(struct vm_area_struct *vma); 293 294 /** 295 * anon_vma_clone - Establishes new anon_vma_chain objects in @dst linking to 296 * all of the anon_vma objects contained within @src anon_vma_chain's. 297 * @dst: The destination VMA with an empty anon_vma_chain. 298 * @src: The source VMA we wish to duplicate. 299 * @operation: The type of operation which resulted in the clone. 300 * 301 * This is the heart of the VMA side of the anon_vma implementation - we invoke 302 * this function whenever we need to set up a new VMA's anon_vma state. 303 * 304 * This is invoked for: 305 * 306 * - VMA Merge, but only when @dst is unfaulted and @src is faulted - meaning we 307 * clone @src into @dst. 308 * - VMA split. 309 * - VMA (m)remap. 310 * - Fork of faulted VMA. 311 * 312 * In all cases other than fork this is simply a duplication. Fork additionally 313 * adds a new active anon_vma. 314 * 315 * ONLY in the case of fork do we try to 'reuse' existing anon_vma's in an 316 * anon_vma hierarchy, reusing anon_vma's which have no VMA associated with them 317 * but do have a single child. This is to avoid waste of memory when repeatedly 318 * forking. 319 * 320 * Returns: 0 on success, -ENOMEM on failure. 321 */ 322 int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src, 323 enum vma_operation operation) 324 { 325 struct anon_vma_chain *avc, *pavc; 326 struct anon_vma *active_anon_vma = src->anon_vma; 327 328 check_anon_vma_clone(dst, src, operation); 329 330 if (!vma_has_anon_rmap(src)) 331 return 0; 332 333 /* 334 * Allocate AVCs. We don't need an anon_vma lock for this as we 335 * are not updating the anon_vma rbtree nor are we changing 336 * anon_vma statistics. 337 * 338 * Either src, dst have the same mm for which we hold an exclusive mmap 339 * write lock, or we are forking and we hold it on src->vm_mm and dst is 340 * not yet accessible to other threads so there's no possibliity of the 341 * unlinked AVC's being observed yet. 342 */ 343 list_for_each_entry(pavc, &src->anon_vma_chain, same_vma) { 344 avc = anon_vma_chain_alloc(GFP_KERNEL); 345 if (!avc) 346 goto enomem_failure; 347 348 anon_vma_chain_assign(dst, avc, pavc->anon_vma); 349 } 350 351 /* 352 * Now link the anon_vma's back to the newly inserted AVCs. 353 * Note that all anon_vma's share the same root. 354 */ 355 anon_vma_lock_write(active_anon_vma); 356 list_for_each_entry_reverse(avc, &dst->anon_vma_chain, same_vma) { 357 struct anon_vma *anon_vma = avc->anon_vma; 358 359 anon_rmap_tree_insert(avc, anon_vma); 360 if (operation == VMA_OP_FORK) 361 maybe_reuse_anon_vma(dst, anon_vma); 362 } 363 364 if (operation != VMA_OP_FORK) 365 dst->anon_vma->num_active_vmas++; 366 367 anon_vma_unlock_write(active_anon_vma); 368 return 0; 369 370 enomem_failure: 371 cleanup_partial_anon_vmas(dst); 372 return -ENOMEM; 373 } 374 375 /* 376 * Attach vma to its own anon_vma, as well as to the anon_vmas that 377 * the corresponding VMA in the parent process is attached to. 378 * Returns 0 on success, non-zero on failure. 379 */ 380 int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) 381 { 382 struct anon_vma_chain *avc; 383 struct anon_vma *anon_vma; 384 int rc; 385 386 /* Don't bother if the parent process has no anon_vma here. */ 387 if (!vma_has_anon_rmap(pvma)) 388 return 0; 389 390 /* Drop inherited anon_vma, we'll reuse existing or allocate new. */ 391 vma->anon_vma = NULL; 392 393 anon_vma = anon_vma_alloc(); 394 if (!anon_vma) 395 return -ENOMEM; 396 avc = anon_vma_chain_alloc(GFP_KERNEL); 397 if (!avc) { 398 put_anon_vma(anon_vma); 399 return -ENOMEM; 400 } 401 402 /* 403 * First, attach the new VMA to the parent VMA's anon_vmas, 404 * so rmap can find non-COWed pages in child processes. 405 */ 406 rc = anon_vma_clone(vma, pvma, VMA_OP_FORK); 407 /* An error arose or an existing anon_vma was reused, all done then. */ 408 if (rc || vma_has_anon_rmap(vma)) { 409 put_anon_vma(anon_vma); 410 anon_vma_chain_free(avc); 411 return rc; 412 } 413 414 /* 415 * OK no reuse, so add our own anon_vma. 416 * 417 * Since it is not linked anywhere we can safely manipulate anon_vma 418 * fields without a lock. 419 */ 420 421 anon_vma->num_active_vmas = 1; 422 /* 423 * The root anon_vma's rwsem is the lock actually used when we 424 * lock any of the anon_vmas in this anon_vma tree. 425 */ 426 anon_vma->root = pvma->anon_vma->root; 427 anon_vma->parent = pvma->anon_vma; 428 /* 429 * With refcounts, an anon_vma can stay around longer than the 430 * process it belongs to. The root anon_vma needs to be pinned until 431 * this anon_vma is freed, because the lock lives in the root. 432 */ 433 get_anon_vma(anon_vma->root); 434 /* Mark this anon_vma as the one where our new (COWed) pages go. */ 435 vma->anon_vma = anon_vma; 436 anon_vma_chain_assign(vma, avc, anon_vma); 437 /* Now let rmap see it. */ 438 anon_vma_lock_write(anon_vma); 439 anon_rmap_tree_insert(avc, anon_vma); 440 anon_vma->parent->num_children++; 441 anon_vma_unlock_write(anon_vma); 442 443 return 0; 444 } 445 446 /* 447 * In the unfortunate case of anon_vma_clone() failing to allocate memory we 448 * have to clean things up. 449 * 450 * Since we allocate anon_vma_chain's before we insert them into the interval 451 * trees, we simply have to free up the AVC's and remove the entries from the 452 * VMA's anon_vma_chain. 453 */ 454 static void cleanup_partial_anon_vmas(struct vm_area_struct *vma) 455 { 456 struct anon_vma_chain *avc, *next; 457 458 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { 459 list_del(&avc->same_vma); 460 anon_vma_chain_free(avc); 461 } 462 463 /* 464 * The anon_vma assigned to this VMA is no longer valid, as we were not 465 * able to correctly clone AVC state. Avoid inconsistent anon_vma tree 466 * state by resetting. 467 */ 468 vma->anon_vma = NULL; 469 } 470 471 /** 472 * unlink_anon_vmas() - remove all links between a VMA and anon_vma's, freeing 473 * anon_vma_chain objects. 474 * @vma: The VMA whose links to anon_vma objects is to be severed. 475 * 476 * As part of the process anon_vma_chain's are freed, 477 * anon_vma->num_children,num_active_vmas is updated as required and, if the 478 * relevant anon_vma references no further VMAs, its reference count is 479 * decremented. 480 */ 481 void unlink_anon_vmas(struct vm_area_struct *vma) 482 { 483 struct anon_vma_chain *avc, *next; 484 struct anon_vma *active_anon_vma = vma->anon_vma; 485 486 /* Always hold mmap lock, read-lock on unmap possibly. */ 487 mmap_assert_locked(vma->vm_mm); 488 489 /* Unfaulted is a no-op. */ 490 if (!active_anon_vma) { 491 VM_WARN_ON_ONCE(!list_empty(&vma->anon_vma_chain)); 492 return; 493 } 494 495 anon_vma_lock_write(active_anon_vma); 496 497 /* 498 * Unlink each anon_vma chained to the VMA. This list is ordered 499 * from newest to oldest, ensuring the root anon_vma gets freed last. 500 */ 501 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { 502 struct anon_vma *anon_vma = avc->anon_vma; 503 504 anon_rmap_tree_remove(avc, anon_vma); 505 506 /* 507 * Leave empty anon_vmas on the list - we'll need 508 * to free them outside the lock. 509 */ 510 if (RB_EMPTY_ROOT(&anon_vma->rb_root.rb_root)) { 511 anon_vma->parent->num_children--; 512 continue; 513 } 514 515 list_del(&avc->same_vma); 516 anon_vma_chain_free(avc); 517 } 518 519 active_anon_vma->num_active_vmas--; 520 /* 521 * vma would still be needed after unlink, and anon_vma will be prepared 522 * when handle fault. 523 */ 524 vma->anon_vma = NULL; 525 anon_vma_unlock_write(active_anon_vma); 526 527 528 /* 529 * Iterate the list once more, it now only contains empty and unlinked 530 * anon_vmas, destroy them. Could not do before due to __put_anon_vma() 531 * needing to write-acquire the anon_vma->root->rwsem. 532 */ 533 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { 534 struct anon_vma *anon_vma = avc->anon_vma; 535 536 VM_WARN_ON(anon_vma->num_children); 537 VM_WARN_ON(anon_vma->num_active_vmas); 538 put_anon_vma(anon_vma); 539 540 list_del(&avc->same_vma); 541 anon_vma_chain_free(avc); 542 } 543 } 544 545 static void anon_vma_ctor(void *data) 546 { 547 struct anon_vma *anon_vma = data; 548 549 init_rwsem(&anon_vma->rwsem); 550 atomic_set(&anon_vma->refcount, 0); 551 anon_vma->rb_root = RB_ROOT_CACHED; 552 } 553 554 void __init anon_vma_init(void) 555 { 556 anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma), 557 0, SLAB_TYPESAFE_BY_RCU|SLAB_PANIC|SLAB_ACCOUNT, 558 anon_vma_ctor); 559 anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain, 560 SLAB_PANIC|SLAB_ACCOUNT); 561 } 562 563 /* 564 * Getting a lock on a stable anon_vma from a page off the LRU is tricky! 565 * 566 * Since there is no serialization what so ever against folio_remove_rmap_*() 567 * the best this function can do is return a refcount increased anon_vma 568 * that might have been relevant to this page. 569 * 570 * The page might have been remapped to a different anon_vma or the anon_vma 571 * returned may already be freed (and even reused). 572 * 573 * In case it was remapped to a different anon_vma, the new anon_vma will be a 574 * child of the old anon_vma, and the anon_vma lifetime rules will therefore 575 * ensure that any anon_vma obtained from the page will still be valid for as 576 * long as we observe folio_mapped() [ hence all those folio_mapped() tests ]. 577 * 578 * All users of this function must be very careful when walking the anon_vma 579 * chain and verify that the page in question is indeed mapped in it 580 * [ something equivalent to page_mapped_in_vma() ]. 581 * 582 * Since anon_vma's slab is SLAB_TYPESAFE_BY_RCU and we know from 583 * folio_remove_rmap_*() that the anon_vma pointer from page->mapping is valid 584 * if there is a mapcount, we can dereference the anon_vma after observing 585 * those. 586 * 587 * NOTE: the caller should hold folio lock when calling this. 588 */ 589 struct anon_vma *folio_get_anon_vma(const struct folio *folio) 590 { 591 struct anon_vma *anon_vma = NULL; 592 unsigned long anon_mapping; 593 594 VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); 595 596 rcu_read_lock(); 597 anon_mapping = (unsigned long)READ_ONCE(folio->mapping); 598 if ((anon_mapping & FOLIO_MAPPING_FLAGS) != FOLIO_MAPPING_ANON) 599 goto out; 600 if (!folio_mapped(folio)) 601 goto out; 602 603 anon_vma = (struct anon_vma *) (anon_mapping - FOLIO_MAPPING_ANON); 604 if (!atomic_inc_not_zero(&anon_vma->refcount)) { 605 anon_vma = NULL; 606 goto out; 607 } 608 609 /* 610 * If this folio is still mapped, then its anon_vma cannot have been 611 * freed. But if it has been unmapped, we have no security against the 612 * anon_vma structure being freed and reused (for another anon_vma: 613 * SLAB_TYPESAFE_BY_RCU guarantees that - so the atomic_inc_not_zero() 614 * above cannot corrupt). 615 */ 616 if (!folio_mapped(folio)) { 617 rcu_read_unlock(); 618 put_anon_vma(anon_vma); 619 return NULL; 620 } 621 out: 622 rcu_read_unlock(); 623 624 return anon_vma; 625 } 626 627 /* 628 * Similar to folio_get_anon_vma() except it locks the anon_vma. 629 * 630 * Its a little more complex as it tries to keep the fast path to a single 631 * atomic op -- the trylock. If we fail the trylock, we fall back to getting a 632 * reference like with folio_get_anon_vma() and then block on the mutex 633 * on !rwc->try_lock case. 634 */ 635 struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio, 636 struct rmap_walk_control *rwc) 637 { 638 struct anon_vma *anon_vma = NULL; 639 struct anon_vma *root_anon_vma; 640 unsigned long anon_mapping; 641 642 VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); 643 644 rcu_read_lock(); 645 anon_mapping = (unsigned long)READ_ONCE(folio->mapping); 646 if ((anon_mapping & FOLIO_MAPPING_FLAGS) != FOLIO_MAPPING_ANON) 647 goto out; 648 if (!folio_mapped(folio)) 649 goto out; 650 651 anon_vma = (struct anon_vma *) (anon_mapping - FOLIO_MAPPING_ANON); 652 root_anon_vma = READ_ONCE(anon_vma->root); 653 if (down_read_trylock(&root_anon_vma->rwsem)) { 654 /* 655 * If the folio is still mapped, then this anon_vma is still 656 * its anon_vma, and holding the mutex ensures that it will 657 * not go away, see anon_vma_free(). 658 */ 659 if (!folio_mapped(folio)) { 660 up_read(&root_anon_vma->rwsem); 661 anon_vma = NULL; 662 } 663 goto out; 664 } 665 666 if (rwc && rwc->try_lock) { 667 anon_vma = NULL; 668 rwc->contended = true; 669 goto out; 670 } 671 672 /* trylock failed, we got to sleep */ 673 if (!atomic_inc_not_zero(&anon_vma->refcount)) { 674 anon_vma = NULL; 675 goto out; 676 } 677 678 if (!folio_mapped(folio)) { 679 rcu_read_unlock(); 680 put_anon_vma(anon_vma); 681 return NULL; 682 } 683 684 /* we pinned the anon_vma, its safe to sleep */ 685 rcu_read_unlock(); 686 anon_vma_lock_read(anon_vma); 687 688 if (atomic_dec_and_test(&anon_vma->refcount)) { 689 /* 690 * Oops, we held the last refcount, release the lock 691 * and bail -- can't simply use put_anon_vma() because 692 * we'll deadlock on the anon_vma_lock_write() recursion. 693 */ 694 anon_vma_unlock_read(anon_vma); 695 __put_anon_vma(anon_vma); 696 anon_vma = NULL; 697 } 698 699 return anon_vma; 700 701 out: 702 rcu_read_unlock(); 703 return anon_vma; 704 } 705 706 #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH 707 /* 708 * Flush TLB entries for recently unmapped pages from remote CPUs. It is 709 * important if a PTE was dirty when it was unmapped that it's flushed 710 * before any IO is initiated on the page to prevent lost writes. Similarly, 711 * it must be flushed before freeing to prevent data leakage. 712 */ 713 void try_to_unmap_flush(void) 714 { 715 struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; 716 717 if (!tlb_ubc->flush_required) 718 return; 719 720 arch_tlbbatch_flush(&tlb_ubc->arch); 721 tlb_ubc->flush_required = false; 722 tlb_ubc->writable = false; 723 } 724 725 /* Flush iff there are potentially writable TLB entries that can race with IO */ 726 void try_to_unmap_flush_dirty(void) 727 { 728 struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; 729 730 if (tlb_ubc->writable) 731 try_to_unmap_flush(); 732 } 733 734 /* 735 * Bits 0-14 of mm->tlb_flush_batched record pending generations. 736 * Bits 16-30 of mm->tlb_flush_batched bit record flushed generations. 737 */ 738 #define TLB_FLUSH_BATCH_FLUSHED_SHIFT 16 739 #define TLB_FLUSH_BATCH_PENDING_MASK \ 740 ((1 << (TLB_FLUSH_BATCH_FLUSHED_SHIFT - 1)) - 1) 741 #define TLB_FLUSH_BATCH_PENDING_LARGE \ 742 (TLB_FLUSH_BATCH_PENDING_MASK / 2) 743 744 static void set_tlb_ubc_flush_pending(struct mm_struct *mm, pte_t pteval, 745 unsigned long start, unsigned long end) 746 { 747 struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; 748 int batch; 749 bool writable = pte_dirty(pteval); 750 751 if (!pte_accessible(mm, pteval)) 752 return; 753 754 arch_tlbbatch_add_pending(&tlb_ubc->arch, mm, start, end); 755 tlb_ubc->flush_required = true; 756 757 /* 758 * Ensure compiler does not re-order the setting of tlb_flush_batched 759 * before the PTE is cleared. 760 */ 761 barrier(); 762 batch = atomic_read(&mm->tlb_flush_batched); 763 retry: 764 if ((batch & TLB_FLUSH_BATCH_PENDING_MASK) > TLB_FLUSH_BATCH_PENDING_LARGE) { 765 /* 766 * Prevent `pending' from catching up with `flushed' because of 767 * overflow. Reset `pending' and `flushed' to be 1 and 0 if 768 * `pending' becomes large. 769 */ 770 if (!atomic_try_cmpxchg(&mm->tlb_flush_batched, &batch, 1)) 771 goto retry; 772 } else { 773 atomic_inc(&mm->tlb_flush_batched); 774 } 775 776 /* 777 * If the PTE was dirty then it's best to assume it's writable. The 778 * caller must use try_to_unmap_flush_dirty() or try_to_unmap_flush() 779 * before the page is queued for IO. 780 */ 781 if (writable) 782 tlb_ubc->writable = true; 783 } 784 785 /* 786 * Returns true if the TLB flush should be deferred to the end of a batch of 787 * unmap operations to reduce IPIs. 788 */ 789 static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) 790 { 791 if (!(flags & TTU_BATCH_FLUSH)) 792 return false; 793 794 return arch_tlbbatch_should_defer(mm); 795 } 796 797 /* 798 * Reclaim unmaps pages under the PTL but do not flush the TLB prior to 799 * releasing the PTL if TLB flushes are batched. It's possible for a parallel 800 * operation such as mprotect or munmap to race between reclaim unmapping 801 * the page and flushing the page. If this race occurs, it potentially allows 802 * access to data via a stale TLB entry. Tracking all mm's that have TLB 803 * batching in flight would be expensive during reclaim so instead track 804 * whether TLB batching occurred in the past and if so then do a flush here 805 * if required. This will cost one additional flush per reclaim cycle paid 806 * by the first operation at risk such as mprotect and mumap. 807 * 808 * This must be called under the PTL so that an access to tlb_flush_batched 809 * that is potentially a "reclaim vs mprotect/munmap/etc" race will synchronise 810 * via the PTL. 811 */ 812 void flush_tlb_batched_pending(struct mm_struct *mm) 813 { 814 int batch = atomic_read(&mm->tlb_flush_batched); 815 int pending = batch & TLB_FLUSH_BATCH_PENDING_MASK; 816 int flushed = batch >> TLB_FLUSH_BATCH_FLUSHED_SHIFT; 817 818 if (pending != flushed) { 819 flush_tlb_mm(mm); 820 /* 821 * If the new TLB flushing is pending during flushing, leave 822 * mm->tlb_flush_batched as is, to avoid losing flushing. 823 */ 824 atomic_cmpxchg(&mm->tlb_flush_batched, batch, 825 pending | (pending << TLB_FLUSH_BATCH_FLUSHED_SHIFT)); 826 } 827 } 828 #else 829 static void set_tlb_ubc_flush_pending(struct mm_struct *mm, pte_t pteval, 830 unsigned long start, unsigned long end) 831 { 832 } 833 834 static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) 835 { 836 return false; 837 } 838 #endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */ 839 840 /** 841 * page_address_in_vma - The virtual address of a page in this VMA. 842 * @folio: The folio containing the page. 843 * @page: The page within the folio. 844 * @vma: The VMA we need to know the address in. 845 * 846 * Calculates the user virtual address of this page in the specified VMA. 847 * It is the caller's responsibility to check the page is actually 848 * within the VMA. There may not currently be a PTE pointing at this 849 * page, but if a page fault occurs at this address, this is the page 850 * which will be accessed. 851 * 852 * Context: Caller should hold a reference to the folio. Caller should 853 * hold a lock (eg the i_mmap_lock or the mmap_lock) which keeps the 854 * VMA from being altered. 855 * 856 * Return: The virtual address corresponding to this page in the VMA. 857 */ 858 unsigned long page_address_in_vma(const struct folio *folio, 859 const struct page *page, const struct vm_area_struct *vma) 860 { 861 if (folio_test_anon(folio)) { 862 struct anon_vma *anon_vma = folio_anon_vma(folio); 863 /* 864 * Note: swapoff's unuse_vma() is more efficient with this 865 * check, and needs it to match anon_vma when KSM is active. 866 */ 867 if (!vma_has_anon_rmap(vma) || !anon_vma || 868 vma->anon_vma->root != anon_vma->root) 869 return -EFAULT; 870 /* KSM folios don't reach here because of the !anon_vma check */ 871 return vma_anon_address(vma, page_pgoff(folio, page), 1); 872 } else if (!vma->vm_file) { 873 return -EFAULT; 874 } else if (vma->vm_file->f_mapping != folio->mapping) { 875 return -EFAULT; 876 } 877 878 return vma_filebacked_address(vma, page_pgoff(folio, page), 1); 879 } 880 881 /* 882 * Returns the actual pmd_t* where we expect 'address' to be mapped from, or 883 * NULL if it doesn't exist. No guarantees / checks on what the pmd_t* 884 * represents. 885 */ 886 pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) 887 { 888 pgd_t *pgd; 889 p4d_t *p4d; 890 pud_t *pud; 891 pmd_t *pmd = NULL; 892 893 pgd = pgd_offset(mm, address); 894 if (!pgd_present(*pgd)) 895 goto out; 896 897 p4d = p4d_offset(pgd, address); 898 if (!p4d_present(*p4d)) 899 goto out; 900 901 pud = pud_offset(p4d, address); 902 if (!pud_present(*pud)) 903 goto out; 904 905 pmd = pmd_offset(pud, address); 906 out: 907 return pmd; 908 } 909 910 struct folio_referenced_arg { 911 int mapcount; 912 int referenced; 913 vma_flags_t vma_flags; 914 struct mem_cgroup *memcg; 915 }; 916 917 /* 918 * arg: folio_referenced_arg will be passed 919 */ 920 static bool folio_referenced_one(struct folio *folio, 921 struct vm_area_struct *vma, unsigned long address, void *arg) 922 { 923 struct folio_referenced_arg *pra = arg; 924 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); 925 int ptes = 0, referenced = 0; 926 unsigned int nr; 927 pte_t pteval; 928 929 while (page_vma_mapped_walk(&pvmw)) { 930 address = pvmw.address; 931 nr = 1; 932 933 if (vma_test(vma, VMA_LOCKED_BIT)) { 934 ptes++; 935 pra->mapcount--; 936 937 /* Only mlock fully mapped pages */ 938 if (pvmw.pte && ptes != pvmw.nr_pages) 939 continue; 940 941 /* 942 * All PTEs must be protected by page table lock in 943 * order to mlock the page. 944 * 945 * If page table boundary has been cross, current ptl 946 * only protect part of ptes. 947 */ 948 if (pvmw.flags & PVMW_PGTABLE_CROSSED) 949 continue; 950 951 /* Restore the mlock which got missed */ 952 mlock_vma_folio(folio, vma); 953 page_vma_mapped_walk_done(&pvmw); 954 vma_flags_set(&pra->vma_flags, VMA_LOCKED_BIT); 955 return false; /* To break the loop */ 956 } 957 958 /* 959 * Skip the non-shared swapbacked folio mapped solely by 960 * the exiting or OOM-reaped process. This avoids redundant 961 * swap-out followed by an immediate unmap. 962 */ 963 if ((!atomic_read(&vma->vm_mm->mm_users) || 964 check_stable_address_space(vma->vm_mm)) && 965 folio_test_anon(folio) && folio_test_swapbacked(folio) && 966 !folio_maybe_mapped_shared(folio)) { 967 pra->referenced = -1; 968 page_vma_mapped_walk_done(&pvmw); 969 return false; 970 } 971 972 if (pvmw.pte) 973 pteval = ptep_get(pvmw.pte); 974 975 if (pvmw.pte && pte_present(pteval) && folio_test_large(folio)) { 976 const unsigned long end_addr = pte_table_clamped_end(address, vma->vm_end); 977 const unsigned int max_nr = (end_addr - address) >> PAGE_SHIFT; 978 979 nr = folio_pte_batch(folio, pvmw.pte, pteval, max_nr); 980 } 981 982 /* 983 * When LRU is switching, we don’t know where the surrounding folios 984 * are. —they could be on active/inactive lists or on MGLRU. So the 985 * simplest approach is to disable this look-around optimization. 986 */ 987 if (lru_gen_enabled() && !lru_gen_switching() && pvmw.pte && 988 pte_present(pteval)) { 989 if (lru_gen_look_around(&pvmw, nr)) 990 referenced++; 991 } else if (pvmw.pte) { 992 bool young = false; 993 994 /* 995 * PFN swap PTEs, such as device-exclusive ones, that 996 * actually map pages are "old" from a CPU perspective. 997 * The MMU notifier takes care of any device aspects. 998 */ 999 if (pte_present(pteval)) 1000 young = clear_flush_young_ptes(vma, address, pvmw.pte, nr); 1001 young |= mmu_notifier_clear_flush_young(vma->vm_mm, address, 1002 address + nr * PAGE_SIZE); 1003 if (young) 1004 referenced++; 1005 } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { 1006 if (pmdp_clear_flush_young_notify(vma, address, 1007 pvmw.pmd)) 1008 referenced++; 1009 } else { 1010 /* unexpected pmd-mapped folio? */ 1011 WARN_ON_ONCE(1); 1012 } 1013 1014 ptes += nr; 1015 pra->mapcount -= nr; 1016 /* 1017 * If we are sure that we batched the entire folio, 1018 * we can just optimize and stop right here. 1019 */ 1020 if (ptes == pvmw.nr_pages) { 1021 page_vma_mapped_walk_done(&pvmw); 1022 break; 1023 } 1024 1025 /* Skip the batched PTEs */ 1026 pvmw.pte += nr - 1; 1027 pvmw.address += (nr - 1) * PAGE_SIZE; 1028 } 1029 1030 if (referenced) 1031 folio_clear_idle(folio); 1032 if (folio_test_clear_young(folio)) 1033 referenced++; 1034 1035 if (referenced) { 1036 vma_flags_t vma_flags = vma->flags; 1037 1038 pra->referenced++; 1039 vma_flags_clear(&vma_flags, VMA_LOCKED_BIT); 1040 vma_flags_set_mask(&pra->vma_flags, vma_flags); 1041 } 1042 1043 if (!pra->mapcount) 1044 return false; /* To break the loop */ 1045 1046 return true; 1047 } 1048 1049 static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg) 1050 { 1051 struct folio_referenced_arg *pra = arg; 1052 struct mem_cgroup *memcg = pra->memcg; 1053 1054 /* 1055 * Ignore references from this mapping if it has no recency. If the 1056 * folio has been used in another mapping, we will catch it; if this 1057 * other mapping is already gone, the unmap path will have set the 1058 * referenced flag or activated the folio in zap_pte_range(). 1059 */ 1060 if (!vma_has_recency(vma)) 1061 return true; 1062 1063 /* 1064 * If we are reclaiming on behalf of a cgroup, skip counting on behalf 1065 * of references from different cgroups. 1066 */ 1067 if (memcg && !mm_match_cgroup(vma->vm_mm, memcg)) 1068 return true; 1069 1070 return false; 1071 } 1072 1073 /** 1074 * folio_referenced() - Test if the folio was referenced. 1075 * @folio: The folio to test. 1076 * @is_locked: Caller holds lock on the folio. 1077 * @memcg: target memory cgroup 1078 * @vma_flags: A combination of all the vma->flags which referenced the folio. 1079 * 1080 * Quick test_and_clear_referenced for all mappings of a folio, 1081 * 1082 * Return: The number of mappings which referenced the folio. Return -1 if 1083 * the function bailed out due to rmap lock contention. 1084 */ 1085 int folio_referenced(struct folio *folio, int is_locked, 1086 struct mem_cgroup *memcg, vma_flags_t *vma_flags) 1087 { 1088 bool we_locked = false; 1089 struct folio_referenced_arg pra = { 1090 .mapcount = folio_mapcount(folio), 1091 .memcg = memcg, 1092 }; 1093 struct rmap_walk_control rwc = { 1094 .rmap_one = folio_referenced_one, 1095 .arg = (void *)&pra, 1096 .anon_lock = folio_lock_anon_vma_read, 1097 .try_lock = true, 1098 .invalid_vma = invalid_folio_referenced_vma, 1099 }; 1100 1101 VM_WARN_ON_ONCE_FOLIO(folio_is_zone_device(folio), folio); 1102 vma_flags_clear_all(vma_flags); 1103 if (!pra.mapcount) 1104 return 0; 1105 1106 if (!folio_raw_mapping(folio)) 1107 return 0; 1108 1109 if (!is_locked) { 1110 we_locked = folio_trylock(folio); 1111 if (!we_locked) 1112 return 1; 1113 } 1114 1115 rmap_walk(folio, &rwc); 1116 vma_flags_set_mask(vma_flags, pra.vma_flags); 1117 1118 if (we_locked) 1119 folio_unlock(folio); 1120 1121 return rwc.contended ? -1 : pra.referenced; 1122 } 1123 1124 static int page_vma_mkclean_one(struct page_vma_mapped_walk *pvmw) 1125 { 1126 int cleaned = 0; 1127 struct vm_area_struct *vma = pvmw->vma; 1128 struct mmu_notifier_range range; 1129 unsigned long address = pvmw->address; 1130 1131 /* 1132 * We have to assume the worse case ie pmd for invalidation. Note that 1133 * the folio can not be freed from this function. 1134 */ 1135 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE, 0, 1136 vma->vm_mm, address, vma_address_end(pvmw)); 1137 mmu_notifier_invalidate_range_start(&range); 1138 1139 while (page_vma_mapped_walk(pvmw)) { 1140 int ret = 0; 1141 1142 address = pvmw->address; 1143 if (pvmw->pte) { 1144 pte_t *pte = pvmw->pte; 1145 pte_t entry = ptep_get(pte); 1146 1147 /* 1148 * PFN swap PTEs, such as device-exclusive ones, that 1149 * actually map pages are clean and not writable from a 1150 * CPU perspective. The MMU notifier takes care of any 1151 * device aspects. 1152 */ 1153 if (!pte_present(entry)) 1154 continue; 1155 if (!pte_dirty(entry) && !pte_write(entry)) 1156 continue; 1157 1158 flush_cache_page(vma, address, pte_pfn(entry)); 1159 entry = ptep_clear_flush(vma, address, pte); 1160 entry = pte_wrprotect(entry); 1161 entry = pte_mkclean(entry); 1162 set_pte_at(vma->vm_mm, address, pte, entry); 1163 ret = 1; 1164 } else { 1165 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1166 pmd_t *pmd = pvmw->pmd; 1167 pmd_t entry = pmdp_get(pmd); 1168 1169 /* 1170 * Please see the comment above (!pte_present). 1171 * A non present PMD is not writable from a CPU 1172 * perspective. 1173 */ 1174 if (!pmd_present(entry)) 1175 continue; 1176 if (!pmd_dirty(entry) && !pmd_write(entry)) 1177 continue; 1178 1179 flush_cache_range(vma, address, 1180 address + HPAGE_PMD_SIZE); 1181 entry = pmdp_invalidate(vma, address, pmd); 1182 entry = pmd_wrprotect(entry); 1183 entry = pmd_mkclean(entry); 1184 set_pmd_at(vma->vm_mm, address, pmd, entry); 1185 ret = 1; 1186 #else 1187 /* unexpected pmd-mapped folio? */ 1188 WARN_ON_ONCE(1); 1189 #endif 1190 } 1191 1192 if (ret) 1193 cleaned++; 1194 } 1195 1196 mmu_notifier_invalidate_range_end(&range); 1197 1198 return cleaned; 1199 } 1200 1201 static bool page_mkclean_one(struct folio *folio, struct vm_area_struct *vma, 1202 unsigned long address, void *arg) 1203 { 1204 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC); 1205 int *cleaned = arg; 1206 1207 *cleaned += page_vma_mkclean_one(&pvmw); 1208 1209 return true; 1210 } 1211 1212 static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg) 1213 { 1214 if (vma->vm_flags & VM_SHARED) 1215 return false; 1216 1217 return true; 1218 } 1219 1220 int folio_mkclean(struct folio *folio) 1221 { 1222 int cleaned = 0; 1223 struct address_space *mapping; 1224 struct rmap_walk_control rwc = { 1225 .arg = (void *)&cleaned, 1226 .rmap_one = page_mkclean_one, 1227 .invalid_vma = invalid_mkclean_vma, 1228 }; 1229 1230 BUG_ON(!folio_test_locked(folio)); 1231 1232 if (!folio_mapped(folio)) 1233 return 0; 1234 1235 mapping = folio_mapping(folio); 1236 if (!mapping) 1237 return 0; 1238 1239 rmap_walk(folio, &rwc); 1240 1241 return cleaned; 1242 } 1243 EXPORT_SYMBOL_GPL(folio_mkclean); 1244 1245 struct wrprotect_file_state { 1246 int cleaned; 1247 pgoff_t pgoff; 1248 unsigned long pfn; 1249 unsigned long nr_pages; 1250 }; 1251 1252 static bool mapping_wrprotect_range_one(struct folio *folio, 1253 struct vm_area_struct *vma, unsigned long address, void *arg) 1254 { 1255 struct wrprotect_file_state *state = (struct wrprotect_file_state *)arg; 1256 struct page_vma_mapped_walk pvmw = { 1257 .pfn = state->pfn, 1258 .nr_pages = state->nr_pages, 1259 .pgoff = state->pgoff, 1260 .vma = vma, 1261 .address = address, 1262 .flags = PVMW_SYNC, 1263 .pgoff_is_anon = false, 1264 }; 1265 1266 state->cleaned += page_vma_mkclean_one(&pvmw); 1267 1268 return true; 1269 } 1270 1271 static void __rmap_walk_file(struct folio *folio, struct address_space *mapping, 1272 pgoff_t pgoff_start, unsigned long nr_pages, 1273 struct rmap_walk_control *rwc, bool locked); 1274 1275 /** 1276 * mapping_wrprotect_range() - Write-protect all mappings in a specified range. 1277 * 1278 * @mapping: The mapping whose reverse mapping should be traversed. 1279 * @pgoff: The page offset at which @pfn is mapped within @mapping. 1280 * @pfn: The PFN of the page mapped in @mapping at @pgoff. 1281 * @nr_pages: The number of physically contiguous base pages spanned. 1282 * 1283 * Traverses the reverse mapping, finding all VMAs which contain a shared 1284 * mapping of the pages in the specified range in @mapping, and write-protects 1285 * them (that is, updates the page tables to mark the mappings read-only such 1286 * that a write protection fault arises when the mappings are written to). 1287 * 1288 * The @pfn value need not refer to a folio, but rather can reference a kernel 1289 * allocation which is mapped into userland. We therefore do not require that 1290 * the page maps to a folio with a valid mapping or index field, rather the 1291 * caller specifies these in @mapping and @pgoff. 1292 * 1293 * Return: the number of write-protected PTEs, or an error. 1294 */ 1295 int mapping_wrprotect_range(struct address_space *mapping, pgoff_t pgoff, 1296 unsigned long pfn, unsigned long nr_pages) 1297 { 1298 struct wrprotect_file_state state = { 1299 .cleaned = 0, 1300 .pgoff = pgoff, 1301 .pfn = pfn, 1302 .nr_pages = nr_pages, 1303 }; 1304 struct rmap_walk_control rwc = { 1305 .arg = (void *)&state, 1306 .rmap_one = mapping_wrprotect_range_one, 1307 .invalid_vma = invalid_mkclean_vma, 1308 }; 1309 1310 if (!mapping) 1311 return 0; 1312 1313 __rmap_walk_file(/* folio = */NULL, mapping, pgoff, nr_pages, &rwc, 1314 /* locked = */false); 1315 1316 return state.cleaned; 1317 } 1318 EXPORT_SYMBOL_GPL(mapping_wrprotect_range); 1319 1320 /** 1321 * pfn_mkclean_range - Cleans the PTEs (including PMDs) mapped with range of 1322 * [@pfn, @pfn + @nr_pages) at the specific offset (@pgoff) 1323 * within the @vma of shared mappings. And since clean PTEs 1324 * should also be readonly, write protects them too. 1325 * @pfn: start pfn. 1326 * @nr_pages: number of physically contiguous pages srarting with @pfn. 1327 * @pgoff: page offset that the @pfn mapped with. 1328 * @vma: vma that @pfn mapped within. 1329 * 1330 * Returns the number of cleaned PTEs (including PMDs). 1331 */ 1332 int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff, 1333 struct vm_area_struct *vma) 1334 { 1335 struct page_vma_mapped_walk pvmw = { 1336 .pfn = pfn, 1337 .nr_pages = nr_pages, 1338 .pgoff = pgoff, 1339 .vma = vma, 1340 .flags = PVMW_SYNC, 1341 .pgoff_is_anon = false, 1342 }; 1343 1344 if (invalid_mkclean_vma(vma, NULL)) 1345 return 0; 1346 1347 pvmw.address = vma_filebacked_address(vma, pgoff, nr_pages); 1348 VM_BUG_ON_VMA(pvmw.address == -EFAULT, vma); 1349 1350 return page_vma_mkclean_one(&pvmw); 1351 } 1352 1353 static void __folio_mod_stat(struct folio *folio, int nr, int nr_pmdmapped) 1354 { 1355 int idx; 1356 1357 if (nr) { 1358 idx = folio_test_anon(folio) ? NR_ANON_MAPPED : NR_FILE_MAPPED; 1359 lruvec_stat_mod_folio(folio, idx, nr); 1360 } 1361 if (nr_pmdmapped) { 1362 if (folio_test_anon(folio)) { 1363 idx = NR_ANON_THPS; 1364 lruvec_stat_mod_folio(folio, idx, nr_pmdmapped); 1365 } else { 1366 /* NR_*_PMDMAPPED are not maintained per-memcg */ 1367 idx = folio_test_swapbacked(folio) ? 1368 NR_SHMEM_PMDMAPPED : NR_FILE_PMDMAPPED; 1369 __mod_node_page_state(folio_pgdat(folio), idx, 1370 nr_pmdmapped); 1371 } 1372 } 1373 } 1374 1375 static __always_inline void __folio_add_rmap(struct folio *folio, 1376 struct page *page, int nr_pages, struct vm_area_struct *vma, 1377 enum pgtable_level level) 1378 { 1379 atomic_t *mapped = &folio->_nr_pages_mapped; 1380 const int orig_nr_pages = nr_pages; 1381 int first = 0, nr = 0, nr_pmdmapped = 0; 1382 1383 __folio_rmap_sanity_checks(folio, page, nr_pages, level); 1384 1385 switch (level) { 1386 case PGTABLE_LEVEL_PTE: 1387 if (!folio_test_large(folio)) { 1388 nr = atomic_inc_and_test(&folio->_mapcount); 1389 break; 1390 } 1391 1392 if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) { 1393 nr = folio_add_return_large_mapcount(folio, orig_nr_pages, vma); 1394 if (nr == orig_nr_pages) 1395 /* Was completely unmapped. */ 1396 nr = folio_large_nr_pages(folio); 1397 else 1398 nr = 0; 1399 break; 1400 } 1401 1402 do { 1403 first += atomic_inc_and_test(&page->_mapcount); 1404 } while (page++, --nr_pages > 0); 1405 1406 if (first && 1407 atomic_add_return_relaxed(first, mapped) < ENTIRELY_MAPPED) 1408 nr = first; 1409 1410 folio_add_large_mapcount(folio, orig_nr_pages, vma); 1411 break; 1412 case PGTABLE_LEVEL_PMD: 1413 case PGTABLE_LEVEL_PUD: 1414 first = atomic_inc_and_test(&folio->_entire_mapcount); 1415 if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) { 1416 if (level == PGTABLE_LEVEL_PMD && first) 1417 nr_pmdmapped = folio_large_nr_pages(folio); 1418 nr = folio_inc_return_large_mapcount(folio, vma); 1419 if (nr == 1) 1420 /* Was completely unmapped. */ 1421 nr = folio_large_nr_pages(folio); 1422 else 1423 nr = 0; 1424 break; 1425 } 1426 1427 if (first) { 1428 nr = atomic_add_return_relaxed(ENTIRELY_MAPPED, mapped); 1429 if (likely(nr < ENTIRELY_MAPPED + ENTIRELY_MAPPED)) { 1430 nr_pages = folio_large_nr_pages(folio); 1431 /* 1432 * We only track PMD mappings of PMD-sized 1433 * folios separately. 1434 */ 1435 if (level == PGTABLE_LEVEL_PMD) 1436 nr_pmdmapped = nr_pages; 1437 nr = nr_pages - (nr & FOLIO_PAGES_MAPPED); 1438 /* Raced ahead of a remove and another add? */ 1439 if (unlikely(nr < 0)) 1440 nr = 0; 1441 } else { 1442 /* Raced ahead of a remove of ENTIRELY_MAPPED */ 1443 nr = 0; 1444 } 1445 } 1446 folio_inc_large_mapcount(folio, vma); 1447 break; 1448 default: 1449 BUILD_BUG(); 1450 } 1451 __folio_mod_stat(folio, nr, nr_pmdmapped); 1452 } 1453 1454 /** 1455 * folio_move_anon_rmap - move a folio to our anon_vma 1456 * @folio: The folio to move to our anon_vma 1457 * @vma: The vma the folio belongs to 1458 * 1459 * When a folio belongs exclusively to one process after a COW event, 1460 * that folio can be moved into the anon_vma that belongs to just that 1461 * process, so the rmap code will not search the parent or sibling processes. 1462 */ 1463 void folio_move_anon_rmap(struct folio *folio, struct vm_area_struct *vma) 1464 { 1465 void *anon_vma = vma->anon_vma; 1466 1467 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 1468 VM_BUG_ON_VMA(!anon_vma, vma); 1469 1470 anon_vma += FOLIO_MAPPING_ANON; 1471 /* 1472 * Ensure that anon_vma and the FOLIO_MAPPING_ANON bit are written 1473 * simultaneously, so a concurrent reader (eg folio_referenced()'s 1474 * folio_test_anon()) will not see one without the other. 1475 */ 1476 WRITE_ONCE(folio->mapping, anon_vma); 1477 } 1478 1479 /** 1480 * __folio_set_anon - set up a new anonymous rmap for a folio 1481 * @folio: The folio to set up the new anonymous rmap for. 1482 * @vma: VM area to add the folio to. 1483 * @address: User virtual address of the mapping 1484 * @exclusive: Whether the folio is exclusive to the process. 1485 */ 1486 static void __folio_set_anon(struct folio *folio, struct vm_area_struct *vma, 1487 unsigned long address, bool exclusive) 1488 { 1489 struct anon_vma *anon_vma = vma->anon_vma; 1490 1491 BUG_ON(!anon_vma); 1492 1493 /* 1494 * If the folio isn't exclusive to this vma, we must use the _oldest_ 1495 * possible anon_vma for the folio mapping! 1496 */ 1497 if (!exclusive) 1498 anon_vma = anon_vma->root; 1499 1500 /* 1501 * page_idle does a lockless/optimistic rmap scan on folio->mapping. 1502 * Make sure the compiler doesn't split the stores of anon_vma and 1503 * the FOLIO_MAPPING_ANON type identifier, otherwise the rmap code 1504 * could mistake the mapping for a struct address_space and crash. 1505 */ 1506 anon_vma = (void *) anon_vma + FOLIO_MAPPING_ANON; 1507 WRITE_ONCE(folio->mapping, (struct address_space *) anon_vma); 1508 folio->index = linear_anon_page_index(vma, address); 1509 } 1510 1511 /** 1512 * __page_check_anon_rmap - sanity check anonymous rmap addition 1513 * @folio: The folio containing @page. 1514 * @page: the page to check the mapping of 1515 * @vma: the vm area in which the mapping is added 1516 * @address: the user virtual address mapped 1517 */ 1518 static void __page_check_anon_rmap(const struct folio *folio, 1519 const struct page *page, struct vm_area_struct *vma, 1520 unsigned long address) 1521 { 1522 /* 1523 * The page's anon-rmap details (mapping and index) are guaranteed to 1524 * be set up correctly at this point. 1525 * 1526 * We have exclusion against folio_add_anon_rmap_*() because the caller 1527 * always holds the page locked. 1528 * 1529 * We have exclusion against folio_add_new_anon_rmap because those pages 1530 * are initially only visible via the pagetables, and the pte is locked 1531 * over the call to folio_add_new_anon_rmap. 1532 */ 1533 VM_BUG_ON_FOLIO(folio_anon_vma(folio)->root != vma->anon_vma->root, 1534 folio); 1535 VM_BUG_ON_PAGE(page_pgoff(folio, page) != 1536 linear_anon_page_index(vma, address), page); 1537 } 1538 1539 static __always_inline void __folio_add_anon_rmap(struct folio *folio, 1540 struct page *page, int nr_pages, struct vm_area_struct *vma, 1541 unsigned long address, rmap_t flags, enum pgtable_level level) 1542 { 1543 int i; 1544 1545 VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); 1546 1547 __folio_add_rmap(folio, page, nr_pages, vma, level); 1548 1549 if (likely(!folio_test_ksm(folio))) 1550 __page_check_anon_rmap(folio, page, vma, address); 1551 1552 if (flags & RMAP_EXCLUSIVE) { 1553 switch (level) { 1554 case PGTABLE_LEVEL_PTE: 1555 for (i = 0; i < nr_pages; i++) 1556 SetPageAnonExclusive(page + i); 1557 break; 1558 case PGTABLE_LEVEL_PMD: 1559 SetPageAnonExclusive(page); 1560 break; 1561 case PGTABLE_LEVEL_PUD: 1562 /* 1563 * Keep the compiler happy, we don't support anonymous 1564 * PUD mappings. 1565 */ 1566 WARN_ON_ONCE(1); 1567 break; 1568 default: 1569 BUILD_BUG(); 1570 } 1571 } 1572 1573 VM_WARN_ON_FOLIO(!folio_test_large(folio) && PageAnonExclusive(page) && 1574 atomic_read(&folio->_mapcount) > 0, folio); 1575 for (i = 0; i < nr_pages; i++) { 1576 struct page *cur_page = page + i; 1577 1578 VM_WARN_ON_FOLIO(folio_test_large(folio) && 1579 folio_entire_mapcount(folio) > 1 && 1580 PageAnonExclusive(cur_page), folio); 1581 if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) 1582 continue; 1583 1584 /* 1585 * While PTE-mapping a THP we have a PMD and a PTE 1586 * mapping. 1587 */ 1588 VM_WARN_ON_FOLIO(atomic_read(&cur_page->_mapcount) > 0 && 1589 PageAnonExclusive(cur_page), folio); 1590 } 1591 1592 /* 1593 * Only mlock it if the folio is fully mapped to the VMA. 1594 * 1595 * Partially mapped folios can be split on reclaim and part outside 1596 * of mlocked VMA can be evicted or freed. 1597 */ 1598 if (folio_nr_pages(folio) == nr_pages) 1599 mlock_vma_folio(folio, vma); 1600 } 1601 1602 /** 1603 * folio_add_anon_rmap_ptes - add PTE mappings to a page range of an anon folio 1604 * @folio: The folio to add the mappings to 1605 * @page: The first page to add 1606 * @nr_pages: The number of pages which will be mapped 1607 * @vma: The vm area in which the mappings are added 1608 * @address: The user virtual address of the first page to map 1609 * @flags: The rmap flags 1610 * 1611 * The page range of folio is defined by [first_page, first_page + nr_pages) 1612 * 1613 * The caller needs to hold the page table lock, and the page must be locked in 1614 * the anon_vma case: to serialize mapping,index checking after setting, 1615 * and to ensure that an anon folio is not being upgraded racily to a KSM folio 1616 * (but KSM folios are never downgraded). 1617 */ 1618 void folio_add_anon_rmap_ptes(struct folio *folio, struct page *page, 1619 int nr_pages, struct vm_area_struct *vma, unsigned long address, 1620 rmap_t flags) 1621 { 1622 __folio_add_anon_rmap(folio, page, nr_pages, vma, address, flags, 1623 PGTABLE_LEVEL_PTE); 1624 } 1625 1626 /** 1627 * folio_add_anon_rmap_pmd - add a PMD mapping to a page range of an anon folio 1628 * @folio: The folio to add the mapping to 1629 * @page: The first page to add 1630 * @vma: The vm area in which the mapping is added 1631 * @address: The user virtual address of the first page to map 1632 * @flags: The rmap flags 1633 * 1634 * The page range of folio is defined by [first_page, first_page + HPAGE_PMD_NR) 1635 * 1636 * The caller needs to hold the page table lock, and the page must be locked in 1637 * the anon_vma case: to serialize mapping,index checking after setting. 1638 */ 1639 void folio_add_anon_rmap_pmd(struct folio *folio, struct page *page, 1640 struct vm_area_struct *vma, unsigned long address, rmap_t flags) 1641 { 1642 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1643 __folio_add_anon_rmap(folio, page, HPAGE_PMD_NR, vma, address, flags, 1644 PGTABLE_LEVEL_PMD); 1645 #else 1646 WARN_ON_ONCE(true); 1647 #endif 1648 } 1649 1650 /** 1651 * folio_add_new_anon_rmap - Add mapping to a new anonymous folio. 1652 * @folio: The folio to add the mapping to. 1653 * @vma: the vm area in which the mapping is added 1654 * @address: the user virtual address mapped 1655 * @flags: The rmap flags 1656 * 1657 * Like folio_add_anon_rmap_*() but must only be called on *new* folios. 1658 * This means the inc-and-test can be bypassed. 1659 * The folio doesn't necessarily need to be locked while it's exclusive 1660 * unless two threads map it concurrently. However, the folio must be 1661 * locked if it's shared. 1662 * 1663 * If the folio is pmd-mappable, it is accounted as a THP. 1664 */ 1665 void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma, 1666 unsigned long address, rmap_t flags) 1667 { 1668 const bool exclusive = flags & RMAP_EXCLUSIVE; 1669 int nr = 1, nr_pmdmapped = 0; 1670 1671 VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio); 1672 VM_WARN_ON_FOLIO(!exclusive && !folio_test_locked(folio), folio); 1673 1674 /* 1675 * VM_DROPPABLE mappings don't swap; instead they're just dropped when 1676 * under memory pressure. 1677 */ 1678 if (!folio_test_swapbacked(folio) && !(vma->vm_flags & VM_DROPPABLE)) 1679 __folio_set_swapbacked(folio); 1680 __folio_set_anon(folio, vma, address, exclusive); 1681 1682 if (likely(!folio_test_large(folio))) { 1683 /* increment count (starts at -1) */ 1684 atomic_set(&folio->_mapcount, 0); 1685 if (exclusive) 1686 SetPageAnonExclusive(&folio->page); 1687 } else if (!folio_test_pmd_mappable(folio)) { 1688 int i; 1689 1690 nr = folio_large_nr_pages(folio); 1691 for (i = 0; i < nr; i++) { 1692 struct page *page = folio_page(folio, i); 1693 1694 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) 1695 /* increment count (starts at -1) */ 1696 atomic_set(&page->_mapcount, 0); 1697 if (exclusive) 1698 SetPageAnonExclusive(page); 1699 } 1700 1701 folio_set_large_mapcount(folio, nr, vma); 1702 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) 1703 atomic_set(&folio->_nr_pages_mapped, nr); 1704 } else { 1705 nr = folio_large_nr_pages(folio); 1706 /* increment count (starts at -1) */ 1707 atomic_set(&folio->_entire_mapcount, 0); 1708 folio_set_large_mapcount(folio, 1, vma); 1709 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) 1710 atomic_set(&folio->_nr_pages_mapped, ENTIRELY_MAPPED); 1711 if (exclusive) 1712 SetPageAnonExclusive(&folio->page); 1713 nr_pmdmapped = nr; 1714 } 1715 1716 VM_WARN_ON_ONCE(address < vma->vm_start || 1717 address + (nr << PAGE_SHIFT) > vma->vm_end); 1718 1719 __folio_mod_stat(folio, nr, nr_pmdmapped); 1720 mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, 1); 1721 } 1722 1723 static __always_inline void __folio_add_file_rmap(struct folio *folio, 1724 struct page *page, int nr_pages, struct vm_area_struct *vma, 1725 enum pgtable_level level) 1726 { 1727 VM_WARN_ON_FOLIO(folio_test_anon(folio), folio); 1728 1729 __folio_add_rmap(folio, page, nr_pages, vma, level); 1730 1731 /* 1732 * Only mlock it if the folio is fully mapped to the VMA. 1733 * 1734 * Partially mapped folios can be split on reclaim and part outside 1735 * of mlocked VMA can be evicted or freed. 1736 */ 1737 if (folio_nr_pages(folio) == nr_pages) 1738 mlock_vma_folio(folio, vma); 1739 } 1740 1741 /** 1742 * folio_add_file_rmap_ptes - add PTE mappings to a page range of a folio 1743 * @folio: The folio to add the mappings to 1744 * @page: The first page to add 1745 * @nr_pages: The number of pages that will be mapped using PTEs 1746 * @vma: The vm area in which the mappings are added 1747 * 1748 * The page range of the folio is defined by [page, page + nr_pages) 1749 * 1750 * The caller needs to hold the page table lock. 1751 */ 1752 void folio_add_file_rmap_ptes(struct folio *folio, struct page *page, 1753 int nr_pages, struct vm_area_struct *vma) 1754 { 1755 __folio_add_file_rmap(folio, page, nr_pages, vma, PGTABLE_LEVEL_PTE); 1756 } 1757 1758 /** 1759 * folio_add_file_rmap_pmd - add a PMD mapping to a page range of a folio 1760 * @folio: The folio to add the mapping to 1761 * @page: The first page to add 1762 * @vma: The vm area in which the mapping is added 1763 * 1764 * The page range of the folio is defined by [page, page + HPAGE_PMD_NR) 1765 * 1766 * The caller needs to hold the page table lock. 1767 */ 1768 void folio_add_file_rmap_pmd(struct folio *folio, struct page *page, 1769 struct vm_area_struct *vma) 1770 { 1771 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1772 __folio_add_file_rmap(folio, page, HPAGE_PMD_NR, vma, PGTABLE_LEVEL_PMD); 1773 #else 1774 WARN_ON_ONCE(true); 1775 #endif 1776 } 1777 1778 /** 1779 * folio_add_file_rmap_pud - add a PUD mapping to a page range of a folio 1780 * @folio: The folio to add the mapping to 1781 * @page: The first page to add 1782 * @vma: The vm area in which the mapping is added 1783 * 1784 * The page range of the folio is defined by [page, page + HPAGE_PUD_NR) 1785 * 1786 * The caller needs to hold the page table lock. 1787 */ 1788 void folio_add_file_rmap_pud(struct folio *folio, struct page *page, 1789 struct vm_area_struct *vma) 1790 { 1791 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ 1792 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) 1793 __folio_add_file_rmap(folio, page, HPAGE_PUD_NR, vma, PGTABLE_LEVEL_PUD); 1794 #else 1795 WARN_ON_ONCE(true); 1796 #endif 1797 } 1798 1799 static __always_inline void __folio_remove_rmap(struct folio *folio, 1800 struct page *page, int nr_pages, struct vm_area_struct *vma, 1801 enum pgtable_level level) 1802 { 1803 atomic_t *mapped = &folio->_nr_pages_mapped; 1804 int last = 0, nr = 0, nr_pmdmapped = 0; 1805 bool partially_mapped = false; 1806 1807 __folio_rmap_sanity_checks(folio, page, nr_pages, level); 1808 1809 switch (level) { 1810 case PGTABLE_LEVEL_PTE: 1811 if (!folio_test_large(folio)) { 1812 nr = atomic_add_negative(-1, &folio->_mapcount); 1813 break; 1814 } 1815 1816 if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) { 1817 nr = folio_sub_return_large_mapcount(folio, nr_pages, vma); 1818 if (!nr) { 1819 /* Now completely unmapped. */ 1820 nr = folio_large_nr_pages(folio); 1821 } else { 1822 partially_mapped = nr < folio_large_nr_pages(folio) && 1823 !folio_entire_mapcount(folio); 1824 nr = 0; 1825 } 1826 break; 1827 } 1828 1829 folio_sub_large_mapcount(folio, nr_pages, vma); 1830 do { 1831 last += atomic_add_negative(-1, &page->_mapcount); 1832 } while (page++, --nr_pages > 0); 1833 1834 if (last && 1835 atomic_sub_return_relaxed(last, mapped) < ENTIRELY_MAPPED) 1836 nr = last; 1837 1838 partially_mapped = nr && atomic_read(mapped); 1839 break; 1840 case PGTABLE_LEVEL_PMD: 1841 case PGTABLE_LEVEL_PUD: 1842 if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) { 1843 last = atomic_add_negative(-1, &folio->_entire_mapcount); 1844 if (level == PGTABLE_LEVEL_PMD && last) 1845 nr_pmdmapped = folio_large_nr_pages(folio); 1846 nr = folio_dec_return_large_mapcount(folio, vma); 1847 if (!nr) { 1848 /* Now completely unmapped. */ 1849 nr = folio_large_nr_pages(folio); 1850 } else { 1851 partially_mapped = last && 1852 nr < folio_large_nr_pages(folio); 1853 nr = 0; 1854 } 1855 break; 1856 } 1857 1858 folio_dec_large_mapcount(folio, vma); 1859 last = atomic_add_negative(-1, &folio->_entire_mapcount); 1860 if (last) { 1861 nr = atomic_sub_return_relaxed(ENTIRELY_MAPPED, mapped); 1862 if (likely(nr < ENTIRELY_MAPPED)) { 1863 nr_pages = folio_large_nr_pages(folio); 1864 if (level == PGTABLE_LEVEL_PMD) 1865 nr_pmdmapped = nr_pages; 1866 nr = nr_pages - nr; 1867 /* Raced ahead of another remove and an add? */ 1868 if (unlikely(nr < 0)) 1869 nr = 0; 1870 } else { 1871 /* An add of ENTIRELY_MAPPED raced ahead */ 1872 nr = 0; 1873 } 1874 } 1875 1876 partially_mapped = nr && nr < nr_pmdmapped; 1877 break; 1878 default: 1879 BUILD_BUG(); 1880 } 1881 1882 /* 1883 * Queue anon large folio for deferred split if at least one page of 1884 * the folio is unmapped and at least one page is still mapped. 1885 * 1886 * Check partially_mapped first to ensure it is a large folio. 1887 * 1888 * Device private folios do not support deferred splitting and 1889 * shrinker based scanning of the folios to free. 1890 */ 1891 if (partially_mapped && folio_test_anon(folio) && 1892 !folio_test_partially_mapped(folio) && 1893 !folio_is_device_private(folio)) 1894 deferred_split_folio(folio, true); 1895 1896 __folio_mod_stat(folio, -nr, -nr_pmdmapped); 1897 1898 /* 1899 * It would be tidy to reset folio_test_anon mapping when fully 1900 * unmapped, but that might overwrite a racing folio_add_anon_rmap_*() 1901 * which increments mapcount after us but sets mapping before us: 1902 * so leave the reset to free_pages_prepare, and remember that 1903 * it's only reliable while mapped. 1904 */ 1905 1906 munlock_vma_folio(folio, vma); 1907 } 1908 1909 /** 1910 * folio_remove_rmap_ptes - remove PTE mappings from a page range of a folio 1911 * @folio: The folio to remove the mappings from 1912 * @page: The first page to remove 1913 * @nr_pages: The number of pages that will be removed from the mapping 1914 * @vma: The vm area from which the mappings are removed 1915 * 1916 * The page range of the folio is defined by [page, page + nr_pages) 1917 * 1918 * The caller needs to hold the page table lock. 1919 */ 1920 void folio_remove_rmap_ptes(struct folio *folio, struct page *page, 1921 int nr_pages, struct vm_area_struct *vma) 1922 { 1923 __folio_remove_rmap(folio, page, nr_pages, vma, PGTABLE_LEVEL_PTE); 1924 } 1925 1926 /** 1927 * folio_remove_rmap_pmd - remove a PMD mapping from a page range of a folio 1928 * @folio: The folio to remove the mapping from 1929 * @page: The first page to remove 1930 * @vma: The vm area from which the mapping is removed 1931 * 1932 * The page range of the folio is defined by [page, page + HPAGE_PMD_NR) 1933 * 1934 * The caller needs to hold the page table lock. 1935 */ 1936 void folio_remove_rmap_pmd(struct folio *folio, struct page *page, 1937 struct vm_area_struct *vma) 1938 { 1939 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1940 __folio_remove_rmap(folio, page, HPAGE_PMD_NR, vma, PGTABLE_LEVEL_PMD); 1941 #else 1942 WARN_ON_ONCE(true); 1943 #endif 1944 } 1945 1946 /** 1947 * folio_remove_rmap_pud - remove a PUD mapping from a page range of a folio 1948 * @folio: The folio to remove the mapping from 1949 * @page: The first page to remove 1950 * @vma: The vm area from which the mapping is removed 1951 * 1952 * The page range of the folio is defined by [page, page + HPAGE_PUD_NR) 1953 * 1954 * The caller needs to hold the page table lock. 1955 */ 1956 void folio_remove_rmap_pud(struct folio *folio, struct page *page, 1957 struct vm_area_struct *vma) 1958 { 1959 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ 1960 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) 1961 __folio_remove_rmap(folio, page, HPAGE_PUD_NR, vma, PGTABLE_LEVEL_PUD); 1962 #else 1963 WARN_ON_ONCE(true); 1964 #endif 1965 } 1966 1967 static inline unsigned int folio_unmap_pte_batch(struct folio *folio, 1968 struct page_vma_mapped_walk *pvmw, 1969 enum ttu_flags flags, pte_t pte) 1970 { 1971 unsigned long end_addr, addr = pvmw->address; 1972 struct vm_area_struct *vma = pvmw->vma; 1973 unsigned int max_nr; 1974 1975 if (flags & TTU_HWPOISON) 1976 return 1; 1977 if (!folio_test_large(folio)) 1978 return 1; 1979 1980 /* We may only batch within a single VMA and a single page table. */ 1981 end_addr = pte_table_clamped_end(addr, vma->vm_end); 1982 max_nr = (end_addr - addr) >> PAGE_SHIFT; 1983 1984 /* We only support lazyfree or file folios batching for now ... */ 1985 if (folio_test_anon(folio) && folio_test_swapbacked(folio)) 1986 return 1; 1987 1988 if (pte_unused(pte)) 1989 return 1; 1990 1991 /* 1992 * If unmap fails, we need to restore the ptes. To avoid accidentally 1993 * upgrading write permissions for ptes that were not originally 1994 * writable, and to avoid losing the soft-dirty bit, use the 1995 * appropriate FPB flags. 1996 */ 1997 return folio_pte_batch_flags(folio, vma, pvmw->pte, &pte, max_nr, 1998 FPB_RESPECT_WRITE | FPB_RESPECT_SOFT_DIRTY); 1999 } 2000 2001 static bool try_to_unmap_poisoned_hugetlb_one(struct folio *folio, 2002 struct vm_area_struct *vma, unsigned long address, void *arg) 2003 { 2004 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); 2005 const unsigned long hsz = huge_page_size(hstate_vma(vma)); 2006 const enum ttu_flags flags = (enum ttu_flags)(long)arg; 2007 struct mm_struct *mm = vma->vm_mm; 2008 struct mmu_notifier_range range; 2009 bool ret = true; 2010 pte_t pteval; 2011 2012 /* 2013 * The try_to_unmap() is only passed a hugetlb folio in the case 2014 * where the hugetlb folio is poisoned. 2015 */ 2016 VM_WARN_ON_ONCE_FOLIO(!folio_test_hwpoison(folio), folio); 2017 VM_WARN_ON_ONCE(!(flags & TTU_HWPOISON)); 2018 2019 range.end = vma_address_end(&pvmw); 2020 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, 2021 address, range.end); 2022 adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end); 2023 mmu_notifier_invalidate_range_start(&range); 2024 2025 /* There is only a single mapping in a VMA. */ 2026 if (!page_vma_mapped_walk(&pvmw)) 2027 goto range_end; 2028 2029 VM_WARN_ON_ONCE(address != pvmw.address); 2030 2031 pteval = huge_ptep_get(mm, address, pvmw.pte); 2032 VM_WARN_ON_ONCE(!pte_present(pteval)); 2033 VM_WARN_ON_ONCE(pte_pfn(pteval) != folio_pfn(folio)); 2034 2035 /* 2036 * huge_pmd_unshare may unmap an entire PMD page. There is no way of 2037 * knowing exactly which PMDs may be cached for this mm, so we must 2038 * flush them all. start/end were already adjusted above to cover this 2039 * range. 2040 */ 2041 flush_cache_range(vma, range.start, range.end); 2042 2043 /* 2044 * To call huge_pmd_unshare, i_mmap_rwsem must be held in write mode. 2045 * Caller needs to explicitly do this outside rmap routines. 2046 * 2047 * We also must hold hugetlb vma_lock in write mode. Lock order dictates 2048 * acquiring vma_lock BEFORE i_mmap_rwsem. We can only try lock here and 2049 * fail if unsuccessful. 2050 */ 2051 if (!folio_test_anon(folio)) { 2052 struct mmu_gather tlb; 2053 2054 VM_WARN_ON_ONCE(!(flags & TTU_RMAP_LOCKED)); 2055 if (!hugetlb_vma_trylock_write(vma)) { 2056 ret = false; 2057 goto walk_done; 2058 } 2059 2060 tlb_gather_mmu_vma(&tlb, vma); 2061 if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) { 2062 hugetlb_vma_unlock_write(vma); 2063 huge_pmd_unshare_flush(&tlb, vma); 2064 tlb_finish_mmu(&tlb); 2065 /* 2066 * The PMD table was unmapped, consequently unmapping 2067 * the folio. 2068 */ 2069 goto walk_done; 2070 } 2071 hugetlb_vma_unlock_write(vma); 2072 tlb_finish_mmu(&tlb); 2073 } 2074 pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); 2075 if (huge_pte_dirty(pteval)) 2076 folio_mark_dirty(folio); 2077 2078 pteval = swp_entry_to_pte(make_hwpoison_entry(folio_page(folio, 0))); 2079 hugetlb_count_sub(folio_nr_pages(folio), mm); 2080 set_huge_pte_at(mm, address, pvmw.pte, pteval, hsz); 2081 hugetlb_remove_rmap(folio); 2082 folio_put_refs(folio, 1); 2083 2084 walk_done: 2085 page_vma_mapped_walk_done(&pvmw); 2086 range_end: 2087 mmu_notifier_invalidate_range_end(&range); 2088 return ret; 2089 } 2090 2091 static bool ttu_anon_lazyfree_folio(struct vm_area_struct *vma, 2092 struct folio *folio, unsigned long nr_pages) 2093 { 2094 int ref_count, map_count; 2095 2096 /* 2097 * Synchronize with gup_pte_range(): 2098 * - clear PTE; barrier; read refcount 2099 * - inc refcount; barrier; read PTE 2100 */ 2101 smp_mb(); 2102 2103 ref_count = folio_ref_count(folio); 2104 map_count = folio_mapcount(folio); 2105 2106 /* 2107 * Order reads for page refcount and dirty flag 2108 * (see comments in __remove_mapping()). 2109 */ 2110 smp_rmb(); 2111 2112 if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) { 2113 /* 2114 * redirtied either using the page table or a previously 2115 * obtained GUP reference. 2116 */ 2117 folio_set_swapbacked(folio); 2118 return false; 2119 } 2120 2121 /* 2122 * Additional references could be due to GUP or speculative lookups. 2123 * GUP users must mark the folio dirty if there was a modification. 2124 * This folio cannot be reclaimed right now either way, so act just 2125 * like nothing happened. We'll come back here later and detect if the 2126 * folio was dirtied when the additional reference is gone. 2127 */ 2128 if (ref_count != 1 + map_count) 2129 return false; 2130 2131 add_mm_counter(vma->vm_mm, MM_ANONPAGES, -nr_pages); 2132 return true; 2133 } 2134 2135 static pte_t swp_pte_prepare(swp_entry_t entry, pte_t old_pte, 2136 bool anon_exclusive) 2137 { 2138 pte_t swp_pte = swp_entry_to_pte(entry); 2139 2140 if (anon_exclusive) 2141 swp_pte = pte_swp_mkexclusive(swp_pte); 2142 2143 if (likely(pte_present(old_pte))) { 2144 if (pte_soft_dirty(old_pte)) 2145 swp_pte = pte_swp_mksoft_dirty(swp_pte); 2146 if (pte_uffd(old_pte)) 2147 swp_pte = pte_swp_mkuffd(swp_pte); 2148 } else { 2149 /* Device-exclusive entry */ 2150 if (pte_swp_soft_dirty(old_pte)) 2151 swp_pte = pte_swp_mksoft_dirty(swp_pte); 2152 if (pte_swp_uffd(old_pte)) 2153 swp_pte = pte_swp_mkuffd(swp_pte); 2154 } 2155 2156 return swp_pte; 2157 } 2158 2159 static bool ttu_anon_swapbacked_folio(struct vm_area_struct *vma, 2160 struct folio *folio, struct page *page, unsigned long address, 2161 pte_t *ptep, pte_t pteval) 2162 { 2163 const bool anon_exclusive = folio_test_anon(folio) && 2164 PageAnonExclusive(page); 2165 swp_entry_t entry = folio_page_swap_entry(folio, page); 2166 struct mm_struct *mm = vma->vm_mm; 2167 2168 if (folio_dup_swap(folio, page) < 0) 2169 return false; 2170 2171 /* 2172 * arch_unmap_one() is expected to be a NOP on 2173 * architectures where we could have PFN swap PTEs, 2174 * so we'll not check/care. 2175 */ 2176 if (arch_unmap_one(mm, vma, address, pteval) < 0) { 2177 folio_put_swap(folio, page); 2178 return false; 2179 } 2180 2181 /* See folio_try_share_anon_rmap(): clear PTE first. */ 2182 if (anon_exclusive && folio_try_share_anon_rmap_pte(folio, page)) { 2183 folio_put_swap(folio, page); 2184 return false; 2185 } 2186 2187 mm_prepare_for_swap_entries(mm); 2188 dec_mm_counter(mm, MM_ANONPAGES); 2189 inc_mm_counter(mm, MM_SWAPENTS); 2190 set_pte_at(mm, address, ptep, 2191 swp_pte_prepare(entry, pteval, anon_exclusive)); 2192 return true; 2193 } 2194 2195 static bool ttu_anon_folio(struct vm_area_struct *vma, struct folio *folio, 2196 struct page *page, unsigned long address, pte_t *ptep, 2197 pte_t pteval, unsigned long nr_pages) 2198 { 2199 /* 2200 * Store the swap location in the pte. 2201 * See handle_pte_fault() ... 2202 */ 2203 if (WARN_ON_ONCE(folio_test_swapbacked(folio) != 2204 folio_test_swapcache(folio))) 2205 return false; 2206 2207 if (!folio_test_swapbacked(folio)) 2208 return ttu_anon_lazyfree_folio(vma, folio, nr_pages); 2209 2210 /* nr_pages > 1 not supported yet */ 2211 return ttu_anon_swapbacked_folio(vma, folio, page, address, ptep, 2212 pteval); 2213 } 2214 2215 /* 2216 * @arg: enum ttu_flags will be passed to this argument 2217 */ 2218 static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, 2219 unsigned long address, void *arg) 2220 { 2221 struct mm_struct *mm = vma->vm_mm; 2222 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); 2223 bool ret = true; 2224 pte_t pteval; 2225 struct page *page; 2226 struct mmu_notifier_range range; 2227 enum ttu_flags flags = (enum ttu_flags)(long)arg; 2228 unsigned long nr_pages = 1, end_addr; 2229 unsigned long pfn; 2230 int ptes = 0; 2231 2232 /* 2233 * When racing against e.g. zap_pte_range() on another cpu, 2234 * in between its ptep_get_and_clear_full() and folio_remove_rmap_*(), 2235 * try_to_unmap() may return before folio_mapped() has become false, 2236 * if page table locking is skipped: use TTU_SYNC to wait for that. 2237 */ 2238 if (flags & TTU_SYNC) 2239 pvmw.flags = PVMW_SYNC; 2240 2241 /* 2242 * For THP, we have to assume the worse case ie pmd for invalidation. 2243 * 2244 * Note that the folio can not be freed in this function as call of 2245 * try_to_unmap() must hold a reference on the folio. 2246 */ 2247 range.end = vma_address_end(&pvmw); 2248 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, 2249 address, range.end); 2250 mmu_notifier_invalidate_range_start(&range); 2251 2252 while (page_vma_mapped_walk(&pvmw)) { 2253 nr_pages = 1; 2254 2255 /* 2256 * If the folio is in an mlock()d vma, we must not swap it out. 2257 * VMA_LOCKONFAULT_BIT alone marks an mlock walk in progress, see 2258 * mlock_vma_pages_range(). 2259 */ 2260 if (!(flags & TTU_IGNORE_MLOCK) && 2261 vma_test_any_mask(vma, VMA_LOCKED_MASK)) { 2262 ptes++; 2263 2264 /* 2265 * Set 'ret' to indicate the page cannot be unmapped. 2266 * 2267 * Do not jump to walk_abort immediately as additional 2268 * iteration might be required to detect fully mapped 2269 * folio an mlock it. 2270 */ 2271 ret = false; 2272 2273 /* Only mlock fully mapped pages */ 2274 if (pvmw.pte && ptes != pvmw.nr_pages) 2275 continue; 2276 2277 /* 2278 * All PTEs must be protected by page table lock in 2279 * order to mlock the page. 2280 * 2281 * If page table boundary has been cross, current ptl 2282 * only protect part of ptes. 2283 */ 2284 if (pvmw.flags & PVMW_PGTABLE_CROSSED) 2285 goto walk_done; 2286 2287 /* Restore the mlock which got missed */ 2288 mlock_vma_folio(folio, vma); 2289 goto walk_done; 2290 } 2291 2292 if (!pvmw.pte) { 2293 if (flags & TTU_SPLIT_HUGE_PMD) { 2294 /* 2295 * We temporarily have to drop the PTL and 2296 * restart so we can process the PTE-mapped THP. 2297 */ 2298 split_huge_pmd_locked(vma, pvmw.address, 2299 pvmw.pmd, false); 2300 flags &= ~TTU_SPLIT_HUGE_PMD; 2301 page_vma_mapped_walk_restart(&pvmw); 2302 continue; 2303 } 2304 } 2305 2306 /* Unexpected PMD-mapped THP? */ 2307 VM_BUG_ON_FOLIO(!pvmw.pte, folio); 2308 2309 address = pvmw.address; 2310 pteval = ptep_get(pvmw.pte); 2311 2312 if (likely(pte_present(pteval))) { 2313 pfn = pte_pfn(pteval); 2314 } else { 2315 /* 2316 * Handle PFN swap PTEs, such as device-exclusive ones, 2317 * that actually map pages. 2318 */ 2319 const softleaf_t entry = softleaf_from_pte(pteval); 2320 2321 pfn = softleaf_to_pfn(entry); 2322 } 2323 2324 page = folio_page(folio, pfn - folio_pfn(folio)); 2325 2326 if (likely(pte_present(pteval))) { 2327 nr_pages = folio_unmap_pte_batch(folio, &pvmw, flags, pteval); 2328 end_addr = address + nr_pages * PAGE_SIZE; 2329 flush_cache_range(vma, address, end_addr); 2330 2331 /* Nuke the page table entry. */ 2332 pteval = get_and_clear_ptes(mm, address, pvmw.pte, nr_pages); 2333 /* 2334 * We clear the PTE but do not flush so potentially 2335 * a remote CPU could still be writing to the folio. 2336 * If the entry was previously clean then the 2337 * architecture must guarantee that a clear->dirty 2338 * transition on a cached TLB entry is written through 2339 * and traps if the PTE is unmapped. 2340 */ 2341 if (should_defer_flush(mm, flags)) 2342 set_tlb_ubc_flush_pending(mm, pteval, address, end_addr); 2343 else 2344 flush_tlb_range(vma, address, end_addr); 2345 if (pte_dirty(pteval)) 2346 folio_mark_dirty(folio); 2347 } else { 2348 pte_clear(mm, address, pvmw.pte); 2349 } 2350 2351 /* 2352 * Now the pte is cleared. If this pte was uffd-wp armed, 2353 * we may want to replace a none pte with a marker pte if 2354 * it's file-backed, so we don't lose the tracking info. 2355 */ 2356 cond_install_uffd_wp_ptes(vma, address, pvmw.pte, pteval, 2357 nr_pages); 2358 2359 /* Update high watermark before we lower rss */ 2360 update_hiwater_rss(mm); 2361 2362 /* unmap_poisoned_folio() only refs order-0 folios */ 2363 if (folio_test_hwpoison(folio) && (flags & TTU_HWPOISON)) { 2364 pteval = swp_entry_to_pte(make_hwpoison_entry(page)); 2365 dec_mm_counter(mm, mm_counter(folio)); 2366 set_pte_at(mm, address, pvmw.pte, pteval); 2367 } else if (likely(pte_present(pteval)) && pte_unused(pteval) && 2368 !userfaultfd_armed(vma)) { 2369 /* 2370 * The guest indicated that the page content is of no 2371 * interest anymore. Simply discard the pte, vmscan 2372 * will take care of the rest. 2373 * A future reference will then fault in a new zero 2374 * page. When userfaultfd is active, we must not drop 2375 * this page though, as its main user (postcopy 2376 * migration) will not expect userfaults on already 2377 * copied pages. 2378 */ 2379 dec_mm_counter(mm, mm_counter(folio)); 2380 } else if (folio_test_anon(folio)) { 2381 if (!ttu_anon_folio(vma, folio, page, address, 2382 pvmw.pte, pteval, nr_pages)) { 2383 set_ptes(mm, address, pvmw.pte, pteval, nr_pages); 2384 goto walk_abort; 2385 } 2386 2387 goto finish_unmap; 2388 } else { 2389 /* 2390 * This is a locked file-backed folio, 2391 * so it cannot be removed from the page 2392 * cache and replaced by a new folio before 2393 * mmu_notifier_invalidate_range_end, so no 2394 * concurrent thread might update its page table 2395 * to point at a new folio while a device is 2396 * still using this folio. 2397 * 2398 * See Documentation/mm/mmu_notifier.rst 2399 */ 2400 add_mm_counter(mm, mm_counter_file(folio), -nr_pages); 2401 } 2402 finish_unmap: 2403 folio_remove_rmap_ptes(folio, page, nr_pages, vma); 2404 if (vma_test_any_mask(vma, VMA_LOCKED_MASK)) 2405 mlock_drain_local(); 2406 folio_put_refs(folio, nr_pages); 2407 2408 /* 2409 * If we are sure that we batched the entire folio and cleared 2410 * all PTEs, we can just optimize and stop right here. 2411 */ 2412 if (nr_pages == folio_nr_pages(folio)) 2413 goto walk_done; 2414 continue; 2415 walk_abort: 2416 ret = false; 2417 walk_done: 2418 page_vma_mapped_walk_done(&pvmw); 2419 break; 2420 } 2421 2422 mmu_notifier_invalidate_range_end(&range); 2423 2424 return ret; 2425 } 2426 2427 static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg) 2428 { 2429 return vma_is_temporary_stack(vma); 2430 } 2431 2432 static int folio_not_mapped(struct folio *folio) 2433 { 2434 return !folio_mapped(folio); 2435 } 2436 2437 /** 2438 * try_to_unmap - Try to remove all page table mappings to a folio. 2439 * @folio: The folio to unmap. 2440 * @flags: action and flags 2441 * 2442 * Tries to remove all the page table entries which are mapping this 2443 * folio. It is the caller's responsibility to check if the folio is 2444 * still mapped if needed (use TTU_SYNC to prevent accounting races). 2445 * 2446 * Context: Caller must hold the folio lock. 2447 */ 2448 void try_to_unmap(struct folio *folio, enum ttu_flags flags) 2449 { 2450 struct rmap_walk_control rwc = { 2451 .rmap_one = folio_test_hugetlb(folio) ? 2452 try_to_unmap_poisoned_hugetlb_one : try_to_unmap_one, 2453 .arg = (void *)flags, 2454 .done = folio_not_mapped, 2455 .anon_lock = folio_lock_anon_vma_read, 2456 }; 2457 2458 if (flags & TTU_RMAP_LOCKED) 2459 rmap_walk_locked(folio, &rwc); 2460 else 2461 rmap_walk(folio, &rwc); 2462 } 2463 2464 /* 2465 * @arg: enum ttu_flags will be passed to this argument. 2466 * 2467 * If TTU_SPLIT_HUGE_PMD is specified any PMD mappings will be split into PTEs 2468 * containing migration entries. 2469 */ 2470 static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, 2471 unsigned long address, void *arg) 2472 { 2473 struct mm_struct *mm = vma->vm_mm; 2474 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); 2475 bool anon_exclusive, writable, ret = true; 2476 pte_t pteval; 2477 struct page *subpage; 2478 struct mmu_notifier_range range; 2479 enum ttu_flags flags = (enum ttu_flags)(long)arg; 2480 unsigned long pfn; 2481 unsigned long hsz = 0; 2482 2483 /* 2484 * When racing against e.g. zap_pte_range() on another cpu, 2485 * in between its ptep_get_and_clear_full() and folio_remove_rmap_*(), 2486 * try_to_migrate() may return before folio_mapped() has become false, 2487 * if page table locking is skipped: use TTU_SYNC to wait for that. 2488 */ 2489 if (flags & TTU_SYNC) 2490 pvmw.flags = PVMW_SYNC; 2491 2492 /* 2493 * For THP, we have to assume the worse case ie pmd for invalidation. 2494 * For hugetlb, it could be much worse if we need to do pud 2495 * invalidation in the case of pmd sharing. 2496 * 2497 * Note that the page can not be free in this function as call of 2498 * try_to_unmap() must hold a reference on the page. 2499 */ 2500 range.end = vma_address_end(&pvmw); 2501 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, 2502 address, range.end); 2503 if (folio_test_hugetlb(folio)) { 2504 /* 2505 * If sharing is possible, start and end will be adjusted 2506 * accordingly. 2507 */ 2508 adjust_range_if_pmd_sharing_possible(vma, &range.start, 2509 &range.end); 2510 2511 /* We need the huge page size for set_huge_pte_at() */ 2512 hsz = huge_page_size(hstate_vma(vma)); 2513 } 2514 mmu_notifier_invalidate_range_start(&range); 2515 2516 while (page_vma_mapped_walk(&pvmw)) { 2517 /* PMD-mapped THP migration entry */ 2518 if (!pvmw.pte) { 2519 __maybe_unused unsigned long pfn; 2520 __maybe_unused pmd_t pmdval; 2521 2522 if (flags & TTU_SPLIT_HUGE_PMD) { 2523 /* 2524 * split_huge_pmd_locked() might leave the 2525 * folio mapped through PTEs. Retry the walk 2526 * so we can detect this scenario and properly 2527 * abort the walk. 2528 */ 2529 split_huge_pmd_locked(vma, pvmw.address, 2530 pvmw.pmd, true); 2531 flags &= ~TTU_SPLIT_HUGE_PMD; 2532 page_vma_mapped_walk_restart(&pvmw); 2533 continue; 2534 } 2535 #ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES 2536 pmdval = pmdp_get(pvmw.pmd); 2537 if (likely(pmd_present(pmdval))) 2538 pfn = pmd_pfn(pmdval); 2539 else 2540 pfn = softleaf_to_pfn(softleaf_from_pmd(pmdval)); 2541 2542 subpage = folio_page(folio, pfn - folio_pfn(folio)); 2543 2544 VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) || 2545 !folio_test_pmd_mappable(folio), folio); 2546 2547 if (set_pmd_migration_entry(&pvmw, subpage)) { 2548 ret = false; 2549 page_vma_mapped_walk_done(&pvmw); 2550 break; 2551 } 2552 continue; 2553 #endif 2554 } 2555 2556 /* Unexpected PMD-mapped THP? */ 2557 VM_BUG_ON_FOLIO(!pvmw.pte, folio); 2558 2559 address = pvmw.address; 2560 if (folio_test_hugetlb(folio)) 2561 pteval = huge_ptep_get(mm, address, pvmw.pte); 2562 else 2563 pteval = ptep_get(pvmw.pte); 2564 if (likely(pte_present(pteval))) { 2565 pfn = pte_pfn(pteval); 2566 } else { 2567 /* 2568 * Handle PFN swap PTEs, such as device-exclusive ones, 2569 * that actually map pages. 2570 */ 2571 const softleaf_t entry = softleaf_from_pte(pteval); 2572 2573 pfn = softleaf_to_pfn(entry); 2574 VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio); 2575 } 2576 2577 subpage = folio_page(folio, pfn - folio_pfn(folio)); 2578 anon_exclusive = folio_test_anon(folio) && 2579 PageAnonExclusive(subpage); 2580 2581 if (folio_test_hugetlb(folio)) { 2582 bool anon = folio_test_anon(folio); 2583 2584 /* 2585 * huge_pmd_unshare may unmap an entire PMD page. 2586 * There is no way of knowing exactly which PMDs may 2587 * be cached for this mm, so we must flush them all. 2588 * start/end were already adjusted above to cover this 2589 * range. 2590 */ 2591 flush_cache_range(vma, range.start, range.end); 2592 2593 /* 2594 * To call huge_pmd_unshare, i_mmap_rwsem must be 2595 * held in write mode. Caller needs to explicitly 2596 * do this outside rmap routines. 2597 * 2598 * We also must hold hugetlb vma_lock in write mode. 2599 * Lock order dictates acquiring vma_lock BEFORE 2600 * i_mmap_rwsem. We can only try lock here and 2601 * fail if unsuccessful. 2602 */ 2603 if (!anon) { 2604 struct mmu_gather tlb; 2605 2606 VM_BUG_ON(!(flags & TTU_RMAP_LOCKED)); 2607 if (!hugetlb_vma_trylock_write(vma)) { 2608 page_vma_mapped_walk_done(&pvmw); 2609 ret = false; 2610 break; 2611 } 2612 2613 tlb_gather_mmu_vma(&tlb, vma); 2614 if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) { 2615 hugetlb_vma_unlock_write(vma); 2616 huge_pmd_unshare_flush(&tlb, vma); 2617 tlb_finish_mmu(&tlb); 2618 /* 2619 * The PMD table was unmapped, 2620 * consequently unmapping the folio. 2621 */ 2622 page_vma_mapped_walk_done(&pvmw); 2623 break; 2624 } 2625 hugetlb_vma_unlock_write(vma); 2626 tlb_finish_mmu(&tlb); 2627 } 2628 /* Nuke the hugetlb page table entry */ 2629 pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); 2630 if (pte_dirty(pteval)) 2631 folio_mark_dirty(folio); 2632 writable = pte_write(pteval); 2633 } else if (likely(pte_present(pteval))) { 2634 flush_cache_page(vma, address, pfn); 2635 /* Nuke the page table entry. */ 2636 if (should_defer_flush(mm, flags)) { 2637 /* 2638 * We clear the PTE but do not flush so potentially 2639 * a remote CPU could still be writing to the folio. 2640 * If the entry was previously clean then the 2641 * architecture must guarantee that a clear->dirty 2642 * transition on a cached TLB entry is written through 2643 * and traps if the PTE is unmapped. 2644 */ 2645 pteval = ptep_get_and_clear(mm, address, pvmw.pte); 2646 2647 set_tlb_ubc_flush_pending(mm, pteval, address, address + PAGE_SIZE); 2648 } else { 2649 pteval = ptep_clear_flush(vma, address, pvmw.pte); 2650 } 2651 if (pte_dirty(pteval)) 2652 folio_mark_dirty(folio); 2653 writable = pte_write(pteval); 2654 } else { 2655 const softleaf_t entry = softleaf_from_pte(pteval); 2656 2657 pte_clear(mm, address, pvmw.pte); 2658 2659 writable = softleaf_is_device_private_write(entry); 2660 } 2661 2662 VM_WARN_ON_FOLIO(writable && folio_test_anon(folio) && 2663 !anon_exclusive, folio); 2664 2665 /* Update high watermark before we lower rss */ 2666 update_hiwater_rss(mm); 2667 2668 if (PageHWPoison(subpage)) { 2669 VM_WARN_ON_FOLIO(folio_is_device_private(folio), folio); 2670 2671 pteval = swp_entry_to_pte(make_hwpoison_entry(subpage)); 2672 if (folio_test_hugetlb(folio)) { 2673 hugetlb_count_sub(folio_nr_pages(folio), mm); 2674 set_huge_pte_at(mm, address, pvmw.pte, pteval, 2675 hsz); 2676 } else { 2677 dec_mm_counter(mm, mm_counter(folio)); 2678 set_pte_at(mm, address, pvmw.pte, pteval); 2679 } 2680 } else if (likely(pte_present(pteval)) && pte_unused(pteval) && 2681 !userfaultfd_armed(vma)) { 2682 /* 2683 * The guest indicated that the page content is of no 2684 * interest anymore. Simply discard the pte, vmscan 2685 * will take care of the rest. 2686 * A future reference will then fault in a new zero 2687 * page. When userfaultfd is active, we must not drop 2688 * this page though, as its main user (postcopy 2689 * migration) will not expect userfaults on already 2690 * copied pages. 2691 */ 2692 dec_mm_counter(mm, mm_counter(folio)); 2693 } else { 2694 swp_entry_t entry; 2695 pte_t swp_pte; 2696 2697 /* 2698 * arch_unmap_one() is expected to be a NOP on 2699 * architectures where we could have PFN swap PTEs, 2700 * so we'll not check/care. 2701 */ 2702 if (arch_unmap_one(mm, vma, address, pteval) < 0) { 2703 if (folio_test_hugetlb(folio)) 2704 set_huge_pte_at(mm, address, pvmw.pte, 2705 pteval, hsz); 2706 else 2707 set_pte_at(mm, address, pvmw.pte, pteval); 2708 ret = false; 2709 page_vma_mapped_walk_done(&pvmw); 2710 break; 2711 } 2712 2713 /* See folio_try_share_anon_rmap_pte(): clear PTE first. */ 2714 if (folio_test_hugetlb(folio)) { 2715 if (anon_exclusive && 2716 hugetlb_try_share_anon_rmap(folio)) { 2717 set_huge_pte_at(mm, address, pvmw.pte, 2718 pteval, hsz); 2719 ret = false; 2720 page_vma_mapped_walk_done(&pvmw); 2721 break; 2722 } 2723 } else if (anon_exclusive && 2724 folio_try_share_anon_rmap_pte(folio, subpage)) { 2725 set_pte_at(mm, address, pvmw.pte, pteval); 2726 ret = false; 2727 page_vma_mapped_walk_done(&pvmw); 2728 break; 2729 } 2730 2731 /* 2732 * Store the pfn of the page in a special migration 2733 * pte. do_swap_page() will wait until the migration 2734 * pte is removed and then restart fault handling. 2735 */ 2736 if (writable) 2737 entry = make_writable_migration_entry( 2738 page_to_pfn(subpage)); 2739 else if (anon_exclusive) 2740 entry = make_readable_exclusive_migration_entry( 2741 page_to_pfn(subpage)); 2742 else 2743 entry = make_readable_migration_entry( 2744 page_to_pfn(subpage)); 2745 if (likely(pte_present(pteval))) { 2746 if (pte_young(pteval)) 2747 entry = make_migration_entry_young(entry); 2748 if (pte_dirty(pteval)) 2749 entry = make_migration_entry_dirty(entry); 2750 swp_pte = swp_entry_to_pte(entry); 2751 if (pte_soft_dirty(pteval)) 2752 swp_pte = pte_swp_mksoft_dirty(swp_pte); 2753 if (pte_uffd(pteval)) 2754 swp_pte = pte_swp_mkuffd(swp_pte); 2755 } else { 2756 swp_pte = swp_entry_to_pte(entry); 2757 if (pte_swp_soft_dirty(pteval)) 2758 swp_pte = pte_swp_mksoft_dirty(swp_pte); 2759 if (pte_swp_uffd(pteval)) 2760 swp_pte = pte_swp_mkuffd(swp_pte); 2761 } 2762 if (folio_test_hugetlb(folio)) 2763 set_huge_pte_at(mm, address, pvmw.pte, swp_pte, 2764 hsz); 2765 else 2766 set_pte_at(mm, address, pvmw.pte, swp_pte); 2767 trace_set_migration_pte(address, pte_val(swp_pte), 2768 folio_order(folio)); 2769 /* 2770 * No need to invalidate here it will synchronize on 2771 * against the special swap migration pte. 2772 */ 2773 } 2774 2775 if (unlikely(folio_test_hugetlb(folio))) 2776 hugetlb_remove_rmap(folio); 2777 else 2778 folio_remove_rmap_pte(folio, subpage, vma); 2779 if (vma_test_any_mask(vma, VMA_LOCKED_MASK)) 2780 mlock_drain_local(); 2781 folio_put(folio); 2782 } 2783 2784 mmu_notifier_invalidate_range_end(&range); 2785 2786 return ret; 2787 } 2788 2789 /** 2790 * try_to_migrate - try to replace all page table mappings with swap entries 2791 * @folio: the folio to replace page table entries for 2792 * @flags: action and flags 2793 * 2794 * Tries to remove all the page table entries which are mapping this folio and 2795 * replace them with special swap entries. Caller must hold the folio lock. 2796 */ 2797 void try_to_migrate(struct folio *folio, enum ttu_flags flags) 2798 { 2799 struct rmap_walk_control rwc = { 2800 .rmap_one = try_to_migrate_one, 2801 .arg = (void *)flags, 2802 .done = folio_not_mapped, 2803 .anon_lock = folio_lock_anon_vma_read, 2804 }; 2805 2806 /* 2807 * Migration always ignores mlock and only supports TTU_RMAP_LOCKED and 2808 * TTU_SPLIT_HUGE_PMD, TTU_SYNC, and TTU_BATCH_FLUSH flags. 2809 */ 2810 if (WARN_ON_ONCE(flags & ~(TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD | 2811 TTU_SYNC | TTU_BATCH_FLUSH))) 2812 return; 2813 2814 if (folio_is_zone_device(folio) && 2815 (!folio_is_device_private(folio) && !folio_is_device_coherent(folio))) 2816 return; 2817 2818 /* 2819 * During exec, a temporary VMA is setup and later moved. 2820 * The VMA is moved under the anon_vma lock but not the 2821 * page tables leading to a race where migration cannot 2822 * find the migration ptes. Rather than increasing the 2823 * locking requirements of exec(), migration skips 2824 * temporary VMAs until after exec() completes. 2825 */ 2826 if (!folio_test_ksm(folio) && folio_test_anon(folio)) 2827 rwc.invalid_vma = invalid_migration_vma; 2828 2829 if (flags & TTU_RMAP_LOCKED) 2830 rmap_walk_locked(folio, &rwc); 2831 else 2832 rmap_walk(folio, &rwc); 2833 } 2834 2835 #ifdef CONFIG_DEVICE_PRIVATE 2836 /** 2837 * make_device_exclusive() - Mark a page for exclusive use by a device 2838 * @mm: mm_struct of associated target process 2839 * @addr: the virtual address to mark for exclusive device access 2840 * @owner: passed to MMU_NOTIFY_EXCLUSIVE range notifier to allow filtering 2841 * @foliop: folio pointer will be stored here on success. 2842 * 2843 * This function looks up the page mapped at the given address, grabs a 2844 * folio reference, locks the folio and replaces the PTE with special 2845 * device-exclusive PFN swap entry, preventing access through the process 2846 * page tables. The function will return with the folio locked and referenced. 2847 * 2848 * On fault, the device-exclusive entries are replaced with the original PTE 2849 * under folio lock, after calling MMU notifiers. 2850 * 2851 * Only anonymous non-hugetlb folios are supported and the VMA must have 2852 * write permissions such that we can fault in the anonymous page writable 2853 * in order to mark it exclusive. The caller must hold the mmap_lock in read 2854 * mode. 2855 * 2856 * A driver using this to program access from a device must use a mmu notifier 2857 * critical section to hold a device specific lock during programming. Once 2858 * programming is complete it should drop the folio lock and reference after 2859 * which point CPU access to the page will revoke the exclusive access. 2860 * 2861 * Notes: 2862 * #. This function always operates on individual PTEs mapping individual 2863 * pages. PMD-sized THPs are first remapped to be mapped by PTEs before 2864 * the conversion happens on a single PTE corresponding to @addr. 2865 * #. While concurrent access through the process page tables is prevented, 2866 * concurrent access through other page references (e.g., earlier GUP 2867 * invocation) is not handled and not supported. 2868 * #. device-exclusive entries are considered "clean" and "old" by core-mm. 2869 * Device drivers must update the folio state when informed by MMU 2870 * notifiers. 2871 * 2872 * Returns: pointer to mapped page on success, otherwise a negative error. 2873 */ 2874 struct page *make_device_exclusive(struct mm_struct *mm, unsigned long addr, 2875 void *owner, struct folio **foliop) 2876 { 2877 struct mmu_notifier_range range; 2878 struct folio *folio, *fw_folio; 2879 struct vm_area_struct *vma; 2880 struct folio_walk fw; 2881 struct page *page; 2882 swp_entry_t entry; 2883 pte_t swp_pte; 2884 int ret; 2885 2886 mmap_assert_locked(mm); 2887 addr = PAGE_ALIGN_DOWN(addr); 2888 2889 /* 2890 * Fault in the page writable and try to lock it; note that if the 2891 * address would already be marked for exclusive use by a device, 2892 * the GUP call would undo that first by triggering a fault. 2893 * 2894 * If any other device would already map this page exclusively, the 2895 * fault will trigger a conversion to an ordinary 2896 * (non-device-exclusive) PTE and issue a MMU_NOTIFY_EXCLUSIVE. 2897 */ 2898 retry: 2899 page = get_user_page_vma_remote(mm, addr, 2900 FOLL_GET | FOLL_WRITE | FOLL_SPLIT_PMD, 2901 &vma); 2902 if (IS_ERR(page)) 2903 return page; 2904 folio = page_folio(page); 2905 2906 if (!folio_test_anon(folio) || folio_test_hugetlb(folio)) { 2907 folio_put(folio); 2908 return ERR_PTR(-EOPNOTSUPP); 2909 } 2910 2911 ret = folio_lock_killable(folio); 2912 if (ret) { 2913 folio_put(folio); 2914 return ERR_PTR(ret); 2915 } 2916 2917 /* 2918 * Inform secondary MMUs that we are going to convert this PTE to 2919 * device-exclusive, such that they unmap it now. Note that the 2920 * caller must filter this event out to prevent livelocks. 2921 */ 2922 mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0, 2923 mm, addr, addr + PAGE_SIZE, owner); 2924 mmu_notifier_invalidate_range_start(&range); 2925 2926 /* 2927 * Let's do a second walk and make sure we still find the same page 2928 * mapped writable. Note that any page of an anonymous folio can 2929 * only be mapped writable using exactly one PTE ("exclusive"), so 2930 * there cannot be other mappings. 2931 */ 2932 fw_folio = folio_walk_start(&fw, vma, addr, 0); 2933 if (fw_folio != folio || fw.page != page || 2934 fw.level != FW_LEVEL_PTE || !pte_write(fw.pte)) { 2935 if (fw_folio) 2936 folio_walk_end(&fw, vma); 2937 mmu_notifier_invalidate_range_end(&range); 2938 folio_unlock(folio); 2939 folio_put(folio); 2940 goto retry; 2941 } 2942 2943 /* Nuke the page table entry so we get the uptodate dirty bit. */ 2944 flush_cache_page(vma, addr, page_to_pfn(page)); 2945 fw.pte = ptep_clear_flush(vma, addr, fw.ptep); 2946 2947 /* Set the dirty flag on the folio now the PTE is gone. */ 2948 if (pte_dirty(fw.pte)) 2949 folio_mark_dirty(folio); 2950 2951 /* 2952 * Store the pfn of the page in a special device-exclusive PFN swap PTE. 2953 * do_swap_page() will trigger the conversion back while holding the 2954 * folio lock. 2955 */ 2956 entry = make_device_exclusive_entry(page_to_pfn(page)); 2957 swp_pte = swp_entry_to_pte(entry); 2958 if (pte_soft_dirty(fw.pte)) 2959 swp_pte = pte_swp_mksoft_dirty(swp_pte); 2960 /* The pte is writable, uffd-wp does not apply. */ 2961 set_pte_at(mm, addr, fw.ptep, swp_pte); 2962 2963 folio_walk_end(&fw, vma); 2964 mmu_notifier_invalidate_range_end(&range); 2965 *foliop = folio; 2966 return page; 2967 } 2968 EXPORT_SYMBOL_GPL(make_device_exclusive); 2969 #endif 2970 2971 void __put_anon_vma(struct anon_vma *anon_vma) 2972 { 2973 struct anon_vma *root = anon_vma->root; 2974 2975 anon_vma_free(anon_vma); 2976 if (root != anon_vma && atomic_dec_and_test(&root->refcount)) 2977 anon_vma_free(root); 2978 } 2979 2980 static struct anon_vma *rmap_walk_anon_lock(const struct folio *folio, 2981 struct rmap_walk_control *rwc) 2982 { 2983 struct anon_vma *anon_vma; 2984 2985 if (rwc->anon_lock) 2986 return rwc->anon_lock(folio, rwc); 2987 2988 /* 2989 * Note: remove_migration_ptes() cannot use folio_lock_anon_vma_read() 2990 * because that depends on folio_mapped(); but not all its usages 2991 * are holding mmap_lock. Users without mmap_lock are required to 2992 * take a reference count to prevent the anon_vma disappearing 2993 */ 2994 anon_vma = folio_anon_vma(folio); 2995 if (!anon_vma) 2996 return NULL; 2997 2998 if (anon_vma_trylock_read(anon_vma)) 2999 goto out; 3000 3001 if (rwc->try_lock) { 3002 anon_vma = NULL; 3003 rwc->contended = true; 3004 goto out; 3005 } 3006 3007 anon_vma_lock_read(anon_vma); 3008 out: 3009 return anon_vma; 3010 } 3011 3012 /* 3013 * rmap_walk_anon - do something to anonymous page using the object-based 3014 * rmap method 3015 * @folio: the folio to be handled 3016 * @rwc: control variable according to each walk type 3017 * @locked: caller holds relevant rmap lock 3018 * 3019 * Find all the mappings of a folio using the mapping pointer and the vma 3020 * chains contained in the anon_vma struct it points to. 3021 */ 3022 static void rmap_walk_anon(struct folio *folio, 3023 struct rmap_walk_control *rwc, bool locked) 3024 { 3025 struct anon_vma *anon_vma; 3026 pgoff_t pgoff_start, pgoff_end; 3027 struct anon_vma_chain *avc; 3028 3029 /* 3030 * The folio lock ensures that folio->mapping can't be changed under us 3031 * to an anon_vma with different root. 3032 */ 3033 VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); 3034 3035 if (locked) { 3036 anon_vma = folio_anon_vma(folio); 3037 /* anon_vma disappear under us? */ 3038 VM_BUG_ON_FOLIO(!anon_vma, folio); 3039 } else { 3040 anon_vma = rmap_walk_anon_lock(folio, rwc); 3041 } 3042 if (!anon_vma) 3043 return; 3044 3045 pgoff_start = folio_pgoff(folio); 3046 pgoff_end = pgoff_start + folio_nr_pages(folio) - 1; 3047 anon_rmap_tree_foreach(avc, anon_vma, pgoff_start, pgoff_end) { 3048 struct vm_area_struct *vma = avc->vma; 3049 const unsigned long address = vma_anon_address(vma, pgoff_start, 3050 folio_nr_pages(folio)); 3051 3052 VM_WARN_ON_ONCE_VMA(address == -EFAULT, vma); 3053 cond_resched(); 3054 3055 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) 3056 continue; 3057 3058 if (!rwc->rmap_one(folio, vma, address, rwc->arg)) 3059 break; 3060 if (rwc->done && rwc->done(folio)) 3061 break; 3062 } 3063 3064 if (!locked) 3065 anon_vma_unlock_read(anon_vma); 3066 } 3067 3068 /** 3069 * __rmap_walk_file() - Traverse the reverse mapping for a file-backed mapping 3070 * of a page mapped within a specified page cache object at a specified offset. 3071 * 3072 * @folio: Either the folio whose mappings to traverse, or if NULL, 3073 * the callbacks specified in @rwc will be configured such 3074 * as to be able to look up mappings correctly. 3075 * @mapping: The page cache object whose mapping VMAs we intend to 3076 * traverse. If @folio is non-NULL, this should be equal to 3077 * folio_mapping(folio). 3078 * @pgoff_start: The offset within @mapping of the page which we are 3079 * looking up. If @folio is non-NULL, this should be equal 3080 * to folio_pgoff(folio). 3081 * @nr_pages: The number of pages mapped by the mapping. If @folio is 3082 * non-NULL, this should be equal to folio_nr_pages(folio). 3083 * @rwc: The reverse mapping walk control object describing how 3084 * the traversal should proceed. 3085 * @locked: Is the @mapping already locked? If not, we acquire the 3086 * lock. 3087 */ 3088 static void __rmap_walk_file(struct folio *folio, struct address_space *mapping, 3089 pgoff_t pgoff_start, unsigned long nr_pages, 3090 struct rmap_walk_control *rwc, bool locked) 3091 { 3092 pgoff_t pgoff_end = pgoff_start + nr_pages - 1; 3093 struct vm_area_struct *vma; 3094 3095 VM_WARN_ON_FOLIO(folio && mapping != folio_mapping(folio), folio); 3096 VM_WARN_ON_FOLIO(folio && pgoff_start != folio_pgoff(folio), folio); 3097 VM_WARN_ON_FOLIO(folio && nr_pages != folio_nr_pages(folio), folio); 3098 3099 if (!locked) { 3100 if (i_mmap_trylock_read(mapping)) 3101 goto lookup; 3102 3103 if (rwc->try_lock) { 3104 rwc->contended = true; 3105 return; 3106 } 3107 3108 i_mmap_lock_read(mapping); 3109 } 3110 lookup: 3111 mapping_rmap_tree_foreach(vma, mapping, pgoff_start, pgoff_end) { 3112 unsigned long address = vma_filebacked_address(vma, pgoff_start, 3113 nr_pages); 3114 3115 VM_BUG_ON_VMA(address == -EFAULT, vma); 3116 cond_resched(); 3117 3118 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) 3119 continue; 3120 3121 if (!rwc->rmap_one(folio, vma, address, rwc->arg)) 3122 goto done; 3123 if (rwc->done && rwc->done(folio)) 3124 goto done; 3125 } 3126 done: 3127 if (!locked) 3128 i_mmap_unlock_read(mapping); 3129 } 3130 3131 /* 3132 * rmap_walk_file - do something to file page using the object-based rmap method 3133 * @folio: the folio to be handled 3134 * @rwc: control variable according to each walk type 3135 * @locked: caller holds relevant rmap lock 3136 * 3137 * Find all the mappings of a folio using the mapping pointer and the vma chains 3138 * contained in the address_space struct it points to. 3139 */ 3140 static void rmap_walk_file(struct folio *folio, 3141 struct rmap_walk_control *rwc, bool locked) 3142 { 3143 /* 3144 * The folio lock not only makes sure that folio->mapping cannot 3145 * suddenly be NULLified by truncation, it makes sure that the structure 3146 * at mapping cannot be freed and reused yet, so we can safely take 3147 * mapping->i_mmap_rwsem. 3148 */ 3149 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 3150 3151 if (!folio->mapping) 3152 return; 3153 3154 __rmap_walk_file(folio, folio->mapping, folio->index, 3155 folio_nr_pages(folio), rwc, locked); 3156 } 3157 3158 void rmap_walk(struct folio *folio, struct rmap_walk_control *rwc) 3159 { 3160 if (unlikely(folio_test_ksm(folio))) 3161 rmap_walk_ksm(folio, rwc); 3162 else if (folio_test_anon(folio)) 3163 rmap_walk_anon(folio, rwc, false); 3164 else 3165 rmap_walk_file(folio, rwc, false); 3166 } 3167 3168 /* Like rmap_walk, but caller holds relevant rmap lock */ 3169 void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc) 3170 { 3171 /* no ksm support for now */ 3172 VM_BUG_ON_FOLIO(folio_test_ksm(folio), folio); 3173 if (folio_test_anon(folio)) 3174 rmap_walk_anon(folio, rwc, true); 3175 else 3176 rmap_walk_file(folio, rwc, true); 3177 } 3178 3179 #ifdef CONFIG_HUGETLB_PAGE 3180 /* 3181 * The following two functions are for anonymous (private mapped) hugepages. 3182 * Unlike common anonymous pages, anonymous hugepages have no accounting code 3183 * and no lru code, because we handle hugepages differently from common pages. 3184 */ 3185 void hugetlb_add_anon_rmap(struct folio *folio, struct vm_area_struct *vma, 3186 unsigned long address, rmap_t flags) 3187 { 3188 VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio); 3189 VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); 3190 3191 atomic_inc(&folio->_entire_mapcount); 3192 atomic_inc(&folio->_large_mapcount); 3193 if (flags & RMAP_EXCLUSIVE) 3194 SetPageAnonExclusive(&folio->page); 3195 VM_WARN_ON_FOLIO(folio_entire_mapcount(folio) > 1 && 3196 PageAnonExclusive(&folio->page), folio); 3197 } 3198 3199 void hugetlb_add_new_anon_rmap(struct folio *folio, 3200 struct vm_area_struct *vma, unsigned long address) 3201 { 3202 VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio); 3203 3204 BUG_ON(address < vma->vm_start || address >= vma->vm_end); 3205 /* increment count (starts at -1) */ 3206 atomic_set(&folio->_entire_mapcount, 0); 3207 atomic_set(&folio->_large_mapcount, 0); 3208 folio_clear_hugetlb_restore_reserve(folio); 3209 __folio_set_anon(folio, vma, address, true); 3210 SetPageAnonExclusive(&folio->page); 3211 } 3212 #endif /* CONFIG_HUGETLB_PAGE */ 3213