11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * mm/rmap.c - physical to virtual reverse mappings 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright 2001, Rik van Riel <riel@conectiva.com.br> 51da177e4SLinus Torvalds * Released under the General Public License (GPL). 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * Simple, low overhead reverse mapping scheme. 81da177e4SLinus Torvalds * Please try to keep this thing as modular as possible. 91da177e4SLinus Torvalds * 101da177e4SLinus Torvalds * Provides methods for unmapping each kind of mapped page: 111da177e4SLinus Torvalds * the anon methods track anonymous pages, and 121da177e4SLinus Torvalds * the file methods track pages belonging to an inode. 131da177e4SLinus Torvalds * 141da177e4SLinus Torvalds * Original design by Rik van Riel <riel@conectiva.com.br> 2001 151da177e4SLinus Torvalds * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004 161da177e4SLinus Torvalds * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004 1798f32602SHugh Dickins * Contributions by Hugh Dickins 2003, 2004 181da177e4SLinus Torvalds */ 191da177e4SLinus Torvalds 201da177e4SLinus Torvalds /* 211da177e4SLinus Torvalds * Lock ordering in mm: 221da177e4SLinus Torvalds * 239608703eSJan Kara * inode->i_rwsem (while writing or truncating, not reading or faulting) 24c1e8d7c6SMichel Lespinasse * mm->mmap_lock 25730633f0SJan Kara * mapping->invalidate_lock (in filemap_fault) 263a47c54fSMike Kravetz * page->flags PG_locked (lock_page) 278d9bfb26SMike Kravetz * hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share, see hugetlbfs below) 2855fd6fccSSuren Baghdasaryan * vma_start_write 29c8c06efaSDavidlohr Bueso * mapping->i_mmap_rwsem 305a505085SIngo Molnar * anon_vma->rwsem 31b8072f09SHugh Dickins * mm->page_table_lock or pte_lock 325d337b91SHugh Dickins * swap_lock (in swap_duplicate, swap_info_get) 331da177e4SLinus Torvalds * mmlist_lock (in mmput, drain_mmlist and others) 34e621900aSMatthew Wilcox (Oracle) * mapping->private_lock (in block_dirty_folio) 35e621900aSMatthew Wilcox (Oracle) * folio_lock_memcg move_lock (in block_dirty_folio) 36b93b0163SMatthew Wilcox * i_pages lock (widely used) 37e809c3feSMatthew Wilcox (Oracle) * lruvec->lru_lock (in folio_lruvec_lock_irq) 38250df6edSDave Chinner * inode->i_lock (in set_page_dirty's __mark_inode_dirty) 39f758eeabSChristoph Hellwig * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty) 401da177e4SLinus Torvalds * sb_lock (within inode_lock in fs/fs-writeback.c) 41b93b0163SMatthew Wilcox * i_pages lock (widely used, in set_page_dirty, 421da177e4SLinus Torvalds * in arch-dependent flush_dcache_mmap_lock, 43f758eeabSChristoph Hellwig * within bdi.wb->list_lock in __sync_single_inode) 446a46079cSAndi Kleen * 459608703eSJan Kara * anon_vma->rwsem,mapping->i_mmap_rwsem (memory_failure, collect_procs_anon) 466a46079cSAndi Kleen * ->tasklist_lock 476a46079cSAndi Kleen * pte map lock 48c0d0381aSMike Kravetz * 498d9bfb26SMike Kravetz * hugetlbfs PageHuge() take locks in this order: 50c0d0381aSMike Kravetz * hugetlb_fault_mutex (hugetlbfs specific page fault mutex) 518d9bfb26SMike Kravetz * vma_lock (hugetlb specific lock for pmd_sharing) 528d9bfb26SMike Kravetz * mapping->i_mmap_rwsem (also used for hugetlb pmd sharing) 53c0d0381aSMike Kravetz * page->flags PG_locked (lock_page) 541da177e4SLinus Torvalds */ 551da177e4SLinus Torvalds 561da177e4SLinus Torvalds #include <linux/mm.h> 576e84f315SIngo Molnar #include <linux/sched/mm.h> 5829930025SIngo Molnar #include <linux/sched/task.h> 591da177e4SLinus Torvalds #include <linux/pagemap.h> 601da177e4SLinus Torvalds #include <linux/swap.h> 611da177e4SLinus Torvalds #include <linux/swapops.h> 621da177e4SLinus Torvalds #include <linux/slab.h> 631da177e4SLinus Torvalds #include <linux/init.h> 645ad64688SHugh Dickins #include <linux/ksm.h> 651da177e4SLinus Torvalds #include <linux/rmap.h> 661da177e4SLinus Torvalds #include <linux/rcupdate.h> 67b95f1b31SPaul Gortmaker #include <linux/export.h> 688a9f3ccdSBalbir Singh #include <linux/memcontrol.h> 69cddb8a5cSAndrea Arcangeli #include <linux/mmu_notifier.h> 7064cdd548SKOSAKI Motohiro #include <linux/migrate.h> 710fe6e20bSNaoya Horiguchi #include <linux/hugetlb.h> 72444f84fdSBen Dooks #include <linux/huge_mm.h> 73ef5d437fSJan Kara #include <linux/backing-dev.h> 7433c3fc71SVladimir Davydov #include <linux/page_idle.h> 75a5430ddaSJérôme Glisse #include <linux/memremap.h> 76bce73e48SChristian Borntraeger #include <linux/userfaultfd_k.h> 77999dad82SPeter Xu #include <linux/mm_inline.h> 781da177e4SLinus Torvalds 791da177e4SLinus Torvalds #include <asm/tlbflush.h> 801da177e4SLinus Torvalds 814cc79b33SAnshuman Khandual #define CREATE_TRACE_POINTS 8272b252aeSMel Gorman #include <trace/events/tlb.h> 834cc79b33SAnshuman Khandual #include <trace/events/migrate.h> 8472b252aeSMel Gorman 85b291f000SNick Piggin #include "internal.h" 86b291f000SNick Piggin 87fdd2e5f8SAdrian Bunk static struct kmem_cache *anon_vma_cachep; 885beb4930SRik van Riel static struct kmem_cache *anon_vma_chain_cachep; 89fdd2e5f8SAdrian Bunk 90fdd2e5f8SAdrian Bunk static inline struct anon_vma *anon_vma_alloc(void) 91fdd2e5f8SAdrian Bunk { 9201d8b20dSPeter Zijlstra struct anon_vma *anon_vma; 9301d8b20dSPeter Zijlstra 9401d8b20dSPeter Zijlstra anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL); 9501d8b20dSPeter Zijlstra if (anon_vma) { 9601d8b20dSPeter Zijlstra atomic_set(&anon_vma->refcount, 1); 972555283eSJann Horn anon_vma->num_children = 0; 982555283eSJann Horn anon_vma->num_active_vmas = 0; 997a3ef208SKonstantin Khlebnikov anon_vma->parent = anon_vma; 10001d8b20dSPeter Zijlstra /* 10101d8b20dSPeter Zijlstra * Initialise the anon_vma root to point to itself. If called 10201d8b20dSPeter Zijlstra * from fork, the root will be reset to the parents anon_vma. 10301d8b20dSPeter Zijlstra */ 10401d8b20dSPeter Zijlstra anon_vma->root = anon_vma; 105fdd2e5f8SAdrian Bunk } 106fdd2e5f8SAdrian Bunk 10701d8b20dSPeter Zijlstra return anon_vma; 10801d8b20dSPeter Zijlstra } 10901d8b20dSPeter Zijlstra 11001d8b20dSPeter Zijlstra static inline void anon_vma_free(struct anon_vma *anon_vma) 111fdd2e5f8SAdrian Bunk { 11201d8b20dSPeter Zijlstra VM_BUG_ON(atomic_read(&anon_vma->refcount)); 11388c22088SPeter Zijlstra 11488c22088SPeter Zijlstra /* 1152f031c6fSMatthew Wilcox (Oracle) * Synchronize against folio_lock_anon_vma_read() such that 11688c22088SPeter Zijlstra * we can safely hold the lock without the anon_vma getting 11788c22088SPeter Zijlstra * freed. 11888c22088SPeter Zijlstra * 11988c22088SPeter Zijlstra * Relies on the full mb implied by the atomic_dec_and_test() from 12088c22088SPeter Zijlstra * put_anon_vma() against the acquire barrier implied by 1212f031c6fSMatthew Wilcox (Oracle) * down_read_trylock() from folio_lock_anon_vma_read(). This orders: 12288c22088SPeter Zijlstra * 1232f031c6fSMatthew Wilcox (Oracle) * folio_lock_anon_vma_read() VS put_anon_vma() 1244fc3f1d6SIngo Molnar * down_read_trylock() atomic_dec_and_test() 12588c22088SPeter Zijlstra * LOCK MB 1264fc3f1d6SIngo Molnar * atomic_read() rwsem_is_locked() 12788c22088SPeter Zijlstra * 12888c22088SPeter Zijlstra * LOCK should suffice since the actual taking of the lock must 12988c22088SPeter Zijlstra * happen _before_ what follows. 13088c22088SPeter Zijlstra */ 1317f39dda9SHugh Dickins might_sleep(); 1325a505085SIngo Molnar if (rwsem_is_locked(&anon_vma->root->rwsem)) { 1334fc3f1d6SIngo Molnar anon_vma_lock_write(anon_vma); 13408b52706SKonstantin Khlebnikov anon_vma_unlock_write(anon_vma); 13588c22088SPeter Zijlstra } 13688c22088SPeter Zijlstra 137fdd2e5f8SAdrian Bunk kmem_cache_free(anon_vma_cachep, anon_vma); 138fdd2e5f8SAdrian Bunk } 1391da177e4SLinus Torvalds 140dd34739cSLinus Torvalds static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp) 1415beb4930SRik van Riel { 142dd34739cSLinus Torvalds return kmem_cache_alloc(anon_vma_chain_cachep, gfp); 1435beb4930SRik van Riel } 1445beb4930SRik van Riel 145e574b5fdSNamhyung Kim static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain) 1465beb4930SRik van Riel { 1475beb4930SRik van Riel kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain); 1485beb4930SRik van Riel } 1495beb4930SRik van Riel 1506583a843SKautuk Consul static void anon_vma_chain_link(struct vm_area_struct *vma, 1516583a843SKautuk Consul struct anon_vma_chain *avc, 1526583a843SKautuk Consul struct anon_vma *anon_vma) 1536583a843SKautuk Consul { 1546583a843SKautuk Consul avc->vma = vma; 1556583a843SKautuk Consul avc->anon_vma = anon_vma; 1566583a843SKautuk Consul list_add(&avc->same_vma, &vma->anon_vma_chain); 157bf181b9fSMichel Lespinasse anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); 1586583a843SKautuk Consul } 1596583a843SKautuk Consul 160d9d332e0SLinus Torvalds /** 161d5a187daSVlastimil Babka * __anon_vma_prepare - attach an anon_vma to a memory region 162d9d332e0SLinus Torvalds * @vma: the memory region in question 163d9d332e0SLinus Torvalds * 164d9d332e0SLinus Torvalds * This makes sure the memory mapping described by 'vma' has 165d9d332e0SLinus Torvalds * an 'anon_vma' attached to it, so that we can associate the 166d9d332e0SLinus Torvalds * anonymous pages mapped into it with that anon_vma. 167d9d332e0SLinus Torvalds * 168d5a187daSVlastimil Babka * The common case will be that we already have one, which 169d5a187daSVlastimil Babka * is handled inline by anon_vma_prepare(). But if 17023a0790aSFigo.zhang * not we either need to find an adjacent mapping that we 171d9d332e0SLinus Torvalds * can re-use the anon_vma from (very common when the only 172d9d332e0SLinus Torvalds * reason for splitting a vma has been mprotect()), or we 173d9d332e0SLinus Torvalds * allocate a new one. 174d9d332e0SLinus Torvalds * 175d9d332e0SLinus Torvalds * Anon-vma allocations are very subtle, because we may have 1762f031c6fSMatthew Wilcox (Oracle) * optimistically looked up an anon_vma in folio_lock_anon_vma_read() 177aaf1f990SMiaohe Lin * and that may actually touch the rwsem even in the newly 178d9d332e0SLinus Torvalds * allocated vma (it depends on RCU to make sure that the 179d9d332e0SLinus Torvalds * anon_vma isn't actually destroyed). 180d9d332e0SLinus Torvalds * 181d9d332e0SLinus Torvalds * As a result, we need to do proper anon_vma locking even 182d9d332e0SLinus Torvalds * for the new allocation. At the same time, we do not want 183d9d332e0SLinus Torvalds * to do any locking for the common case of already having 184d9d332e0SLinus Torvalds * an anon_vma. 185d9d332e0SLinus Torvalds * 186c1e8d7c6SMichel Lespinasse * This must be called with the mmap_lock held for reading. 187d9d332e0SLinus Torvalds */ 188d5a187daSVlastimil Babka int __anon_vma_prepare(struct vm_area_struct *vma) 1891da177e4SLinus Torvalds { 190d5a187daSVlastimil Babka struct mm_struct *mm = vma->vm_mm; 191d5a187daSVlastimil Babka struct anon_vma *anon_vma, *allocated; 1925beb4930SRik van Riel struct anon_vma_chain *avc; 1931da177e4SLinus Torvalds 1941da177e4SLinus Torvalds might_sleep(); 1951da177e4SLinus Torvalds 196dd34739cSLinus Torvalds avc = anon_vma_chain_alloc(GFP_KERNEL); 1975beb4930SRik van Riel if (!avc) 1985beb4930SRik van Riel goto out_enomem; 1995beb4930SRik van Riel 2001da177e4SLinus Torvalds anon_vma = find_mergeable_anon_vma(vma); 2011da177e4SLinus Torvalds allocated = NULL; 202d9d332e0SLinus Torvalds if (!anon_vma) { 2031da177e4SLinus Torvalds anon_vma = anon_vma_alloc(); 2041da177e4SLinus Torvalds if (unlikely(!anon_vma)) 2055beb4930SRik van Riel goto out_enomem_free_avc; 2062555283eSJann Horn anon_vma->num_children++; /* self-parent link for new root */ 2071da177e4SLinus Torvalds allocated = anon_vma; 2081da177e4SLinus Torvalds } 2091da177e4SLinus Torvalds 2104fc3f1d6SIngo Molnar anon_vma_lock_write(anon_vma); 2111da177e4SLinus Torvalds /* page_table_lock to protect against threads */ 2121da177e4SLinus Torvalds spin_lock(&mm->page_table_lock); 2131da177e4SLinus Torvalds if (likely(!vma->anon_vma)) { 2141da177e4SLinus Torvalds vma->anon_vma = anon_vma; 2156583a843SKautuk Consul anon_vma_chain_link(vma, avc, anon_vma); 2162555283eSJann Horn anon_vma->num_active_vmas++; 2171da177e4SLinus Torvalds allocated = NULL; 21831f2b0ebSOleg Nesterov avc = NULL; 2191da177e4SLinus Torvalds } 2201da177e4SLinus Torvalds spin_unlock(&mm->page_table_lock); 22108b52706SKonstantin Khlebnikov anon_vma_unlock_write(anon_vma); 22231f2b0ebSOleg Nesterov 22331f2b0ebSOleg Nesterov if (unlikely(allocated)) 22401d8b20dSPeter Zijlstra put_anon_vma(allocated); 22531f2b0ebSOleg Nesterov if (unlikely(avc)) 2265beb4930SRik van Riel anon_vma_chain_free(avc); 227d5a187daSVlastimil Babka 2281da177e4SLinus Torvalds return 0; 2295beb4930SRik van Riel 2305beb4930SRik van Riel out_enomem_free_avc: 2315beb4930SRik van Riel anon_vma_chain_free(avc); 2325beb4930SRik van Riel out_enomem: 2335beb4930SRik van Riel return -ENOMEM; 2341da177e4SLinus Torvalds } 2351da177e4SLinus Torvalds 236bb4aa396SLinus Torvalds /* 237bb4aa396SLinus Torvalds * This is a useful helper function for locking the anon_vma root as 238bb4aa396SLinus Torvalds * we traverse the vma->anon_vma_chain, looping over anon_vma's that 239bb4aa396SLinus Torvalds * have the same vma. 240bb4aa396SLinus Torvalds * 241bb4aa396SLinus Torvalds * Such anon_vma's should have the same root, so you'd expect to see 242bb4aa396SLinus Torvalds * just a single mutex_lock for the whole traversal. 243bb4aa396SLinus Torvalds */ 244bb4aa396SLinus Torvalds static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma) 245bb4aa396SLinus Torvalds { 246bb4aa396SLinus Torvalds struct anon_vma *new_root = anon_vma->root; 247bb4aa396SLinus Torvalds if (new_root != root) { 248bb4aa396SLinus Torvalds if (WARN_ON_ONCE(root)) 2495a505085SIngo Molnar up_write(&root->rwsem); 250bb4aa396SLinus Torvalds root = new_root; 2515a505085SIngo Molnar down_write(&root->rwsem); 252bb4aa396SLinus Torvalds } 253bb4aa396SLinus Torvalds return root; 254bb4aa396SLinus Torvalds } 255bb4aa396SLinus Torvalds 256bb4aa396SLinus Torvalds static inline void unlock_anon_vma_root(struct anon_vma *root) 257bb4aa396SLinus Torvalds { 258bb4aa396SLinus Torvalds if (root) 2595a505085SIngo Molnar up_write(&root->rwsem); 260bb4aa396SLinus Torvalds } 261bb4aa396SLinus Torvalds 2625beb4930SRik van Riel /* 2635beb4930SRik van Riel * Attach the anon_vmas from src to dst. 2645beb4930SRik van Riel * Returns 0 on success, -ENOMEM on failure. 2657a3ef208SKonstantin Khlebnikov * 2660503ea8fSLiam R. Howlett * anon_vma_clone() is called by vma_expand(), vma_merge(), __split_vma(), 2670503ea8fSLiam R. Howlett * copy_vma() and anon_vma_fork(). The first four want an exact copy of src, 2680503ea8fSLiam R. Howlett * while the last one, anon_vma_fork(), may try to reuse an existing anon_vma to 2690503ea8fSLiam R. Howlett * prevent endless growth of anon_vma. Since dst->anon_vma is set to NULL before 2700503ea8fSLiam R. Howlett * call, we can identify this case by checking (!dst->anon_vma && 2710503ea8fSLiam R. Howlett * src->anon_vma). 27247b390d2SWei Yang * 27347b390d2SWei Yang * If (!dst->anon_vma && src->anon_vma) is true, this function tries to find 27447b390d2SWei Yang * and reuse existing anon_vma which has no vmas and only one child anon_vma. 27547b390d2SWei Yang * This prevents degradation of anon_vma hierarchy to endless linear chain in 27647b390d2SWei Yang * case of constantly forking task. On the other hand, an anon_vma with more 27747b390d2SWei Yang * than one child isn't reused even if there was no alive vma, thus rmap 27847b390d2SWei Yang * walker has a good chance of avoiding scanning the whole hierarchy when it 27947b390d2SWei Yang * searches where page is mapped. 2805beb4930SRik van Riel */ 2815beb4930SRik van Riel int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src) 2825beb4930SRik van Riel { 2835beb4930SRik van Riel struct anon_vma_chain *avc, *pavc; 284bb4aa396SLinus Torvalds struct anon_vma *root = NULL; 2855beb4930SRik van Riel 286646d87b4SLinus Torvalds list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) { 287bb4aa396SLinus Torvalds struct anon_vma *anon_vma; 288bb4aa396SLinus Torvalds 289dd34739cSLinus Torvalds avc = anon_vma_chain_alloc(GFP_NOWAIT | __GFP_NOWARN); 290dd34739cSLinus Torvalds if (unlikely(!avc)) { 291dd34739cSLinus Torvalds unlock_anon_vma_root(root); 292dd34739cSLinus Torvalds root = NULL; 293dd34739cSLinus Torvalds avc = anon_vma_chain_alloc(GFP_KERNEL); 2945beb4930SRik van Riel if (!avc) 2955beb4930SRik van Riel goto enomem_failure; 296dd34739cSLinus Torvalds } 297bb4aa396SLinus Torvalds anon_vma = pavc->anon_vma; 298bb4aa396SLinus Torvalds root = lock_anon_vma_root(root, anon_vma); 299bb4aa396SLinus Torvalds anon_vma_chain_link(dst, avc, anon_vma); 3007a3ef208SKonstantin Khlebnikov 3017a3ef208SKonstantin Khlebnikov /* 3022555283eSJann Horn * Reuse existing anon_vma if it has no vma and only one 3032555283eSJann Horn * anon_vma child. 3047a3ef208SKonstantin Khlebnikov * 3052555283eSJann Horn * Root anon_vma is never reused: 3067a3ef208SKonstantin Khlebnikov * it has self-parent reference and at least one child. 3077a3ef208SKonstantin Khlebnikov */ 30847b390d2SWei Yang if (!dst->anon_vma && src->anon_vma && 3092555283eSJann Horn anon_vma->num_children < 2 && 3102555283eSJann Horn anon_vma->num_active_vmas == 0) 3117a3ef208SKonstantin Khlebnikov dst->anon_vma = anon_vma; 3125beb4930SRik van Riel } 3137a3ef208SKonstantin Khlebnikov if (dst->anon_vma) 3142555283eSJann Horn dst->anon_vma->num_active_vmas++; 315bb4aa396SLinus Torvalds unlock_anon_vma_root(root); 3165beb4930SRik van Riel return 0; 3175beb4930SRik van Riel 3185beb4930SRik van Riel enomem_failure: 3193fe89b3eSLeon Yu /* 320d8e454ebSMa Wupeng * dst->anon_vma is dropped here otherwise its num_active_vmas can 321d8e454ebSMa Wupeng * be incorrectly decremented in unlink_anon_vmas(). 3223fe89b3eSLeon Yu * We can safely do this because callers of anon_vma_clone() don't care 3233fe89b3eSLeon Yu * about dst->anon_vma if anon_vma_clone() failed. 3243fe89b3eSLeon Yu */ 3253fe89b3eSLeon Yu dst->anon_vma = NULL; 3265beb4930SRik van Riel unlink_anon_vmas(dst); 3275beb4930SRik van Riel return -ENOMEM; 3281da177e4SLinus Torvalds } 3291da177e4SLinus Torvalds 3305beb4930SRik van Riel /* 3315beb4930SRik van Riel * Attach vma to its own anon_vma, as well as to the anon_vmas that 3325beb4930SRik van Riel * the corresponding VMA in the parent process is attached to. 3335beb4930SRik van Riel * Returns 0 on success, non-zero on failure. 3345beb4930SRik van Riel */ 3355beb4930SRik van Riel int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) 3361da177e4SLinus Torvalds { 3375beb4930SRik van Riel struct anon_vma_chain *avc; 3385beb4930SRik van Riel struct anon_vma *anon_vma; 339c4ea95d7SDaniel Forrest int error; 3405beb4930SRik van Riel 3415beb4930SRik van Riel /* Don't bother if the parent process has no anon_vma here. */ 3425beb4930SRik van Riel if (!pvma->anon_vma) 3435beb4930SRik van Riel return 0; 3445beb4930SRik van Riel 3457a3ef208SKonstantin Khlebnikov /* Drop inherited anon_vma, we'll reuse existing or allocate new. */ 3467a3ef208SKonstantin Khlebnikov vma->anon_vma = NULL; 3477a3ef208SKonstantin Khlebnikov 3485beb4930SRik van Riel /* 3495beb4930SRik van Riel * First, attach the new VMA to the parent VMA's anon_vmas, 3505beb4930SRik van Riel * so rmap can find non-COWed pages in child processes. 3515beb4930SRik van Riel */ 352c4ea95d7SDaniel Forrest error = anon_vma_clone(vma, pvma); 353c4ea95d7SDaniel Forrest if (error) 354c4ea95d7SDaniel Forrest return error; 3555beb4930SRik van Riel 3567a3ef208SKonstantin Khlebnikov /* An existing anon_vma has been reused, all done then. */ 3577a3ef208SKonstantin Khlebnikov if (vma->anon_vma) 3587a3ef208SKonstantin Khlebnikov return 0; 3597a3ef208SKonstantin Khlebnikov 3605beb4930SRik van Riel /* Then add our own anon_vma. */ 3615beb4930SRik van Riel anon_vma = anon_vma_alloc(); 3625beb4930SRik van Riel if (!anon_vma) 3635beb4930SRik van Riel goto out_error; 3642555283eSJann Horn anon_vma->num_active_vmas++; 365dd34739cSLinus Torvalds avc = anon_vma_chain_alloc(GFP_KERNEL); 3665beb4930SRik van Riel if (!avc) 3675beb4930SRik van Riel goto out_error_free_anon_vma; 3685c341ee1SRik van Riel 3695c341ee1SRik van Riel /* 370aaf1f990SMiaohe Lin * The root anon_vma's rwsem is the lock actually used when we 3715c341ee1SRik van Riel * lock any of the anon_vmas in this anon_vma tree. 3725c341ee1SRik van Riel */ 3735c341ee1SRik van Riel anon_vma->root = pvma->anon_vma->root; 3747a3ef208SKonstantin Khlebnikov anon_vma->parent = pvma->anon_vma; 37576545066SRik van Riel /* 37601d8b20dSPeter Zijlstra * With refcounts, an anon_vma can stay around longer than the 37701d8b20dSPeter Zijlstra * process it belongs to. The root anon_vma needs to be pinned until 37801d8b20dSPeter Zijlstra * this anon_vma is freed, because the lock lives in the root. 37976545066SRik van Riel */ 38076545066SRik van Riel get_anon_vma(anon_vma->root); 3815beb4930SRik van Riel /* Mark this anon_vma as the one where our new (COWed) pages go. */ 3825beb4930SRik van Riel vma->anon_vma = anon_vma; 3834fc3f1d6SIngo Molnar anon_vma_lock_write(anon_vma); 3845c341ee1SRik van Riel anon_vma_chain_link(vma, avc, anon_vma); 3852555283eSJann Horn anon_vma->parent->num_children++; 38608b52706SKonstantin Khlebnikov anon_vma_unlock_write(anon_vma); 3875beb4930SRik van Riel 3885beb4930SRik van Riel return 0; 3895beb4930SRik van Riel 3905beb4930SRik van Riel out_error_free_anon_vma: 39101d8b20dSPeter Zijlstra put_anon_vma(anon_vma); 3925beb4930SRik van Riel out_error: 3934946d54cSRik van Riel unlink_anon_vmas(vma); 3945beb4930SRik van Riel return -ENOMEM; 3955beb4930SRik van Riel } 3965beb4930SRik van Riel 3975beb4930SRik van Riel void unlink_anon_vmas(struct vm_area_struct *vma) 3985beb4930SRik van Riel { 3995beb4930SRik van Riel struct anon_vma_chain *avc, *next; 400eee2acbaSPeter Zijlstra struct anon_vma *root = NULL; 4015beb4930SRik van Riel 4025c341ee1SRik van Riel /* 4035c341ee1SRik van Riel * Unlink each anon_vma chained to the VMA. This list is ordered 4045c341ee1SRik van Riel * from newest to oldest, ensuring the root anon_vma gets freed last. 4055c341ee1SRik van Riel */ 4065beb4930SRik van Riel list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { 407eee2acbaSPeter Zijlstra struct anon_vma *anon_vma = avc->anon_vma; 408eee2acbaSPeter Zijlstra 409eee2acbaSPeter Zijlstra root = lock_anon_vma_root(root, anon_vma); 410bf181b9fSMichel Lespinasse anon_vma_interval_tree_remove(avc, &anon_vma->rb_root); 411eee2acbaSPeter Zijlstra 412eee2acbaSPeter Zijlstra /* 413eee2acbaSPeter Zijlstra * Leave empty anon_vmas on the list - we'll need 414eee2acbaSPeter Zijlstra * to free them outside the lock. 415eee2acbaSPeter Zijlstra */ 416f808c13fSDavidlohr Bueso if (RB_EMPTY_ROOT(&anon_vma->rb_root.rb_root)) { 4172555283eSJann Horn anon_vma->parent->num_children--; 418eee2acbaSPeter Zijlstra continue; 4197a3ef208SKonstantin Khlebnikov } 420eee2acbaSPeter Zijlstra 421eee2acbaSPeter Zijlstra list_del(&avc->same_vma); 422eee2acbaSPeter Zijlstra anon_vma_chain_free(avc); 423eee2acbaSPeter Zijlstra } 424ee8ab190SLi Xinhai if (vma->anon_vma) { 4252555283eSJann Horn vma->anon_vma->num_active_vmas--; 426ee8ab190SLi Xinhai 427ee8ab190SLi Xinhai /* 428ee8ab190SLi Xinhai * vma would still be needed after unlink, and anon_vma will be prepared 429ee8ab190SLi Xinhai * when handle fault. 430ee8ab190SLi Xinhai */ 431ee8ab190SLi Xinhai vma->anon_vma = NULL; 432ee8ab190SLi Xinhai } 433eee2acbaSPeter Zijlstra unlock_anon_vma_root(root); 434eee2acbaSPeter Zijlstra 435eee2acbaSPeter Zijlstra /* 436eee2acbaSPeter Zijlstra * Iterate the list once more, it now only contains empty and unlinked 437eee2acbaSPeter Zijlstra * anon_vmas, destroy them. Could not do before due to __put_anon_vma() 4385a505085SIngo Molnar * needing to write-acquire the anon_vma->root->rwsem. 439eee2acbaSPeter Zijlstra */ 440eee2acbaSPeter Zijlstra list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { 441eee2acbaSPeter Zijlstra struct anon_vma *anon_vma = avc->anon_vma; 442eee2acbaSPeter Zijlstra 4432555283eSJann Horn VM_WARN_ON(anon_vma->num_children); 4442555283eSJann Horn VM_WARN_ON(anon_vma->num_active_vmas); 445eee2acbaSPeter Zijlstra put_anon_vma(anon_vma); 446eee2acbaSPeter Zijlstra 4475beb4930SRik van Riel list_del(&avc->same_vma); 4485beb4930SRik van Riel anon_vma_chain_free(avc); 4495beb4930SRik van Riel } 4505beb4930SRik van Riel } 4515beb4930SRik van Riel 45251cc5068SAlexey Dobriyan static void anon_vma_ctor(void *data) 4531da177e4SLinus Torvalds { 4541da177e4SLinus Torvalds struct anon_vma *anon_vma = data; 4551da177e4SLinus Torvalds 4565a505085SIngo Molnar init_rwsem(&anon_vma->rwsem); 45783813267SPeter Zijlstra atomic_set(&anon_vma->refcount, 0); 458f808c13fSDavidlohr Bueso anon_vma->rb_root = RB_ROOT_CACHED; 4591da177e4SLinus Torvalds } 4601da177e4SLinus Torvalds 4611da177e4SLinus Torvalds void __init anon_vma_init(void) 4621da177e4SLinus Torvalds { 4631da177e4SLinus Torvalds anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma), 4645f0d5a3aSPaul E. McKenney 0, SLAB_TYPESAFE_BY_RCU|SLAB_PANIC|SLAB_ACCOUNT, 4655d097056SVladimir Davydov anon_vma_ctor); 4665d097056SVladimir Davydov anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain, 4675d097056SVladimir Davydov SLAB_PANIC|SLAB_ACCOUNT); 4681da177e4SLinus Torvalds } 4691da177e4SLinus Torvalds 4701da177e4SLinus Torvalds /* 4716111e4caSPeter Zijlstra * Getting a lock on a stable anon_vma from a page off the LRU is tricky! 4726111e4caSPeter Zijlstra * 4734d8f7418SDavid Hildenbrand * Since there is no serialization what so ever against folio_remove_rmap_*() 474ad8a20cfSMiaohe Lin * the best this function can do is return a refcount increased anon_vma 475ad8a20cfSMiaohe Lin * that might have been relevant to this page. 4766111e4caSPeter Zijlstra * 4776111e4caSPeter Zijlstra * The page might have been remapped to a different anon_vma or the anon_vma 4786111e4caSPeter Zijlstra * returned may already be freed (and even reused). 4796111e4caSPeter Zijlstra * 480bc658c96SPeter Zijlstra * In case it was remapped to a different anon_vma, the new anon_vma will be a 481bc658c96SPeter Zijlstra * child of the old anon_vma, and the anon_vma lifetime rules will therefore 482bc658c96SPeter Zijlstra * ensure that any anon_vma obtained from the page will still be valid for as 483bc658c96SPeter Zijlstra * long as we observe page_mapped() [ hence all those page_mapped() tests ]. 484bc658c96SPeter Zijlstra * 4856111e4caSPeter Zijlstra * All users of this function must be very careful when walking the anon_vma 4866111e4caSPeter Zijlstra * chain and verify that the page in question is indeed mapped in it 4876111e4caSPeter Zijlstra * [ something equivalent to page_mapped_in_vma() ]. 4886111e4caSPeter Zijlstra * 489091e4299SMiles Chen * Since anon_vma's slab is SLAB_TYPESAFE_BY_RCU and we know from 4904d8f7418SDavid Hildenbrand * folio_remove_rmap_*() that the anon_vma pointer from page->mapping is valid 491091e4299SMiles Chen * if there is a mapcount, we can dereference the anon_vma after observing 492091e4299SMiles Chen * those. 493adef4406SAndrea Arcangeli * 494adef4406SAndrea Arcangeli * NOTE: the caller should normally hold folio lock when calling this. If 495adef4406SAndrea Arcangeli * not, the caller needs to double check the anon_vma didn't change after 496adef4406SAndrea Arcangeli * taking the anon_vma lock for either read or write (UFFDIO_MOVE can modify it 497adef4406SAndrea Arcangeli * concurrently without folio lock protection). See folio_lock_anon_vma_read() 498adef4406SAndrea Arcangeli * which has already covered that, and comment above remap_pages(). 4991da177e4SLinus Torvalds */ 50029eea9b5SMatthew Wilcox (Oracle) struct anon_vma *folio_get_anon_vma(struct folio *folio) 5011da177e4SLinus Torvalds { 502746b18d4SPeter Zijlstra struct anon_vma *anon_vma = NULL; 5031da177e4SLinus Torvalds unsigned long anon_mapping; 5041da177e4SLinus Torvalds 5051da177e4SLinus Torvalds rcu_read_lock(); 50629eea9b5SMatthew Wilcox (Oracle) anon_mapping = (unsigned long)READ_ONCE(folio->mapping); 5073ca7b3c5SHugh Dickins if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) 5081da177e4SLinus Torvalds goto out; 50929eea9b5SMatthew Wilcox (Oracle) if (!folio_mapped(folio)) 5101da177e4SLinus Torvalds goto out; 5111da177e4SLinus Torvalds 5121da177e4SLinus Torvalds anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); 513746b18d4SPeter Zijlstra if (!atomic_inc_not_zero(&anon_vma->refcount)) { 514746b18d4SPeter Zijlstra anon_vma = NULL; 515746b18d4SPeter Zijlstra goto out; 516746b18d4SPeter Zijlstra } 517f1819427SHugh Dickins 518f1819427SHugh Dickins /* 51929eea9b5SMatthew Wilcox (Oracle) * If this folio is still mapped, then its anon_vma cannot have been 520746b18d4SPeter Zijlstra * freed. But if it has been unmapped, we have no security against the 521746b18d4SPeter Zijlstra * anon_vma structure being freed and reused (for another anon_vma: 5225f0d5a3aSPaul E. McKenney * SLAB_TYPESAFE_BY_RCU guarantees that - so the atomic_inc_not_zero() 523746b18d4SPeter Zijlstra * above cannot corrupt). 524f1819427SHugh Dickins */ 52529eea9b5SMatthew Wilcox (Oracle) if (!folio_mapped(folio)) { 5267f39dda9SHugh Dickins rcu_read_unlock(); 527746b18d4SPeter Zijlstra put_anon_vma(anon_vma); 5287f39dda9SHugh Dickins return NULL; 529746b18d4SPeter Zijlstra } 5301da177e4SLinus Torvalds out: 5311da177e4SLinus Torvalds rcu_read_unlock(); 532746b18d4SPeter Zijlstra 533746b18d4SPeter Zijlstra return anon_vma; 534746b18d4SPeter Zijlstra } 535746b18d4SPeter Zijlstra 53688c22088SPeter Zijlstra /* 53729eea9b5SMatthew Wilcox (Oracle) * Similar to folio_get_anon_vma() except it locks the anon_vma. 53888c22088SPeter Zijlstra * 53988c22088SPeter Zijlstra * Its a little more complex as it tries to keep the fast path to a single 54088c22088SPeter Zijlstra * atomic op -- the trylock. If we fail the trylock, we fall back to getting a 54129eea9b5SMatthew Wilcox (Oracle) * reference like with folio_get_anon_vma() and then block on the mutex 5426d4675e6SMinchan Kim * on !rwc->try_lock case. 54388c22088SPeter Zijlstra */ 5446d4675e6SMinchan Kim struct anon_vma *folio_lock_anon_vma_read(struct folio *folio, 5456d4675e6SMinchan Kim struct rmap_walk_control *rwc) 546746b18d4SPeter Zijlstra { 54788c22088SPeter Zijlstra struct anon_vma *anon_vma = NULL; 548eee0f252SHugh Dickins struct anon_vma *root_anon_vma; 54988c22088SPeter Zijlstra unsigned long anon_mapping; 550746b18d4SPeter Zijlstra 551880a99b6SAndrea Arcangeli retry: 55288c22088SPeter Zijlstra rcu_read_lock(); 5539595d769SMatthew Wilcox (Oracle) anon_mapping = (unsigned long)READ_ONCE(folio->mapping); 55488c22088SPeter Zijlstra if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) 55588c22088SPeter Zijlstra goto out; 5569595d769SMatthew Wilcox (Oracle) if (!folio_mapped(folio)) 55788c22088SPeter Zijlstra goto out; 55888c22088SPeter Zijlstra 55988c22088SPeter Zijlstra anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); 5604db0c3c2SJason Low root_anon_vma = READ_ONCE(anon_vma->root); 5614fc3f1d6SIngo Molnar if (down_read_trylock(&root_anon_vma->rwsem)) { 56288c22088SPeter Zijlstra /* 563880a99b6SAndrea Arcangeli * folio_move_anon_rmap() might have changed the anon_vma as we 564880a99b6SAndrea Arcangeli * might not hold the folio lock here. 565880a99b6SAndrea Arcangeli */ 566880a99b6SAndrea Arcangeli if (unlikely((unsigned long)READ_ONCE(folio->mapping) != 567880a99b6SAndrea Arcangeli anon_mapping)) { 568880a99b6SAndrea Arcangeli up_read(&root_anon_vma->rwsem); 569880a99b6SAndrea Arcangeli rcu_read_unlock(); 570880a99b6SAndrea Arcangeli goto retry; 571880a99b6SAndrea Arcangeli } 572880a99b6SAndrea Arcangeli 573880a99b6SAndrea Arcangeli /* 5749595d769SMatthew Wilcox (Oracle) * If the folio is still mapped, then this anon_vma is still 575eee0f252SHugh Dickins * its anon_vma, and holding the mutex ensures that it will 576bc658c96SPeter Zijlstra * not go away, see anon_vma_free(). 57788c22088SPeter Zijlstra */ 5789595d769SMatthew Wilcox (Oracle) if (!folio_mapped(folio)) { 5794fc3f1d6SIngo Molnar up_read(&root_anon_vma->rwsem); 58088c22088SPeter Zijlstra anon_vma = NULL; 58188c22088SPeter Zijlstra } 58288c22088SPeter Zijlstra goto out; 58388c22088SPeter Zijlstra } 58488c22088SPeter Zijlstra 5856d4675e6SMinchan Kim if (rwc && rwc->try_lock) { 5866d4675e6SMinchan Kim anon_vma = NULL; 5876d4675e6SMinchan Kim rwc->contended = true; 5886d4675e6SMinchan Kim goto out; 5896d4675e6SMinchan Kim } 5906d4675e6SMinchan Kim 59188c22088SPeter Zijlstra /* trylock failed, we got to sleep */ 59288c22088SPeter Zijlstra if (!atomic_inc_not_zero(&anon_vma->refcount)) { 59388c22088SPeter Zijlstra anon_vma = NULL; 59488c22088SPeter Zijlstra goto out; 59588c22088SPeter Zijlstra } 59688c22088SPeter Zijlstra 5979595d769SMatthew Wilcox (Oracle) if (!folio_mapped(folio)) { 5987f39dda9SHugh Dickins rcu_read_unlock(); 59988c22088SPeter Zijlstra put_anon_vma(anon_vma); 6007f39dda9SHugh Dickins return NULL; 60188c22088SPeter Zijlstra } 60288c22088SPeter Zijlstra 60388c22088SPeter Zijlstra /* we pinned the anon_vma, its safe to sleep */ 60488c22088SPeter Zijlstra rcu_read_unlock(); 6054fc3f1d6SIngo Molnar anon_vma_lock_read(anon_vma); 606746b18d4SPeter Zijlstra 607880a99b6SAndrea Arcangeli /* 608880a99b6SAndrea Arcangeli * folio_move_anon_rmap() might have changed the anon_vma as we might 609880a99b6SAndrea Arcangeli * not hold the folio lock here. 610880a99b6SAndrea Arcangeli */ 611880a99b6SAndrea Arcangeli if (unlikely((unsigned long)READ_ONCE(folio->mapping) != 612880a99b6SAndrea Arcangeli anon_mapping)) { 613880a99b6SAndrea Arcangeli anon_vma_unlock_read(anon_vma); 614880a99b6SAndrea Arcangeli put_anon_vma(anon_vma); 615880a99b6SAndrea Arcangeli anon_vma = NULL; 616880a99b6SAndrea Arcangeli goto retry; 617880a99b6SAndrea Arcangeli } 618880a99b6SAndrea Arcangeli 61988c22088SPeter Zijlstra if (atomic_dec_and_test(&anon_vma->refcount)) { 62088c22088SPeter Zijlstra /* 62188c22088SPeter Zijlstra * Oops, we held the last refcount, release the lock 62288c22088SPeter Zijlstra * and bail -- can't simply use put_anon_vma() because 6234fc3f1d6SIngo Molnar * we'll deadlock on the anon_vma_lock_write() recursion. 62488c22088SPeter Zijlstra */ 6254fc3f1d6SIngo Molnar anon_vma_unlock_read(anon_vma); 62688c22088SPeter Zijlstra __put_anon_vma(anon_vma); 62788c22088SPeter Zijlstra anon_vma = NULL; 62888c22088SPeter Zijlstra } 62988c22088SPeter Zijlstra 63088c22088SPeter Zijlstra return anon_vma; 63188c22088SPeter Zijlstra 63288c22088SPeter Zijlstra out: 63388c22088SPeter Zijlstra rcu_read_unlock(); 634746b18d4SPeter Zijlstra return anon_vma; 63534bbd704SOleg Nesterov } 63634bbd704SOleg Nesterov 63772b252aeSMel Gorman #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH 63872b252aeSMel Gorman /* 63972b252aeSMel Gorman * Flush TLB entries for recently unmapped pages from remote CPUs. It is 64072b252aeSMel Gorman * important if a PTE was dirty when it was unmapped that it's flushed 64172b252aeSMel Gorman * before any IO is initiated on the page to prevent lost writes. Similarly, 64272b252aeSMel Gorman * it must be flushed before freeing to prevent data leakage. 64372b252aeSMel Gorman */ 64472b252aeSMel Gorman void try_to_unmap_flush(void) 64572b252aeSMel Gorman { 64672b252aeSMel Gorman struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; 64772b252aeSMel Gorman 64872b252aeSMel Gorman if (!tlb_ubc->flush_required) 64972b252aeSMel Gorman return; 65072b252aeSMel Gorman 651e73ad5ffSAndy Lutomirski arch_tlbbatch_flush(&tlb_ubc->arch); 65272b252aeSMel Gorman tlb_ubc->flush_required = false; 653d950c947SMel Gorman tlb_ubc->writable = false; 65472b252aeSMel Gorman } 65572b252aeSMel Gorman 656d950c947SMel Gorman /* Flush iff there are potentially writable TLB entries that can race with IO */ 657d950c947SMel Gorman void try_to_unmap_flush_dirty(void) 658d950c947SMel Gorman { 659d950c947SMel Gorman struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; 660d950c947SMel Gorman 661d950c947SMel Gorman if (tlb_ubc->writable) 662d950c947SMel Gorman try_to_unmap_flush(); 663d950c947SMel Gorman } 664d950c947SMel Gorman 6655ee2fa2fSHuang Ying /* 6665ee2fa2fSHuang Ying * Bits 0-14 of mm->tlb_flush_batched record pending generations. 6675ee2fa2fSHuang Ying * Bits 16-30 of mm->tlb_flush_batched bit record flushed generations. 6685ee2fa2fSHuang Ying */ 6695ee2fa2fSHuang Ying #define TLB_FLUSH_BATCH_FLUSHED_SHIFT 16 6705ee2fa2fSHuang Ying #define TLB_FLUSH_BATCH_PENDING_MASK \ 6715ee2fa2fSHuang Ying ((1 << (TLB_FLUSH_BATCH_FLUSHED_SHIFT - 1)) - 1) 6725ee2fa2fSHuang Ying #define TLB_FLUSH_BATCH_PENDING_LARGE \ 6735ee2fa2fSHuang Ying (TLB_FLUSH_BATCH_PENDING_MASK / 2) 6745ee2fa2fSHuang Ying 675f73419bbSBarry Song static void set_tlb_ubc_flush_pending(struct mm_struct *mm, pte_t pteval, 676f73419bbSBarry Song unsigned long uaddr) 67772b252aeSMel Gorman { 67872b252aeSMel Gorman struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; 679bdeb9188SUros Bizjak int batch; 6804d4b6d66SHuang Ying bool writable = pte_dirty(pteval); 6814d4b6d66SHuang Ying 6824d4b6d66SHuang Ying if (!pte_accessible(mm, pteval)) 6834d4b6d66SHuang Ying return; 68472b252aeSMel Gorman 685f73419bbSBarry Song arch_tlbbatch_add_pending(&tlb_ubc->arch, mm, uaddr); 68672b252aeSMel Gorman tlb_ubc->flush_required = true; 687d950c947SMel Gorman 688d950c947SMel Gorman /* 6893ea27719SMel Gorman * Ensure compiler does not re-order the setting of tlb_flush_batched 6903ea27719SMel Gorman * before the PTE is cleared. 6913ea27719SMel Gorman */ 6923ea27719SMel Gorman barrier(); 6935ee2fa2fSHuang Ying batch = atomic_read(&mm->tlb_flush_batched); 6945ee2fa2fSHuang Ying retry: 6955ee2fa2fSHuang Ying if ((batch & TLB_FLUSH_BATCH_PENDING_MASK) > TLB_FLUSH_BATCH_PENDING_LARGE) { 6965ee2fa2fSHuang Ying /* 6975ee2fa2fSHuang Ying * Prevent `pending' from catching up with `flushed' because of 6985ee2fa2fSHuang Ying * overflow. Reset `pending' and `flushed' to be 1 and 0 if 6995ee2fa2fSHuang Ying * `pending' becomes large. 7005ee2fa2fSHuang Ying */ 701bdeb9188SUros Bizjak if (!atomic_try_cmpxchg(&mm->tlb_flush_batched, &batch, 1)) 7025ee2fa2fSHuang Ying goto retry; 7035ee2fa2fSHuang Ying } else { 7045ee2fa2fSHuang Ying atomic_inc(&mm->tlb_flush_batched); 7055ee2fa2fSHuang Ying } 7063ea27719SMel Gorman 7073ea27719SMel Gorman /* 708d950c947SMel Gorman * If the PTE was dirty then it's best to assume it's writable. The 709d950c947SMel Gorman * caller must use try_to_unmap_flush_dirty() or try_to_unmap_flush() 710d950c947SMel Gorman * before the page is queued for IO. 711d950c947SMel Gorman */ 712d950c947SMel Gorman if (writable) 713d950c947SMel Gorman tlb_ubc->writable = true; 71472b252aeSMel Gorman } 71572b252aeSMel Gorman 71672b252aeSMel Gorman /* 71772b252aeSMel Gorman * Returns true if the TLB flush should be deferred to the end of a batch of 71872b252aeSMel Gorman * unmap operations to reduce IPIs. 71972b252aeSMel Gorman */ 72072b252aeSMel Gorman static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) 72172b252aeSMel Gorman { 72272b252aeSMel Gorman if (!(flags & TTU_BATCH_FLUSH)) 72372b252aeSMel Gorman return false; 72472b252aeSMel Gorman 72565c8d30eSAnshuman Khandual return arch_tlbbatch_should_defer(mm); 72672b252aeSMel Gorman } 7273ea27719SMel Gorman 7283ea27719SMel Gorman /* 7293ea27719SMel Gorman * Reclaim unmaps pages under the PTL but do not flush the TLB prior to 7303ea27719SMel Gorman * releasing the PTL if TLB flushes are batched. It's possible for a parallel 7313ea27719SMel Gorman * operation such as mprotect or munmap to race between reclaim unmapping 7323ea27719SMel Gorman * the page and flushing the page. If this race occurs, it potentially allows 7333ea27719SMel Gorman * access to data via a stale TLB entry. Tracking all mm's that have TLB 7343ea27719SMel Gorman * batching in flight would be expensive during reclaim so instead track 7353ea27719SMel Gorman * whether TLB batching occurred in the past and if so then do a flush here 7363ea27719SMel Gorman * if required. This will cost one additional flush per reclaim cycle paid 7373ea27719SMel Gorman * by the first operation at risk such as mprotect and mumap. 7383ea27719SMel Gorman * 7393ea27719SMel Gorman * This must be called under the PTL so that an access to tlb_flush_batched 7403ea27719SMel Gorman * that is potentially a "reclaim vs mprotect/munmap/etc" race will synchronise 7413ea27719SMel Gorman * via the PTL. 7423ea27719SMel Gorman */ 7433ea27719SMel Gorman void flush_tlb_batched_pending(struct mm_struct *mm) 7443ea27719SMel Gorman { 7455ee2fa2fSHuang Ying int batch = atomic_read(&mm->tlb_flush_batched); 7465ee2fa2fSHuang Ying int pending = batch & TLB_FLUSH_BATCH_PENDING_MASK; 7475ee2fa2fSHuang Ying int flushed = batch >> TLB_FLUSH_BATCH_FLUSHED_SHIFT; 7483ea27719SMel Gorman 7495ee2fa2fSHuang Ying if (pending != flushed) { 750db6c1f6fSYicong Yang arch_flush_tlb_batched_pending(mm); 7513ea27719SMel Gorman /* 7525ee2fa2fSHuang Ying * If the new TLB flushing is pending during flushing, leave 7535ee2fa2fSHuang Ying * mm->tlb_flush_batched as is, to avoid losing flushing. 7543ea27719SMel Gorman */ 7555ee2fa2fSHuang Ying atomic_cmpxchg(&mm->tlb_flush_batched, batch, 7565ee2fa2fSHuang Ying pending | (pending << TLB_FLUSH_BATCH_FLUSHED_SHIFT)); 7573ea27719SMel Gorman } 7583ea27719SMel Gorman } 75972b252aeSMel Gorman #else 760f73419bbSBarry Song static void set_tlb_ubc_flush_pending(struct mm_struct *mm, pte_t pteval, 761f73419bbSBarry Song unsigned long uaddr) 76272b252aeSMel Gorman { 76372b252aeSMel Gorman } 76472b252aeSMel Gorman 76572b252aeSMel Gorman static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) 76672b252aeSMel Gorman { 76772b252aeSMel Gorman return false; 76872b252aeSMel Gorman } 76972b252aeSMel Gorman #endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */ 77072b252aeSMel Gorman 7711da177e4SLinus Torvalds /* 772bf89c8c8SHuang Shijie * At what user virtual address is page expected in vma? 773ab941e0fSNaoya Horiguchi * Caller should check the page is actually part of the vma. 7741da177e4SLinus Torvalds */ 7751da177e4SLinus Torvalds unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) 7761da177e4SLinus Torvalds { 777e05b3453SMatthew Wilcox (Oracle) struct folio *folio = page_folio(page); 778e05b3453SMatthew Wilcox (Oracle) if (folio_test_anon(folio)) { 779e05b3453SMatthew Wilcox (Oracle) struct anon_vma *page__anon_vma = folio_anon_vma(folio); 7804829b906SHugh Dickins /* 7814829b906SHugh Dickins * Note: swapoff's unuse_vma() is more efficient with this 7824829b906SHugh Dickins * check, and needs it to match anon_vma when KSM is active. 7834829b906SHugh Dickins */ 7844829b906SHugh Dickins if (!vma->anon_vma || !page__anon_vma || 7854829b906SHugh Dickins vma->anon_vma->root != page__anon_vma->root) 78621d0d443SAndrea Arcangeli return -EFAULT; 78731657170SJue Wang } else if (!vma->vm_file) { 7881da177e4SLinus Torvalds return -EFAULT; 789e05b3453SMatthew Wilcox (Oracle) } else if (vma->vm_file->f_mapping != folio->mapping) { 7901da177e4SLinus Torvalds return -EFAULT; 79131657170SJue Wang } 792494334e4SHugh Dickins 793494334e4SHugh Dickins return vma_address(page, vma); 7941da177e4SLinus Torvalds } 7951da177e4SLinus Torvalds 79650722804SZach O'Keefe /* 79750722804SZach O'Keefe * Returns the actual pmd_t* where we expect 'address' to be mapped from, or 79850722804SZach O'Keefe * NULL if it doesn't exist. No guarantees / checks on what the pmd_t* 79950722804SZach O'Keefe * represents. 80050722804SZach O'Keefe */ 8016219049aSBob Liu pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) 8026219049aSBob Liu { 8036219049aSBob Liu pgd_t *pgd; 804c2febafcSKirill A. Shutemov p4d_t *p4d; 8056219049aSBob Liu pud_t *pud; 8066219049aSBob Liu pmd_t *pmd = NULL; 8076219049aSBob Liu 8086219049aSBob Liu pgd = pgd_offset(mm, address); 8096219049aSBob Liu if (!pgd_present(*pgd)) 8106219049aSBob Liu goto out; 8116219049aSBob Liu 812c2febafcSKirill A. Shutemov p4d = p4d_offset(pgd, address); 813c2febafcSKirill A. Shutemov if (!p4d_present(*p4d)) 814c2febafcSKirill A. Shutemov goto out; 815c2febafcSKirill A. Shutemov 816c2febafcSKirill A. Shutemov pud = pud_offset(p4d, address); 8176219049aSBob Liu if (!pud_present(*pud)) 8186219049aSBob Liu goto out; 8196219049aSBob Liu 8206219049aSBob Liu pmd = pmd_offset(pud, address); 8216219049aSBob Liu out: 8226219049aSBob Liu return pmd; 8236219049aSBob Liu } 8246219049aSBob Liu 825b3ac0413SMatthew Wilcox (Oracle) struct folio_referenced_arg { 8269f32624bSJoonsoo Kim int mapcount; 8279f32624bSJoonsoo Kim int referenced; 8289f32624bSJoonsoo Kim unsigned long vm_flags; 8299f32624bSJoonsoo Kim struct mem_cgroup *memcg; 8309f32624bSJoonsoo Kim }; 8311acbc3f9SYin Fengwei 83281b4082dSNikita Danilov /* 833b3ac0413SMatthew Wilcox (Oracle) * arg: folio_referenced_arg will be passed 8341da177e4SLinus Torvalds */ 8352f031c6fSMatthew Wilcox (Oracle) static bool folio_referenced_one(struct folio *folio, 8362f031c6fSMatthew Wilcox (Oracle) struct vm_area_struct *vma, unsigned long address, void *arg) 8371da177e4SLinus Torvalds { 838b3ac0413SMatthew Wilcox (Oracle) struct folio_referenced_arg *pra = arg; 839b3ac0413SMatthew Wilcox (Oracle) DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); 8408749cfeaSVladimir Davydov int referenced = 0; 8411acbc3f9SYin Fengwei unsigned long start = address, ptes = 0; 8422da28bfdSAndrea Arcangeli 8438eaededeSKirill A. Shutemov while (page_vma_mapped_walk(&pvmw)) { 8448eaededeSKirill A. Shutemov address = pvmw.address; 8452da28bfdSAndrea Arcangeli 8461acbc3f9SYin Fengwei if (vma->vm_flags & VM_LOCKED) { 8471acbc3f9SYin Fengwei if (!folio_test_large(folio) || !pvmw.pte) { 84847d4f3eeSHugh Dickins /* Restore the mlock which got missed */ 8491acbc3f9SYin Fengwei mlock_vma_folio(folio, vma); 8508eaededeSKirill A. Shutemov page_vma_mapped_walk_done(&pvmw); 8519f32624bSJoonsoo Kim pra->vm_flags |= VM_LOCKED; 852e4b82222SMinchan Kim return false; /* To break the loop */ 8532da28bfdSAndrea Arcangeli } 8541acbc3f9SYin Fengwei /* 8551acbc3f9SYin Fengwei * For large folio fully mapped to VMA, will 8561acbc3f9SYin Fengwei * be handled after the pvmw loop. 8571acbc3f9SYin Fengwei * 8581acbc3f9SYin Fengwei * For large folio cross VMA boundaries, it's 8591acbc3f9SYin Fengwei * expected to be picked by page reclaim. But 8601acbc3f9SYin Fengwei * should skip reference of pages which are in 8611acbc3f9SYin Fengwei * the range of VM_LOCKED vma. As page reclaim 8621acbc3f9SYin Fengwei * should just count the reference of pages out 8631acbc3f9SYin Fengwei * the range of VM_LOCKED vma. 8641acbc3f9SYin Fengwei */ 8651acbc3f9SYin Fengwei ptes++; 8661acbc3f9SYin Fengwei pra->mapcount--; 8671acbc3f9SYin Fengwei continue; 8681acbc3f9SYin Fengwei } 8692da28bfdSAndrea Arcangeli 8708eaededeSKirill A. Shutemov if (pvmw.pte) { 871c33c7948SRyan Roberts if (lru_gen_enabled() && 872c33c7948SRyan Roberts pte_young(ptep_get(pvmw.pte))) { 873018ee47fSYu Zhao lru_gen_look_around(&pvmw); 874018ee47fSYu Zhao referenced++; 875018ee47fSYu Zhao } 876018ee47fSYu Zhao 8778eaededeSKirill A. Shutemov if (ptep_clear_flush_young_notify(vma, address, 8788788f678SYu Zhao pvmw.pte)) 8791da177e4SLinus Torvalds referenced++; 8808749cfeaSVladimir Davydov } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { 8818eaededeSKirill A. Shutemov if (pmdp_clear_flush_young_notify(vma, address, 8828eaededeSKirill A. Shutemov pvmw.pmd)) 8838749cfeaSVladimir Davydov referenced++; 8848749cfeaSVladimir Davydov } else { 885b3ac0413SMatthew Wilcox (Oracle) /* unexpected pmd-mapped folio? */ 8868749cfeaSVladimir Davydov WARN_ON_ONCE(1); 8878749cfeaSVladimir Davydov } 8888eaededeSKirill A. Shutemov 8898eaededeSKirill A. Shutemov pra->mapcount--; 8908eaededeSKirill A. Shutemov } 89171e3aac0SAndrea Arcangeli 8921acbc3f9SYin Fengwei if ((vma->vm_flags & VM_LOCKED) && 8931acbc3f9SYin Fengwei folio_test_large(folio) && 8941acbc3f9SYin Fengwei folio_within_vma(folio, vma)) { 8951acbc3f9SYin Fengwei unsigned long s_align, e_align; 8961acbc3f9SYin Fengwei 8971acbc3f9SYin Fengwei s_align = ALIGN_DOWN(start, PMD_SIZE); 8981acbc3f9SYin Fengwei e_align = ALIGN_DOWN(start + folio_size(folio) - 1, PMD_SIZE); 8991acbc3f9SYin Fengwei 9001acbc3f9SYin Fengwei /* folio doesn't cross page table boundary and fully mapped */ 9011acbc3f9SYin Fengwei if ((s_align == e_align) && (ptes == folio_nr_pages(folio))) { 9021acbc3f9SYin Fengwei /* Restore the mlock which got missed */ 9031acbc3f9SYin Fengwei mlock_vma_folio(folio, vma); 9041acbc3f9SYin Fengwei pra->vm_flags |= VM_LOCKED; 9051acbc3f9SYin Fengwei return false; /* To break the loop */ 9061acbc3f9SYin Fengwei } 9071acbc3f9SYin Fengwei } 9081acbc3f9SYin Fengwei 90933c3fc71SVladimir Davydov if (referenced) 910b3ac0413SMatthew Wilcox (Oracle) folio_clear_idle(folio); 911b3ac0413SMatthew Wilcox (Oracle) if (folio_test_clear_young(folio)) 91233c3fc71SVladimir Davydov referenced++; 91333c3fc71SVladimir Davydov 9149f32624bSJoonsoo Kim if (referenced) { 9159f32624bSJoonsoo Kim pra->referenced++; 91647d4f3eeSHugh Dickins pra->vm_flags |= vma->vm_flags & ~VM_LOCKED; 9171da177e4SLinus Torvalds } 9181da177e4SLinus Torvalds 9199f32624bSJoonsoo Kim if (!pra->mapcount) 920e4b82222SMinchan Kim return false; /* To break the loop */ 9219f32624bSJoonsoo Kim 922e4b82222SMinchan Kim return true; 9239f32624bSJoonsoo Kim } 9249f32624bSJoonsoo Kim 925b3ac0413SMatthew Wilcox (Oracle) static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg) 9261da177e4SLinus Torvalds { 927b3ac0413SMatthew Wilcox (Oracle) struct folio_referenced_arg *pra = arg; 9289f32624bSJoonsoo Kim struct mem_cgroup *memcg = pra->memcg; 9291da177e4SLinus Torvalds 9308788f678SYu Zhao /* 9318788f678SYu Zhao * Ignore references from this mapping if it has no recency. If the 9328788f678SYu Zhao * folio has been used in another mapping, we will catch it; if this 9338788f678SYu Zhao * other mapping is already gone, the unmap path will have set the 9348788f678SYu Zhao * referenced flag or activated the folio in zap_pte_range(). 9358788f678SYu Zhao */ 9368788f678SYu Zhao if (!vma_has_recency(vma)) 9378788f678SYu Zhao return true; 9388788f678SYu Zhao 9398788f678SYu Zhao /* 9408788f678SYu Zhao * If we are reclaiming on behalf of a cgroup, skip counting on behalf 9418788f678SYu Zhao * of references from different cgroups. 9428788f678SYu Zhao */ 9438788f678SYu Zhao if (memcg && !mm_match_cgroup(vma->vm_mm, memcg)) 9449f32624bSJoonsoo Kim return true; 9451da177e4SLinus Torvalds 9469f32624bSJoonsoo Kim return false; 9471da177e4SLinus Torvalds } 9481da177e4SLinus Torvalds 9491da177e4SLinus Torvalds /** 950b3ac0413SMatthew Wilcox (Oracle) * folio_referenced() - Test if the folio was referenced. 951b3ac0413SMatthew Wilcox (Oracle) * @folio: The folio to test. 952b3ac0413SMatthew Wilcox (Oracle) * @is_locked: Caller holds lock on the folio. 95372835c86SJohannes Weiner * @memcg: target memory cgroup 954b3ac0413SMatthew Wilcox (Oracle) * @vm_flags: A combination of all the vma->vm_flags which referenced the folio. 9551da177e4SLinus Torvalds * 956b3ac0413SMatthew Wilcox (Oracle) * Quick test_and_clear_referenced for all mappings of a folio, 957b3ac0413SMatthew Wilcox (Oracle) * 9586d4675e6SMinchan Kim * Return: The number of mappings which referenced the folio. Return -1 if 9596d4675e6SMinchan Kim * the function bailed out due to rmap lock contention. 9601da177e4SLinus Torvalds */ 961b3ac0413SMatthew Wilcox (Oracle) int folio_referenced(struct folio *folio, int is_locked, 962b3ac0413SMatthew Wilcox (Oracle) struct mem_cgroup *memcg, unsigned long *vm_flags) 9631da177e4SLinus Torvalds { 9645ad64688SHugh Dickins int we_locked = 0; 965b3ac0413SMatthew Wilcox (Oracle) struct folio_referenced_arg pra = { 966b3ac0413SMatthew Wilcox (Oracle) .mapcount = folio_mapcount(folio), 9679f32624bSJoonsoo Kim .memcg = memcg, 9689f32624bSJoonsoo Kim }; 9699f32624bSJoonsoo Kim struct rmap_walk_control rwc = { 970b3ac0413SMatthew Wilcox (Oracle) .rmap_one = folio_referenced_one, 9719f32624bSJoonsoo Kim .arg = (void *)&pra, 9722f031c6fSMatthew Wilcox (Oracle) .anon_lock = folio_lock_anon_vma_read, 9736d4675e6SMinchan Kim .try_lock = true, 9748788f678SYu Zhao .invalid_vma = invalid_folio_referenced_vma, 9759f32624bSJoonsoo Kim }; 9761da177e4SLinus Torvalds 9776fe6b7e3SWu Fengguang *vm_flags = 0; 978059d8442SHuang Shijie if (!pra.mapcount) 9799f32624bSJoonsoo Kim return 0; 9809f32624bSJoonsoo Kim 981b3ac0413SMatthew Wilcox (Oracle) if (!folio_raw_mapping(folio)) 9829f32624bSJoonsoo Kim return 0; 9839f32624bSJoonsoo Kim 984b3ac0413SMatthew Wilcox (Oracle) if (!is_locked && (!folio_test_anon(folio) || folio_test_ksm(folio))) { 985b3ac0413SMatthew Wilcox (Oracle) we_locked = folio_trylock(folio); 9869f32624bSJoonsoo Kim if (!we_locked) 9879f32624bSJoonsoo Kim return 1; 9885ad64688SHugh Dickins } 9899f32624bSJoonsoo Kim 9902f031c6fSMatthew Wilcox (Oracle) rmap_walk(folio, &rwc); 9919f32624bSJoonsoo Kim *vm_flags = pra.vm_flags; 9929f32624bSJoonsoo Kim 9935ad64688SHugh Dickins if (we_locked) 994b3ac0413SMatthew Wilcox (Oracle) folio_unlock(folio); 9959f32624bSJoonsoo Kim 9966d4675e6SMinchan Kim return rwc.contended ? -1 : pra.referenced; 9971da177e4SLinus Torvalds } 9981da177e4SLinus Torvalds 9996a8e0596SMuchun Song static int page_vma_mkclean_one(struct page_vma_mapped_walk *pvmw) 1000d08b3851SPeter Zijlstra { 10016a8e0596SMuchun Song int cleaned = 0; 10026a8e0596SMuchun Song struct vm_area_struct *vma = pvmw->vma; 1003ac46d4f3SJérôme Glisse struct mmu_notifier_range range; 10046a8e0596SMuchun Song unsigned long address = pvmw->address; 1005d08b3851SPeter Zijlstra 1006369ea824SJérôme Glisse /* 1007369ea824SJérôme Glisse * We have to assume the worse case ie pmd for invalidation. Note that 1008e83c09a2SMatthew Wilcox (Oracle) * the folio can not be freed from this function. 1009369ea824SJérôme Glisse */ 10107d4a8be0SAlistair Popple mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE, 0, 10117d4a8be0SAlistair Popple vma->vm_mm, address, vma_address_end(pvmw)); 1012ac46d4f3SJérôme Glisse mmu_notifier_invalidate_range_start(&range); 1013369ea824SJérôme Glisse 10146a8e0596SMuchun Song while (page_vma_mapped_walk(pvmw)) { 1015f27176cfSKirill A. Shutemov int ret = 0; 1016369ea824SJérôme Glisse 10176a8e0596SMuchun Song address = pvmw->address; 10186a8e0596SMuchun Song if (pvmw->pte) { 10196a8e0596SMuchun Song pte_t *pte = pvmw->pte; 1020c33c7948SRyan Roberts pte_t entry = ptep_get(pte); 1021f27176cfSKirill A. Shutemov 1022c33c7948SRyan Roberts if (!pte_dirty(entry) && !pte_write(entry)) 1023f27176cfSKirill A. Shutemov continue; 1024d08b3851SPeter Zijlstra 1025c33c7948SRyan Roberts flush_cache_page(vma, address, pte_pfn(entry)); 1026785373b4SLinus Torvalds entry = ptep_clear_flush(vma, address, pte); 1027d08b3851SPeter Zijlstra entry = pte_wrprotect(entry); 1028c2fda5feSPeter Zijlstra entry = pte_mkclean(entry); 1029785373b4SLinus Torvalds set_pte_at(vma->vm_mm, address, pte, entry); 1030d08b3851SPeter Zijlstra ret = 1; 1031f27176cfSKirill A. Shutemov } else { 1032396bcc52SMatthew Wilcox (Oracle) #ifdef CONFIG_TRANSPARENT_HUGEPAGE 10336a8e0596SMuchun Song pmd_t *pmd = pvmw->pmd; 1034f27176cfSKirill A. Shutemov pmd_t entry; 1035d08b3851SPeter Zijlstra 1036f27176cfSKirill A. Shutemov if (!pmd_dirty(*pmd) && !pmd_write(*pmd)) 1037f27176cfSKirill A. Shutemov continue; 1038f27176cfSKirill A. Shutemov 10397f9c9b60SMuchun Song flush_cache_range(vma, address, 10407f9c9b60SMuchun Song address + HPAGE_PMD_SIZE); 1041024eee0eSAneesh Kumar K.V entry = pmdp_invalidate(vma, address, pmd); 1042f27176cfSKirill A. Shutemov entry = pmd_wrprotect(entry); 1043f27176cfSKirill A. Shutemov entry = pmd_mkclean(entry); 1044785373b4SLinus Torvalds set_pmd_at(vma->vm_mm, address, pmd, entry); 1045f27176cfSKirill A. Shutemov ret = 1; 1046f27176cfSKirill A. Shutemov #else 1047e83c09a2SMatthew Wilcox (Oracle) /* unexpected pmd-mapped folio? */ 1048f27176cfSKirill A. Shutemov WARN_ON_ONCE(1); 1049f27176cfSKirill A. Shutemov #endif 1050f27176cfSKirill A. Shutemov } 10512ec74c3eSSagi Grimberg 10520f10851eSJérôme Glisse if (ret) 10536a8e0596SMuchun Song cleaned++; 10549853a407SJoonsoo Kim } 1055f27176cfSKirill A. Shutemov 1056ac46d4f3SJérôme Glisse mmu_notifier_invalidate_range_end(&range); 1057369ea824SJérôme Glisse 10586a8e0596SMuchun Song return cleaned; 10596a8e0596SMuchun Song } 10606a8e0596SMuchun Song 10616a8e0596SMuchun Song static bool page_mkclean_one(struct folio *folio, struct vm_area_struct *vma, 10626a8e0596SMuchun Song unsigned long address, void *arg) 10636a8e0596SMuchun Song { 10646a8e0596SMuchun Song DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC); 10656a8e0596SMuchun Song int *cleaned = arg; 10666a8e0596SMuchun Song 10676a8e0596SMuchun Song *cleaned += page_vma_mkclean_one(&pvmw); 10686a8e0596SMuchun Song 1069e4b82222SMinchan Kim return true; 1070d08b3851SPeter Zijlstra } 1071d08b3851SPeter Zijlstra 10729853a407SJoonsoo Kim static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg) 1073d08b3851SPeter Zijlstra { 10749853a407SJoonsoo Kim if (vma->vm_flags & VM_SHARED) 1075871beb8cSFengguang Wu return false; 1076d08b3851SPeter Zijlstra 1077871beb8cSFengguang Wu return true; 1078d08b3851SPeter Zijlstra } 1079d08b3851SPeter Zijlstra 1080d9c08e22SMatthew Wilcox (Oracle) int folio_mkclean(struct folio *folio) 1081d08b3851SPeter Zijlstra { 10829853a407SJoonsoo Kim int cleaned = 0; 10839853a407SJoonsoo Kim struct address_space *mapping; 10849853a407SJoonsoo Kim struct rmap_walk_control rwc = { 10859853a407SJoonsoo Kim .arg = (void *)&cleaned, 10869853a407SJoonsoo Kim .rmap_one = page_mkclean_one, 10879853a407SJoonsoo Kim .invalid_vma = invalid_mkclean_vma, 10889853a407SJoonsoo Kim }; 1089d08b3851SPeter Zijlstra 1090d9c08e22SMatthew Wilcox (Oracle) BUG_ON(!folio_test_locked(folio)); 1091d08b3851SPeter Zijlstra 1092d9c08e22SMatthew Wilcox (Oracle) if (!folio_mapped(folio)) 10939853a407SJoonsoo Kim return 0; 1094d08b3851SPeter Zijlstra 1095d9c08e22SMatthew Wilcox (Oracle) mapping = folio_mapping(folio); 10969853a407SJoonsoo Kim if (!mapping) 10979853a407SJoonsoo Kim return 0; 10989853a407SJoonsoo Kim 10992f031c6fSMatthew Wilcox (Oracle) rmap_walk(folio, &rwc); 11009853a407SJoonsoo Kim 11019853a407SJoonsoo Kim return cleaned; 1102d08b3851SPeter Zijlstra } 1103d9c08e22SMatthew Wilcox (Oracle) EXPORT_SYMBOL_GPL(folio_mkclean); 1104d08b3851SPeter Zijlstra 11051da177e4SLinus Torvalds /** 11066a8e0596SMuchun Song * pfn_mkclean_range - Cleans the PTEs (including PMDs) mapped with range of 11076a8e0596SMuchun Song * [@pfn, @pfn + @nr_pages) at the specific offset (@pgoff) 11086a8e0596SMuchun Song * within the @vma of shared mappings. And since clean PTEs 11096a8e0596SMuchun Song * should also be readonly, write protects them too. 11106a8e0596SMuchun Song * @pfn: start pfn. 11116a8e0596SMuchun Song * @nr_pages: number of physically contiguous pages srarting with @pfn. 11126a8e0596SMuchun Song * @pgoff: page offset that the @pfn mapped with. 11136a8e0596SMuchun Song * @vma: vma that @pfn mapped within. 11146a8e0596SMuchun Song * 11156a8e0596SMuchun Song * Returns the number of cleaned PTEs (including PMDs). 11166a8e0596SMuchun Song */ 11176a8e0596SMuchun Song int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff, 11186a8e0596SMuchun Song struct vm_area_struct *vma) 11196a8e0596SMuchun Song { 11206a8e0596SMuchun Song struct page_vma_mapped_walk pvmw = { 11216a8e0596SMuchun Song .pfn = pfn, 11226a8e0596SMuchun Song .nr_pages = nr_pages, 11236a8e0596SMuchun Song .pgoff = pgoff, 11246a8e0596SMuchun Song .vma = vma, 11256a8e0596SMuchun Song .flags = PVMW_SYNC, 11266a8e0596SMuchun Song }; 11276a8e0596SMuchun Song 11286a8e0596SMuchun Song if (invalid_mkclean_vma(vma, NULL)) 11296a8e0596SMuchun Song return 0; 11306a8e0596SMuchun Song 11316a8e0596SMuchun Song pvmw.address = vma_pgoff_address(pgoff, nr_pages, vma); 11326a8e0596SMuchun Song VM_BUG_ON_VMA(pvmw.address == -EFAULT, vma); 11336a8e0596SMuchun Song 11346a8e0596SMuchun Song return page_vma_mkclean_one(&pvmw); 11356a8e0596SMuchun Song } 11366a8e0596SMuchun Song 1137b14224fbSMatthew Wilcox (Oracle) int folio_total_mapcount(struct folio *folio) 11389bd3155eSHugh Dickins { 1139b14224fbSMatthew Wilcox (Oracle) int mapcount = folio_entire_mapcount(folio); 1140b14224fbSMatthew Wilcox (Oracle) int nr_pages; 1141cb67f428SHugh Dickins int i; 1142cb67f428SHugh Dickins 1143b14224fbSMatthew Wilcox (Oracle) /* In the common case, avoid the loop when no pages mapped by PTE */ 1144eec20426SMatthew Wilcox (Oracle) if (folio_nr_pages_mapped(folio) == 0) 1145be5ef2d9SHugh Dickins return mapcount; 1146be5ef2d9SHugh Dickins /* 1147b14224fbSMatthew Wilcox (Oracle) * Add all the PTE mappings of those pages mapped by PTE. 1148b14224fbSMatthew Wilcox (Oracle) * Limit the loop to folio_nr_pages_mapped()? 1149be5ef2d9SHugh Dickins * Perhaps: given all the raciness, that may be a good or a bad idea. 1150be5ef2d9SHugh Dickins */ 1151b14224fbSMatthew Wilcox (Oracle) nr_pages = folio_nr_pages(folio); 1152b14224fbSMatthew Wilcox (Oracle) for (i = 0; i < nr_pages; i++) 1153b14224fbSMatthew Wilcox (Oracle) mapcount += atomic_read(&folio_page(folio, i)->_mapcount); 1154be5ef2d9SHugh Dickins 1155be5ef2d9SHugh Dickins /* But each of those _mapcounts was based on -1 */ 1156b14224fbSMatthew Wilcox (Oracle) mapcount += nr_pages; 1157be5ef2d9SHugh Dickins return mapcount; 1158cb67f428SHugh Dickins } 1159cb67f428SHugh Dickins 116096fd7495SDavid Hildenbrand static __always_inline unsigned int __folio_add_rmap(struct folio *folio, 116196fd7495SDavid Hildenbrand struct page *page, int nr_pages, enum rmap_level level, 116296fd7495SDavid Hildenbrand int *nr_pmdmapped) 116396fd7495SDavid Hildenbrand { 116496fd7495SDavid Hildenbrand atomic_t *mapped = &folio->_nr_pages_mapped; 116596fd7495SDavid Hildenbrand int first, nr = 0; 116696fd7495SDavid Hildenbrand 116796fd7495SDavid Hildenbrand __folio_rmap_sanity_checks(folio, page, nr_pages, level); 116896fd7495SDavid Hildenbrand 116996fd7495SDavid Hildenbrand switch (level) { 117096fd7495SDavid Hildenbrand case RMAP_LEVEL_PTE: 117196fd7495SDavid Hildenbrand do { 117296fd7495SDavid Hildenbrand first = atomic_inc_and_test(&page->_mapcount); 117396fd7495SDavid Hildenbrand if (first && folio_test_large(folio)) { 117496fd7495SDavid Hildenbrand first = atomic_inc_return_relaxed(mapped); 1175e78a13fdSDavid Hildenbrand first = (first < ENTIRELY_MAPPED); 117696fd7495SDavid Hildenbrand } 117796fd7495SDavid Hildenbrand 117896fd7495SDavid Hildenbrand if (first) 117996fd7495SDavid Hildenbrand nr++; 118096fd7495SDavid Hildenbrand } while (page++, --nr_pages > 0); 118196fd7495SDavid Hildenbrand break; 118296fd7495SDavid Hildenbrand case RMAP_LEVEL_PMD: 118396fd7495SDavid Hildenbrand first = atomic_inc_and_test(&folio->_entire_mapcount); 118496fd7495SDavid Hildenbrand if (first) { 1185e78a13fdSDavid Hildenbrand nr = atomic_add_return_relaxed(ENTIRELY_MAPPED, mapped); 1186e78a13fdSDavid Hildenbrand if (likely(nr < ENTIRELY_MAPPED + ENTIRELY_MAPPED)) { 118796fd7495SDavid Hildenbrand *nr_pmdmapped = folio_nr_pages(folio); 118896fd7495SDavid Hildenbrand nr = *nr_pmdmapped - (nr & FOLIO_PAGES_MAPPED); 118996fd7495SDavid Hildenbrand /* Raced ahead of a remove and another add? */ 119096fd7495SDavid Hildenbrand if (unlikely(nr < 0)) 119196fd7495SDavid Hildenbrand nr = 0; 119296fd7495SDavid Hildenbrand } else { 1193e78a13fdSDavid Hildenbrand /* Raced ahead of a remove of ENTIRELY_MAPPED */ 119496fd7495SDavid Hildenbrand nr = 0; 119596fd7495SDavid Hildenbrand } 119696fd7495SDavid Hildenbrand } 119796fd7495SDavid Hildenbrand break; 119896fd7495SDavid Hildenbrand } 119996fd7495SDavid Hildenbrand return nr; 120096fd7495SDavid Hildenbrand } 120196fd7495SDavid Hildenbrand 12026a8e0596SMuchun Song /** 120306968625SDavid Hildenbrand * folio_move_anon_rmap - move a folio to our anon_vma 120406968625SDavid Hildenbrand * @folio: The folio to move to our anon_vma 120506968625SDavid Hildenbrand * @vma: The vma the folio belongs to 1206c44b6743SRik van Riel * 120706968625SDavid Hildenbrand * When a folio belongs exclusively to one process after a COW event, 120806968625SDavid Hildenbrand * that folio can be moved into the anon_vma that belongs to just that 120906968625SDavid Hildenbrand * process, so the rmap code will not search the parent or sibling processes. 1210c44b6743SRik van Riel */ 121106968625SDavid Hildenbrand void folio_move_anon_rmap(struct folio *folio, struct vm_area_struct *vma) 1212c44b6743SRik van Riel { 1213595af4c9SMatthew Wilcox (Oracle) void *anon_vma = vma->anon_vma; 1214c44b6743SRik van Riel 1215595af4c9SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 121681d1b09cSSasha Levin VM_BUG_ON_VMA(!anon_vma, vma); 1217c44b6743SRik van Riel 1218595af4c9SMatthew Wilcox (Oracle) anon_vma += PAGE_MAPPING_ANON; 1219414e2fb8SVladimir Davydov /* 1220414e2fb8SVladimir Davydov * Ensure that anon_vma and the PAGE_MAPPING_ANON bit are written 1221b3ac0413SMatthew Wilcox (Oracle) * simultaneously, so a concurrent reader (eg folio_referenced()'s 1222b3ac0413SMatthew Wilcox (Oracle) * folio_test_anon()) will not see one without the other. 1223414e2fb8SVladimir Davydov */ 1224595af4c9SMatthew Wilcox (Oracle) WRITE_ONCE(folio->mapping, anon_vma); 1225c44b6743SRik van Riel } 1226c44b6743SRik van Riel 1227c44b6743SRik van Riel /** 1228c66db8c0SDavid Hildenbrand * __folio_set_anon - set up a new anonymous rmap for a folio 1229c66db8c0SDavid Hildenbrand * @folio: The folio to set up the new anonymous rmap for. 1230c66db8c0SDavid Hildenbrand * @vma: VM area to add the folio to. 12314e1c1975SAndi Kleen * @address: User virtual address of the mapping 1232c66db8c0SDavid Hildenbrand * @exclusive: Whether the folio is exclusive to the process. 12331da177e4SLinus Torvalds */ 1234c66db8c0SDavid Hildenbrand static void __folio_set_anon(struct folio *folio, struct vm_area_struct *vma, 1235c66db8c0SDavid Hildenbrand unsigned long address, bool exclusive) 12361da177e4SLinus Torvalds { 1237e8a03febSRik van Riel struct anon_vma *anon_vma = vma->anon_vma; 12382822c1aaSNick Piggin 1239e8a03febSRik van Riel BUG_ON(!anon_vma); 1240ea90002bSLinus Torvalds 1241ea90002bSLinus Torvalds /* 1242c66db8c0SDavid Hildenbrand * If the folio isn't exclusive to this vma, we must use the _oldest_ 1243c66db8c0SDavid Hildenbrand * possible anon_vma for the folio mapping! 1244ea90002bSLinus Torvalds */ 12454e1c1975SAndi Kleen if (!exclusive) 1246288468c3SAndrea Arcangeli anon_vma = anon_vma->root; 1247ea90002bSLinus Torvalds 124816f5e707SAlex Shi /* 12495b4bd90fSMatthew Wilcox (Oracle) * page_idle does a lockless/optimistic rmap scan on folio->mapping. 125016f5e707SAlex Shi * Make sure the compiler doesn't split the stores of anon_vma and 125116f5e707SAlex Shi * the PAGE_MAPPING_ANON type identifier, otherwise the rmap code 125216f5e707SAlex Shi * could mistake the mapping for a struct address_space and crash. 125316f5e707SAlex Shi */ 12541da177e4SLinus Torvalds anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 12555b4bd90fSMatthew Wilcox (Oracle) WRITE_ONCE(folio->mapping, (struct address_space *) anon_vma); 12565b4bd90fSMatthew Wilcox (Oracle) folio->index = linear_page_index(vma, address); 12571da177e4SLinus Torvalds } 12589617d95eSNick Piggin 12599617d95eSNick Piggin /** 126043d8eac4SRandy Dunlap * __page_check_anon_rmap - sanity check anonymous rmap addition 1261dba438bdSMatthew Wilcox (Oracle) * @folio: The folio containing @page. 1262dba438bdSMatthew Wilcox (Oracle) * @page: the page to check the mapping of 1263c97a9e10SNick Piggin * @vma: the vm area in which the mapping is added 1264c97a9e10SNick Piggin * @address: the user virtual address mapped 1265c97a9e10SNick Piggin */ 1266dba438bdSMatthew Wilcox (Oracle) static void __page_check_anon_rmap(struct folio *folio, struct page *page, 1267c97a9e10SNick Piggin struct vm_area_struct *vma, unsigned long address) 1268c97a9e10SNick Piggin { 1269c97a9e10SNick Piggin /* 1270c97a9e10SNick Piggin * The page's anon-rmap details (mapping and index) are guaranteed to 1271c97a9e10SNick Piggin * be set up correctly at this point. 1272c97a9e10SNick Piggin * 127384f0169eSDavid Hildenbrand * We have exclusion against folio_add_anon_rmap_*() because the caller 127490aaca85SMiaohe Lin * always holds the page locked. 1275c97a9e10SNick Piggin * 1276cb9089baSMatthew Wilcox (Oracle) * We have exclusion against folio_add_new_anon_rmap because those pages 1277c97a9e10SNick Piggin * are initially only visible via the pagetables, and the pte is locked 1278cb9089baSMatthew Wilcox (Oracle) * over the call to folio_add_new_anon_rmap. 1279c97a9e10SNick Piggin */ 1280e05b3453SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(folio_anon_vma(folio)->root != vma->anon_vma->root, 1281e05b3453SMatthew Wilcox (Oracle) folio); 128230c46382SYang Shi VM_BUG_ON_PAGE(page_to_pgoff(page) != linear_page_index(vma, address), 128330c46382SYang Shi page); 1284c97a9e10SNick Piggin } 1285c97a9e10SNick Piggin 12868bd51300SDavid Hildenbrand static __always_inline void __folio_add_anon_rmap(struct folio *folio, 12878bd51300SDavid Hildenbrand struct page *page, int nr_pages, struct vm_area_struct *vma, 12888bd51300SDavid Hildenbrand unsigned long address, rmap_t flags, enum rmap_level level) 12898bd51300SDavid Hildenbrand { 12908bd51300SDavid Hildenbrand int i, nr, nr_pmdmapped = 0; 12918bd51300SDavid Hildenbrand 12928bd51300SDavid Hildenbrand nr = __folio_add_rmap(folio, page, nr_pages, level, &nr_pmdmapped); 12939bd3155eSHugh Dickins if (nr_pmdmapped) 1294ee0800c2SMatthew Wilcox (Oracle) __lruvec_stat_mod_folio(folio, NR_ANON_THPS, nr_pmdmapped); 12959bd3155eSHugh Dickins if (nr) 1296ee0800c2SMatthew Wilcox (Oracle) __lruvec_stat_mod_folio(folio, NR_ANON_MAPPED, nr); 12975ad64688SHugh Dickins 1298c5c54003SDavid Hildenbrand if (unlikely(!folio_test_anon(folio))) { 1299c5c54003SDavid Hildenbrand VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); 1300a1f34ee1SDavid Hildenbrand /* 1301a1f34ee1SDavid Hildenbrand * For a PTE-mapped large folio, we only know that the single 1302a1f34ee1SDavid Hildenbrand * PTE is exclusive. Further, __folio_set_anon() might not get 1303a1f34ee1SDavid Hildenbrand * folio->index right when not given the address of the head 1304a1f34ee1SDavid Hildenbrand * page. 1305a1f34ee1SDavid Hildenbrand */ 13068bd51300SDavid Hildenbrand VM_WARN_ON_FOLIO(folio_test_large(folio) && 13078bd51300SDavid Hildenbrand level != RMAP_LEVEL_PMD, folio); 1308c66db8c0SDavid Hildenbrand __folio_set_anon(folio, vma, address, 130914f9135dSDavid Hildenbrand !!(flags & RMAP_EXCLUSIVE)); 1310c5c54003SDavid Hildenbrand } else if (likely(!folio_test_ksm(folio))) { 1311dba438bdSMatthew Wilcox (Oracle) __page_check_anon_rmap(folio, page, vma, address); 1312c7c3dec1SJohannes Weiner } 13138bd51300SDavid Hildenbrand 13148bd51300SDavid Hildenbrand if (flags & RMAP_EXCLUSIVE) { 13158bd51300SDavid Hildenbrand switch (level) { 13168bd51300SDavid Hildenbrand case RMAP_LEVEL_PTE: 13178bd51300SDavid Hildenbrand for (i = 0; i < nr_pages; i++) 13188bd51300SDavid Hildenbrand SetPageAnonExclusive(page + i); 13198bd51300SDavid Hildenbrand break; 13208bd51300SDavid Hildenbrand case RMAP_LEVEL_PMD: 1321c66db8c0SDavid Hildenbrand SetPageAnonExclusive(page); 13228bd51300SDavid Hildenbrand break; 13238bd51300SDavid Hildenbrand } 13248bd51300SDavid Hildenbrand } 13258bd51300SDavid Hildenbrand for (i = 0; i < nr_pages; i++) { 13268bd51300SDavid Hildenbrand struct page *cur_page = page + i; 13278bd51300SDavid Hildenbrand 1328132b180fSDavid Hildenbrand /* While PTE-mapping a THP we have a PMD and a PTE mapping. */ 13298bd51300SDavid Hildenbrand VM_WARN_ON_FOLIO((atomic_read(&cur_page->_mapcount) > 0 || 13308bd51300SDavid Hildenbrand (folio_test_large(folio) && 13318bd51300SDavid Hildenbrand folio_entire_mapcount(folio) > 1)) && 13328bd51300SDavid Hildenbrand PageAnonExclusive(cur_page), folio); 13338bd51300SDavid Hildenbrand } 1334cea86fe2SHugh Dickins 13351acbc3f9SYin Fengwei /* 13361acbc3f9SYin Fengwei * For large folio, only mlock it if it's fully mapped to VMA. It's 13371acbc3f9SYin Fengwei * not easy to check whether the large folio is fully mapped to VMA 13381acbc3f9SYin Fengwei * here. Only mlock normal 4K folio and leave page reclaim to handle 13391acbc3f9SYin Fengwei * large folio. 13401acbc3f9SYin Fengwei */ 13411acbc3f9SYin Fengwei if (!folio_test_large(folio)) 13421acbc3f9SYin Fengwei mlock_vma_folio(folio, vma); 13431da177e4SLinus Torvalds } 13441da177e4SLinus Torvalds 134543d8eac4SRandy Dunlap /** 13468bd51300SDavid Hildenbrand * folio_add_anon_rmap_ptes - add PTE mappings to a page range of an anon folio 13478bd51300SDavid Hildenbrand * @folio: The folio to add the mappings to 13488bd51300SDavid Hildenbrand * @page: The first page to add 13498bd51300SDavid Hildenbrand * @nr_pages: The number of pages which will be mapped 13508bd51300SDavid Hildenbrand * @vma: The vm area in which the mappings are added 13518bd51300SDavid Hildenbrand * @address: The user virtual address of the first page to map 13528bd51300SDavid Hildenbrand * @flags: The rmap flags 13538bd51300SDavid Hildenbrand * 13548bd51300SDavid Hildenbrand * The page range of folio is defined by [first_page, first_page + nr_pages) 13558bd51300SDavid Hildenbrand * 13568bd51300SDavid Hildenbrand * The caller needs to hold the page table lock, and the page must be locked in 13578bd51300SDavid Hildenbrand * the anon_vma case: to serialize mapping,index checking after setting, 13588bd51300SDavid Hildenbrand * and to ensure that an anon folio is not being upgraded racily to a KSM folio 13598bd51300SDavid Hildenbrand * (but KSM folios are never downgraded). 13608bd51300SDavid Hildenbrand */ 13618bd51300SDavid Hildenbrand void folio_add_anon_rmap_ptes(struct folio *folio, struct page *page, 13628bd51300SDavid Hildenbrand int nr_pages, struct vm_area_struct *vma, unsigned long address, 13638bd51300SDavid Hildenbrand rmap_t flags) 13648bd51300SDavid Hildenbrand { 13658bd51300SDavid Hildenbrand __folio_add_anon_rmap(folio, page, nr_pages, vma, address, flags, 13668bd51300SDavid Hildenbrand RMAP_LEVEL_PTE); 13678bd51300SDavid Hildenbrand } 13688bd51300SDavid Hildenbrand 13698bd51300SDavid Hildenbrand /** 13708bd51300SDavid Hildenbrand * folio_add_anon_rmap_pmd - add a PMD mapping to a page range of an anon folio 13718bd51300SDavid Hildenbrand * @folio: The folio to add the mapping to 13728bd51300SDavid Hildenbrand * @page: The first page to add 13738bd51300SDavid Hildenbrand * @vma: The vm area in which the mapping is added 13748bd51300SDavid Hildenbrand * @address: The user virtual address of the first page to map 13758bd51300SDavid Hildenbrand * @flags: The rmap flags 13768bd51300SDavid Hildenbrand * 13778bd51300SDavid Hildenbrand * The page range of folio is defined by [first_page, first_page + HPAGE_PMD_NR) 13788bd51300SDavid Hildenbrand * 13798bd51300SDavid Hildenbrand * The caller needs to hold the page table lock, and the page must be locked in 13808bd51300SDavid Hildenbrand * the anon_vma case: to serialize mapping,index checking after setting. 13818bd51300SDavid Hildenbrand */ 13828bd51300SDavid Hildenbrand void folio_add_anon_rmap_pmd(struct folio *folio, struct page *page, 13838bd51300SDavid Hildenbrand struct vm_area_struct *vma, unsigned long address, rmap_t flags) 13848bd51300SDavid Hildenbrand { 13858bd51300SDavid Hildenbrand #ifdef CONFIG_TRANSPARENT_HUGEPAGE 13868bd51300SDavid Hildenbrand __folio_add_anon_rmap(folio, page, HPAGE_PMD_NR, vma, address, flags, 13878bd51300SDavid Hildenbrand RMAP_LEVEL_PMD); 13888bd51300SDavid Hildenbrand #else 13898bd51300SDavid Hildenbrand WARN_ON_ONCE(true); 13908bd51300SDavid Hildenbrand #endif 13918bd51300SDavid Hildenbrand } 13928bd51300SDavid Hildenbrand 13938bd51300SDavid Hildenbrand /** 13944d510f3dSMatthew Wilcox (Oracle) * folio_add_new_anon_rmap - Add mapping to a new anonymous folio. 13954d510f3dSMatthew Wilcox (Oracle) * @folio: The folio to add the mapping to. 13969617d95eSNick Piggin * @vma: the vm area in which the mapping is added 13979617d95eSNick Piggin * @address: the user virtual address mapped 139840f2bbf7SDavid Hildenbrand * 139984f0169eSDavid Hildenbrand * Like folio_add_anon_rmap_*() but must only be called on *new* folios. 14009617d95eSNick Piggin * This means the inc-and-test can be bypassed. 14014d510f3dSMatthew Wilcox (Oracle) * The folio does not have to be locked. 14024d510f3dSMatthew Wilcox (Oracle) * 1403372cbd4dSRyan Roberts * If the folio is pmd-mappable, it is accounted as a THP. As the folio 14044d510f3dSMatthew Wilcox (Oracle) * is new, it's assumed to be mapped exclusively by a single process. 14059617d95eSNick Piggin */ 14064d510f3dSMatthew Wilcox (Oracle) void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma, 14074d510f3dSMatthew Wilcox (Oracle) unsigned long address) 14089617d95eSNick Piggin { 1409372cbd4dSRyan Roberts int nr = folio_nr_pages(folio); 1410d281ee61SKirill A. Shutemov 1411a4ea1864SDavid Hildenbrand VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio); 1412372cbd4dSRyan Roberts VM_BUG_ON_VMA(address < vma->vm_start || 1413372cbd4dSRyan Roberts address + (nr << PAGE_SHIFT) > vma->vm_end, vma); 14144d510f3dSMatthew Wilcox (Oracle) __folio_set_swapbacked(folio); 1415372cbd4dSRyan Roberts __folio_set_anon(folio, vma, address, true); 1416d8dd5e97SHugh Dickins 1417372cbd4dSRyan Roberts if (likely(!folio_test_large(folio))) { 1418d8dd5e97SHugh Dickins /* increment count (starts at -1) */ 14194d510f3dSMatthew Wilcox (Oracle) atomic_set(&folio->_mapcount, 0); 1420372cbd4dSRyan Roberts SetPageAnonExclusive(&folio->page); 1421372cbd4dSRyan Roberts } else if (!folio_test_pmd_mappable(folio)) { 1422372cbd4dSRyan Roberts int i; 1423372cbd4dSRyan Roberts 1424372cbd4dSRyan Roberts for (i = 0; i < nr; i++) { 1425372cbd4dSRyan Roberts struct page *page = folio_page(folio, i); 1426372cbd4dSRyan Roberts 1427372cbd4dSRyan Roberts /* increment count (starts at -1) */ 1428372cbd4dSRyan Roberts atomic_set(&page->_mapcount, 0); 1429372cbd4dSRyan Roberts SetPageAnonExclusive(page); 1430372cbd4dSRyan Roberts } 1431372cbd4dSRyan Roberts 1432372cbd4dSRyan Roberts atomic_set(&folio->_nr_pages_mapped, nr); 1433d8dd5e97SHugh Dickins } else { 143453f9263bSKirill A. Shutemov /* increment count (starts at -1) */ 14354d510f3dSMatthew Wilcox (Oracle) atomic_set(&folio->_entire_mapcount, 0); 1436e78a13fdSDavid Hildenbrand atomic_set(&folio->_nr_pages_mapped, ENTIRELY_MAPPED); 1437372cbd4dSRyan Roberts SetPageAnonExclusive(&folio->page); 14384d510f3dSMatthew Wilcox (Oracle) __lruvec_stat_mod_folio(folio, NR_ANON_THPS, nr); 1439d281ee61SKirill A. Shutemov } 1440d8dd5e97SHugh Dickins 14414d510f3dSMatthew Wilcox (Oracle) __lruvec_stat_mod_folio(folio, NR_ANON_MAPPED, nr); 14429617d95eSNick Piggin } 14439617d95eSNick Piggin 144468f03208SDavid Hildenbrand static __always_inline void __folio_add_file_rmap(struct folio *folio, 144568f03208SDavid Hildenbrand struct page *page, int nr_pages, struct vm_area_struct *vma, 144668f03208SDavid Hildenbrand enum rmap_level level) 14471da177e4SLinus Torvalds { 144896fd7495SDavid Hildenbrand int nr, nr_pmdmapped = 0; 1449dd78feddSKirill A. Shutemov 145068f03208SDavid Hildenbrand VM_WARN_ON_FOLIO(folio_test_anon(folio), folio); 14519bd3155eSHugh Dickins 145296fd7495SDavid Hildenbrand nr = __folio_add_rmap(folio, page, nr_pages, level, &nr_pmdmapped); 14539bd3155eSHugh Dickins if (nr_pmdmapped) 1454eb01a2adSMatthew Wilcox (Oracle) __lruvec_stat_mod_folio(folio, folio_test_swapbacked(folio) ? 14559bd3155eSHugh Dickins NR_SHMEM_PMDMAPPED : NR_FILE_PMDMAPPED, nr_pmdmapped); 14565d543f13SHugh Dickins if (nr) 1457eb01a2adSMatthew Wilcox (Oracle) __lruvec_stat_mod_folio(folio, NR_FILE_MAPPED, nr); 1458cea86fe2SHugh Dickins 145984f0169eSDavid Hildenbrand /* See comments in folio_add_anon_rmap_*() */ 14601acbc3f9SYin Fengwei if (!folio_test_large(folio)) 14611acbc3f9SYin Fengwei mlock_vma_folio(folio, vma); 14621da177e4SLinus Torvalds } 14631da177e4SLinus Torvalds 14641da177e4SLinus Torvalds /** 146568f03208SDavid Hildenbrand * folio_add_file_rmap_ptes - add PTE mappings to a page range of a folio 146668f03208SDavid Hildenbrand * @folio: The folio to add the mappings to 146768f03208SDavid Hildenbrand * @page: The first page to add 146868f03208SDavid Hildenbrand * @nr_pages: The number of pages that will be mapped using PTEs 146968f03208SDavid Hildenbrand * @vma: The vm area in which the mappings are added 147068f03208SDavid Hildenbrand * 147168f03208SDavid Hildenbrand * The page range of the folio is defined by [page, page + nr_pages) 147268f03208SDavid Hildenbrand * 147368f03208SDavid Hildenbrand * The caller needs to hold the page table lock. 147468f03208SDavid Hildenbrand */ 147568f03208SDavid Hildenbrand void folio_add_file_rmap_ptes(struct folio *folio, struct page *page, 147668f03208SDavid Hildenbrand int nr_pages, struct vm_area_struct *vma) 147768f03208SDavid Hildenbrand { 147868f03208SDavid Hildenbrand __folio_add_file_rmap(folio, page, nr_pages, vma, RMAP_LEVEL_PTE); 147968f03208SDavid Hildenbrand } 148068f03208SDavid Hildenbrand 148168f03208SDavid Hildenbrand /** 148268f03208SDavid Hildenbrand * folio_add_file_rmap_pmd - add a PMD mapping to a page range of a folio 148368f03208SDavid Hildenbrand * @folio: The folio to add the mapping to 148468f03208SDavid Hildenbrand * @page: The first page to add 148568f03208SDavid Hildenbrand * @vma: The vm area in which the mapping is added 148668f03208SDavid Hildenbrand * 148768f03208SDavid Hildenbrand * The page range of the folio is defined by [page, page + HPAGE_PMD_NR) 148868f03208SDavid Hildenbrand * 148968f03208SDavid Hildenbrand * The caller needs to hold the page table lock. 149068f03208SDavid Hildenbrand */ 149168f03208SDavid Hildenbrand void folio_add_file_rmap_pmd(struct folio *folio, struct page *page, 149268f03208SDavid Hildenbrand struct vm_area_struct *vma) 149368f03208SDavid Hildenbrand { 149468f03208SDavid Hildenbrand #ifdef CONFIG_TRANSPARENT_HUGEPAGE 149568f03208SDavid Hildenbrand __folio_add_file_rmap(folio, page, HPAGE_PMD_NR, vma, RMAP_LEVEL_PMD); 149668f03208SDavid Hildenbrand #else 149768f03208SDavid Hildenbrand WARN_ON_ONCE(true); 149868f03208SDavid Hildenbrand #endif 149968f03208SDavid Hildenbrand } 150068f03208SDavid Hildenbrand 1501b06dc281SDavid Hildenbrand static __always_inline void __folio_remove_rmap(struct folio *folio, 1502b06dc281SDavid Hildenbrand struct page *page, int nr_pages, struct vm_area_struct *vma, 1503b06dc281SDavid Hildenbrand enum rmap_level level) 1504b06dc281SDavid Hildenbrand { 150562beb906SMatthew Wilcox (Oracle) atomic_t *mapped = &folio->_nr_pages_mapped; 1506b06dc281SDavid Hildenbrand int last, nr = 0, nr_pmdmapped = 0; 150762beb906SMatthew Wilcox (Oracle) enum node_stat_item idx; 15089bd3155eSHugh Dickins 1509b06dc281SDavid Hildenbrand __folio_rmap_sanity_checks(folio, page, nr_pages, level); 15109bd3155eSHugh Dickins 1511b06dc281SDavid Hildenbrand switch (level) { 1512b06dc281SDavid Hildenbrand case RMAP_LEVEL_PTE: 1513b06dc281SDavid Hildenbrand do { 1514d8dd5e97SHugh Dickins last = atomic_add_negative(-1, &page->_mapcount); 151562beb906SMatthew Wilcox (Oracle) if (last && folio_test_large(folio)) { 1516b06dc281SDavid Hildenbrand last = atomic_dec_return_relaxed(mapped); 1517e78a13fdSDavid Hildenbrand last = (last < ENTIRELY_MAPPED); 1518cb67f428SHugh Dickins } 1519be5ef2d9SHugh Dickins 1520b06dc281SDavid Hildenbrand if (last) 1521b06dc281SDavid Hildenbrand nr++; 1522b06dc281SDavid Hildenbrand } while (page++, --nr_pages > 0); 1523b06dc281SDavid Hildenbrand break; 1524b06dc281SDavid Hildenbrand case RMAP_LEVEL_PMD: 152562beb906SMatthew Wilcox (Oracle) last = atomic_add_negative(-1, &folio->_entire_mapcount); 1526be5ef2d9SHugh Dickins if (last) { 1527e78a13fdSDavid Hildenbrand nr = atomic_sub_return_relaxed(ENTIRELY_MAPPED, mapped); 1528e78a13fdSDavid Hildenbrand if (likely(nr < ENTIRELY_MAPPED)) { 152962beb906SMatthew Wilcox (Oracle) nr_pmdmapped = folio_nr_pages(folio); 1530eec20426SMatthew Wilcox (Oracle) nr = nr_pmdmapped - (nr & FOLIO_PAGES_MAPPED); 15316287b7daSHugh Dickins /* Raced ahead of another remove and an add? */ 15326287b7daSHugh Dickins if (unlikely(nr < 0)) 15336287b7daSHugh Dickins nr = 0; 15346287b7daSHugh Dickins } else { 1535e78a13fdSDavid Hildenbrand /* An add of ENTIRELY_MAPPED raced ahead */ 15366287b7daSHugh Dickins nr = 0; 15376287b7daSHugh Dickins } 1538be5ef2d9SHugh Dickins } 1539b06dc281SDavid Hildenbrand break; 1540be5ef2d9SHugh Dickins } 1541cb67f428SHugh Dickins 15429bd3155eSHugh Dickins if (nr_pmdmapped) { 154362beb906SMatthew Wilcox (Oracle) if (folio_test_anon(folio)) 154462beb906SMatthew Wilcox (Oracle) idx = NR_ANON_THPS; 154562beb906SMatthew Wilcox (Oracle) else if (folio_test_swapbacked(folio)) 154662beb906SMatthew Wilcox (Oracle) idx = NR_SHMEM_PMDMAPPED; 154762beb906SMatthew Wilcox (Oracle) else 154862beb906SMatthew Wilcox (Oracle) idx = NR_FILE_PMDMAPPED; 154962beb906SMatthew Wilcox (Oracle) __lruvec_stat_mod_folio(folio, idx, -nr_pmdmapped); 15509bd3155eSHugh Dickins } 15519bd3155eSHugh Dickins if (nr) { 155262beb906SMatthew Wilcox (Oracle) idx = folio_test_anon(folio) ? NR_ANON_MAPPED : NR_FILE_MAPPED; 155362beb906SMatthew Wilcox (Oracle) __lruvec_stat_mod_folio(folio, idx, -nr); 155462beb906SMatthew Wilcox (Oracle) 15559bd3155eSHugh Dickins /* 15567dc7c5efSRyan Roberts * Queue anon large folio for deferred split if at least one 155762beb906SMatthew Wilcox (Oracle) * page of the folio is unmapped and at least one page 155862beb906SMatthew Wilcox (Oracle) * is still mapped. 15599bd3155eSHugh Dickins */ 15607dc7c5efSRyan Roberts if (folio_test_large(folio) && folio_test_anon(folio)) 1561b06dc281SDavid Hildenbrand if (level == RMAP_LEVEL_PTE || nr < nr_pmdmapped) 1562f158ed61SMatthew Wilcox (Oracle) deferred_split_folio(folio); 15639bd3155eSHugh Dickins } 15649a982250SKirill A. Shutemov 156516f8c5b2SHugh Dickins /* 1566672aa27dSMatthew Wilcox (Oracle) * It would be tidy to reset folio_test_anon mapping when fully 156784f0169eSDavid Hildenbrand * unmapped, but that might overwrite a racing folio_add_anon_rmap_*() 1568672aa27dSMatthew Wilcox (Oracle) * which increments mapcount after us but sets mapping before us: 1569672aa27dSMatthew Wilcox (Oracle) * so leave the reset to free_pages_prepare, and remember that 1570672aa27dSMatthew Wilcox (Oracle) * it's only reliable while mapped. 15711da177e4SLinus Torvalds */ 15729bd3155eSHugh Dickins 15731acbc3f9SYin Fengwei munlock_vma_folio(folio, vma); 15741da177e4SLinus Torvalds } 15751da177e4SLinus Torvalds 1576b06dc281SDavid Hildenbrand /** 1577b06dc281SDavid Hildenbrand * folio_remove_rmap_ptes - remove PTE mappings from a page range of a folio 1578b06dc281SDavid Hildenbrand * @folio: The folio to remove the mappings from 1579b06dc281SDavid Hildenbrand * @page: The first page to remove 1580b06dc281SDavid Hildenbrand * @nr_pages: The number of pages that will be removed from the mapping 1581b06dc281SDavid Hildenbrand * @vma: The vm area from which the mappings are removed 1582b06dc281SDavid Hildenbrand * 1583b06dc281SDavid Hildenbrand * The page range of the folio is defined by [page, page + nr_pages) 1584b06dc281SDavid Hildenbrand * 1585b06dc281SDavid Hildenbrand * The caller needs to hold the page table lock. 1586b06dc281SDavid Hildenbrand */ 1587b06dc281SDavid Hildenbrand void folio_remove_rmap_ptes(struct folio *folio, struct page *page, 1588b06dc281SDavid Hildenbrand int nr_pages, struct vm_area_struct *vma) 1589b06dc281SDavid Hildenbrand { 1590b06dc281SDavid Hildenbrand __folio_remove_rmap(folio, page, nr_pages, vma, RMAP_LEVEL_PTE); 1591b06dc281SDavid Hildenbrand } 1592b06dc281SDavid Hildenbrand 1593b06dc281SDavid Hildenbrand /** 1594b06dc281SDavid Hildenbrand * folio_remove_rmap_pmd - remove a PMD mapping from a page range of a folio 1595b06dc281SDavid Hildenbrand * @folio: The folio to remove the mapping from 1596b06dc281SDavid Hildenbrand * @page: The first page to remove 1597b06dc281SDavid Hildenbrand * @vma: The vm area from which the mapping is removed 1598b06dc281SDavid Hildenbrand * 1599b06dc281SDavid Hildenbrand * The page range of the folio is defined by [page, page + HPAGE_PMD_NR) 1600b06dc281SDavid Hildenbrand * 1601b06dc281SDavid Hildenbrand * The caller needs to hold the page table lock. 1602b06dc281SDavid Hildenbrand */ 1603b06dc281SDavid Hildenbrand void folio_remove_rmap_pmd(struct folio *folio, struct page *page, 1604b06dc281SDavid Hildenbrand struct vm_area_struct *vma) 1605b06dc281SDavid Hildenbrand { 1606b06dc281SDavid Hildenbrand #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1607b06dc281SDavid Hildenbrand __folio_remove_rmap(folio, page, HPAGE_PMD_NR, vma, RMAP_LEVEL_PMD); 1608b06dc281SDavid Hildenbrand #else 1609b06dc281SDavid Hildenbrand WARN_ON_ONCE(true); 1610b06dc281SDavid Hildenbrand #endif 1611b06dc281SDavid Hildenbrand } 1612b06dc281SDavid Hildenbrand 16131da177e4SLinus Torvalds /* 161452629506SJoonsoo Kim * @arg: enum ttu_flags will be passed to this argument 16151da177e4SLinus Torvalds */ 16162f031c6fSMatthew Wilcox (Oracle) static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, 161752629506SJoonsoo Kim unsigned long address, void *arg) 16181da177e4SLinus Torvalds { 16191da177e4SLinus Torvalds struct mm_struct *mm = vma->vm_mm; 1620869f7ee6SMatthew Wilcox (Oracle) DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); 16211da177e4SLinus Torvalds pte_t pteval; 1622c7ab0d2fSKirill A. Shutemov struct page *subpage; 16236c287605SDavid Hildenbrand bool anon_exclusive, ret = true; 1624ac46d4f3SJérôme Glisse struct mmu_notifier_range range; 16254708f318SPalmer Dabbelt enum ttu_flags flags = (enum ttu_flags)(long)arg; 1626c33c7948SRyan Roberts unsigned long pfn; 1627935d4f0cSRyan Roberts unsigned long hsz = 0; 16281da177e4SLinus Torvalds 1629732ed558SHugh Dickins /* 1630732ed558SHugh Dickins * When racing against e.g. zap_pte_range() on another cpu, 1631ca1a0746SDavid Hildenbrand * in between its ptep_get_and_clear_full() and folio_remove_rmap_*(), 16321fb08ac6SYang Shi * try_to_unmap() may return before page_mapped() has become false, 1633732ed558SHugh Dickins * if page table locking is skipped: use TTU_SYNC to wait for that. 1634732ed558SHugh Dickins */ 1635732ed558SHugh Dickins if (flags & TTU_SYNC) 1636732ed558SHugh Dickins pvmw.flags = PVMW_SYNC; 1637732ed558SHugh Dickins 1638a98a2f0cSAlistair Popple if (flags & TTU_SPLIT_HUGE_PMD) 1639af28a988SMatthew Wilcox (Oracle) split_huge_pmd_address(vma, address, false, folio); 1640fec89c10SKirill A. Shutemov 1641369ea824SJérôme Glisse /* 1642017b1660SMike Kravetz * For THP, we have to assume the worse case ie pmd for invalidation. 1643017b1660SMike Kravetz * For hugetlb, it could be much worse if we need to do pud 1644017b1660SMike Kravetz * invalidation in the case of pmd sharing. 1645017b1660SMike Kravetz * 1646869f7ee6SMatthew Wilcox (Oracle) * Note that the folio can not be freed in this function as call of 1647869f7ee6SMatthew Wilcox (Oracle) * try_to_unmap() must hold a reference on the folio. 1648369ea824SJérôme Glisse */ 16492aff7a47SMatthew Wilcox (Oracle) range.end = vma_address_end(&pvmw); 16507d4a8be0SAlistair Popple mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, 1651494334e4SHugh Dickins address, range.end); 1652869f7ee6SMatthew Wilcox (Oracle) if (folio_test_hugetlb(folio)) { 1653017b1660SMike Kravetz /* 1654017b1660SMike Kravetz * If sharing is possible, start and end will be adjusted 1655017b1660SMike Kravetz * accordingly. 1656017b1660SMike Kravetz */ 1657ac46d4f3SJérôme Glisse adjust_range_if_pmd_sharing_possible(vma, &range.start, 1658ac46d4f3SJérôme Glisse &range.end); 1659935d4f0cSRyan Roberts 1660935d4f0cSRyan Roberts /* We need the huge page size for set_huge_pte_at() */ 1661935d4f0cSRyan Roberts hsz = huge_page_size(hstate_vma(vma)); 1662017b1660SMike Kravetz } 1663ac46d4f3SJérôme Glisse mmu_notifier_invalidate_range_start(&range); 1664369ea824SJérôme Glisse 1665c7ab0d2fSKirill A. Shutemov while (page_vma_mapped_walk(&pvmw)) { 1666cea86fe2SHugh Dickins /* Unexpected PMD-mapped THP? */ 1667869f7ee6SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(!pvmw.pte, folio); 1668cea86fe2SHugh Dickins 16691da177e4SLinus Torvalds /* 1670869f7ee6SMatthew Wilcox (Oracle) * If the folio is in an mlock()d vma, we must not swap it out. 16711da177e4SLinus Torvalds */ 1672efdb6720SHugh Dickins if (!(flags & TTU_IGNORE_MLOCK) && 1673efdb6720SHugh Dickins (vma->vm_flags & VM_LOCKED)) { 1674cea86fe2SHugh Dickins /* Restore the mlock which got missed */ 16751acbc3f9SYin Fengwei if (!folio_test_large(folio)) 16761acbc3f9SYin Fengwei mlock_vma_folio(folio, vma); 1677c7ab0d2fSKirill A. Shutemov page_vma_mapped_walk_done(&pvmw); 1678efdb6720SHugh Dickins ret = false; 1679c7ab0d2fSKirill A. Shutemov break; 1680b87537d9SHugh Dickins } 1681c7ab0d2fSKirill A. Shutemov 1682c33c7948SRyan Roberts pfn = pte_pfn(ptep_get(pvmw.pte)); 1683c33c7948SRyan Roberts subpage = folio_page(folio, pfn - folio_pfn(folio)); 1684785373b4SLinus Torvalds address = pvmw.address; 16856c287605SDavid Hildenbrand anon_exclusive = folio_test_anon(folio) && 16866c287605SDavid Hildenbrand PageAnonExclusive(subpage); 1687785373b4SLinus Torvalds 1688dfc7ab57SBaolin Wang if (folio_test_hugetlb(folio)) { 16890506c31dSBaolin Wang bool anon = folio_test_anon(folio); 16900506c31dSBaolin Wang 1691017b1660SMike Kravetz /* 1692a00a8759SBaolin Wang * The try_to_unmap() is only passed a hugetlb page 1693a00a8759SBaolin Wang * in the case where the hugetlb page is poisoned. 1694a00a8759SBaolin Wang */ 1695a00a8759SBaolin Wang VM_BUG_ON_PAGE(!PageHWPoison(subpage), subpage); 1696a00a8759SBaolin Wang /* 169754205e9cSBaolin Wang * huge_pmd_unshare may unmap an entire PMD page. 169854205e9cSBaolin Wang * There is no way of knowing exactly which PMDs may 169954205e9cSBaolin Wang * be cached for this mm, so we must flush them all. 170054205e9cSBaolin Wang * start/end were already adjusted above to cover this 170154205e9cSBaolin Wang * range. 1702017b1660SMike Kravetz */ 1703ac46d4f3SJérôme Glisse flush_cache_range(vma, range.start, range.end); 170454205e9cSBaolin Wang 1705dfc7ab57SBaolin Wang /* 1706dfc7ab57SBaolin Wang * To call huge_pmd_unshare, i_mmap_rwsem must be 1707dfc7ab57SBaolin Wang * held in write mode. Caller needs to explicitly 1708dfc7ab57SBaolin Wang * do this outside rmap routines. 170940549ba8SMike Kravetz * 171040549ba8SMike Kravetz * We also must hold hugetlb vma_lock in write mode. 171140549ba8SMike Kravetz * Lock order dictates acquiring vma_lock BEFORE 171240549ba8SMike Kravetz * i_mmap_rwsem. We can only try lock here and fail 171340549ba8SMike Kravetz * if unsuccessful. 1714dfc7ab57SBaolin Wang */ 171540549ba8SMike Kravetz if (!anon) { 171640549ba8SMike Kravetz VM_BUG_ON(!(flags & TTU_RMAP_LOCKED)); 171740549ba8SMike Kravetz if (!hugetlb_vma_trylock_write(vma)) { 171840549ba8SMike Kravetz page_vma_mapped_walk_done(&pvmw); 171940549ba8SMike Kravetz ret = false; 172040549ba8SMike Kravetz break; 172140549ba8SMike Kravetz } 172240549ba8SMike Kravetz if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) { 172340549ba8SMike Kravetz hugetlb_vma_unlock_write(vma); 172440549ba8SMike Kravetz flush_tlb_range(vma, 172540549ba8SMike Kravetz range.start, range.end); 1726017b1660SMike Kravetz /* 172740549ba8SMike Kravetz * The ref count of the PMD page was 172840549ba8SMike Kravetz * dropped which is part of the way map 172940549ba8SMike Kravetz * counting is done for shared PMDs. 173040549ba8SMike Kravetz * Return 'true' here. When there is 173140549ba8SMike Kravetz * no other sharing, huge_pmd_unshare 173240549ba8SMike Kravetz * returns false and we will unmap the 173340549ba8SMike Kravetz * actual page and drop map count 1734017b1660SMike Kravetz * to zero. 1735017b1660SMike Kravetz */ 1736017b1660SMike Kravetz page_vma_mapped_walk_done(&pvmw); 1737017b1660SMike Kravetz break; 1738017b1660SMike Kravetz } 173940549ba8SMike Kravetz hugetlb_vma_unlock_write(vma); 174040549ba8SMike Kravetz } 1741a00a8759SBaolin Wang pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); 174254205e9cSBaolin Wang } else { 1743c33c7948SRyan Roberts flush_cache_page(vma, address, pfn); 1744088b8aa5SDavid Hildenbrand /* Nuke the page table entry. */ 1745088b8aa5SDavid Hildenbrand if (should_defer_flush(mm, flags)) { 174672b252aeSMel Gorman /* 1747c7ab0d2fSKirill A. Shutemov * We clear the PTE but do not flush so potentially 1748869f7ee6SMatthew Wilcox (Oracle) * a remote CPU could still be writing to the folio. 1749c7ab0d2fSKirill A. Shutemov * If the entry was previously clean then the 1750c7ab0d2fSKirill A. Shutemov * architecture must guarantee that a clear->dirty 1751c7ab0d2fSKirill A. Shutemov * transition on a cached TLB entry is written through 1752c7ab0d2fSKirill A. Shutemov * and traps if the PTE is unmapped. 175372b252aeSMel Gorman */ 1754785373b4SLinus Torvalds pteval = ptep_get_and_clear(mm, address, pvmw.pte); 175572b252aeSMel Gorman 1756f73419bbSBarry Song set_tlb_ubc_flush_pending(mm, pteval, address); 175772b252aeSMel Gorman } else { 1758785373b4SLinus Torvalds pteval = ptep_clear_flush(vma, address, pvmw.pte); 175972b252aeSMel Gorman } 1760a00a8759SBaolin Wang } 17611da177e4SLinus Torvalds 1762999dad82SPeter Xu /* 1763999dad82SPeter Xu * Now the pte is cleared. If this pte was uffd-wp armed, 1764999dad82SPeter Xu * we may want to replace a none pte with a marker pte if 1765999dad82SPeter Xu * it's file-backed, so we don't lose the tracking info. 1766999dad82SPeter Xu */ 1767999dad82SPeter Xu pte_install_uffd_wp_if_needed(vma, address, pvmw.pte, pteval); 1768999dad82SPeter Xu 1769869f7ee6SMatthew Wilcox (Oracle) /* Set the dirty flag on the folio now the pte is gone. */ 17701da177e4SLinus Torvalds if (pte_dirty(pteval)) 1771869f7ee6SMatthew Wilcox (Oracle) folio_mark_dirty(folio); 17721da177e4SLinus Torvalds 1773365e9c87SHugh Dickins /* Update high watermark before we lower rss */ 1774365e9c87SHugh Dickins update_hiwater_rss(mm); 1775365e9c87SHugh Dickins 17766da6b1d4SNaoya Horiguchi if (PageHWPoison(subpage) && (flags & TTU_HWPOISON)) { 17775fd27b8eSPunit Agrawal pteval = swp_entry_to_pte(make_hwpoison_entry(subpage)); 1778869f7ee6SMatthew Wilcox (Oracle) if (folio_test_hugetlb(folio)) { 1779869f7ee6SMatthew Wilcox (Oracle) hugetlb_count_sub(folio_nr_pages(folio), mm); 1780935d4f0cSRyan Roberts set_huge_pte_at(mm, address, pvmw.pte, pteval, 1781935d4f0cSRyan Roberts hsz); 17825d317b2bSNaoya Horiguchi } else { 1783a23f517bSKefeng Wang dec_mm_counter(mm, mm_counter(folio)); 1784785373b4SLinus Torvalds set_pte_at(mm, address, pvmw.pte, pteval); 17855f24ae58SNaoya Horiguchi } 1786c7ab0d2fSKirill A. Shutemov 1787bce73e48SChristian Borntraeger } else if (pte_unused(pteval) && !userfaultfd_armed(vma)) { 178845961722SKonstantin Weitz /* 178945961722SKonstantin Weitz * The guest indicated that the page content is of no 179045961722SKonstantin Weitz * interest anymore. Simply discard the pte, vmscan 179145961722SKonstantin Weitz * will take care of the rest. 1792bce73e48SChristian Borntraeger * A future reference will then fault in a new zero 1793bce73e48SChristian Borntraeger * page. When userfaultfd is active, we must not drop 1794bce73e48SChristian Borntraeger * this page though, as its main user (postcopy 1795bce73e48SChristian Borntraeger * migration) will not expect userfaults on already 1796bce73e48SChristian Borntraeger * copied pages. 179745961722SKonstantin Weitz */ 1798a23f517bSKefeng Wang dec_mm_counter(mm, mm_counter(folio)); 1799869f7ee6SMatthew Wilcox (Oracle) } else if (folio_test_anon(folio)) { 1800cfeed8ffSDavid Hildenbrand swp_entry_t entry = page_swap_entry(subpage); 1801179ef71cSCyrill Gorcunov pte_t swp_pte; 18021da177e4SLinus Torvalds /* 18031da177e4SLinus Torvalds * Store the swap location in the pte. 18041da177e4SLinus Torvalds * See handle_pte_fault() ... 18051da177e4SLinus Torvalds */ 1806869f7ee6SMatthew Wilcox (Oracle) if (unlikely(folio_test_swapbacked(folio) != 1807869f7ee6SMatthew Wilcox (Oracle) folio_test_swapcache(folio))) { 1808eb94a878SMinchan Kim WARN_ON_ONCE(1); 180983612a94SMinchan Kim ret = false; 1810eb94a878SMinchan Kim page_vma_mapped_walk_done(&pvmw); 1811eb94a878SMinchan Kim break; 1812eb94a878SMinchan Kim } 1813854e9ed0SMinchan Kim 1814802a3a92SShaohua Li /* MADV_FREE page check */ 1815869f7ee6SMatthew Wilcox (Oracle) if (!folio_test_swapbacked(folio)) { 18166c8e2a25SMauricio Faria de Oliveira int ref_count, map_count; 18176c8e2a25SMauricio Faria de Oliveira 18186c8e2a25SMauricio Faria de Oliveira /* 18196c8e2a25SMauricio Faria de Oliveira * Synchronize with gup_pte_range(): 18206c8e2a25SMauricio Faria de Oliveira * - clear PTE; barrier; read refcount 18216c8e2a25SMauricio Faria de Oliveira * - inc refcount; barrier; read PTE 18226c8e2a25SMauricio Faria de Oliveira */ 18236c8e2a25SMauricio Faria de Oliveira smp_mb(); 18246c8e2a25SMauricio Faria de Oliveira 18256c8e2a25SMauricio Faria de Oliveira ref_count = folio_ref_count(folio); 18266c8e2a25SMauricio Faria de Oliveira map_count = folio_mapcount(folio); 18276c8e2a25SMauricio Faria de Oliveira 18286c8e2a25SMauricio Faria de Oliveira /* 18296c8e2a25SMauricio Faria de Oliveira * Order reads for page refcount and dirty flag 18306c8e2a25SMauricio Faria de Oliveira * (see comments in __remove_mapping()). 18316c8e2a25SMauricio Faria de Oliveira */ 18326c8e2a25SMauricio Faria de Oliveira smp_rmb(); 18336c8e2a25SMauricio Faria de Oliveira 18346c8e2a25SMauricio Faria de Oliveira /* 18356c8e2a25SMauricio Faria de Oliveira * The only page refs must be one from isolation 18366c8e2a25SMauricio Faria de Oliveira * plus the rmap(s) (dropped by discard:). 18376c8e2a25SMauricio Faria de Oliveira */ 18386c8e2a25SMauricio Faria de Oliveira if (ref_count == 1 + map_count && 18396c8e2a25SMauricio Faria de Oliveira !folio_test_dirty(folio)) { 1840854e9ed0SMinchan Kim dec_mm_counter(mm, MM_ANONPAGES); 1841854e9ed0SMinchan Kim goto discard; 1842854e9ed0SMinchan Kim } 1843854e9ed0SMinchan Kim 1844802a3a92SShaohua Li /* 1845869f7ee6SMatthew Wilcox (Oracle) * If the folio was redirtied, it cannot be 1846802a3a92SShaohua Li * discarded. Remap the page to page table. 1847802a3a92SShaohua Li */ 1848785373b4SLinus Torvalds set_pte_at(mm, address, pvmw.pte, pteval); 1849869f7ee6SMatthew Wilcox (Oracle) folio_set_swapbacked(folio); 1850e4b82222SMinchan Kim ret = false; 1851802a3a92SShaohua Li page_vma_mapped_walk_done(&pvmw); 1852802a3a92SShaohua Li break; 1853802a3a92SShaohua Li } 1854802a3a92SShaohua Li 1855570a335bSHugh Dickins if (swap_duplicate(entry) < 0) { 1856785373b4SLinus Torvalds set_pte_at(mm, address, pvmw.pte, pteval); 1857e4b82222SMinchan Kim ret = false; 1858c7ab0d2fSKirill A. Shutemov page_vma_mapped_walk_done(&pvmw); 1859c7ab0d2fSKirill A. Shutemov break; 1860570a335bSHugh Dickins } 1861ca827d55SKhalid Aziz if (arch_unmap_one(mm, vma, address, pteval) < 0) { 1862322842eaSDavid Hildenbrand swap_free(entry); 1863ca827d55SKhalid Aziz set_pte_at(mm, address, pvmw.pte, pteval); 1864ca827d55SKhalid Aziz ret = false; 1865ca827d55SKhalid Aziz page_vma_mapped_walk_done(&pvmw); 1866ca827d55SKhalid Aziz break; 1867ca827d55SKhalid Aziz } 1868088b8aa5SDavid Hildenbrand 1869e3b4b137SDavid Hildenbrand /* See folio_try_share_anon_rmap(): clear PTE first. */ 18706c287605SDavid Hildenbrand if (anon_exclusive && 1871e3b4b137SDavid Hildenbrand folio_try_share_anon_rmap_pte(folio, subpage)) { 18726c287605SDavid Hildenbrand swap_free(entry); 18736c287605SDavid Hildenbrand set_pte_at(mm, address, pvmw.pte, pteval); 18746c287605SDavid Hildenbrand ret = false; 18756c287605SDavid Hildenbrand page_vma_mapped_walk_done(&pvmw); 18766c287605SDavid Hildenbrand break; 18776c287605SDavid Hildenbrand } 18781da177e4SLinus Torvalds if (list_empty(&mm->mmlist)) { 18791da177e4SLinus Torvalds spin_lock(&mmlist_lock); 1880f412ac08SHugh Dickins if (list_empty(&mm->mmlist)) 18811da177e4SLinus Torvalds list_add(&mm->mmlist, &init_mm.mmlist); 18821da177e4SLinus Torvalds spin_unlock(&mmlist_lock); 18831da177e4SLinus Torvalds } 1884d559db08SKAMEZAWA Hiroyuki dec_mm_counter(mm, MM_ANONPAGES); 1885b084d435SKAMEZAWA Hiroyuki inc_mm_counter(mm, MM_SWAPENTS); 1886179ef71cSCyrill Gorcunov swp_pte = swp_entry_to_pte(entry); 18871493a191SDavid Hildenbrand if (anon_exclusive) 18881493a191SDavid Hildenbrand swp_pte = pte_swp_mkexclusive(swp_pte); 1889179ef71cSCyrill Gorcunov if (pte_soft_dirty(pteval)) 1890179ef71cSCyrill Gorcunov swp_pte = pte_swp_mksoft_dirty(swp_pte); 1891f45ec5ffSPeter Xu if (pte_uffd_wp(pteval)) 1892f45ec5ffSPeter Xu swp_pte = pte_swp_mkuffd_wp(swp_pte); 1893785373b4SLinus Torvalds set_pte_at(mm, address, pvmw.pte, swp_pte); 18940f10851eSJérôme Glisse } else { 18950f10851eSJérôme Glisse /* 1896869f7ee6SMatthew Wilcox (Oracle) * This is a locked file-backed folio, 1897869f7ee6SMatthew Wilcox (Oracle) * so it cannot be removed from the page 1898869f7ee6SMatthew Wilcox (Oracle) * cache and replaced by a new folio before 1899869f7ee6SMatthew Wilcox (Oracle) * mmu_notifier_invalidate_range_end, so no 1900869f7ee6SMatthew Wilcox (Oracle) * concurrent thread might update its page table 1901869f7ee6SMatthew Wilcox (Oracle) * to point at a new folio while a device is 1902869f7ee6SMatthew Wilcox (Oracle) * still using this folio. 19030f10851eSJérôme Glisse * 1904ee65728eSMike Rapoport * See Documentation/mm/mmu_notifier.rst 19050f10851eSJérôme Glisse */ 19066b27cc6cSKefeng Wang dec_mm_counter(mm, mm_counter_file(folio)); 19070f10851eSJérôme Glisse } 19080f10851eSJérôme Glisse discard: 1909e135826bSDavid Hildenbrand if (unlikely(folio_test_hugetlb(folio))) 1910e135826bSDavid Hildenbrand hugetlb_remove_rmap(folio); 1911e135826bSDavid Hildenbrand else 1912ca1a0746SDavid Hildenbrand folio_remove_rmap_pte(folio, subpage, vma); 1913b7435507SHugh Dickins if (vma->vm_flags & VM_LOCKED) 191496f97c43SLorenzo Stoakes mlock_drain_local(); 1915869f7ee6SMatthew Wilcox (Oracle) folio_put(folio); 1916c7ab0d2fSKirill A. Shutemov } 1917369ea824SJérôme Glisse 1918ac46d4f3SJérôme Glisse mmu_notifier_invalidate_range_end(&range); 1919369ea824SJérôme Glisse 1920caed0f48SKOSAKI Motohiro return ret; 19211da177e4SLinus Torvalds } 19221da177e4SLinus Torvalds 192352629506SJoonsoo Kim static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg) 192452629506SJoonsoo Kim { 1925222100eeSAnshuman Khandual return vma_is_temporary_stack(vma); 192652629506SJoonsoo Kim } 192752629506SJoonsoo Kim 1928f3ad032cSKefeng Wang static int folio_not_mapped(struct folio *folio) 192952629506SJoonsoo Kim { 19302f031c6fSMatthew Wilcox (Oracle) return !folio_mapped(folio); 19312a52bcbcSKirill A. Shutemov } 193252629506SJoonsoo Kim 19331da177e4SLinus Torvalds /** 1934869f7ee6SMatthew Wilcox (Oracle) * try_to_unmap - Try to remove all page table mappings to a folio. 1935869f7ee6SMatthew Wilcox (Oracle) * @folio: The folio to unmap. 193614fa31b8SAndi Kleen * @flags: action and flags 19371da177e4SLinus Torvalds * 19381da177e4SLinus Torvalds * Tries to remove all the page table entries which are mapping this 1939869f7ee6SMatthew Wilcox (Oracle) * folio. It is the caller's responsibility to check if the folio is 1940869f7ee6SMatthew Wilcox (Oracle) * still mapped if needed (use TTU_SYNC to prevent accounting races). 19411da177e4SLinus Torvalds * 1942869f7ee6SMatthew Wilcox (Oracle) * Context: Caller must hold the folio lock. 19431da177e4SLinus Torvalds */ 1944869f7ee6SMatthew Wilcox (Oracle) void try_to_unmap(struct folio *folio, enum ttu_flags flags) 19451da177e4SLinus Torvalds { 194652629506SJoonsoo Kim struct rmap_walk_control rwc = { 194752629506SJoonsoo Kim .rmap_one = try_to_unmap_one, 1948802a3a92SShaohua Li .arg = (void *)flags, 1949f3ad032cSKefeng Wang .done = folio_not_mapped, 19502f031c6fSMatthew Wilcox (Oracle) .anon_lock = folio_lock_anon_vma_read, 195152629506SJoonsoo Kim }; 19521da177e4SLinus Torvalds 1953a98a2f0cSAlistair Popple if (flags & TTU_RMAP_LOCKED) 19542f031c6fSMatthew Wilcox (Oracle) rmap_walk_locked(folio, &rwc); 1955a98a2f0cSAlistair Popple else 19562f031c6fSMatthew Wilcox (Oracle) rmap_walk(folio, &rwc); 1957a98a2f0cSAlistair Popple } 1958a98a2f0cSAlistair Popple 1959a98a2f0cSAlistair Popple /* 1960a98a2f0cSAlistair Popple * @arg: enum ttu_flags will be passed to this argument. 1961a98a2f0cSAlistair Popple * 1962a98a2f0cSAlistair Popple * If TTU_SPLIT_HUGE_PMD is specified any PMD mappings will be split into PTEs 196364b586d1SHugh Dickins * containing migration entries. 1964a98a2f0cSAlistair Popple */ 19652f031c6fSMatthew Wilcox (Oracle) static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, 1966a98a2f0cSAlistair Popple unsigned long address, void *arg) 1967a98a2f0cSAlistair Popple { 1968a98a2f0cSAlistair Popple struct mm_struct *mm = vma->vm_mm; 19694b8554c5SMatthew Wilcox (Oracle) DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); 1970a98a2f0cSAlistair Popple pte_t pteval; 1971a98a2f0cSAlistair Popple struct page *subpage; 19726c287605SDavid Hildenbrand bool anon_exclusive, ret = true; 1973a98a2f0cSAlistair Popple struct mmu_notifier_range range; 1974a98a2f0cSAlistair Popple enum ttu_flags flags = (enum ttu_flags)(long)arg; 1975c33c7948SRyan Roberts unsigned long pfn; 1976935d4f0cSRyan Roberts unsigned long hsz = 0; 1977a98a2f0cSAlistair Popple 1978a98a2f0cSAlistair Popple /* 1979a98a2f0cSAlistair Popple * When racing against e.g. zap_pte_range() on another cpu, 1980ca1a0746SDavid Hildenbrand * in between its ptep_get_and_clear_full() and folio_remove_rmap_*(), 1981a98a2f0cSAlistair Popple * try_to_migrate() may return before page_mapped() has become false, 1982a98a2f0cSAlistair Popple * if page table locking is skipped: use TTU_SYNC to wait for that. 1983a98a2f0cSAlistair Popple */ 1984a98a2f0cSAlistair Popple if (flags & TTU_SYNC) 1985a98a2f0cSAlistair Popple pvmw.flags = PVMW_SYNC; 1986a98a2f0cSAlistair Popple 1987a98a2f0cSAlistair Popple /* 1988a98a2f0cSAlistair Popple * unmap_page() in mm/huge_memory.c is the only user of migration with 1989a98a2f0cSAlistair Popple * TTU_SPLIT_HUGE_PMD and it wants to freeze. 1990a98a2f0cSAlistair Popple */ 1991a98a2f0cSAlistair Popple if (flags & TTU_SPLIT_HUGE_PMD) 1992af28a988SMatthew Wilcox (Oracle) split_huge_pmd_address(vma, address, true, folio); 1993a98a2f0cSAlistair Popple 1994a98a2f0cSAlistair Popple /* 1995a98a2f0cSAlistair Popple * For THP, we have to assume the worse case ie pmd for invalidation. 1996a98a2f0cSAlistair Popple * For hugetlb, it could be much worse if we need to do pud 1997a98a2f0cSAlistair Popple * invalidation in the case of pmd sharing. 1998a98a2f0cSAlistair Popple * 1999a98a2f0cSAlistair Popple * Note that the page can not be free in this function as call of 2000a98a2f0cSAlistair Popple * try_to_unmap() must hold a reference on the page. 2001a98a2f0cSAlistair Popple */ 20022aff7a47SMatthew Wilcox (Oracle) range.end = vma_address_end(&pvmw); 20037d4a8be0SAlistair Popple mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, 2004a98a2f0cSAlistair Popple address, range.end); 20054b8554c5SMatthew Wilcox (Oracle) if (folio_test_hugetlb(folio)) { 2006a98a2f0cSAlistair Popple /* 2007a98a2f0cSAlistair Popple * If sharing is possible, start and end will be adjusted 2008a98a2f0cSAlistair Popple * accordingly. 2009a98a2f0cSAlistair Popple */ 2010a98a2f0cSAlistair Popple adjust_range_if_pmd_sharing_possible(vma, &range.start, 2011a98a2f0cSAlistair Popple &range.end); 2012935d4f0cSRyan Roberts 2013935d4f0cSRyan Roberts /* We need the huge page size for set_huge_pte_at() */ 2014935d4f0cSRyan Roberts hsz = huge_page_size(hstate_vma(vma)); 2015a98a2f0cSAlistair Popple } 2016a98a2f0cSAlistair Popple mmu_notifier_invalidate_range_start(&range); 2017a98a2f0cSAlistair Popple 2018a98a2f0cSAlistair Popple while (page_vma_mapped_walk(&pvmw)) { 2019a98a2f0cSAlistair Popple #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 2020a98a2f0cSAlistair Popple /* PMD-mapped THP migration entry */ 2021a98a2f0cSAlistair Popple if (!pvmw.pte) { 20224b8554c5SMatthew Wilcox (Oracle) subpage = folio_page(folio, 20234b8554c5SMatthew Wilcox (Oracle) pmd_pfn(*pvmw.pmd) - folio_pfn(folio)); 20244b8554c5SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) || 20254b8554c5SMatthew Wilcox (Oracle) !folio_test_pmd_mappable(folio), folio); 2026a98a2f0cSAlistair Popple 20277f5abe60SDavid Hildenbrand if (set_pmd_migration_entry(&pvmw, subpage)) { 20287f5abe60SDavid Hildenbrand ret = false; 20297f5abe60SDavid Hildenbrand page_vma_mapped_walk_done(&pvmw); 20307f5abe60SDavid Hildenbrand break; 20317f5abe60SDavid Hildenbrand } 2032a98a2f0cSAlistair Popple continue; 2033a98a2f0cSAlistair Popple } 2034a98a2f0cSAlistair Popple #endif 2035a98a2f0cSAlistair Popple 2036a98a2f0cSAlistair Popple /* Unexpected PMD-mapped THP? */ 20374b8554c5SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(!pvmw.pte, folio); 2038a98a2f0cSAlistair Popple 2039c33c7948SRyan Roberts pfn = pte_pfn(ptep_get(pvmw.pte)); 2040c33c7948SRyan Roberts 20411118234eSDavid Hildenbrand if (folio_is_zone_device(folio)) { 20421118234eSDavid Hildenbrand /* 20431118234eSDavid Hildenbrand * Our PTE is a non-present device exclusive entry and 20441118234eSDavid Hildenbrand * calculating the subpage as for the common case would 20451118234eSDavid Hildenbrand * result in an invalid pointer. 20461118234eSDavid Hildenbrand * 20471118234eSDavid Hildenbrand * Since only PAGE_SIZE pages can currently be 20481118234eSDavid Hildenbrand * migrated, just set it to page. This will need to be 20491118234eSDavid Hildenbrand * changed when hugepage migrations to device private 20501118234eSDavid Hildenbrand * memory are supported. 20511118234eSDavid Hildenbrand */ 20521118234eSDavid Hildenbrand VM_BUG_ON_FOLIO(folio_nr_pages(folio) > 1, folio); 20531118234eSDavid Hildenbrand subpage = &folio->page; 20541118234eSDavid Hildenbrand } else { 2055c33c7948SRyan Roberts subpage = folio_page(folio, pfn - folio_pfn(folio)); 20561118234eSDavid Hildenbrand } 2057a98a2f0cSAlistair Popple address = pvmw.address; 20586c287605SDavid Hildenbrand anon_exclusive = folio_test_anon(folio) && 20596c287605SDavid Hildenbrand PageAnonExclusive(subpage); 2060a98a2f0cSAlistair Popple 2061dfc7ab57SBaolin Wang if (folio_test_hugetlb(folio)) { 20620506c31dSBaolin Wang bool anon = folio_test_anon(folio); 20630506c31dSBaolin Wang 2064a98a2f0cSAlistair Popple /* 206554205e9cSBaolin Wang * huge_pmd_unshare may unmap an entire PMD page. 206654205e9cSBaolin Wang * There is no way of knowing exactly which PMDs may 206754205e9cSBaolin Wang * be cached for this mm, so we must flush them all. 206854205e9cSBaolin Wang * start/end were already adjusted above to cover this 206954205e9cSBaolin Wang * range. 2070a98a2f0cSAlistair Popple */ 2071a98a2f0cSAlistair Popple flush_cache_range(vma, range.start, range.end); 207254205e9cSBaolin Wang 2073dfc7ab57SBaolin Wang /* 2074dfc7ab57SBaolin Wang * To call huge_pmd_unshare, i_mmap_rwsem must be 2075dfc7ab57SBaolin Wang * held in write mode. Caller needs to explicitly 2076dfc7ab57SBaolin Wang * do this outside rmap routines. 207740549ba8SMike Kravetz * 207840549ba8SMike Kravetz * We also must hold hugetlb vma_lock in write mode. 207940549ba8SMike Kravetz * Lock order dictates acquiring vma_lock BEFORE 208040549ba8SMike Kravetz * i_mmap_rwsem. We can only try lock here and 208140549ba8SMike Kravetz * fail if unsuccessful. 2082dfc7ab57SBaolin Wang */ 208340549ba8SMike Kravetz if (!anon) { 208440549ba8SMike Kravetz VM_BUG_ON(!(flags & TTU_RMAP_LOCKED)); 208540549ba8SMike Kravetz if (!hugetlb_vma_trylock_write(vma)) { 208640549ba8SMike Kravetz page_vma_mapped_walk_done(&pvmw); 208740549ba8SMike Kravetz ret = false; 208840549ba8SMike Kravetz break; 208940549ba8SMike Kravetz } 209040549ba8SMike Kravetz if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) { 209140549ba8SMike Kravetz hugetlb_vma_unlock_write(vma); 209240549ba8SMike Kravetz flush_tlb_range(vma, 209340549ba8SMike Kravetz range.start, range.end); 2094a98a2f0cSAlistair Popple 2095a98a2f0cSAlistair Popple /* 209640549ba8SMike Kravetz * The ref count of the PMD page was 209740549ba8SMike Kravetz * dropped which is part of the way map 209840549ba8SMike Kravetz * counting is done for shared PMDs. 209940549ba8SMike Kravetz * Return 'true' here. When there is 210040549ba8SMike Kravetz * no other sharing, huge_pmd_unshare 210140549ba8SMike Kravetz * returns false and we will unmap the 210240549ba8SMike Kravetz * actual page and drop map count 2103a98a2f0cSAlistair Popple * to zero. 2104a98a2f0cSAlistair Popple */ 2105a98a2f0cSAlistair Popple page_vma_mapped_walk_done(&pvmw); 2106a98a2f0cSAlistair Popple break; 2107a98a2f0cSAlistair Popple } 210840549ba8SMike Kravetz hugetlb_vma_unlock_write(vma); 210940549ba8SMike Kravetz } 21105d4af619SBaolin Wang /* Nuke the hugetlb page table entry */ 21115d4af619SBaolin Wang pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); 211254205e9cSBaolin Wang } else { 2113c33c7948SRyan Roberts flush_cache_page(vma, address, pfn); 2114a98a2f0cSAlistair Popple /* Nuke the page table entry. */ 21157e12beb8SHuang Ying if (should_defer_flush(mm, flags)) { 21167e12beb8SHuang Ying /* 21177e12beb8SHuang Ying * We clear the PTE but do not flush so potentially 21187e12beb8SHuang Ying * a remote CPU could still be writing to the folio. 21197e12beb8SHuang Ying * If the entry was previously clean then the 21207e12beb8SHuang Ying * architecture must guarantee that a clear->dirty 21217e12beb8SHuang Ying * transition on a cached TLB entry is written through 21227e12beb8SHuang Ying * and traps if the PTE is unmapped. 21237e12beb8SHuang Ying */ 21247e12beb8SHuang Ying pteval = ptep_get_and_clear(mm, address, pvmw.pte); 21257e12beb8SHuang Ying 2126f73419bbSBarry Song set_tlb_ubc_flush_pending(mm, pteval, address); 21277e12beb8SHuang Ying } else { 2128a98a2f0cSAlistair Popple pteval = ptep_clear_flush(vma, address, pvmw.pte); 21295d4af619SBaolin Wang } 21307e12beb8SHuang Ying } 2131a98a2f0cSAlistair Popple 21324b8554c5SMatthew Wilcox (Oracle) /* Set the dirty flag on the folio now the pte is gone. */ 2133a98a2f0cSAlistair Popple if (pte_dirty(pteval)) 21344b8554c5SMatthew Wilcox (Oracle) folio_mark_dirty(folio); 2135a98a2f0cSAlistair Popple 2136a98a2f0cSAlistair Popple /* Update high watermark before we lower rss */ 2137a98a2f0cSAlistair Popple update_hiwater_rss(mm); 2138a98a2f0cSAlistair Popple 2139f25cbb7aSAlex Sierra if (folio_is_device_private(folio)) { 21404b8554c5SMatthew Wilcox (Oracle) unsigned long pfn = folio_pfn(folio); 2141a98a2f0cSAlistair Popple swp_entry_t entry; 2142a98a2f0cSAlistair Popple pte_t swp_pte; 2143a98a2f0cSAlistair Popple 21446c287605SDavid Hildenbrand if (anon_exclusive) 2145e3b4b137SDavid Hildenbrand WARN_ON_ONCE(folio_try_share_anon_rmap_pte(folio, 2146e3b4b137SDavid Hildenbrand subpage)); 21476c287605SDavid Hildenbrand 2148a98a2f0cSAlistair Popple /* 2149a98a2f0cSAlistair Popple * Store the pfn of the page in a special migration 2150a98a2f0cSAlistair Popple * pte. do_swap_page() will wait until the migration 2151a98a2f0cSAlistair Popple * pte is removed and then restart fault handling. 2152a98a2f0cSAlistair Popple */ 21533d88705cSAlistair Popple entry = pte_to_swp_entry(pteval); 21543d88705cSAlistair Popple if (is_writable_device_private_entry(entry)) 21553d88705cSAlistair Popple entry = make_writable_migration_entry(pfn); 21566c287605SDavid Hildenbrand else if (anon_exclusive) 21576c287605SDavid Hildenbrand entry = make_readable_exclusive_migration_entry(pfn); 21583d88705cSAlistair Popple else 21593d88705cSAlistair Popple entry = make_readable_migration_entry(pfn); 2160a98a2f0cSAlistair Popple swp_pte = swp_entry_to_pte(entry); 2161a98a2f0cSAlistair Popple 2162a98a2f0cSAlistair Popple /* 2163a98a2f0cSAlistair Popple * pteval maps a zone device page and is therefore 2164a98a2f0cSAlistair Popple * a swap pte. 2165a98a2f0cSAlistair Popple */ 2166a98a2f0cSAlistair Popple if (pte_swp_soft_dirty(pteval)) 2167a98a2f0cSAlistair Popple swp_pte = pte_swp_mksoft_dirty(swp_pte); 2168a98a2f0cSAlistair Popple if (pte_swp_uffd_wp(pteval)) 2169a98a2f0cSAlistair Popple swp_pte = pte_swp_mkuffd_wp(swp_pte); 2170a98a2f0cSAlistair Popple set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte); 21714cc79b33SAnshuman Khandual trace_set_migration_pte(pvmw.address, pte_val(swp_pte), 2172*059ab7beSMatthew Wilcox (Oracle) folio_order(folio)); 2173a98a2f0cSAlistair Popple /* 2174a98a2f0cSAlistair Popple * No need to invalidate here it will synchronize on 2175a98a2f0cSAlistair Popple * against the special swap migration pte. 2176a98a2f0cSAlistair Popple */ 2177da358d5cSMatthew Wilcox (Oracle) } else if (PageHWPoison(subpage)) { 2178a98a2f0cSAlistair Popple pteval = swp_entry_to_pte(make_hwpoison_entry(subpage)); 21794b8554c5SMatthew Wilcox (Oracle) if (folio_test_hugetlb(folio)) { 21804b8554c5SMatthew Wilcox (Oracle) hugetlb_count_sub(folio_nr_pages(folio), mm); 2181935d4f0cSRyan Roberts set_huge_pte_at(mm, address, pvmw.pte, pteval, 2182935d4f0cSRyan Roberts hsz); 2183a98a2f0cSAlistair Popple } else { 2184a23f517bSKefeng Wang dec_mm_counter(mm, mm_counter(folio)); 2185a98a2f0cSAlistair Popple set_pte_at(mm, address, pvmw.pte, pteval); 2186a98a2f0cSAlistair Popple } 2187a98a2f0cSAlistair Popple 2188a98a2f0cSAlistair Popple } else if (pte_unused(pteval) && !userfaultfd_armed(vma)) { 2189a98a2f0cSAlistair Popple /* 2190a98a2f0cSAlistair Popple * The guest indicated that the page content is of no 2191a98a2f0cSAlistair Popple * interest anymore. Simply discard the pte, vmscan 2192a98a2f0cSAlistair Popple * will take care of the rest. 2193a98a2f0cSAlistair Popple * A future reference will then fault in a new zero 2194a98a2f0cSAlistair Popple * page. When userfaultfd is active, we must not drop 2195a98a2f0cSAlistair Popple * this page though, as its main user (postcopy 2196a98a2f0cSAlistair Popple * migration) will not expect userfaults on already 2197a98a2f0cSAlistair Popple * copied pages. 2198a98a2f0cSAlistair Popple */ 2199a23f517bSKefeng Wang dec_mm_counter(mm, mm_counter(folio)); 2200a98a2f0cSAlistair Popple } else { 2201a98a2f0cSAlistair Popple swp_entry_t entry; 2202a98a2f0cSAlistair Popple pte_t swp_pte; 2203a98a2f0cSAlistair Popple 2204a98a2f0cSAlistair Popple if (arch_unmap_one(mm, vma, address, pteval) < 0) { 22055d4af619SBaolin Wang if (folio_test_hugetlb(folio)) 2206935d4f0cSRyan Roberts set_huge_pte_at(mm, address, pvmw.pte, 2207935d4f0cSRyan Roberts pteval, hsz); 22085d4af619SBaolin Wang else 2209a98a2f0cSAlistair Popple set_pte_at(mm, address, pvmw.pte, pteval); 2210a98a2f0cSAlistair Popple ret = false; 2211a98a2f0cSAlistair Popple page_vma_mapped_walk_done(&pvmw); 2212a98a2f0cSAlistair Popple break; 2213a98a2f0cSAlistair Popple } 22146c287605SDavid Hildenbrand VM_BUG_ON_PAGE(pte_write(pteval) && folio_test_anon(folio) && 22156c287605SDavid Hildenbrand !anon_exclusive, subpage); 2216088b8aa5SDavid Hildenbrand 2217e3b4b137SDavid Hildenbrand /* See folio_try_share_anon_rmap_pte(): clear PTE first. */ 22180c2ec32bSDavid Hildenbrand if (folio_test_hugetlb(folio)) { 22196c287605SDavid Hildenbrand if (anon_exclusive && 22200c2ec32bSDavid Hildenbrand hugetlb_try_share_anon_rmap(folio)) { 2221935d4f0cSRyan Roberts set_huge_pte_at(mm, address, pvmw.pte, 2222935d4f0cSRyan Roberts pteval, hsz); 22230c2ec32bSDavid Hildenbrand ret = false; 22240c2ec32bSDavid Hildenbrand page_vma_mapped_walk_done(&pvmw); 22250c2ec32bSDavid Hildenbrand break; 22260c2ec32bSDavid Hildenbrand } 22270c2ec32bSDavid Hildenbrand } else if (anon_exclusive && 2228e3b4b137SDavid Hildenbrand folio_try_share_anon_rmap_pte(folio, subpage)) { 22296c287605SDavid Hildenbrand set_pte_at(mm, address, pvmw.pte, pteval); 22306c287605SDavid Hildenbrand ret = false; 22316c287605SDavid Hildenbrand page_vma_mapped_walk_done(&pvmw); 22326c287605SDavid Hildenbrand break; 22336c287605SDavid Hildenbrand } 2234a98a2f0cSAlistair Popple 2235a98a2f0cSAlistair Popple /* 2236a98a2f0cSAlistair Popple * Store the pfn of the page in a special migration 2237a98a2f0cSAlistair Popple * pte. do_swap_page() will wait until the migration 2238a98a2f0cSAlistair Popple * pte is removed and then restart fault handling. 2239a98a2f0cSAlistair Popple */ 2240a98a2f0cSAlistair Popple if (pte_write(pteval)) 2241a98a2f0cSAlistair Popple entry = make_writable_migration_entry( 2242a98a2f0cSAlistair Popple page_to_pfn(subpage)); 22436c287605SDavid Hildenbrand else if (anon_exclusive) 22446c287605SDavid Hildenbrand entry = make_readable_exclusive_migration_entry( 22456c287605SDavid Hildenbrand page_to_pfn(subpage)); 2246a98a2f0cSAlistair Popple else 2247a98a2f0cSAlistair Popple entry = make_readable_migration_entry( 2248a98a2f0cSAlistair Popple page_to_pfn(subpage)); 22492e346877SPeter Xu if (pte_young(pteval)) 22502e346877SPeter Xu entry = make_migration_entry_young(entry); 22512e346877SPeter Xu if (pte_dirty(pteval)) 22522e346877SPeter Xu entry = make_migration_entry_dirty(entry); 2253a98a2f0cSAlistair Popple swp_pte = swp_entry_to_pte(entry); 2254a98a2f0cSAlistair Popple if (pte_soft_dirty(pteval)) 2255a98a2f0cSAlistair Popple swp_pte = pte_swp_mksoft_dirty(swp_pte); 2256a98a2f0cSAlistair Popple if (pte_uffd_wp(pteval)) 2257a98a2f0cSAlistair Popple swp_pte = pte_swp_mkuffd_wp(swp_pte); 22585d4af619SBaolin Wang if (folio_test_hugetlb(folio)) 2259935d4f0cSRyan Roberts set_huge_pte_at(mm, address, pvmw.pte, swp_pte, 2260935d4f0cSRyan Roberts hsz); 22615d4af619SBaolin Wang else 2262a98a2f0cSAlistair Popple set_pte_at(mm, address, pvmw.pte, swp_pte); 22634cc79b33SAnshuman Khandual trace_set_migration_pte(address, pte_val(swp_pte), 2264*059ab7beSMatthew Wilcox (Oracle) folio_order(folio)); 2265a98a2f0cSAlistair Popple /* 2266a98a2f0cSAlistair Popple * No need to invalidate here it will synchronize on 2267a98a2f0cSAlistair Popple * against the special swap migration pte. 2268a98a2f0cSAlistair Popple */ 2269a98a2f0cSAlistair Popple } 2270a98a2f0cSAlistair Popple 2271e135826bSDavid Hildenbrand if (unlikely(folio_test_hugetlb(folio))) 2272e135826bSDavid Hildenbrand hugetlb_remove_rmap(folio); 2273e135826bSDavid Hildenbrand else 2274ca1a0746SDavid Hildenbrand folio_remove_rmap_pte(folio, subpage, vma); 2275b7435507SHugh Dickins if (vma->vm_flags & VM_LOCKED) 227696f97c43SLorenzo Stoakes mlock_drain_local(); 22774b8554c5SMatthew Wilcox (Oracle) folio_put(folio); 2278a98a2f0cSAlistair Popple } 2279a98a2f0cSAlistair Popple 2280a98a2f0cSAlistair Popple mmu_notifier_invalidate_range_end(&range); 2281a98a2f0cSAlistair Popple 2282a98a2f0cSAlistair Popple return ret; 2283a98a2f0cSAlistair Popple } 2284a98a2f0cSAlistair Popple 2285a98a2f0cSAlistair Popple /** 2286a98a2f0cSAlistair Popple * try_to_migrate - try to replace all page table mappings with swap entries 22874b8554c5SMatthew Wilcox (Oracle) * @folio: the folio to replace page table entries for 2288a98a2f0cSAlistair Popple * @flags: action and flags 2289a98a2f0cSAlistair Popple * 22904b8554c5SMatthew Wilcox (Oracle) * Tries to remove all the page table entries which are mapping this folio and 22914b8554c5SMatthew Wilcox (Oracle) * replace them with special swap entries. Caller must hold the folio lock. 2292a98a2f0cSAlistair Popple */ 22934b8554c5SMatthew Wilcox (Oracle) void try_to_migrate(struct folio *folio, enum ttu_flags flags) 2294a98a2f0cSAlistair Popple { 2295a98a2f0cSAlistair Popple struct rmap_walk_control rwc = { 2296a98a2f0cSAlistair Popple .rmap_one = try_to_migrate_one, 2297a98a2f0cSAlistair Popple .arg = (void *)flags, 2298f3ad032cSKefeng Wang .done = folio_not_mapped, 22992f031c6fSMatthew Wilcox (Oracle) .anon_lock = folio_lock_anon_vma_read, 2300a98a2f0cSAlistair Popple }; 2301a98a2f0cSAlistair Popple 2302a98a2f0cSAlistair Popple /* 2303a98a2f0cSAlistair Popple * Migration always ignores mlock and only supports TTU_RMAP_LOCKED and 23047e12beb8SHuang Ying * TTU_SPLIT_HUGE_PMD, TTU_SYNC, and TTU_BATCH_FLUSH flags. 2305a98a2f0cSAlistair Popple */ 2306a98a2f0cSAlistair Popple if (WARN_ON_ONCE(flags & ~(TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD | 23077e12beb8SHuang Ying TTU_SYNC | TTU_BATCH_FLUSH))) 2308a98a2f0cSAlistair Popple return; 2309a98a2f0cSAlistair Popple 2310f25cbb7aSAlex Sierra if (folio_is_zone_device(folio) && 2311f25cbb7aSAlex Sierra (!folio_is_device_private(folio) && !folio_is_device_coherent(folio))) 23126c855fceSHugh Dickins return; 23136c855fceSHugh Dickins 231452629506SJoonsoo Kim /* 231552629506SJoonsoo Kim * During exec, a temporary VMA is setup and later moved. 231652629506SJoonsoo Kim * The VMA is moved under the anon_vma lock but not the 231752629506SJoonsoo Kim * page tables leading to a race where migration cannot 231852629506SJoonsoo Kim * find the migration ptes. Rather than increasing the 231952629506SJoonsoo Kim * locking requirements of exec(), migration skips 232052629506SJoonsoo Kim * temporary VMAs until after exec() completes. 232152629506SJoonsoo Kim */ 23224b8554c5SMatthew Wilcox (Oracle) if (!folio_test_ksm(folio) && folio_test_anon(folio)) 232352629506SJoonsoo Kim rwc.invalid_vma = invalid_migration_vma; 232452629506SJoonsoo Kim 23252a52bcbcSKirill A. Shutemov if (flags & TTU_RMAP_LOCKED) 23262f031c6fSMatthew Wilcox (Oracle) rmap_walk_locked(folio, &rwc); 23272a52bcbcSKirill A. Shutemov else 23282f031c6fSMatthew Wilcox (Oracle) rmap_walk(folio, &rwc); 2329b291f000SNick Piggin } 2330e9995ef9SHugh Dickins 2331b756a3b5SAlistair Popple #ifdef CONFIG_DEVICE_PRIVATE 2332b756a3b5SAlistair Popple struct make_exclusive_args { 2333b756a3b5SAlistair Popple struct mm_struct *mm; 2334b756a3b5SAlistair Popple unsigned long address; 2335b756a3b5SAlistair Popple void *owner; 2336b756a3b5SAlistair Popple bool valid; 2337b756a3b5SAlistair Popple }; 2338b756a3b5SAlistair Popple 23392f031c6fSMatthew Wilcox (Oracle) static bool page_make_device_exclusive_one(struct folio *folio, 2340b756a3b5SAlistair Popple struct vm_area_struct *vma, unsigned long address, void *priv) 2341b756a3b5SAlistair Popple { 2342b756a3b5SAlistair Popple struct mm_struct *mm = vma->vm_mm; 23430d251485SMatthew Wilcox (Oracle) DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); 2344b756a3b5SAlistair Popple struct make_exclusive_args *args = priv; 2345b756a3b5SAlistair Popple pte_t pteval; 2346b756a3b5SAlistair Popple struct page *subpage; 2347b756a3b5SAlistair Popple bool ret = true; 2348b756a3b5SAlistair Popple struct mmu_notifier_range range; 2349b756a3b5SAlistair Popple swp_entry_t entry; 2350b756a3b5SAlistair Popple pte_t swp_pte; 2351c33c7948SRyan Roberts pte_t ptent; 2352b756a3b5SAlistair Popple 23537d4a8be0SAlistair Popple mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0, 2354b756a3b5SAlistair Popple vma->vm_mm, address, min(vma->vm_end, 23550d251485SMatthew Wilcox (Oracle) address + folio_size(folio)), 23560d251485SMatthew Wilcox (Oracle) args->owner); 2357b756a3b5SAlistair Popple mmu_notifier_invalidate_range_start(&range); 2358b756a3b5SAlistair Popple 2359b756a3b5SAlistair Popple while (page_vma_mapped_walk(&pvmw)) { 2360b756a3b5SAlistair Popple /* Unexpected PMD-mapped THP? */ 23610d251485SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(!pvmw.pte, folio); 2362b756a3b5SAlistair Popple 2363c33c7948SRyan Roberts ptent = ptep_get(pvmw.pte); 2364c33c7948SRyan Roberts if (!pte_present(ptent)) { 2365b756a3b5SAlistair Popple ret = false; 2366b756a3b5SAlistair Popple page_vma_mapped_walk_done(&pvmw); 2367b756a3b5SAlistair Popple break; 2368b756a3b5SAlistair Popple } 2369b756a3b5SAlistair Popple 23700d251485SMatthew Wilcox (Oracle) subpage = folio_page(folio, 2371c33c7948SRyan Roberts pte_pfn(ptent) - folio_pfn(folio)); 2372b756a3b5SAlistair Popple address = pvmw.address; 2373b756a3b5SAlistair Popple 2374b756a3b5SAlistair Popple /* Nuke the page table entry. */ 2375c33c7948SRyan Roberts flush_cache_page(vma, address, pte_pfn(ptent)); 2376b756a3b5SAlistair Popple pteval = ptep_clear_flush(vma, address, pvmw.pte); 2377b756a3b5SAlistair Popple 23780d251485SMatthew Wilcox (Oracle) /* Set the dirty flag on the folio now the pte is gone. */ 2379b756a3b5SAlistair Popple if (pte_dirty(pteval)) 23800d251485SMatthew Wilcox (Oracle) folio_mark_dirty(folio); 2381b756a3b5SAlistair Popple 2382b756a3b5SAlistair Popple /* 2383b756a3b5SAlistair Popple * Check that our target page is still mapped at the expected 2384b756a3b5SAlistair Popple * address. 2385b756a3b5SAlistair Popple */ 2386b756a3b5SAlistair Popple if (args->mm == mm && args->address == address && 2387b756a3b5SAlistair Popple pte_write(pteval)) 2388b756a3b5SAlistair Popple args->valid = true; 2389b756a3b5SAlistair Popple 2390b756a3b5SAlistair Popple /* 2391b756a3b5SAlistair Popple * Store the pfn of the page in a special migration 2392b756a3b5SAlistair Popple * pte. do_swap_page() will wait until the migration 2393b756a3b5SAlistair Popple * pte is removed and then restart fault handling. 2394b756a3b5SAlistair Popple */ 2395b756a3b5SAlistair Popple if (pte_write(pteval)) 2396b756a3b5SAlistair Popple entry = make_writable_device_exclusive_entry( 2397b756a3b5SAlistair Popple page_to_pfn(subpage)); 2398b756a3b5SAlistair Popple else 2399b756a3b5SAlistair Popple entry = make_readable_device_exclusive_entry( 2400b756a3b5SAlistair Popple page_to_pfn(subpage)); 2401b756a3b5SAlistair Popple swp_pte = swp_entry_to_pte(entry); 2402b756a3b5SAlistair Popple if (pte_soft_dirty(pteval)) 2403b756a3b5SAlistair Popple swp_pte = pte_swp_mksoft_dirty(swp_pte); 2404b756a3b5SAlistair Popple if (pte_uffd_wp(pteval)) 2405b756a3b5SAlistair Popple swp_pte = pte_swp_mkuffd_wp(swp_pte); 2406b756a3b5SAlistair Popple 2407b756a3b5SAlistair Popple set_pte_at(mm, address, pvmw.pte, swp_pte); 2408b756a3b5SAlistair Popple 2409b756a3b5SAlistair Popple /* 2410b756a3b5SAlistair Popple * There is a reference on the page for the swap entry which has 2411b756a3b5SAlistair Popple * been removed, so shouldn't take another. 2412b756a3b5SAlistair Popple */ 2413ca1a0746SDavid Hildenbrand folio_remove_rmap_pte(folio, subpage, vma); 2414b756a3b5SAlistair Popple } 2415b756a3b5SAlistair Popple 2416b756a3b5SAlistair Popple mmu_notifier_invalidate_range_end(&range); 2417b756a3b5SAlistair Popple 2418b756a3b5SAlistair Popple return ret; 2419b756a3b5SAlistair Popple } 2420b756a3b5SAlistair Popple 2421b756a3b5SAlistair Popple /** 24220d251485SMatthew Wilcox (Oracle) * folio_make_device_exclusive - Mark the folio exclusively owned by a device. 24230d251485SMatthew Wilcox (Oracle) * @folio: The folio to replace page table entries for. 24240d251485SMatthew Wilcox (Oracle) * @mm: The mm_struct where the folio is expected to be mapped. 24250d251485SMatthew Wilcox (Oracle) * @address: Address where the folio is expected to be mapped. 2426b756a3b5SAlistair Popple * @owner: passed to MMU_NOTIFY_EXCLUSIVE range notifier callbacks 2427b756a3b5SAlistair Popple * 24280d251485SMatthew Wilcox (Oracle) * Tries to remove all the page table entries which are mapping this 24290d251485SMatthew Wilcox (Oracle) * folio and replace them with special device exclusive swap entries to 24300d251485SMatthew Wilcox (Oracle) * grant a device exclusive access to the folio. 2431b756a3b5SAlistair Popple * 24320d251485SMatthew Wilcox (Oracle) * Context: Caller must hold the folio lock. 24330d251485SMatthew Wilcox (Oracle) * Return: false if the page is still mapped, or if it could not be unmapped 2434b756a3b5SAlistair Popple * from the expected address. Otherwise returns true (success). 2435b756a3b5SAlistair Popple */ 24360d251485SMatthew Wilcox (Oracle) static bool folio_make_device_exclusive(struct folio *folio, 24370d251485SMatthew Wilcox (Oracle) struct mm_struct *mm, unsigned long address, void *owner) 2438b756a3b5SAlistair Popple { 2439b756a3b5SAlistair Popple struct make_exclusive_args args = { 2440b756a3b5SAlistair Popple .mm = mm, 2441b756a3b5SAlistair Popple .address = address, 2442b756a3b5SAlistair Popple .owner = owner, 2443b756a3b5SAlistair Popple .valid = false, 2444b756a3b5SAlistair Popple }; 2445b756a3b5SAlistair Popple struct rmap_walk_control rwc = { 2446b756a3b5SAlistair Popple .rmap_one = page_make_device_exclusive_one, 2447f3ad032cSKefeng Wang .done = folio_not_mapped, 24482f031c6fSMatthew Wilcox (Oracle) .anon_lock = folio_lock_anon_vma_read, 2449b756a3b5SAlistair Popple .arg = &args, 2450b756a3b5SAlistair Popple }; 2451b756a3b5SAlistair Popple 2452b756a3b5SAlistair Popple /* 24530d251485SMatthew Wilcox (Oracle) * Restrict to anonymous folios for now to avoid potential writeback 24540d251485SMatthew Wilcox (Oracle) * issues. 2455b756a3b5SAlistair Popple */ 24560d251485SMatthew Wilcox (Oracle) if (!folio_test_anon(folio)) 2457b756a3b5SAlistair Popple return false; 2458b756a3b5SAlistair Popple 24592f031c6fSMatthew Wilcox (Oracle) rmap_walk(folio, &rwc); 2460b756a3b5SAlistair Popple 24610d251485SMatthew Wilcox (Oracle) return args.valid && !folio_mapcount(folio); 2462b756a3b5SAlistair Popple } 2463b756a3b5SAlistair Popple 2464b756a3b5SAlistair Popple /** 2465b756a3b5SAlistair Popple * make_device_exclusive_range() - Mark a range for exclusive use by a device 2466dd062302SAdrian Huang * @mm: mm_struct of associated target process 2467b756a3b5SAlistair Popple * @start: start of the region to mark for exclusive device access 2468b756a3b5SAlistair Popple * @end: end address of region 2469b756a3b5SAlistair Popple * @pages: returns the pages which were successfully marked for exclusive access 2470b756a3b5SAlistair Popple * @owner: passed to MMU_NOTIFY_EXCLUSIVE range notifier to allow filtering 2471b756a3b5SAlistair Popple * 2472b756a3b5SAlistair Popple * Returns: number of pages found in the range by GUP. A page is marked for 2473b756a3b5SAlistair Popple * exclusive access only if the page pointer is non-NULL. 2474b756a3b5SAlistair Popple * 2475b756a3b5SAlistair Popple * This function finds ptes mapping page(s) to the given address range, locks 2476b756a3b5SAlistair Popple * them and replaces mappings with special swap entries preventing userspace CPU 2477b756a3b5SAlistair Popple * access. On fault these entries are replaced with the original mapping after 2478b756a3b5SAlistair Popple * calling MMU notifiers. 2479b756a3b5SAlistair Popple * 2480b756a3b5SAlistair Popple * A driver using this to program access from a device must use a mmu notifier 2481b756a3b5SAlistair Popple * critical section to hold a device specific lock during programming. Once 2482b756a3b5SAlistair Popple * programming is complete it should drop the page lock and reference after 2483b756a3b5SAlistair Popple * which point CPU access to the page will revoke the exclusive access. 2484b756a3b5SAlistair Popple */ 2485b756a3b5SAlistair Popple int make_device_exclusive_range(struct mm_struct *mm, unsigned long start, 2486b756a3b5SAlistair Popple unsigned long end, struct page **pages, 2487b756a3b5SAlistair Popple void *owner) 2488b756a3b5SAlistair Popple { 2489b756a3b5SAlistair Popple long npages = (end - start) >> PAGE_SHIFT; 2490b756a3b5SAlistair Popple long i; 2491b756a3b5SAlistair Popple 2492b756a3b5SAlistair Popple npages = get_user_pages_remote(mm, start, npages, 2493b756a3b5SAlistair Popple FOLL_GET | FOLL_WRITE | FOLL_SPLIT_PMD, 2494ca5e8632SLorenzo Stoakes pages, NULL); 2495b756a3b5SAlistair Popple if (npages < 0) 2496b756a3b5SAlistair Popple return npages; 2497b756a3b5SAlistair Popple 2498b756a3b5SAlistair Popple for (i = 0; i < npages; i++, start += PAGE_SIZE) { 24990d251485SMatthew Wilcox (Oracle) struct folio *folio = page_folio(pages[i]); 25000d251485SMatthew Wilcox (Oracle) if (PageTail(pages[i]) || !folio_trylock(folio)) { 25010d251485SMatthew Wilcox (Oracle) folio_put(folio); 2502b756a3b5SAlistair Popple pages[i] = NULL; 2503b756a3b5SAlistair Popple continue; 2504b756a3b5SAlistair Popple } 2505b756a3b5SAlistair Popple 25060d251485SMatthew Wilcox (Oracle) if (!folio_make_device_exclusive(folio, mm, start, owner)) { 25070d251485SMatthew Wilcox (Oracle) folio_unlock(folio); 25080d251485SMatthew Wilcox (Oracle) folio_put(folio); 2509b756a3b5SAlistair Popple pages[i] = NULL; 2510b756a3b5SAlistair Popple } 2511b756a3b5SAlistair Popple } 2512b756a3b5SAlistair Popple 2513b756a3b5SAlistair Popple return npages; 2514b756a3b5SAlistair Popple } 2515b756a3b5SAlistair Popple EXPORT_SYMBOL_GPL(make_device_exclusive_range); 2516b756a3b5SAlistair Popple #endif 2517b756a3b5SAlistair Popple 251801d8b20dSPeter Zijlstra void __put_anon_vma(struct anon_vma *anon_vma) 251976545066SRik van Riel { 252076545066SRik van Riel struct anon_vma *root = anon_vma->root; 252176545066SRik van Riel 2522624483f3SAndrey Ryabinin anon_vma_free(anon_vma); 252301d8b20dSPeter Zijlstra if (root != anon_vma && atomic_dec_and_test(&root->refcount)) 252476545066SRik van Riel anon_vma_free(root); 252576545066SRik van Riel } 252676545066SRik van Riel 25272f031c6fSMatthew Wilcox (Oracle) static struct anon_vma *rmap_walk_anon_lock(struct folio *folio, 25286d4675e6SMinchan Kim struct rmap_walk_control *rwc) 2529faecd8ddSJoonsoo Kim { 2530faecd8ddSJoonsoo Kim struct anon_vma *anon_vma; 2531faecd8ddSJoonsoo Kim 25320dd1c7bbSJoonsoo Kim if (rwc->anon_lock) 25336d4675e6SMinchan Kim return rwc->anon_lock(folio, rwc); 25340dd1c7bbSJoonsoo Kim 2535faecd8ddSJoonsoo Kim /* 25362f031c6fSMatthew Wilcox (Oracle) * Note: remove_migration_ptes() cannot use folio_lock_anon_vma_read() 2537faecd8ddSJoonsoo Kim * because that depends on page_mapped(); but not all its usages 2538c1e8d7c6SMichel Lespinasse * are holding mmap_lock. Users without mmap_lock are required to 2539faecd8ddSJoonsoo Kim * take a reference count to prevent the anon_vma disappearing 2540faecd8ddSJoonsoo Kim */ 2541e05b3453SMatthew Wilcox (Oracle) anon_vma = folio_anon_vma(folio); 2542faecd8ddSJoonsoo Kim if (!anon_vma) 2543faecd8ddSJoonsoo Kim return NULL; 2544faecd8ddSJoonsoo Kim 25456d4675e6SMinchan Kim if (anon_vma_trylock_read(anon_vma)) 25466d4675e6SMinchan Kim goto out; 25476d4675e6SMinchan Kim 25486d4675e6SMinchan Kim if (rwc->try_lock) { 25496d4675e6SMinchan Kim anon_vma = NULL; 25506d4675e6SMinchan Kim rwc->contended = true; 25516d4675e6SMinchan Kim goto out; 25526d4675e6SMinchan Kim } 25536d4675e6SMinchan Kim 2554faecd8ddSJoonsoo Kim anon_vma_lock_read(anon_vma); 25556d4675e6SMinchan Kim out: 2556faecd8ddSJoonsoo Kim return anon_vma; 2557faecd8ddSJoonsoo Kim } 2558faecd8ddSJoonsoo Kim 2559e9995ef9SHugh Dickins /* 2560e8351ac9SJoonsoo Kim * rmap_walk_anon - do something to anonymous page using the object-based 2561e8351ac9SJoonsoo Kim * rmap method 256289be82b4SKemeng Shi * @folio: the folio to be handled 2563e8351ac9SJoonsoo Kim * @rwc: control variable according to each walk type 256489be82b4SKemeng Shi * @locked: caller holds relevant rmap lock 2565e8351ac9SJoonsoo Kim * 256689be82b4SKemeng Shi * Find all the mappings of a folio using the mapping pointer and the vma 256789be82b4SKemeng Shi * chains contained in the anon_vma struct it points to. 2568e9995ef9SHugh Dickins */ 256984fbbe21SMatthew Wilcox (Oracle) static void rmap_walk_anon(struct folio *folio, 25706d4675e6SMinchan Kim struct rmap_walk_control *rwc, bool locked) 2571e9995ef9SHugh Dickins { 2572e9995ef9SHugh Dickins struct anon_vma *anon_vma; 2573a8fa41adSKirill A. Shutemov pgoff_t pgoff_start, pgoff_end; 25745beb4930SRik van Riel struct anon_vma_chain *avc; 2575e9995ef9SHugh Dickins 2576b9773199SKirill A. Shutemov if (locked) { 2577e05b3453SMatthew Wilcox (Oracle) anon_vma = folio_anon_vma(folio); 2578b9773199SKirill A. Shutemov /* anon_vma disappear under us? */ 2579e05b3453SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(!anon_vma, folio); 2580b9773199SKirill A. Shutemov } else { 25812f031c6fSMatthew Wilcox (Oracle) anon_vma = rmap_walk_anon_lock(folio, rwc); 2582b9773199SKirill A. Shutemov } 2583e9995ef9SHugh Dickins if (!anon_vma) 25841df631aeSMinchan Kim return; 2585faecd8ddSJoonsoo Kim 25862f031c6fSMatthew Wilcox (Oracle) pgoff_start = folio_pgoff(folio); 25872f031c6fSMatthew Wilcox (Oracle) pgoff_end = pgoff_start + folio_nr_pages(folio) - 1; 2588a8fa41adSKirill A. Shutemov anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, 2589a8fa41adSKirill A. Shutemov pgoff_start, pgoff_end) { 25905beb4930SRik van Riel struct vm_area_struct *vma = avc->vma; 25912f031c6fSMatthew Wilcox (Oracle) unsigned long address = vma_address(&folio->page, vma); 25920dd1c7bbSJoonsoo Kim 2593494334e4SHugh Dickins VM_BUG_ON_VMA(address == -EFAULT, vma); 2594ad12695fSAndrea Arcangeli cond_resched(); 2595ad12695fSAndrea Arcangeli 25960dd1c7bbSJoonsoo Kim if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) 25970dd1c7bbSJoonsoo Kim continue; 25980dd1c7bbSJoonsoo Kim 25992f031c6fSMatthew Wilcox (Oracle) if (!rwc->rmap_one(folio, vma, address, rwc->arg)) 2600e9995ef9SHugh Dickins break; 26012f031c6fSMatthew Wilcox (Oracle) if (rwc->done && rwc->done(folio)) 26020dd1c7bbSJoonsoo Kim break; 2603e9995ef9SHugh Dickins } 2604b9773199SKirill A. Shutemov 2605b9773199SKirill A. Shutemov if (!locked) 26064fc3f1d6SIngo Molnar anon_vma_unlock_read(anon_vma); 2607e9995ef9SHugh Dickins } 2608e9995ef9SHugh Dickins 2609e8351ac9SJoonsoo Kim /* 2610e8351ac9SJoonsoo Kim * rmap_walk_file - do something to file page using the object-based rmap method 261189be82b4SKemeng Shi * @folio: the folio to be handled 2612e8351ac9SJoonsoo Kim * @rwc: control variable according to each walk type 261389be82b4SKemeng Shi * @locked: caller holds relevant rmap lock 2614e8351ac9SJoonsoo Kim * 261589be82b4SKemeng Shi * Find all the mappings of a folio using the mapping pointer and the vma chains 2616e8351ac9SJoonsoo Kim * contained in the address_space struct it points to. 2617e8351ac9SJoonsoo Kim */ 261884fbbe21SMatthew Wilcox (Oracle) static void rmap_walk_file(struct folio *folio, 26196d4675e6SMinchan Kim struct rmap_walk_control *rwc, bool locked) 2620e9995ef9SHugh Dickins { 26212f031c6fSMatthew Wilcox (Oracle) struct address_space *mapping = folio_mapping(folio); 2622a8fa41adSKirill A. Shutemov pgoff_t pgoff_start, pgoff_end; 2623e9995ef9SHugh Dickins struct vm_area_struct *vma; 2624e9995ef9SHugh Dickins 26259f32624bSJoonsoo Kim /* 26269f32624bSJoonsoo Kim * The page lock not only makes sure that page->mapping cannot 26279f32624bSJoonsoo Kim * suddenly be NULLified by truncation, it makes sure that the 26289f32624bSJoonsoo Kim * structure at mapping cannot be freed and reused yet, 2629c8c06efaSDavidlohr Bueso * so we can safely take mapping->i_mmap_rwsem. 26309f32624bSJoonsoo Kim */ 26312f031c6fSMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 26329f32624bSJoonsoo Kim 2633e9995ef9SHugh Dickins if (!mapping) 26341df631aeSMinchan Kim return; 26353dec0ba0SDavidlohr Bueso 26362f031c6fSMatthew Wilcox (Oracle) pgoff_start = folio_pgoff(folio); 26372f031c6fSMatthew Wilcox (Oracle) pgoff_end = pgoff_start + folio_nr_pages(folio) - 1; 26386d4675e6SMinchan Kim if (!locked) { 26396d4675e6SMinchan Kim if (i_mmap_trylock_read(mapping)) 26406d4675e6SMinchan Kim goto lookup; 26416d4675e6SMinchan Kim 26426d4675e6SMinchan Kim if (rwc->try_lock) { 26436d4675e6SMinchan Kim rwc->contended = true; 26446d4675e6SMinchan Kim return; 26456d4675e6SMinchan Kim } 26466d4675e6SMinchan Kim 26473dec0ba0SDavidlohr Bueso i_mmap_lock_read(mapping); 26486d4675e6SMinchan Kim } 26496d4675e6SMinchan Kim lookup: 2650a8fa41adSKirill A. Shutemov vma_interval_tree_foreach(vma, &mapping->i_mmap, 2651a8fa41adSKirill A. Shutemov pgoff_start, pgoff_end) { 26522f031c6fSMatthew Wilcox (Oracle) unsigned long address = vma_address(&folio->page, vma); 26530dd1c7bbSJoonsoo Kim 2654494334e4SHugh Dickins VM_BUG_ON_VMA(address == -EFAULT, vma); 2655ad12695fSAndrea Arcangeli cond_resched(); 2656ad12695fSAndrea Arcangeli 26570dd1c7bbSJoonsoo Kim if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) 26580dd1c7bbSJoonsoo Kim continue; 26590dd1c7bbSJoonsoo Kim 26602f031c6fSMatthew Wilcox (Oracle) if (!rwc->rmap_one(folio, vma, address, rwc->arg)) 26610dd1c7bbSJoonsoo Kim goto done; 26622f031c6fSMatthew Wilcox (Oracle) if (rwc->done && rwc->done(folio)) 26630dd1c7bbSJoonsoo Kim goto done; 2664e9995ef9SHugh Dickins } 26650dd1c7bbSJoonsoo Kim 26660dd1c7bbSJoonsoo Kim done: 2667b9773199SKirill A. Shutemov if (!locked) 26683dec0ba0SDavidlohr Bueso i_mmap_unlock_read(mapping); 2669e9995ef9SHugh Dickins } 2670e9995ef9SHugh Dickins 26716d4675e6SMinchan Kim void rmap_walk(struct folio *folio, struct rmap_walk_control *rwc) 2672e9995ef9SHugh Dickins { 26732f031c6fSMatthew Wilcox (Oracle) if (unlikely(folio_test_ksm(folio))) 26742f031c6fSMatthew Wilcox (Oracle) rmap_walk_ksm(folio, rwc); 26752f031c6fSMatthew Wilcox (Oracle) else if (folio_test_anon(folio)) 26762f031c6fSMatthew Wilcox (Oracle) rmap_walk_anon(folio, rwc, false); 2677e9995ef9SHugh Dickins else 26782f031c6fSMatthew Wilcox (Oracle) rmap_walk_file(folio, rwc, false); 2679b9773199SKirill A. Shutemov } 2680b9773199SKirill A. Shutemov 2681b9773199SKirill A. Shutemov /* Like rmap_walk, but caller holds relevant rmap lock */ 26826d4675e6SMinchan Kim void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc) 2683b9773199SKirill A. Shutemov { 2684b9773199SKirill A. Shutemov /* no ksm support for now */ 26852f031c6fSMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(folio_test_ksm(folio), folio); 26862f031c6fSMatthew Wilcox (Oracle) if (folio_test_anon(folio)) 26872f031c6fSMatthew Wilcox (Oracle) rmap_walk_anon(folio, rwc, true); 2688b9773199SKirill A. Shutemov else 26892f031c6fSMatthew Wilcox (Oracle) rmap_walk_file(folio, rwc, true); 2690e9995ef9SHugh Dickins } 26910fe6e20bSNaoya Horiguchi 2692e3390f67SNaoya Horiguchi #ifdef CONFIG_HUGETLB_PAGE 26930fe6e20bSNaoya Horiguchi /* 2694451b9514SKirill Tkhai * The following two functions are for anonymous (private mapped) hugepages. 26950fe6e20bSNaoya Horiguchi * Unlike common anonymous pages, anonymous hugepages have no accounting code 26960fe6e20bSNaoya Horiguchi * and no lru code, because we handle hugepages differently from common pages. 26970fe6e20bSNaoya Horiguchi */ 26989d5fafd5SDavid Hildenbrand void hugetlb_add_anon_rmap(struct folio *folio, struct vm_area_struct *vma, 269928c5209dSDavid Hildenbrand unsigned long address, rmap_t flags) 27000fe6e20bSNaoya Horiguchi { 2701a4ea1864SDavid Hildenbrand VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio); 2702c5c54003SDavid Hildenbrand VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); 2703c5c54003SDavid Hildenbrand 2704132b180fSDavid Hildenbrand atomic_inc(&folio->_entire_mapcount); 2705c66db8c0SDavid Hildenbrand if (flags & RMAP_EXCLUSIVE) 270609c55050SDavid Hildenbrand SetPageAnonExclusive(&folio->page); 2707132b180fSDavid Hildenbrand VM_WARN_ON_FOLIO(folio_entire_mapcount(folio) > 1 && 270809c55050SDavid Hildenbrand PageAnonExclusive(&folio->page), folio); 27090fe6e20bSNaoya Horiguchi } 27100fe6e20bSNaoya Horiguchi 27119d5fafd5SDavid Hildenbrand void hugetlb_add_new_anon_rmap(struct folio *folio, 27120fe6e20bSNaoya Horiguchi struct vm_area_struct *vma, unsigned long address) 27130fe6e20bSNaoya Horiguchi { 2714a4ea1864SDavid Hildenbrand VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio); 2715a4ea1864SDavid Hildenbrand 27160fe6e20bSNaoya Horiguchi BUG_ON(address < vma->vm_start || address >= vma->vm_end); 2717cb67f428SHugh Dickins /* increment count (starts at -1) */ 2718db4e5dbdSMatthew Wilcox (Oracle) atomic_set(&folio->_entire_mapcount, 0); 2719db4e5dbdSMatthew Wilcox (Oracle) folio_clear_hugetlb_restore_reserve(folio); 2720c66db8c0SDavid Hildenbrand __folio_set_anon(folio, vma, address, true); 2721c66db8c0SDavid Hildenbrand SetPageAnonExclusive(&folio->page); 27220fe6e20bSNaoya Horiguchi } 2723e3390f67SNaoya Horiguchi #endif /* CONFIG_HUGETLB_PAGE */ 2724