11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * mm/rmap.c - physical to virtual reverse mappings 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright 2001, Rik van Riel <riel@conectiva.com.br> 51da177e4SLinus Torvalds * Released under the General Public License (GPL). 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * Simple, low overhead reverse mapping scheme. 81da177e4SLinus Torvalds * Please try to keep this thing as modular as possible. 91da177e4SLinus Torvalds * 101da177e4SLinus Torvalds * Provides methods for unmapping each kind of mapped page: 111da177e4SLinus Torvalds * the anon methods track anonymous pages, and 121da177e4SLinus Torvalds * the file methods track pages belonging to an inode. 131da177e4SLinus Torvalds * 141da177e4SLinus Torvalds * Original design by Rik van Riel <riel@conectiva.com.br> 2001 151da177e4SLinus Torvalds * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004 161da177e4SLinus Torvalds * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004 1798f32602SHugh Dickins * Contributions by Hugh Dickins 2003, 2004 181da177e4SLinus Torvalds */ 191da177e4SLinus Torvalds 201da177e4SLinus Torvalds /* 211da177e4SLinus Torvalds * Lock ordering in mm: 221da177e4SLinus Torvalds * 239608703eSJan Kara * inode->i_rwsem (while writing or truncating, not reading or faulting) 24c1e8d7c6SMichel Lespinasse * mm->mmap_lock 25730633f0SJan Kara * mapping->invalidate_lock (in filemap_fault) 269608703eSJan Kara * page->flags PG_locked (lock_page) * (see hugetlbfs below) 2788f306b6SKirill A. Shutemov * hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share) 28c8c06efaSDavidlohr Bueso * mapping->i_mmap_rwsem 29c0d0381aSMike Kravetz * hugetlb_fault_mutex (hugetlbfs specific page fault mutex) 305a505085SIngo Molnar * anon_vma->rwsem 31b8072f09SHugh Dickins * mm->page_table_lock or pte_lock 325d337b91SHugh Dickins * swap_lock (in swap_duplicate, swap_info_get) 331da177e4SLinus Torvalds * mmlist_lock (in mmput, drain_mmlist and others) 34e621900aSMatthew Wilcox (Oracle) * mapping->private_lock (in block_dirty_folio) 35e621900aSMatthew Wilcox (Oracle) * folio_lock_memcg move_lock (in block_dirty_folio) 36b93b0163SMatthew Wilcox * i_pages lock (widely used) 37e809c3feSMatthew Wilcox (Oracle) * lruvec->lru_lock (in folio_lruvec_lock_irq) 38250df6edSDave Chinner * inode->i_lock (in set_page_dirty's __mark_inode_dirty) 39f758eeabSChristoph Hellwig * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty) 401da177e4SLinus Torvalds * sb_lock (within inode_lock in fs/fs-writeback.c) 41b93b0163SMatthew Wilcox * i_pages lock (widely used, in set_page_dirty, 421da177e4SLinus Torvalds * in arch-dependent flush_dcache_mmap_lock, 43f758eeabSChristoph Hellwig * within bdi.wb->list_lock in __sync_single_inode) 446a46079cSAndi Kleen * 459608703eSJan Kara * anon_vma->rwsem,mapping->i_mmap_rwsem (memory_failure, collect_procs_anon) 466a46079cSAndi Kleen * ->tasklist_lock 476a46079cSAndi Kleen * pte map lock 48c0d0381aSMike Kravetz * 49c0d0381aSMike Kravetz * * hugetlbfs PageHuge() pages take locks in this order: 50c0d0381aSMike Kravetz * mapping->i_mmap_rwsem 51c0d0381aSMike Kravetz * hugetlb_fault_mutex (hugetlbfs specific page fault mutex) 52c0d0381aSMike Kravetz * page->flags PG_locked (lock_page) 531da177e4SLinus Torvalds */ 541da177e4SLinus Torvalds 551da177e4SLinus Torvalds #include <linux/mm.h> 566e84f315SIngo Molnar #include <linux/sched/mm.h> 5729930025SIngo Molnar #include <linux/sched/task.h> 581da177e4SLinus Torvalds #include <linux/pagemap.h> 591da177e4SLinus Torvalds #include <linux/swap.h> 601da177e4SLinus Torvalds #include <linux/swapops.h> 611da177e4SLinus Torvalds #include <linux/slab.h> 621da177e4SLinus Torvalds #include <linux/init.h> 635ad64688SHugh Dickins #include <linux/ksm.h> 641da177e4SLinus Torvalds #include <linux/rmap.h> 651da177e4SLinus Torvalds #include <linux/rcupdate.h> 66b95f1b31SPaul Gortmaker #include <linux/export.h> 678a9f3ccdSBalbir Singh #include <linux/memcontrol.h> 68cddb8a5cSAndrea Arcangeli #include <linux/mmu_notifier.h> 6964cdd548SKOSAKI Motohiro #include <linux/migrate.h> 700fe6e20bSNaoya Horiguchi #include <linux/hugetlb.h> 71444f84fdSBen Dooks #include <linux/huge_mm.h> 72ef5d437fSJan Kara #include <linux/backing-dev.h> 7333c3fc71SVladimir Davydov #include <linux/page_idle.h> 74a5430ddaSJérôme Glisse #include <linux/memremap.h> 75bce73e48SChristian Borntraeger #include <linux/userfaultfd_k.h> 761da177e4SLinus Torvalds 771da177e4SLinus Torvalds #include <asm/tlbflush.h> 781da177e4SLinus Torvalds 79*4cc79b33SAnshuman Khandual #define CREATE_TRACE_POINTS 8072b252aeSMel Gorman #include <trace/events/tlb.h> 81*4cc79b33SAnshuman Khandual #include <trace/events/migrate.h> 8272b252aeSMel Gorman 83b291f000SNick Piggin #include "internal.h" 84b291f000SNick Piggin 85fdd2e5f8SAdrian Bunk static struct kmem_cache *anon_vma_cachep; 865beb4930SRik van Riel static struct kmem_cache *anon_vma_chain_cachep; 87fdd2e5f8SAdrian Bunk 88fdd2e5f8SAdrian Bunk static inline struct anon_vma *anon_vma_alloc(void) 89fdd2e5f8SAdrian Bunk { 9001d8b20dSPeter Zijlstra struct anon_vma *anon_vma; 9101d8b20dSPeter Zijlstra 9201d8b20dSPeter Zijlstra anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL); 9301d8b20dSPeter Zijlstra if (anon_vma) { 9401d8b20dSPeter Zijlstra atomic_set(&anon_vma->refcount, 1); 957a3ef208SKonstantin Khlebnikov anon_vma->degree = 1; /* Reference for first vma */ 967a3ef208SKonstantin Khlebnikov anon_vma->parent = anon_vma; 9701d8b20dSPeter Zijlstra /* 9801d8b20dSPeter Zijlstra * Initialise the anon_vma root to point to itself. If called 9901d8b20dSPeter Zijlstra * from fork, the root will be reset to the parents anon_vma. 10001d8b20dSPeter Zijlstra */ 10101d8b20dSPeter Zijlstra anon_vma->root = anon_vma; 102fdd2e5f8SAdrian Bunk } 103fdd2e5f8SAdrian Bunk 10401d8b20dSPeter Zijlstra return anon_vma; 10501d8b20dSPeter Zijlstra } 10601d8b20dSPeter Zijlstra 10701d8b20dSPeter Zijlstra static inline void anon_vma_free(struct anon_vma *anon_vma) 108fdd2e5f8SAdrian Bunk { 10901d8b20dSPeter Zijlstra VM_BUG_ON(atomic_read(&anon_vma->refcount)); 11088c22088SPeter Zijlstra 11188c22088SPeter Zijlstra /* 1122f031c6fSMatthew Wilcox (Oracle) * Synchronize against folio_lock_anon_vma_read() such that 11388c22088SPeter Zijlstra * we can safely hold the lock without the anon_vma getting 11488c22088SPeter Zijlstra * freed. 11588c22088SPeter Zijlstra * 11688c22088SPeter Zijlstra * Relies on the full mb implied by the atomic_dec_and_test() from 11788c22088SPeter Zijlstra * put_anon_vma() against the acquire barrier implied by 1182f031c6fSMatthew Wilcox (Oracle) * down_read_trylock() from folio_lock_anon_vma_read(). This orders: 11988c22088SPeter Zijlstra * 1202f031c6fSMatthew Wilcox (Oracle) * folio_lock_anon_vma_read() VS put_anon_vma() 1214fc3f1d6SIngo Molnar * down_read_trylock() atomic_dec_and_test() 12288c22088SPeter Zijlstra * LOCK MB 1234fc3f1d6SIngo Molnar * atomic_read() rwsem_is_locked() 12488c22088SPeter Zijlstra * 12588c22088SPeter Zijlstra * LOCK should suffice since the actual taking of the lock must 12688c22088SPeter Zijlstra * happen _before_ what follows. 12788c22088SPeter Zijlstra */ 1287f39dda9SHugh Dickins might_sleep(); 1295a505085SIngo Molnar if (rwsem_is_locked(&anon_vma->root->rwsem)) { 1304fc3f1d6SIngo Molnar anon_vma_lock_write(anon_vma); 13108b52706SKonstantin Khlebnikov anon_vma_unlock_write(anon_vma); 13288c22088SPeter Zijlstra } 13388c22088SPeter Zijlstra 134fdd2e5f8SAdrian Bunk kmem_cache_free(anon_vma_cachep, anon_vma); 135fdd2e5f8SAdrian Bunk } 1361da177e4SLinus Torvalds 137dd34739cSLinus Torvalds static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp) 1385beb4930SRik van Riel { 139dd34739cSLinus Torvalds return kmem_cache_alloc(anon_vma_chain_cachep, gfp); 1405beb4930SRik van Riel } 1415beb4930SRik van Riel 142e574b5fdSNamhyung Kim static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain) 1435beb4930SRik van Riel { 1445beb4930SRik van Riel kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain); 1455beb4930SRik van Riel } 1465beb4930SRik van Riel 1476583a843SKautuk Consul static void anon_vma_chain_link(struct vm_area_struct *vma, 1486583a843SKautuk Consul struct anon_vma_chain *avc, 1496583a843SKautuk Consul struct anon_vma *anon_vma) 1506583a843SKautuk Consul { 1516583a843SKautuk Consul avc->vma = vma; 1526583a843SKautuk Consul avc->anon_vma = anon_vma; 1536583a843SKautuk Consul list_add(&avc->same_vma, &vma->anon_vma_chain); 154bf181b9fSMichel Lespinasse anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); 1556583a843SKautuk Consul } 1566583a843SKautuk Consul 157d9d332e0SLinus Torvalds /** 158d5a187daSVlastimil Babka * __anon_vma_prepare - attach an anon_vma to a memory region 159d9d332e0SLinus Torvalds * @vma: the memory region in question 160d9d332e0SLinus Torvalds * 161d9d332e0SLinus Torvalds * This makes sure the memory mapping described by 'vma' has 162d9d332e0SLinus Torvalds * an 'anon_vma' attached to it, so that we can associate the 163d9d332e0SLinus Torvalds * anonymous pages mapped into it with that anon_vma. 164d9d332e0SLinus Torvalds * 165d5a187daSVlastimil Babka * The common case will be that we already have one, which 166d5a187daSVlastimil Babka * is handled inline by anon_vma_prepare(). But if 16723a0790aSFigo.zhang * not we either need to find an adjacent mapping that we 168d9d332e0SLinus Torvalds * can re-use the anon_vma from (very common when the only 169d9d332e0SLinus Torvalds * reason for splitting a vma has been mprotect()), or we 170d9d332e0SLinus Torvalds * allocate a new one. 171d9d332e0SLinus Torvalds * 172d9d332e0SLinus Torvalds * Anon-vma allocations are very subtle, because we may have 1732f031c6fSMatthew Wilcox (Oracle) * optimistically looked up an anon_vma in folio_lock_anon_vma_read() 174aaf1f990SMiaohe Lin * and that may actually touch the rwsem even in the newly 175d9d332e0SLinus Torvalds * allocated vma (it depends on RCU to make sure that the 176d9d332e0SLinus Torvalds * anon_vma isn't actually destroyed). 177d9d332e0SLinus Torvalds * 178d9d332e0SLinus Torvalds * As a result, we need to do proper anon_vma locking even 179d9d332e0SLinus Torvalds * for the new allocation. At the same time, we do not want 180d9d332e0SLinus Torvalds * to do any locking for the common case of already having 181d9d332e0SLinus Torvalds * an anon_vma. 182d9d332e0SLinus Torvalds * 183c1e8d7c6SMichel Lespinasse * This must be called with the mmap_lock held for reading. 184d9d332e0SLinus Torvalds */ 185d5a187daSVlastimil Babka int __anon_vma_prepare(struct vm_area_struct *vma) 1861da177e4SLinus Torvalds { 187d5a187daSVlastimil Babka struct mm_struct *mm = vma->vm_mm; 188d5a187daSVlastimil Babka struct anon_vma *anon_vma, *allocated; 1895beb4930SRik van Riel struct anon_vma_chain *avc; 1901da177e4SLinus Torvalds 1911da177e4SLinus Torvalds might_sleep(); 1921da177e4SLinus Torvalds 193dd34739cSLinus Torvalds avc = anon_vma_chain_alloc(GFP_KERNEL); 1945beb4930SRik van Riel if (!avc) 1955beb4930SRik van Riel goto out_enomem; 1965beb4930SRik van Riel 1971da177e4SLinus Torvalds anon_vma = find_mergeable_anon_vma(vma); 1981da177e4SLinus Torvalds allocated = NULL; 199d9d332e0SLinus Torvalds if (!anon_vma) { 2001da177e4SLinus Torvalds anon_vma = anon_vma_alloc(); 2011da177e4SLinus Torvalds if (unlikely(!anon_vma)) 2025beb4930SRik van Riel goto out_enomem_free_avc; 2031da177e4SLinus Torvalds allocated = anon_vma; 2041da177e4SLinus Torvalds } 2051da177e4SLinus Torvalds 2064fc3f1d6SIngo Molnar anon_vma_lock_write(anon_vma); 2071da177e4SLinus Torvalds /* page_table_lock to protect against threads */ 2081da177e4SLinus Torvalds spin_lock(&mm->page_table_lock); 2091da177e4SLinus Torvalds if (likely(!vma->anon_vma)) { 2101da177e4SLinus Torvalds vma->anon_vma = anon_vma; 2116583a843SKautuk Consul anon_vma_chain_link(vma, avc, anon_vma); 2127a3ef208SKonstantin Khlebnikov /* vma reference or self-parent link for new root */ 2137a3ef208SKonstantin Khlebnikov anon_vma->degree++; 2141da177e4SLinus Torvalds allocated = NULL; 21531f2b0ebSOleg Nesterov avc = NULL; 2161da177e4SLinus Torvalds } 2171da177e4SLinus Torvalds spin_unlock(&mm->page_table_lock); 21808b52706SKonstantin Khlebnikov anon_vma_unlock_write(anon_vma); 21931f2b0ebSOleg Nesterov 22031f2b0ebSOleg Nesterov if (unlikely(allocated)) 22101d8b20dSPeter Zijlstra put_anon_vma(allocated); 22231f2b0ebSOleg Nesterov if (unlikely(avc)) 2235beb4930SRik van Riel anon_vma_chain_free(avc); 224d5a187daSVlastimil Babka 2251da177e4SLinus Torvalds return 0; 2265beb4930SRik van Riel 2275beb4930SRik van Riel out_enomem_free_avc: 2285beb4930SRik van Riel anon_vma_chain_free(avc); 2295beb4930SRik van Riel out_enomem: 2305beb4930SRik van Riel return -ENOMEM; 2311da177e4SLinus Torvalds } 2321da177e4SLinus Torvalds 233bb4aa396SLinus Torvalds /* 234bb4aa396SLinus Torvalds * This is a useful helper function for locking the anon_vma root as 235bb4aa396SLinus Torvalds * we traverse the vma->anon_vma_chain, looping over anon_vma's that 236bb4aa396SLinus Torvalds * have the same vma. 237bb4aa396SLinus Torvalds * 238bb4aa396SLinus Torvalds * Such anon_vma's should have the same root, so you'd expect to see 239bb4aa396SLinus Torvalds * just a single mutex_lock for the whole traversal. 240bb4aa396SLinus Torvalds */ 241bb4aa396SLinus Torvalds static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma) 242bb4aa396SLinus Torvalds { 243bb4aa396SLinus Torvalds struct anon_vma *new_root = anon_vma->root; 244bb4aa396SLinus Torvalds if (new_root != root) { 245bb4aa396SLinus Torvalds if (WARN_ON_ONCE(root)) 2465a505085SIngo Molnar up_write(&root->rwsem); 247bb4aa396SLinus Torvalds root = new_root; 2485a505085SIngo Molnar down_write(&root->rwsem); 249bb4aa396SLinus Torvalds } 250bb4aa396SLinus Torvalds return root; 251bb4aa396SLinus Torvalds } 252bb4aa396SLinus Torvalds 253bb4aa396SLinus Torvalds static inline void unlock_anon_vma_root(struct anon_vma *root) 254bb4aa396SLinus Torvalds { 255bb4aa396SLinus Torvalds if (root) 2565a505085SIngo Molnar up_write(&root->rwsem); 257bb4aa396SLinus Torvalds } 258bb4aa396SLinus Torvalds 2595beb4930SRik van Riel /* 2605beb4930SRik van Riel * Attach the anon_vmas from src to dst. 2615beb4930SRik van Riel * Returns 0 on success, -ENOMEM on failure. 2627a3ef208SKonstantin Khlebnikov * 263cb152a1aSShijie Luo * anon_vma_clone() is called by __vma_adjust(), __split_vma(), copy_vma() and 26447b390d2SWei Yang * anon_vma_fork(). The first three want an exact copy of src, while the last 26547b390d2SWei Yang * one, anon_vma_fork(), may try to reuse an existing anon_vma to prevent 26647b390d2SWei Yang * endless growth of anon_vma. Since dst->anon_vma is set to NULL before call, 26747b390d2SWei Yang * we can identify this case by checking (!dst->anon_vma && src->anon_vma). 26847b390d2SWei Yang * 26947b390d2SWei Yang * If (!dst->anon_vma && src->anon_vma) is true, this function tries to find 27047b390d2SWei Yang * and reuse existing anon_vma which has no vmas and only one child anon_vma. 27147b390d2SWei Yang * This prevents degradation of anon_vma hierarchy to endless linear chain in 27247b390d2SWei Yang * case of constantly forking task. On the other hand, an anon_vma with more 27347b390d2SWei Yang * than one child isn't reused even if there was no alive vma, thus rmap 27447b390d2SWei Yang * walker has a good chance of avoiding scanning the whole hierarchy when it 27547b390d2SWei Yang * searches where page is mapped. 2765beb4930SRik van Riel */ 2775beb4930SRik van Riel int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src) 2785beb4930SRik van Riel { 2795beb4930SRik van Riel struct anon_vma_chain *avc, *pavc; 280bb4aa396SLinus Torvalds struct anon_vma *root = NULL; 2815beb4930SRik van Riel 282646d87b4SLinus Torvalds list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) { 283bb4aa396SLinus Torvalds struct anon_vma *anon_vma; 284bb4aa396SLinus Torvalds 285dd34739cSLinus Torvalds avc = anon_vma_chain_alloc(GFP_NOWAIT | __GFP_NOWARN); 286dd34739cSLinus Torvalds if (unlikely(!avc)) { 287dd34739cSLinus Torvalds unlock_anon_vma_root(root); 288dd34739cSLinus Torvalds root = NULL; 289dd34739cSLinus Torvalds avc = anon_vma_chain_alloc(GFP_KERNEL); 2905beb4930SRik van Riel if (!avc) 2915beb4930SRik van Riel goto enomem_failure; 292dd34739cSLinus Torvalds } 293bb4aa396SLinus Torvalds anon_vma = pavc->anon_vma; 294bb4aa396SLinus Torvalds root = lock_anon_vma_root(root, anon_vma); 295bb4aa396SLinus Torvalds anon_vma_chain_link(dst, avc, anon_vma); 2967a3ef208SKonstantin Khlebnikov 2977a3ef208SKonstantin Khlebnikov /* 2987a3ef208SKonstantin Khlebnikov * Reuse existing anon_vma if its degree lower than two, 2997a3ef208SKonstantin Khlebnikov * that means it has no vma and only one anon_vma child. 3007a3ef208SKonstantin Khlebnikov * 3017a3ef208SKonstantin Khlebnikov * Do not chose parent anon_vma, otherwise first child 3027a3ef208SKonstantin Khlebnikov * will always reuse it. Root anon_vma is never reused: 3037a3ef208SKonstantin Khlebnikov * it has self-parent reference and at least one child. 3047a3ef208SKonstantin Khlebnikov */ 30547b390d2SWei Yang if (!dst->anon_vma && src->anon_vma && 30647b390d2SWei Yang anon_vma != src->anon_vma && anon_vma->degree < 2) 3077a3ef208SKonstantin Khlebnikov dst->anon_vma = anon_vma; 3085beb4930SRik van Riel } 3097a3ef208SKonstantin Khlebnikov if (dst->anon_vma) 3107a3ef208SKonstantin Khlebnikov dst->anon_vma->degree++; 311bb4aa396SLinus Torvalds unlock_anon_vma_root(root); 3125beb4930SRik van Riel return 0; 3135beb4930SRik van Riel 3145beb4930SRik van Riel enomem_failure: 3153fe89b3eSLeon Yu /* 3163fe89b3eSLeon Yu * dst->anon_vma is dropped here otherwise its degree can be incorrectly 3173fe89b3eSLeon Yu * decremented in unlink_anon_vmas(). 3183fe89b3eSLeon Yu * We can safely do this because callers of anon_vma_clone() don't care 3193fe89b3eSLeon Yu * about dst->anon_vma if anon_vma_clone() failed. 3203fe89b3eSLeon Yu */ 3213fe89b3eSLeon Yu dst->anon_vma = NULL; 3225beb4930SRik van Riel unlink_anon_vmas(dst); 3235beb4930SRik van Riel return -ENOMEM; 3241da177e4SLinus Torvalds } 3251da177e4SLinus Torvalds 3265beb4930SRik van Riel /* 3275beb4930SRik van Riel * Attach vma to its own anon_vma, as well as to the anon_vmas that 3285beb4930SRik van Riel * the corresponding VMA in the parent process is attached to. 3295beb4930SRik van Riel * Returns 0 on success, non-zero on failure. 3305beb4930SRik van Riel */ 3315beb4930SRik van Riel int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) 3321da177e4SLinus Torvalds { 3335beb4930SRik van Riel struct anon_vma_chain *avc; 3345beb4930SRik van Riel struct anon_vma *anon_vma; 335c4ea95d7SDaniel Forrest int error; 3365beb4930SRik van Riel 3375beb4930SRik van Riel /* Don't bother if the parent process has no anon_vma here. */ 3385beb4930SRik van Riel if (!pvma->anon_vma) 3395beb4930SRik van Riel return 0; 3405beb4930SRik van Riel 3417a3ef208SKonstantin Khlebnikov /* Drop inherited anon_vma, we'll reuse existing or allocate new. */ 3427a3ef208SKonstantin Khlebnikov vma->anon_vma = NULL; 3437a3ef208SKonstantin Khlebnikov 3445beb4930SRik van Riel /* 3455beb4930SRik van Riel * First, attach the new VMA to the parent VMA's anon_vmas, 3465beb4930SRik van Riel * so rmap can find non-COWed pages in child processes. 3475beb4930SRik van Riel */ 348c4ea95d7SDaniel Forrest error = anon_vma_clone(vma, pvma); 349c4ea95d7SDaniel Forrest if (error) 350c4ea95d7SDaniel Forrest return error; 3515beb4930SRik van Riel 3527a3ef208SKonstantin Khlebnikov /* An existing anon_vma has been reused, all done then. */ 3537a3ef208SKonstantin Khlebnikov if (vma->anon_vma) 3547a3ef208SKonstantin Khlebnikov return 0; 3557a3ef208SKonstantin Khlebnikov 3565beb4930SRik van Riel /* Then add our own anon_vma. */ 3575beb4930SRik van Riel anon_vma = anon_vma_alloc(); 3585beb4930SRik van Riel if (!anon_vma) 3595beb4930SRik van Riel goto out_error; 360dd34739cSLinus Torvalds avc = anon_vma_chain_alloc(GFP_KERNEL); 3615beb4930SRik van Riel if (!avc) 3625beb4930SRik van Riel goto out_error_free_anon_vma; 3635c341ee1SRik van Riel 3645c341ee1SRik van Riel /* 365aaf1f990SMiaohe Lin * The root anon_vma's rwsem is the lock actually used when we 3665c341ee1SRik van Riel * lock any of the anon_vmas in this anon_vma tree. 3675c341ee1SRik van Riel */ 3685c341ee1SRik van Riel anon_vma->root = pvma->anon_vma->root; 3697a3ef208SKonstantin Khlebnikov anon_vma->parent = pvma->anon_vma; 37076545066SRik van Riel /* 37101d8b20dSPeter Zijlstra * With refcounts, an anon_vma can stay around longer than the 37201d8b20dSPeter Zijlstra * process it belongs to. The root anon_vma needs to be pinned until 37301d8b20dSPeter Zijlstra * this anon_vma is freed, because the lock lives in the root. 37476545066SRik van Riel */ 37576545066SRik van Riel get_anon_vma(anon_vma->root); 3765beb4930SRik van Riel /* Mark this anon_vma as the one where our new (COWed) pages go. */ 3775beb4930SRik van Riel vma->anon_vma = anon_vma; 3784fc3f1d6SIngo Molnar anon_vma_lock_write(anon_vma); 3795c341ee1SRik van Riel anon_vma_chain_link(vma, avc, anon_vma); 3807a3ef208SKonstantin Khlebnikov anon_vma->parent->degree++; 38108b52706SKonstantin Khlebnikov anon_vma_unlock_write(anon_vma); 3825beb4930SRik van Riel 3835beb4930SRik van Riel return 0; 3845beb4930SRik van Riel 3855beb4930SRik van Riel out_error_free_anon_vma: 38601d8b20dSPeter Zijlstra put_anon_vma(anon_vma); 3875beb4930SRik van Riel out_error: 3884946d54cSRik van Riel unlink_anon_vmas(vma); 3895beb4930SRik van Riel return -ENOMEM; 3905beb4930SRik van Riel } 3915beb4930SRik van Riel 3925beb4930SRik van Riel void unlink_anon_vmas(struct vm_area_struct *vma) 3935beb4930SRik van Riel { 3945beb4930SRik van Riel struct anon_vma_chain *avc, *next; 395eee2acbaSPeter Zijlstra struct anon_vma *root = NULL; 3965beb4930SRik van Riel 3975c341ee1SRik van Riel /* 3985c341ee1SRik van Riel * Unlink each anon_vma chained to the VMA. This list is ordered 3995c341ee1SRik van Riel * from newest to oldest, ensuring the root anon_vma gets freed last. 4005c341ee1SRik van Riel */ 4015beb4930SRik van Riel list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { 402eee2acbaSPeter Zijlstra struct anon_vma *anon_vma = avc->anon_vma; 403eee2acbaSPeter Zijlstra 404eee2acbaSPeter Zijlstra root = lock_anon_vma_root(root, anon_vma); 405bf181b9fSMichel Lespinasse anon_vma_interval_tree_remove(avc, &anon_vma->rb_root); 406eee2acbaSPeter Zijlstra 407eee2acbaSPeter Zijlstra /* 408eee2acbaSPeter Zijlstra * Leave empty anon_vmas on the list - we'll need 409eee2acbaSPeter Zijlstra * to free them outside the lock. 410eee2acbaSPeter Zijlstra */ 411f808c13fSDavidlohr Bueso if (RB_EMPTY_ROOT(&anon_vma->rb_root.rb_root)) { 4127a3ef208SKonstantin Khlebnikov anon_vma->parent->degree--; 413eee2acbaSPeter Zijlstra continue; 4147a3ef208SKonstantin Khlebnikov } 415eee2acbaSPeter Zijlstra 416eee2acbaSPeter Zijlstra list_del(&avc->same_vma); 417eee2acbaSPeter Zijlstra anon_vma_chain_free(avc); 418eee2acbaSPeter Zijlstra } 419ee8ab190SLi Xinhai if (vma->anon_vma) { 4207a3ef208SKonstantin Khlebnikov vma->anon_vma->degree--; 421ee8ab190SLi Xinhai 422ee8ab190SLi Xinhai /* 423ee8ab190SLi Xinhai * vma would still be needed after unlink, and anon_vma will be prepared 424ee8ab190SLi Xinhai * when handle fault. 425ee8ab190SLi Xinhai */ 426ee8ab190SLi Xinhai vma->anon_vma = NULL; 427ee8ab190SLi Xinhai } 428eee2acbaSPeter Zijlstra unlock_anon_vma_root(root); 429eee2acbaSPeter Zijlstra 430eee2acbaSPeter Zijlstra /* 431eee2acbaSPeter Zijlstra * Iterate the list once more, it now only contains empty and unlinked 432eee2acbaSPeter Zijlstra * anon_vmas, destroy them. Could not do before due to __put_anon_vma() 4335a505085SIngo Molnar * needing to write-acquire the anon_vma->root->rwsem. 434eee2acbaSPeter Zijlstra */ 435eee2acbaSPeter Zijlstra list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { 436eee2acbaSPeter Zijlstra struct anon_vma *anon_vma = avc->anon_vma; 437eee2acbaSPeter Zijlstra 438e4c5800aSKonstantin Khlebnikov VM_WARN_ON(anon_vma->degree); 439eee2acbaSPeter Zijlstra put_anon_vma(anon_vma); 440eee2acbaSPeter Zijlstra 4415beb4930SRik van Riel list_del(&avc->same_vma); 4425beb4930SRik van Riel anon_vma_chain_free(avc); 4435beb4930SRik van Riel } 4445beb4930SRik van Riel } 4455beb4930SRik van Riel 44651cc5068SAlexey Dobriyan static void anon_vma_ctor(void *data) 4471da177e4SLinus Torvalds { 4481da177e4SLinus Torvalds struct anon_vma *anon_vma = data; 4491da177e4SLinus Torvalds 4505a505085SIngo Molnar init_rwsem(&anon_vma->rwsem); 45183813267SPeter Zijlstra atomic_set(&anon_vma->refcount, 0); 452f808c13fSDavidlohr Bueso anon_vma->rb_root = RB_ROOT_CACHED; 4531da177e4SLinus Torvalds } 4541da177e4SLinus Torvalds 4551da177e4SLinus Torvalds void __init anon_vma_init(void) 4561da177e4SLinus Torvalds { 4571da177e4SLinus Torvalds anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma), 4585f0d5a3aSPaul E. McKenney 0, SLAB_TYPESAFE_BY_RCU|SLAB_PANIC|SLAB_ACCOUNT, 4595d097056SVladimir Davydov anon_vma_ctor); 4605d097056SVladimir Davydov anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain, 4615d097056SVladimir Davydov SLAB_PANIC|SLAB_ACCOUNT); 4621da177e4SLinus Torvalds } 4631da177e4SLinus Torvalds 4641da177e4SLinus Torvalds /* 4656111e4caSPeter Zijlstra * Getting a lock on a stable anon_vma from a page off the LRU is tricky! 4666111e4caSPeter Zijlstra * 4676111e4caSPeter Zijlstra * Since there is no serialization what so ever against page_remove_rmap() 468ad8a20cfSMiaohe Lin * the best this function can do is return a refcount increased anon_vma 469ad8a20cfSMiaohe Lin * that might have been relevant to this page. 4706111e4caSPeter Zijlstra * 4716111e4caSPeter Zijlstra * The page might have been remapped to a different anon_vma or the anon_vma 4726111e4caSPeter Zijlstra * returned may already be freed (and even reused). 4736111e4caSPeter Zijlstra * 474bc658c96SPeter Zijlstra * In case it was remapped to a different anon_vma, the new anon_vma will be a 475bc658c96SPeter Zijlstra * child of the old anon_vma, and the anon_vma lifetime rules will therefore 476bc658c96SPeter Zijlstra * ensure that any anon_vma obtained from the page will still be valid for as 477bc658c96SPeter Zijlstra * long as we observe page_mapped() [ hence all those page_mapped() tests ]. 478bc658c96SPeter Zijlstra * 4796111e4caSPeter Zijlstra * All users of this function must be very careful when walking the anon_vma 4806111e4caSPeter Zijlstra * chain and verify that the page in question is indeed mapped in it 4816111e4caSPeter Zijlstra * [ something equivalent to page_mapped_in_vma() ]. 4826111e4caSPeter Zijlstra * 483091e4299SMiles Chen * Since anon_vma's slab is SLAB_TYPESAFE_BY_RCU and we know from 484091e4299SMiles Chen * page_remove_rmap() that the anon_vma pointer from page->mapping is valid 485091e4299SMiles Chen * if there is a mapcount, we can dereference the anon_vma after observing 486091e4299SMiles Chen * those. 4871da177e4SLinus Torvalds */ 488746b18d4SPeter Zijlstra struct anon_vma *page_get_anon_vma(struct page *page) 4891da177e4SLinus Torvalds { 490746b18d4SPeter Zijlstra struct anon_vma *anon_vma = NULL; 4911da177e4SLinus Torvalds unsigned long anon_mapping; 4921da177e4SLinus Torvalds 4931da177e4SLinus Torvalds rcu_read_lock(); 4944db0c3c2SJason Low anon_mapping = (unsigned long)READ_ONCE(page->mapping); 4953ca7b3c5SHugh Dickins if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) 4961da177e4SLinus Torvalds goto out; 4971da177e4SLinus Torvalds if (!page_mapped(page)) 4981da177e4SLinus Torvalds goto out; 4991da177e4SLinus Torvalds 5001da177e4SLinus Torvalds anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); 501746b18d4SPeter Zijlstra if (!atomic_inc_not_zero(&anon_vma->refcount)) { 502746b18d4SPeter Zijlstra anon_vma = NULL; 503746b18d4SPeter Zijlstra goto out; 504746b18d4SPeter Zijlstra } 505f1819427SHugh Dickins 506f1819427SHugh Dickins /* 507f1819427SHugh Dickins * If this page is still mapped, then its anon_vma cannot have been 508746b18d4SPeter Zijlstra * freed. But if it has been unmapped, we have no security against the 509746b18d4SPeter Zijlstra * anon_vma structure being freed and reused (for another anon_vma: 5105f0d5a3aSPaul E. McKenney * SLAB_TYPESAFE_BY_RCU guarantees that - so the atomic_inc_not_zero() 511746b18d4SPeter Zijlstra * above cannot corrupt). 512f1819427SHugh Dickins */ 513746b18d4SPeter Zijlstra if (!page_mapped(page)) { 5147f39dda9SHugh Dickins rcu_read_unlock(); 515746b18d4SPeter Zijlstra put_anon_vma(anon_vma); 5167f39dda9SHugh Dickins return NULL; 517746b18d4SPeter Zijlstra } 5181da177e4SLinus Torvalds out: 5191da177e4SLinus Torvalds rcu_read_unlock(); 520746b18d4SPeter Zijlstra 521746b18d4SPeter Zijlstra return anon_vma; 522746b18d4SPeter Zijlstra } 523746b18d4SPeter Zijlstra 52488c22088SPeter Zijlstra /* 52588c22088SPeter Zijlstra * Similar to page_get_anon_vma() except it locks the anon_vma. 52688c22088SPeter Zijlstra * 52788c22088SPeter Zijlstra * Its a little more complex as it tries to keep the fast path to a single 52888c22088SPeter Zijlstra * atomic op -- the trylock. If we fail the trylock, we fall back to getting a 52988c22088SPeter Zijlstra * reference like with page_get_anon_vma() and then block on the mutex. 53088c22088SPeter Zijlstra */ 5319595d769SMatthew Wilcox (Oracle) struct anon_vma *folio_lock_anon_vma_read(struct folio *folio) 532746b18d4SPeter Zijlstra { 53388c22088SPeter Zijlstra struct anon_vma *anon_vma = NULL; 534eee0f252SHugh Dickins struct anon_vma *root_anon_vma; 53588c22088SPeter Zijlstra unsigned long anon_mapping; 536746b18d4SPeter Zijlstra 53788c22088SPeter Zijlstra rcu_read_lock(); 5389595d769SMatthew Wilcox (Oracle) anon_mapping = (unsigned long)READ_ONCE(folio->mapping); 53988c22088SPeter Zijlstra if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) 54088c22088SPeter Zijlstra goto out; 5419595d769SMatthew Wilcox (Oracle) if (!folio_mapped(folio)) 54288c22088SPeter Zijlstra goto out; 54388c22088SPeter Zijlstra 54488c22088SPeter Zijlstra anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); 5454db0c3c2SJason Low root_anon_vma = READ_ONCE(anon_vma->root); 5464fc3f1d6SIngo Molnar if (down_read_trylock(&root_anon_vma->rwsem)) { 54788c22088SPeter Zijlstra /* 5489595d769SMatthew Wilcox (Oracle) * If the folio is still mapped, then this anon_vma is still 549eee0f252SHugh Dickins * its anon_vma, and holding the mutex ensures that it will 550bc658c96SPeter Zijlstra * not go away, see anon_vma_free(). 55188c22088SPeter Zijlstra */ 5529595d769SMatthew Wilcox (Oracle) if (!folio_mapped(folio)) { 5534fc3f1d6SIngo Molnar up_read(&root_anon_vma->rwsem); 55488c22088SPeter Zijlstra anon_vma = NULL; 55588c22088SPeter Zijlstra } 55688c22088SPeter Zijlstra goto out; 55788c22088SPeter Zijlstra } 55888c22088SPeter Zijlstra 55988c22088SPeter Zijlstra /* trylock failed, we got to sleep */ 56088c22088SPeter Zijlstra if (!atomic_inc_not_zero(&anon_vma->refcount)) { 56188c22088SPeter Zijlstra anon_vma = NULL; 56288c22088SPeter Zijlstra goto out; 56388c22088SPeter Zijlstra } 56488c22088SPeter Zijlstra 5659595d769SMatthew Wilcox (Oracle) if (!folio_mapped(folio)) { 5667f39dda9SHugh Dickins rcu_read_unlock(); 56788c22088SPeter Zijlstra put_anon_vma(anon_vma); 5687f39dda9SHugh Dickins return NULL; 56988c22088SPeter Zijlstra } 57088c22088SPeter Zijlstra 57188c22088SPeter Zijlstra /* we pinned the anon_vma, its safe to sleep */ 57288c22088SPeter Zijlstra rcu_read_unlock(); 5734fc3f1d6SIngo Molnar anon_vma_lock_read(anon_vma); 574746b18d4SPeter Zijlstra 57588c22088SPeter Zijlstra if (atomic_dec_and_test(&anon_vma->refcount)) { 57688c22088SPeter Zijlstra /* 57788c22088SPeter Zijlstra * Oops, we held the last refcount, release the lock 57888c22088SPeter Zijlstra * and bail -- can't simply use put_anon_vma() because 5794fc3f1d6SIngo Molnar * we'll deadlock on the anon_vma_lock_write() recursion. 58088c22088SPeter Zijlstra */ 5814fc3f1d6SIngo Molnar anon_vma_unlock_read(anon_vma); 58288c22088SPeter Zijlstra __put_anon_vma(anon_vma); 58388c22088SPeter Zijlstra anon_vma = NULL; 58488c22088SPeter Zijlstra } 58588c22088SPeter Zijlstra 58688c22088SPeter Zijlstra return anon_vma; 58788c22088SPeter Zijlstra 58888c22088SPeter Zijlstra out: 58988c22088SPeter Zijlstra rcu_read_unlock(); 590746b18d4SPeter Zijlstra return anon_vma; 59134bbd704SOleg Nesterov } 59234bbd704SOleg Nesterov 5934fc3f1d6SIngo Molnar void page_unlock_anon_vma_read(struct anon_vma *anon_vma) 59434bbd704SOleg Nesterov { 5954fc3f1d6SIngo Molnar anon_vma_unlock_read(anon_vma); 5961da177e4SLinus Torvalds } 5971da177e4SLinus Torvalds 59872b252aeSMel Gorman #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH 59972b252aeSMel Gorman /* 60072b252aeSMel Gorman * Flush TLB entries for recently unmapped pages from remote CPUs. It is 60172b252aeSMel Gorman * important if a PTE was dirty when it was unmapped that it's flushed 60272b252aeSMel Gorman * before any IO is initiated on the page to prevent lost writes. Similarly, 60372b252aeSMel Gorman * it must be flushed before freeing to prevent data leakage. 60472b252aeSMel Gorman */ 60572b252aeSMel Gorman void try_to_unmap_flush(void) 60672b252aeSMel Gorman { 60772b252aeSMel Gorman struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; 60872b252aeSMel Gorman 60972b252aeSMel Gorman if (!tlb_ubc->flush_required) 61072b252aeSMel Gorman return; 61172b252aeSMel Gorman 612e73ad5ffSAndy Lutomirski arch_tlbbatch_flush(&tlb_ubc->arch); 61372b252aeSMel Gorman tlb_ubc->flush_required = false; 614d950c947SMel Gorman tlb_ubc->writable = false; 61572b252aeSMel Gorman } 61672b252aeSMel Gorman 617d950c947SMel Gorman /* Flush iff there are potentially writable TLB entries that can race with IO */ 618d950c947SMel Gorman void try_to_unmap_flush_dirty(void) 619d950c947SMel Gorman { 620d950c947SMel Gorman struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; 621d950c947SMel Gorman 622d950c947SMel Gorman if (tlb_ubc->writable) 623d950c947SMel Gorman try_to_unmap_flush(); 624d950c947SMel Gorman } 625d950c947SMel Gorman 6265ee2fa2fSHuang Ying /* 6275ee2fa2fSHuang Ying * Bits 0-14 of mm->tlb_flush_batched record pending generations. 6285ee2fa2fSHuang Ying * Bits 16-30 of mm->tlb_flush_batched bit record flushed generations. 6295ee2fa2fSHuang Ying */ 6305ee2fa2fSHuang Ying #define TLB_FLUSH_BATCH_FLUSHED_SHIFT 16 6315ee2fa2fSHuang Ying #define TLB_FLUSH_BATCH_PENDING_MASK \ 6325ee2fa2fSHuang Ying ((1 << (TLB_FLUSH_BATCH_FLUSHED_SHIFT - 1)) - 1) 6335ee2fa2fSHuang Ying #define TLB_FLUSH_BATCH_PENDING_LARGE \ 6345ee2fa2fSHuang Ying (TLB_FLUSH_BATCH_PENDING_MASK / 2) 6355ee2fa2fSHuang Ying 636c7ab0d2fSKirill A. Shutemov static void set_tlb_ubc_flush_pending(struct mm_struct *mm, bool writable) 63772b252aeSMel Gorman { 63872b252aeSMel Gorman struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; 6395ee2fa2fSHuang Ying int batch, nbatch; 64072b252aeSMel Gorman 641e73ad5ffSAndy Lutomirski arch_tlbbatch_add_mm(&tlb_ubc->arch, mm); 64272b252aeSMel Gorman tlb_ubc->flush_required = true; 643d950c947SMel Gorman 644d950c947SMel Gorman /* 6453ea27719SMel Gorman * Ensure compiler does not re-order the setting of tlb_flush_batched 6463ea27719SMel Gorman * before the PTE is cleared. 6473ea27719SMel Gorman */ 6483ea27719SMel Gorman barrier(); 6495ee2fa2fSHuang Ying batch = atomic_read(&mm->tlb_flush_batched); 6505ee2fa2fSHuang Ying retry: 6515ee2fa2fSHuang Ying if ((batch & TLB_FLUSH_BATCH_PENDING_MASK) > TLB_FLUSH_BATCH_PENDING_LARGE) { 6525ee2fa2fSHuang Ying /* 6535ee2fa2fSHuang Ying * Prevent `pending' from catching up with `flushed' because of 6545ee2fa2fSHuang Ying * overflow. Reset `pending' and `flushed' to be 1 and 0 if 6555ee2fa2fSHuang Ying * `pending' becomes large. 6565ee2fa2fSHuang Ying */ 6575ee2fa2fSHuang Ying nbatch = atomic_cmpxchg(&mm->tlb_flush_batched, batch, 1); 6585ee2fa2fSHuang Ying if (nbatch != batch) { 6595ee2fa2fSHuang Ying batch = nbatch; 6605ee2fa2fSHuang Ying goto retry; 6615ee2fa2fSHuang Ying } 6625ee2fa2fSHuang Ying } else { 6635ee2fa2fSHuang Ying atomic_inc(&mm->tlb_flush_batched); 6645ee2fa2fSHuang Ying } 6653ea27719SMel Gorman 6663ea27719SMel Gorman /* 667d950c947SMel Gorman * If the PTE was dirty then it's best to assume it's writable. The 668d950c947SMel Gorman * caller must use try_to_unmap_flush_dirty() or try_to_unmap_flush() 669d950c947SMel Gorman * before the page is queued for IO. 670d950c947SMel Gorman */ 671d950c947SMel Gorman if (writable) 672d950c947SMel Gorman tlb_ubc->writable = true; 67372b252aeSMel Gorman } 67472b252aeSMel Gorman 67572b252aeSMel Gorman /* 67672b252aeSMel Gorman * Returns true if the TLB flush should be deferred to the end of a batch of 67772b252aeSMel Gorman * unmap operations to reduce IPIs. 67872b252aeSMel Gorman */ 67972b252aeSMel Gorman static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) 68072b252aeSMel Gorman { 68172b252aeSMel Gorman bool should_defer = false; 68272b252aeSMel Gorman 68372b252aeSMel Gorman if (!(flags & TTU_BATCH_FLUSH)) 68472b252aeSMel Gorman return false; 68572b252aeSMel Gorman 68672b252aeSMel Gorman /* If remote CPUs need to be flushed then defer batch the flush */ 68772b252aeSMel Gorman if (cpumask_any_but(mm_cpumask(mm), get_cpu()) < nr_cpu_ids) 68872b252aeSMel Gorman should_defer = true; 68972b252aeSMel Gorman put_cpu(); 69072b252aeSMel Gorman 69172b252aeSMel Gorman return should_defer; 69272b252aeSMel Gorman } 6933ea27719SMel Gorman 6943ea27719SMel Gorman /* 6953ea27719SMel Gorman * Reclaim unmaps pages under the PTL but do not flush the TLB prior to 6963ea27719SMel Gorman * releasing the PTL if TLB flushes are batched. It's possible for a parallel 6973ea27719SMel Gorman * operation such as mprotect or munmap to race between reclaim unmapping 6983ea27719SMel Gorman * the page and flushing the page. If this race occurs, it potentially allows 6993ea27719SMel Gorman * access to data via a stale TLB entry. Tracking all mm's that have TLB 7003ea27719SMel Gorman * batching in flight would be expensive during reclaim so instead track 7013ea27719SMel Gorman * whether TLB batching occurred in the past and if so then do a flush here 7023ea27719SMel Gorman * if required. This will cost one additional flush per reclaim cycle paid 7033ea27719SMel Gorman * by the first operation at risk such as mprotect and mumap. 7043ea27719SMel Gorman * 7053ea27719SMel Gorman * This must be called under the PTL so that an access to tlb_flush_batched 7063ea27719SMel Gorman * that is potentially a "reclaim vs mprotect/munmap/etc" race will synchronise 7073ea27719SMel Gorman * via the PTL. 7083ea27719SMel Gorman */ 7093ea27719SMel Gorman void flush_tlb_batched_pending(struct mm_struct *mm) 7103ea27719SMel Gorman { 7115ee2fa2fSHuang Ying int batch = atomic_read(&mm->tlb_flush_batched); 7125ee2fa2fSHuang Ying int pending = batch & TLB_FLUSH_BATCH_PENDING_MASK; 7135ee2fa2fSHuang Ying int flushed = batch >> TLB_FLUSH_BATCH_FLUSHED_SHIFT; 7143ea27719SMel Gorman 7155ee2fa2fSHuang Ying if (pending != flushed) { 7165ee2fa2fSHuang Ying flush_tlb_mm(mm); 7173ea27719SMel Gorman /* 7185ee2fa2fSHuang Ying * If the new TLB flushing is pending during flushing, leave 7195ee2fa2fSHuang Ying * mm->tlb_flush_batched as is, to avoid losing flushing. 7203ea27719SMel Gorman */ 7215ee2fa2fSHuang Ying atomic_cmpxchg(&mm->tlb_flush_batched, batch, 7225ee2fa2fSHuang Ying pending | (pending << TLB_FLUSH_BATCH_FLUSHED_SHIFT)); 7233ea27719SMel Gorman } 7243ea27719SMel Gorman } 72572b252aeSMel Gorman #else 726c7ab0d2fSKirill A. Shutemov static void set_tlb_ubc_flush_pending(struct mm_struct *mm, bool writable) 72772b252aeSMel Gorman { 72872b252aeSMel Gorman } 72972b252aeSMel Gorman 73072b252aeSMel Gorman static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) 73172b252aeSMel Gorman { 73272b252aeSMel Gorman return false; 73372b252aeSMel Gorman } 73472b252aeSMel Gorman #endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */ 73572b252aeSMel Gorman 7361da177e4SLinus Torvalds /* 737bf89c8c8SHuang Shijie * At what user virtual address is page expected in vma? 738ab941e0fSNaoya Horiguchi * Caller should check the page is actually part of the vma. 7391da177e4SLinus Torvalds */ 7401da177e4SLinus Torvalds unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) 7411da177e4SLinus Torvalds { 742e05b3453SMatthew Wilcox (Oracle) struct folio *folio = page_folio(page); 743e05b3453SMatthew Wilcox (Oracle) if (folio_test_anon(folio)) { 744e05b3453SMatthew Wilcox (Oracle) struct anon_vma *page__anon_vma = folio_anon_vma(folio); 7454829b906SHugh Dickins /* 7464829b906SHugh Dickins * Note: swapoff's unuse_vma() is more efficient with this 7474829b906SHugh Dickins * check, and needs it to match anon_vma when KSM is active. 7484829b906SHugh Dickins */ 7494829b906SHugh Dickins if (!vma->anon_vma || !page__anon_vma || 7504829b906SHugh Dickins vma->anon_vma->root != page__anon_vma->root) 75121d0d443SAndrea Arcangeli return -EFAULT; 75231657170SJue Wang } else if (!vma->vm_file) { 7531da177e4SLinus Torvalds return -EFAULT; 754e05b3453SMatthew Wilcox (Oracle) } else if (vma->vm_file->f_mapping != folio->mapping) { 7551da177e4SLinus Torvalds return -EFAULT; 75631657170SJue Wang } 757494334e4SHugh Dickins 758494334e4SHugh Dickins return vma_address(page, vma); 7591da177e4SLinus Torvalds } 7601da177e4SLinus Torvalds 7616219049aSBob Liu pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) 7626219049aSBob Liu { 7636219049aSBob Liu pgd_t *pgd; 764c2febafcSKirill A. Shutemov p4d_t *p4d; 7656219049aSBob Liu pud_t *pud; 7666219049aSBob Liu pmd_t *pmd = NULL; 767f72e7dcdSHugh Dickins pmd_t pmde; 7686219049aSBob Liu 7696219049aSBob Liu pgd = pgd_offset(mm, address); 7706219049aSBob Liu if (!pgd_present(*pgd)) 7716219049aSBob Liu goto out; 7726219049aSBob Liu 773c2febafcSKirill A. Shutemov p4d = p4d_offset(pgd, address); 774c2febafcSKirill A. Shutemov if (!p4d_present(*p4d)) 775c2febafcSKirill A. Shutemov goto out; 776c2febafcSKirill A. Shutemov 777c2febafcSKirill A. Shutemov pud = pud_offset(p4d, address); 7786219049aSBob Liu if (!pud_present(*pud)) 7796219049aSBob Liu goto out; 7806219049aSBob Liu 7816219049aSBob Liu pmd = pmd_offset(pud, address); 782f72e7dcdSHugh Dickins /* 7838809aa2dSAneesh Kumar K.V * Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at() 784f72e7dcdSHugh Dickins * without holding anon_vma lock for write. So when looking for a 785f72e7dcdSHugh Dickins * genuine pmde (in which to find pte), test present and !THP together. 786f72e7dcdSHugh Dickins */ 787e37c6982SChristian Borntraeger pmde = *pmd; 788e37c6982SChristian Borntraeger barrier(); 789f72e7dcdSHugh Dickins if (!pmd_present(pmde) || pmd_trans_huge(pmde)) 7906219049aSBob Liu pmd = NULL; 7916219049aSBob Liu out: 7926219049aSBob Liu return pmd; 7936219049aSBob Liu } 7946219049aSBob Liu 795b3ac0413SMatthew Wilcox (Oracle) struct folio_referenced_arg { 7969f32624bSJoonsoo Kim int mapcount; 7979f32624bSJoonsoo Kim int referenced; 7989f32624bSJoonsoo Kim unsigned long vm_flags; 7999f32624bSJoonsoo Kim struct mem_cgroup *memcg; 8009f32624bSJoonsoo Kim }; 80181b4082dSNikita Danilov /* 802b3ac0413SMatthew Wilcox (Oracle) * arg: folio_referenced_arg will be passed 8031da177e4SLinus Torvalds */ 8042f031c6fSMatthew Wilcox (Oracle) static bool folio_referenced_one(struct folio *folio, 8052f031c6fSMatthew Wilcox (Oracle) struct vm_area_struct *vma, unsigned long address, void *arg) 8061da177e4SLinus Torvalds { 807b3ac0413SMatthew Wilcox (Oracle) struct folio_referenced_arg *pra = arg; 808b3ac0413SMatthew Wilcox (Oracle) DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); 8098749cfeaSVladimir Davydov int referenced = 0; 8102da28bfdSAndrea Arcangeli 8118eaededeSKirill A. Shutemov while (page_vma_mapped_walk(&pvmw)) { 8128eaededeSKirill A. Shutemov address = pvmw.address; 8132da28bfdSAndrea Arcangeli 81447d4f3eeSHugh Dickins if ((vma->vm_flags & VM_LOCKED) && 815b3ac0413SMatthew Wilcox (Oracle) (!folio_test_large(folio) || !pvmw.pte)) { 81647d4f3eeSHugh Dickins /* Restore the mlock which got missed */ 817b3ac0413SMatthew Wilcox (Oracle) mlock_vma_folio(folio, vma, !pvmw.pte); 8188eaededeSKirill A. Shutemov page_vma_mapped_walk_done(&pvmw); 8199f32624bSJoonsoo Kim pra->vm_flags |= VM_LOCKED; 820e4b82222SMinchan Kim return false; /* To break the loop */ 8212da28bfdSAndrea Arcangeli } 8222da28bfdSAndrea Arcangeli 8238eaededeSKirill A. Shutemov if (pvmw.pte) { 8248eaededeSKirill A. Shutemov if (ptep_clear_flush_young_notify(vma, address, 8258eaededeSKirill A. Shutemov pvmw.pte)) { 8264917e5d0SJohannes Weiner /* 8278eaededeSKirill A. Shutemov * Don't treat a reference through 8288eaededeSKirill A. Shutemov * a sequentially read mapping as such. 829b3ac0413SMatthew Wilcox (Oracle) * If the folio has been used in another mapping, 8308eaededeSKirill A. Shutemov * we will catch it; if this other mapping is 8318eaededeSKirill A. Shutemov * already gone, the unmap path will have set 832b3ac0413SMatthew Wilcox (Oracle) * the referenced flag or activated the folio. 8334917e5d0SJohannes Weiner */ 83464363aadSJoe Perches if (likely(!(vma->vm_flags & VM_SEQ_READ))) 8351da177e4SLinus Torvalds referenced++; 8364917e5d0SJohannes Weiner } 8378749cfeaSVladimir Davydov } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { 8388eaededeSKirill A. Shutemov if (pmdp_clear_flush_young_notify(vma, address, 8398eaededeSKirill A. Shutemov pvmw.pmd)) 8408749cfeaSVladimir Davydov referenced++; 8418749cfeaSVladimir Davydov } else { 842b3ac0413SMatthew Wilcox (Oracle) /* unexpected pmd-mapped folio? */ 8438749cfeaSVladimir Davydov WARN_ON_ONCE(1); 8448749cfeaSVladimir Davydov } 8458eaededeSKirill A. Shutemov 8468eaededeSKirill A. Shutemov pra->mapcount--; 8478eaededeSKirill A. Shutemov } 84871e3aac0SAndrea Arcangeli 84933c3fc71SVladimir Davydov if (referenced) 850b3ac0413SMatthew Wilcox (Oracle) folio_clear_idle(folio); 851b3ac0413SMatthew Wilcox (Oracle) if (folio_test_clear_young(folio)) 85233c3fc71SVladimir Davydov referenced++; 85333c3fc71SVladimir Davydov 8549f32624bSJoonsoo Kim if (referenced) { 8559f32624bSJoonsoo Kim pra->referenced++; 85647d4f3eeSHugh Dickins pra->vm_flags |= vma->vm_flags & ~VM_LOCKED; 8571da177e4SLinus Torvalds } 8581da177e4SLinus Torvalds 8599f32624bSJoonsoo Kim if (!pra->mapcount) 860e4b82222SMinchan Kim return false; /* To break the loop */ 8619f32624bSJoonsoo Kim 862e4b82222SMinchan Kim return true; 8639f32624bSJoonsoo Kim } 8649f32624bSJoonsoo Kim 865b3ac0413SMatthew Wilcox (Oracle) static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg) 8661da177e4SLinus Torvalds { 867b3ac0413SMatthew Wilcox (Oracle) struct folio_referenced_arg *pra = arg; 8689f32624bSJoonsoo Kim struct mem_cgroup *memcg = pra->memcg; 8691da177e4SLinus Torvalds 8709f32624bSJoonsoo Kim if (!mm_match_cgroup(vma->vm_mm, memcg)) 8719f32624bSJoonsoo Kim return true; 8721da177e4SLinus Torvalds 8739f32624bSJoonsoo Kim return false; 8741da177e4SLinus Torvalds } 8751da177e4SLinus Torvalds 8761da177e4SLinus Torvalds /** 877b3ac0413SMatthew Wilcox (Oracle) * folio_referenced() - Test if the folio was referenced. 878b3ac0413SMatthew Wilcox (Oracle) * @folio: The folio to test. 879b3ac0413SMatthew Wilcox (Oracle) * @is_locked: Caller holds lock on the folio. 88072835c86SJohannes Weiner * @memcg: target memory cgroup 881b3ac0413SMatthew Wilcox (Oracle) * @vm_flags: A combination of all the vma->vm_flags which referenced the folio. 8821da177e4SLinus Torvalds * 883b3ac0413SMatthew Wilcox (Oracle) * Quick test_and_clear_referenced for all mappings of a folio, 884b3ac0413SMatthew Wilcox (Oracle) * 885b3ac0413SMatthew Wilcox (Oracle) * Return: The number of mappings which referenced the folio. 8861da177e4SLinus Torvalds */ 887b3ac0413SMatthew Wilcox (Oracle) int folio_referenced(struct folio *folio, int is_locked, 888b3ac0413SMatthew Wilcox (Oracle) struct mem_cgroup *memcg, unsigned long *vm_flags) 8891da177e4SLinus Torvalds { 8905ad64688SHugh Dickins int we_locked = 0; 891b3ac0413SMatthew Wilcox (Oracle) struct folio_referenced_arg pra = { 892b3ac0413SMatthew Wilcox (Oracle) .mapcount = folio_mapcount(folio), 8939f32624bSJoonsoo Kim .memcg = memcg, 8949f32624bSJoonsoo Kim }; 8959f32624bSJoonsoo Kim struct rmap_walk_control rwc = { 896b3ac0413SMatthew Wilcox (Oracle) .rmap_one = folio_referenced_one, 8979f32624bSJoonsoo Kim .arg = (void *)&pra, 8982f031c6fSMatthew Wilcox (Oracle) .anon_lock = folio_lock_anon_vma_read, 8999f32624bSJoonsoo Kim }; 9001da177e4SLinus Torvalds 9016fe6b7e3SWu Fengguang *vm_flags = 0; 902059d8442SHuang Shijie if (!pra.mapcount) 9039f32624bSJoonsoo Kim return 0; 9049f32624bSJoonsoo Kim 905b3ac0413SMatthew Wilcox (Oracle) if (!folio_raw_mapping(folio)) 9069f32624bSJoonsoo Kim return 0; 9079f32624bSJoonsoo Kim 908b3ac0413SMatthew Wilcox (Oracle) if (!is_locked && (!folio_test_anon(folio) || folio_test_ksm(folio))) { 909b3ac0413SMatthew Wilcox (Oracle) we_locked = folio_trylock(folio); 9109f32624bSJoonsoo Kim if (!we_locked) 9119f32624bSJoonsoo Kim return 1; 9125ad64688SHugh Dickins } 9139f32624bSJoonsoo Kim 9149f32624bSJoonsoo Kim /* 9159f32624bSJoonsoo Kim * If we are reclaiming on behalf of a cgroup, skip 9169f32624bSJoonsoo Kim * counting on behalf of references from different 9179f32624bSJoonsoo Kim * cgroups 9189f32624bSJoonsoo Kim */ 9199f32624bSJoonsoo Kim if (memcg) { 920b3ac0413SMatthew Wilcox (Oracle) rwc.invalid_vma = invalid_folio_referenced_vma; 9215ad64688SHugh Dickins } 9229f32624bSJoonsoo Kim 9232f031c6fSMatthew Wilcox (Oracle) rmap_walk(folio, &rwc); 9249f32624bSJoonsoo Kim *vm_flags = pra.vm_flags; 9259f32624bSJoonsoo Kim 9265ad64688SHugh Dickins if (we_locked) 927b3ac0413SMatthew Wilcox (Oracle) folio_unlock(folio); 9289f32624bSJoonsoo Kim 9299f32624bSJoonsoo Kim return pra.referenced; 9301da177e4SLinus Torvalds } 9311da177e4SLinus Torvalds 9322f031c6fSMatthew Wilcox (Oracle) static bool page_mkclean_one(struct folio *folio, struct vm_area_struct *vma, 9339853a407SJoonsoo Kim unsigned long address, void *arg) 934d08b3851SPeter Zijlstra { 935e83c09a2SMatthew Wilcox (Oracle) DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC); 936ac46d4f3SJérôme Glisse struct mmu_notifier_range range; 9379853a407SJoonsoo Kim int *cleaned = arg; 938d08b3851SPeter Zijlstra 939369ea824SJérôme Glisse /* 940369ea824SJérôme Glisse * We have to assume the worse case ie pmd for invalidation. Note that 941e83c09a2SMatthew Wilcox (Oracle) * the folio can not be freed from this function. 942369ea824SJérôme Glisse */ 9437269f999SJérôme Glisse mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE, 9447269f999SJérôme Glisse 0, vma, vma->vm_mm, address, 9452aff7a47SMatthew Wilcox (Oracle) vma_address_end(&pvmw)); 946ac46d4f3SJérôme Glisse mmu_notifier_invalidate_range_start(&range); 947369ea824SJérôme Glisse 948f27176cfSKirill A. Shutemov while (page_vma_mapped_walk(&pvmw)) { 949f27176cfSKirill A. Shutemov int ret = 0; 950369ea824SJérôme Glisse 9511f18b296SYueHaibing address = pvmw.address; 952f27176cfSKirill A. Shutemov if (pvmw.pte) { 953c2fda5feSPeter Zijlstra pte_t entry; 954f27176cfSKirill A. Shutemov pte_t *pte = pvmw.pte; 955f27176cfSKirill A. Shutemov 956f27176cfSKirill A. Shutemov if (!pte_dirty(*pte) && !pte_write(*pte)) 957f27176cfSKirill A. Shutemov continue; 958d08b3851SPeter Zijlstra 959785373b4SLinus Torvalds flush_cache_page(vma, address, pte_pfn(*pte)); 960785373b4SLinus Torvalds entry = ptep_clear_flush(vma, address, pte); 961d08b3851SPeter Zijlstra entry = pte_wrprotect(entry); 962c2fda5feSPeter Zijlstra entry = pte_mkclean(entry); 963785373b4SLinus Torvalds set_pte_at(vma->vm_mm, address, pte, entry); 964d08b3851SPeter Zijlstra ret = 1; 965f27176cfSKirill A. Shutemov } else { 966396bcc52SMatthew Wilcox (Oracle) #ifdef CONFIG_TRANSPARENT_HUGEPAGE 967f27176cfSKirill A. Shutemov pmd_t *pmd = pvmw.pmd; 968f27176cfSKirill A. Shutemov pmd_t entry; 969d08b3851SPeter Zijlstra 970f27176cfSKirill A. Shutemov if (!pmd_dirty(*pmd) && !pmd_write(*pmd)) 971f27176cfSKirill A. Shutemov continue; 972f27176cfSKirill A. Shutemov 973e83c09a2SMatthew Wilcox (Oracle) flush_cache_page(vma, address, folio_pfn(folio)); 974024eee0eSAneesh Kumar K.V entry = pmdp_invalidate(vma, address, pmd); 975f27176cfSKirill A. Shutemov entry = pmd_wrprotect(entry); 976f27176cfSKirill A. Shutemov entry = pmd_mkclean(entry); 977785373b4SLinus Torvalds set_pmd_at(vma->vm_mm, address, pmd, entry); 978f27176cfSKirill A. Shutemov ret = 1; 979f27176cfSKirill A. Shutemov #else 980e83c09a2SMatthew Wilcox (Oracle) /* unexpected pmd-mapped folio? */ 981f27176cfSKirill A. Shutemov WARN_ON_ONCE(1); 982f27176cfSKirill A. Shutemov #endif 983f27176cfSKirill A. Shutemov } 9842ec74c3eSSagi Grimberg 9850f10851eSJérôme Glisse /* 9860f10851eSJérôme Glisse * No need to call mmu_notifier_invalidate_range() as we are 9870f10851eSJérôme Glisse * downgrading page table protection not changing it to point 9880f10851eSJérôme Glisse * to a new page. 9890f10851eSJérôme Glisse * 990ad56b738SMike Rapoport * See Documentation/vm/mmu_notifier.rst 9910f10851eSJérôme Glisse */ 9920f10851eSJérôme Glisse if (ret) 9939853a407SJoonsoo Kim (*cleaned)++; 9949853a407SJoonsoo Kim } 995f27176cfSKirill A. Shutemov 996ac46d4f3SJérôme Glisse mmu_notifier_invalidate_range_end(&range); 997369ea824SJérôme Glisse 998e4b82222SMinchan Kim return true; 999d08b3851SPeter Zijlstra } 1000d08b3851SPeter Zijlstra 10019853a407SJoonsoo Kim static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg) 1002d08b3851SPeter Zijlstra { 10039853a407SJoonsoo Kim if (vma->vm_flags & VM_SHARED) 1004871beb8cSFengguang Wu return false; 1005d08b3851SPeter Zijlstra 1006871beb8cSFengguang Wu return true; 1007d08b3851SPeter Zijlstra } 1008d08b3851SPeter Zijlstra 1009d9c08e22SMatthew Wilcox (Oracle) int folio_mkclean(struct folio *folio) 1010d08b3851SPeter Zijlstra { 10119853a407SJoonsoo Kim int cleaned = 0; 10129853a407SJoonsoo Kim struct address_space *mapping; 10139853a407SJoonsoo Kim struct rmap_walk_control rwc = { 10149853a407SJoonsoo Kim .arg = (void *)&cleaned, 10159853a407SJoonsoo Kim .rmap_one = page_mkclean_one, 10169853a407SJoonsoo Kim .invalid_vma = invalid_mkclean_vma, 10179853a407SJoonsoo Kim }; 1018d08b3851SPeter Zijlstra 1019d9c08e22SMatthew Wilcox (Oracle) BUG_ON(!folio_test_locked(folio)); 1020d08b3851SPeter Zijlstra 1021d9c08e22SMatthew Wilcox (Oracle) if (!folio_mapped(folio)) 10229853a407SJoonsoo Kim return 0; 1023d08b3851SPeter Zijlstra 1024d9c08e22SMatthew Wilcox (Oracle) mapping = folio_mapping(folio); 10259853a407SJoonsoo Kim if (!mapping) 10269853a407SJoonsoo Kim return 0; 10279853a407SJoonsoo Kim 10282f031c6fSMatthew Wilcox (Oracle) rmap_walk(folio, &rwc); 10299853a407SJoonsoo Kim 10309853a407SJoonsoo Kim return cleaned; 1031d08b3851SPeter Zijlstra } 1032d9c08e22SMatthew Wilcox (Oracle) EXPORT_SYMBOL_GPL(folio_mkclean); 1033d08b3851SPeter Zijlstra 10341da177e4SLinus Torvalds /** 1035c44b6743SRik van Riel * page_move_anon_rmap - move a page to our anon_vma 1036c44b6743SRik van Riel * @page: the page to move to our anon_vma 1037c44b6743SRik van Riel * @vma: the vma the page belongs to 1038c44b6743SRik van Riel * 1039c44b6743SRik van Riel * When a page belongs exclusively to one process after a COW event, 1040c44b6743SRik van Riel * that page can be moved into the anon_vma that belongs to just that 1041c44b6743SRik van Riel * process, so the rmap code will not search the parent or sibling 1042c44b6743SRik van Riel * processes. 1043c44b6743SRik van Riel */ 10445a49973dSHugh Dickins void page_move_anon_rmap(struct page *page, struct vm_area_struct *vma) 1045c44b6743SRik van Riel { 1046c44b6743SRik van Riel struct anon_vma *anon_vma = vma->anon_vma; 1047c44b6743SRik van Riel 10485a49973dSHugh Dickins page = compound_head(page); 10495a49973dSHugh Dickins 1050309381feSSasha Levin VM_BUG_ON_PAGE(!PageLocked(page), page); 105181d1b09cSSasha Levin VM_BUG_ON_VMA(!anon_vma, vma); 1052c44b6743SRik van Riel 1053c44b6743SRik van Riel anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 1054414e2fb8SVladimir Davydov /* 1055414e2fb8SVladimir Davydov * Ensure that anon_vma and the PAGE_MAPPING_ANON bit are written 1056b3ac0413SMatthew Wilcox (Oracle) * simultaneously, so a concurrent reader (eg folio_referenced()'s 1057b3ac0413SMatthew Wilcox (Oracle) * folio_test_anon()) will not see one without the other. 1058414e2fb8SVladimir Davydov */ 1059414e2fb8SVladimir Davydov WRITE_ONCE(page->mapping, (struct address_space *) anon_vma); 1060c44b6743SRik van Riel } 1061c44b6743SRik van Riel 1062c44b6743SRik van Riel /** 106343d8eac4SRandy Dunlap * __page_set_anon_rmap - set up new anonymous rmap 1064451b9514SKirill Tkhai * @page: Page or Hugepage to add to rmap 10654e1c1975SAndi Kleen * @vma: VM area to add page to. 10664e1c1975SAndi Kleen * @address: User virtual address of the mapping 1067e8a03febSRik van Riel * @exclusive: the page is exclusively owned by the current process 10681da177e4SLinus Torvalds */ 10699617d95eSNick Piggin static void __page_set_anon_rmap(struct page *page, 1070e8a03febSRik van Riel struct vm_area_struct *vma, unsigned long address, int exclusive) 10711da177e4SLinus Torvalds { 1072e8a03febSRik van Riel struct anon_vma *anon_vma = vma->anon_vma; 10732822c1aaSNick Piggin 1074e8a03febSRik van Riel BUG_ON(!anon_vma); 1075ea90002bSLinus Torvalds 10764e1c1975SAndi Kleen if (PageAnon(page)) 10774e1c1975SAndi Kleen return; 10784e1c1975SAndi Kleen 1079ea90002bSLinus Torvalds /* 1080e8a03febSRik van Riel * If the page isn't exclusively mapped into this vma, 1081e8a03febSRik van Riel * we must use the _oldest_ possible anon_vma for the 1082e8a03febSRik van Riel * page mapping! 1083ea90002bSLinus Torvalds */ 10844e1c1975SAndi Kleen if (!exclusive) 1085288468c3SAndrea Arcangeli anon_vma = anon_vma->root; 1086ea90002bSLinus Torvalds 108716f5e707SAlex Shi /* 108816f5e707SAlex Shi * page_idle does a lockless/optimistic rmap scan on page->mapping. 108916f5e707SAlex Shi * Make sure the compiler doesn't split the stores of anon_vma and 109016f5e707SAlex Shi * the PAGE_MAPPING_ANON type identifier, otherwise the rmap code 109116f5e707SAlex Shi * could mistake the mapping for a struct address_space and crash. 109216f5e707SAlex Shi */ 10931da177e4SLinus Torvalds anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 109416f5e707SAlex Shi WRITE_ONCE(page->mapping, (struct address_space *) anon_vma); 10954d7670e0SNick Piggin page->index = linear_page_index(vma, address); 10961da177e4SLinus Torvalds } 10979617d95eSNick Piggin 10989617d95eSNick Piggin /** 109943d8eac4SRandy Dunlap * __page_check_anon_rmap - sanity check anonymous rmap addition 1100c97a9e10SNick Piggin * @page: the page to add the mapping to 1101c97a9e10SNick Piggin * @vma: the vm area in which the mapping is added 1102c97a9e10SNick Piggin * @address: the user virtual address mapped 1103c97a9e10SNick Piggin */ 1104c97a9e10SNick Piggin static void __page_check_anon_rmap(struct page *page, 1105c97a9e10SNick Piggin struct vm_area_struct *vma, unsigned long address) 1106c97a9e10SNick Piggin { 1107e05b3453SMatthew Wilcox (Oracle) struct folio *folio = page_folio(page); 1108c97a9e10SNick Piggin /* 1109c97a9e10SNick Piggin * The page's anon-rmap details (mapping and index) are guaranteed to 1110c97a9e10SNick Piggin * be set up correctly at this point. 1111c97a9e10SNick Piggin * 1112c97a9e10SNick Piggin * We have exclusion against page_add_anon_rmap because the caller 111390aaca85SMiaohe Lin * always holds the page locked. 1114c97a9e10SNick Piggin * 1115c97a9e10SNick Piggin * We have exclusion against page_add_new_anon_rmap because those pages 1116c97a9e10SNick Piggin * are initially only visible via the pagetables, and the pte is locked 1117c97a9e10SNick Piggin * over the call to page_add_new_anon_rmap. 1118c97a9e10SNick Piggin */ 1119e05b3453SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(folio_anon_vma(folio)->root != vma->anon_vma->root, 1120e05b3453SMatthew Wilcox (Oracle) folio); 112130c46382SYang Shi VM_BUG_ON_PAGE(page_to_pgoff(page) != linear_page_index(vma, address), 112230c46382SYang Shi page); 1123c97a9e10SNick Piggin } 1124c97a9e10SNick Piggin 1125c97a9e10SNick Piggin /** 11269617d95eSNick Piggin * page_add_anon_rmap - add pte mapping to an anonymous page 11279617d95eSNick Piggin * @page: the page to add the mapping to 11289617d95eSNick Piggin * @vma: the vm area in which the mapping is added 11299617d95eSNick Piggin * @address: the user virtual address mapped 1130d281ee61SKirill A. Shutemov * @compound: charge the page as compound or small page 11319617d95eSNick Piggin * 11325ad64688SHugh Dickins * The caller needs to hold the pte lock, and the page must be locked in 113380e14822SHugh Dickins * the anon_vma case: to serialize mapping,index checking after setting, 113480e14822SHugh Dickins * and to ensure that PageAnon is not being upgraded racily to PageKsm 113580e14822SHugh Dickins * (but PageKsm is never downgraded to PageAnon). 11369617d95eSNick Piggin */ 11379617d95eSNick Piggin void page_add_anon_rmap(struct page *page, 1138d281ee61SKirill A. Shutemov struct vm_area_struct *vma, unsigned long address, bool compound) 11399617d95eSNick Piggin { 1140d281ee61SKirill A. Shutemov do_page_add_anon_rmap(page, vma, address, compound ? RMAP_COMPOUND : 0); 1141ad8c2ee8SRik van Riel } 1142ad8c2ee8SRik van Riel 1143ad8c2ee8SRik van Riel /* 1144ad8c2ee8SRik van Riel * Special version of the above for do_swap_page, which often runs 1145ad8c2ee8SRik van Riel * into pages that are exclusively owned by the current process. 1146ad8c2ee8SRik van Riel * Everybody else should continue to use page_add_anon_rmap above. 1147ad8c2ee8SRik van Riel */ 1148ad8c2ee8SRik van Riel void do_page_add_anon_rmap(struct page *page, 1149d281ee61SKirill A. Shutemov struct vm_area_struct *vma, unsigned long address, int flags) 1150ad8c2ee8SRik van Riel { 1151d281ee61SKirill A. Shutemov bool compound = flags & RMAP_COMPOUND; 115253f9263bSKirill A. Shutemov bool first; 115353f9263bSKirill A. Shutemov 1154be5d0a74SJohannes Weiner if (unlikely(PageKsm(page))) 1155be5d0a74SJohannes Weiner lock_page_memcg(page); 1156be5d0a74SJohannes Weiner else 1157be5d0a74SJohannes Weiner VM_BUG_ON_PAGE(!PageLocked(page), page); 1158be5d0a74SJohannes Weiner 115953f9263bSKirill A. Shutemov if (compound) { 116053f9263bSKirill A. Shutemov atomic_t *mapcount; 1161e9b61f19SKirill A. Shutemov VM_BUG_ON_PAGE(!PageLocked(page), page); 116253f9263bSKirill A. Shutemov VM_BUG_ON_PAGE(!PageTransHuge(page), page); 116353f9263bSKirill A. Shutemov mapcount = compound_mapcount_ptr(page); 116453f9263bSKirill A. Shutemov first = atomic_inc_and_test(mapcount); 116553f9263bSKirill A. Shutemov } else { 116653f9263bSKirill A. Shutemov first = atomic_inc_and_test(&page->_mapcount); 116753f9263bSKirill A. Shutemov } 116853f9263bSKirill A. Shutemov 116953f9263bSKirill A. Shutemov if (first) { 11706c357848SMatthew Wilcox (Oracle) int nr = compound ? thp_nr_pages(page) : 1; 1171bea04b07SJianyu Zhan /* 1172bea04b07SJianyu Zhan * We use the irq-unsafe __{inc|mod}_zone_page_stat because 1173bea04b07SJianyu Zhan * these counters are not modified in interrupt context, and 1174bea04b07SJianyu Zhan * pte lock(a spinlock) is held, which implies preemption 1175bea04b07SJianyu Zhan * disabled. 1176bea04b07SJianyu Zhan */ 117765c45377SKirill A. Shutemov if (compound) 117869473e5dSMuchun Song __mod_lruvec_page_state(page, NR_ANON_THPS, nr); 1179be5d0a74SJohannes Weiner __mod_lruvec_page_state(page, NR_ANON_MAPPED, nr); 118079134171SAndrea Arcangeli } 11815ad64688SHugh Dickins 1182cea86fe2SHugh Dickins if (unlikely(PageKsm(page))) 1183be5d0a74SJohannes Weiner unlock_page_memcg(page); 118453f9263bSKirill A. Shutemov 11855dbe0af4SHugh Dickins /* address might be in next vma when migration races vma_adjust */ 1186cea86fe2SHugh Dickins else if (first) 1187d281ee61SKirill A. Shutemov __page_set_anon_rmap(page, vma, address, 1188d281ee61SKirill A. Shutemov flags & RMAP_EXCLUSIVE); 118969029cd5SKAMEZAWA Hiroyuki else 1190c97a9e10SNick Piggin __page_check_anon_rmap(page, vma, address); 1191cea86fe2SHugh Dickins 1192cea86fe2SHugh Dickins mlock_vma_page(page, vma, compound); 11931da177e4SLinus Torvalds } 11941da177e4SLinus Torvalds 119543d8eac4SRandy Dunlap /** 11969617d95eSNick Piggin * page_add_new_anon_rmap - add pte mapping to a new anonymous page 11979617d95eSNick Piggin * @page: the page to add the mapping to 11989617d95eSNick Piggin * @vma: the vm area in which the mapping is added 11999617d95eSNick Piggin * @address: the user virtual address mapped 1200d281ee61SKirill A. Shutemov * @compound: charge the page as compound or small page 12019617d95eSNick Piggin * 12029617d95eSNick Piggin * Same as page_add_anon_rmap but must only be called on *new* pages. 12039617d95eSNick Piggin * This means the inc-and-test can be bypassed. 1204c97a9e10SNick Piggin * Page does not have to be locked. 12059617d95eSNick Piggin */ 12069617d95eSNick Piggin void page_add_new_anon_rmap(struct page *page, 1207d281ee61SKirill A. Shutemov struct vm_area_struct *vma, unsigned long address, bool compound) 12089617d95eSNick Piggin { 12096c357848SMatthew Wilcox (Oracle) int nr = compound ? thp_nr_pages(page) : 1; 1210d281ee61SKirill A. Shutemov 121181d1b09cSSasha Levin VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma); 1212fa9949daSHugh Dickins __SetPageSwapBacked(page); 1213d281ee61SKirill A. Shutemov if (compound) { 1214d281ee61SKirill A. Shutemov VM_BUG_ON_PAGE(!PageTransHuge(page), page); 121553f9263bSKirill A. Shutemov /* increment count (starts at -1) */ 121653f9263bSKirill A. Shutemov atomic_set(compound_mapcount_ptr(page), 0); 121747e29d32SJohn Hubbard atomic_set(compound_pincount_ptr(page), 0); 121847e29d32SJohn Hubbard 121969473e5dSMuchun Song __mod_lruvec_page_state(page, NR_ANON_THPS, nr); 122053f9263bSKirill A. Shutemov } else { 122153f9263bSKirill A. Shutemov /* Anon THP always mapped first with PMD */ 122253f9263bSKirill A. Shutemov VM_BUG_ON_PAGE(PageTransCompound(page), page); 122353f9263bSKirill A. Shutemov /* increment count (starts at -1) */ 122453f9263bSKirill A. Shutemov atomic_set(&page->_mapcount, 0); 1225d281ee61SKirill A. Shutemov } 1226be5d0a74SJohannes Weiner __mod_lruvec_page_state(page, NR_ANON_MAPPED, nr); 1227e8a03febSRik van Riel __page_set_anon_rmap(page, vma, address, 1); 12289617d95eSNick Piggin } 12299617d95eSNick Piggin 12301da177e4SLinus Torvalds /** 12311da177e4SLinus Torvalds * page_add_file_rmap - add pte mapping to a file page 12321da177e4SLinus Torvalds * @page: the page to add the mapping to 1233cea86fe2SHugh Dickins * @vma: the vm area in which the mapping is added 1234e8b098fcSMike Rapoport * @compound: charge the page as compound or small page 12351da177e4SLinus Torvalds * 1236b8072f09SHugh Dickins * The caller needs to hold the pte lock. 12371da177e4SLinus Torvalds */ 1238cea86fe2SHugh Dickins void page_add_file_rmap(struct page *page, 1239cea86fe2SHugh Dickins struct vm_area_struct *vma, bool compound) 12401da177e4SLinus Torvalds { 12415d543f13SHugh Dickins int i, nr = 0; 1242dd78feddSKirill A. Shutemov 1243dd78feddSKirill A. Shutemov VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page); 124462cccb8cSJohannes Weiner lock_page_memcg(page); 1245dd78feddSKirill A. Shutemov if (compound && PageTransHuge(page)) { 1246a1528e21SMuchun Song int nr_pages = thp_nr_pages(page); 1247a1528e21SMuchun Song 12485d543f13SHugh Dickins for (i = 0; i < nr_pages; i++) { 1249dd78feddSKirill A. Shutemov if (atomic_inc_and_test(&page[i]._mapcount)) 1250dd78feddSKirill A. Shutemov nr++; 1251d69b042fSBalbir Singh } 1252dd78feddSKirill A. Shutemov if (!atomic_inc_and_test(compound_mapcount_ptr(page))) 1253dd78feddSKirill A. Shutemov goto out; 1254bd55b0c2SHugh Dickins 1255bd55b0c2SHugh Dickins /* 1256bd55b0c2SHugh Dickins * It is racy to ClearPageDoubleMap in page_remove_file_rmap(); 1257bd55b0c2SHugh Dickins * but page lock is held by all page_add_file_rmap() compound 1258bd55b0c2SHugh Dickins * callers, and SetPageDoubleMap below warns if !PageLocked: 1259bd55b0c2SHugh Dickins * so here is a place that DoubleMap can be safely cleared. 1260bd55b0c2SHugh Dickins */ 1261bd55b0c2SHugh Dickins VM_WARN_ON_ONCE(!PageLocked(page)); 1262bd55b0c2SHugh Dickins if (nr == nr_pages && PageDoubleMap(page)) 1263bd55b0c2SHugh Dickins ClearPageDoubleMap(page); 1264bd55b0c2SHugh Dickins 126599cb0dbdSSong Liu if (PageSwapBacked(page)) 1266a1528e21SMuchun Song __mod_lruvec_page_state(page, NR_SHMEM_PMDMAPPED, 1267a1528e21SMuchun Song nr_pages); 126899cb0dbdSSong Liu else 1269380780e7SMuchun Song __mod_lruvec_page_state(page, NR_FILE_PMDMAPPED, 1270380780e7SMuchun Song nr_pages); 1271dd78feddSKirill A. Shutemov } else { 1272c8efc390SKirill A. Shutemov if (PageTransCompound(page) && page_mapping(page)) { 1273c8efc390SKirill A. Shutemov VM_WARN_ON_ONCE(!PageLocked(page)); 1274cea86fe2SHugh Dickins SetPageDoubleMap(compound_head(page)); 12759a73f61bSKirill A. Shutemov } 12765d543f13SHugh Dickins if (atomic_inc_and_test(&page->_mapcount)) 12775d543f13SHugh Dickins nr++; 1278dd78feddSKirill A. Shutemov } 1279dd78feddSKirill A. Shutemov out: 12805d543f13SHugh Dickins if (nr) 12815d543f13SHugh Dickins __mod_lruvec_page_state(page, NR_FILE_MAPPED, nr); 128262cccb8cSJohannes Weiner unlock_page_memcg(page); 1283cea86fe2SHugh Dickins 1284cea86fe2SHugh Dickins mlock_vma_page(page, vma, compound); 12851da177e4SLinus Torvalds } 12861da177e4SLinus Torvalds 1287dd78feddSKirill A. Shutemov static void page_remove_file_rmap(struct page *page, bool compound) 12888186eb6aSJohannes Weiner { 12895d543f13SHugh Dickins int i, nr = 0; 1290dd78feddSKirill A. Shutemov 129157dea93aSSteve Capper VM_BUG_ON_PAGE(compound && !PageHead(page), page); 12928186eb6aSJohannes Weiner 129353f9263bSKirill A. Shutemov /* Hugepages are not counted in NR_FILE_MAPPED for now. */ 129453f9263bSKirill A. Shutemov if (unlikely(PageHuge(page))) { 129553f9263bSKirill A. Shutemov /* hugetlb pages are always mapped with pmds */ 129653f9263bSKirill A. Shutemov atomic_dec(compound_mapcount_ptr(page)); 1297be5d0a74SJohannes Weiner return; 129853f9263bSKirill A. Shutemov } 129953f9263bSKirill A. Shutemov 13008186eb6aSJohannes Weiner /* page still mapped by someone else? */ 1301dd78feddSKirill A. Shutemov if (compound && PageTransHuge(page)) { 1302a1528e21SMuchun Song int nr_pages = thp_nr_pages(page); 1303a1528e21SMuchun Song 13045d543f13SHugh Dickins for (i = 0; i < nr_pages; i++) { 1305dd78feddSKirill A. Shutemov if (atomic_add_negative(-1, &page[i]._mapcount)) 1306dd78feddSKirill A. Shutemov nr++; 1307dd78feddSKirill A. Shutemov } 1308dd78feddSKirill A. Shutemov if (!atomic_add_negative(-1, compound_mapcount_ptr(page))) 13095d543f13SHugh Dickins goto out; 131099cb0dbdSSong Liu if (PageSwapBacked(page)) 1311a1528e21SMuchun Song __mod_lruvec_page_state(page, NR_SHMEM_PMDMAPPED, 1312a1528e21SMuchun Song -nr_pages); 131399cb0dbdSSong Liu else 1314380780e7SMuchun Song __mod_lruvec_page_state(page, NR_FILE_PMDMAPPED, 1315380780e7SMuchun Song -nr_pages); 1316dd78feddSKirill A. Shutemov } else { 13175d543f13SHugh Dickins if (atomic_add_negative(-1, &page->_mapcount)) 13185d543f13SHugh Dickins nr++; 1319dd78feddSKirill A. Shutemov } 13205d543f13SHugh Dickins out: 13215d543f13SHugh Dickins if (nr) 132200f3ca2cSJohannes Weiner __mod_lruvec_page_state(page, NR_FILE_MAPPED, -nr); 13238186eb6aSJohannes Weiner } 13248186eb6aSJohannes Weiner 132553f9263bSKirill A. Shutemov static void page_remove_anon_compound_rmap(struct page *page) 132653f9263bSKirill A. Shutemov { 132753f9263bSKirill A. Shutemov int i, nr; 132853f9263bSKirill A. Shutemov 132953f9263bSKirill A. Shutemov if (!atomic_add_negative(-1, compound_mapcount_ptr(page))) 133053f9263bSKirill A. Shutemov return; 133153f9263bSKirill A. Shutemov 133253f9263bSKirill A. Shutemov /* Hugepages are not counted in NR_ANON_PAGES for now. */ 133353f9263bSKirill A. Shutemov if (unlikely(PageHuge(page))) 133453f9263bSKirill A. Shutemov return; 133553f9263bSKirill A. Shutemov 133653f9263bSKirill A. Shutemov if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) 133753f9263bSKirill A. Shutemov return; 133853f9263bSKirill A. Shutemov 133969473e5dSMuchun Song __mod_lruvec_page_state(page, NR_ANON_THPS, -thp_nr_pages(page)); 134053f9263bSKirill A. Shutemov 134153f9263bSKirill A. Shutemov if (TestClearPageDoubleMap(page)) { 134253f9263bSKirill A. Shutemov /* 134353f9263bSKirill A. Shutemov * Subpages can be mapped with PTEs too. Check how many of 1344f1fe80d4SKirill A. Shutemov * them are still mapped. 134553f9263bSKirill A. Shutemov */ 13465eaf35abSMatthew Wilcox (Oracle) for (i = 0, nr = 0; i < thp_nr_pages(page); i++) { 134753f9263bSKirill A. Shutemov if (atomic_add_negative(-1, &page[i]._mapcount)) 134853f9263bSKirill A. Shutemov nr++; 134953f9263bSKirill A. Shutemov } 1350f1fe80d4SKirill A. Shutemov 1351f1fe80d4SKirill A. Shutemov /* 1352f1fe80d4SKirill A. Shutemov * Queue the page for deferred split if at least one small 1353f1fe80d4SKirill A. Shutemov * page of the compound page is unmapped, but at least one 1354f1fe80d4SKirill A. Shutemov * small page is still mapped. 1355f1fe80d4SKirill A. Shutemov */ 13565eaf35abSMatthew Wilcox (Oracle) if (nr && nr < thp_nr_pages(page)) 1357f1fe80d4SKirill A. Shutemov deferred_split_huge_page(page); 135853f9263bSKirill A. Shutemov } else { 13595eaf35abSMatthew Wilcox (Oracle) nr = thp_nr_pages(page); 136053f9263bSKirill A. Shutemov } 136153f9263bSKirill A. Shutemov 1362f1fe80d4SKirill A. Shutemov if (nr) 1363be5d0a74SJohannes Weiner __mod_lruvec_page_state(page, NR_ANON_MAPPED, -nr); 136453f9263bSKirill A. Shutemov } 136553f9263bSKirill A. Shutemov 13661da177e4SLinus Torvalds /** 13671da177e4SLinus Torvalds * page_remove_rmap - take down pte mapping from a page 13681da177e4SLinus Torvalds * @page: page to remove mapping from 1369cea86fe2SHugh Dickins * @vma: the vm area from which the mapping is removed 1370d281ee61SKirill A. Shutemov * @compound: uncharge the page as compound or small page 13711da177e4SLinus Torvalds * 1372b8072f09SHugh Dickins * The caller needs to hold the pte lock. 13731da177e4SLinus Torvalds */ 1374cea86fe2SHugh Dickins void page_remove_rmap(struct page *page, 1375cea86fe2SHugh Dickins struct vm_area_struct *vma, bool compound) 13761da177e4SLinus Torvalds { 1377be5d0a74SJohannes Weiner lock_page_memcg(page); 137889c06bd5SKAMEZAWA Hiroyuki 1379be5d0a74SJohannes Weiner if (!PageAnon(page)) { 1380be5d0a74SJohannes Weiner page_remove_file_rmap(page, compound); 1381be5d0a74SJohannes Weiner goto out; 1382be5d0a74SJohannes Weiner } 1383be5d0a74SJohannes Weiner 1384be5d0a74SJohannes Weiner if (compound) { 1385be5d0a74SJohannes Weiner page_remove_anon_compound_rmap(page); 1386be5d0a74SJohannes Weiner goto out; 1387be5d0a74SJohannes Weiner } 138853f9263bSKirill A. Shutemov 1389b904dcfeSKOSAKI Motohiro /* page still mapped by someone else? */ 1390b904dcfeSKOSAKI Motohiro if (!atomic_add_negative(-1, &page->_mapcount)) 1391be5d0a74SJohannes Weiner goto out; 13928186eb6aSJohannes Weiner 13931da177e4SLinus Torvalds /* 1394bea04b07SJianyu Zhan * We use the irq-unsafe __{inc|mod}_zone_page_stat because 1395bea04b07SJianyu Zhan * these counters are not modified in interrupt context, and 1396bea04b07SJianyu Zhan * pte lock(a spinlock) is held, which implies preemption disabled. 13970fe6e20bSNaoya Horiguchi */ 1398be5d0a74SJohannes Weiner __dec_lruvec_page_state(page, NR_ANON_MAPPED); 13998186eb6aSJohannes Weiner 14009a982250SKirill A. Shutemov if (PageTransCompound(page)) 14019a982250SKirill A. Shutemov deferred_split_huge_page(compound_head(page)); 14029a982250SKirill A. Shutemov 140316f8c5b2SHugh Dickins /* 14041da177e4SLinus Torvalds * It would be tidy to reset the PageAnon mapping here, 14051da177e4SLinus Torvalds * but that might overwrite a racing page_add_anon_rmap 14061da177e4SLinus Torvalds * which increments mapcount after us but sets mapping 14072d4894b5SMel Gorman * before us: so leave the reset to free_unref_page, 14081da177e4SLinus Torvalds * and remember that it's only reliable while mapped. 14091da177e4SLinus Torvalds * Leaving it set also helps swapoff to reinstate ptes 14101da177e4SLinus Torvalds * faster for those pages still in swapcache. 14111da177e4SLinus Torvalds */ 1412be5d0a74SJohannes Weiner out: 1413be5d0a74SJohannes Weiner unlock_page_memcg(page); 1414cea86fe2SHugh Dickins 1415cea86fe2SHugh Dickins munlock_vma_page(page, vma, compound); 14161da177e4SLinus Torvalds } 14171da177e4SLinus Torvalds 14181da177e4SLinus Torvalds /* 141952629506SJoonsoo Kim * @arg: enum ttu_flags will be passed to this argument 14201da177e4SLinus Torvalds */ 14212f031c6fSMatthew Wilcox (Oracle) static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, 142252629506SJoonsoo Kim unsigned long address, void *arg) 14231da177e4SLinus Torvalds { 14241da177e4SLinus Torvalds struct mm_struct *mm = vma->vm_mm; 1425869f7ee6SMatthew Wilcox (Oracle) DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); 14261da177e4SLinus Torvalds pte_t pteval; 1427c7ab0d2fSKirill A. Shutemov struct page *subpage; 1428785373b4SLinus Torvalds bool ret = true; 1429ac46d4f3SJérôme Glisse struct mmu_notifier_range range; 14304708f318SPalmer Dabbelt enum ttu_flags flags = (enum ttu_flags)(long)arg; 14311da177e4SLinus Torvalds 1432732ed558SHugh Dickins /* 1433732ed558SHugh Dickins * When racing against e.g. zap_pte_range() on another cpu, 1434732ed558SHugh Dickins * in between its ptep_get_and_clear_full() and page_remove_rmap(), 14351fb08ac6SYang Shi * try_to_unmap() may return before page_mapped() has become false, 1436732ed558SHugh Dickins * if page table locking is skipped: use TTU_SYNC to wait for that. 1437732ed558SHugh Dickins */ 1438732ed558SHugh Dickins if (flags & TTU_SYNC) 1439732ed558SHugh Dickins pvmw.flags = PVMW_SYNC; 1440732ed558SHugh Dickins 1441a98a2f0cSAlistair Popple if (flags & TTU_SPLIT_HUGE_PMD) 1442af28a988SMatthew Wilcox (Oracle) split_huge_pmd_address(vma, address, false, folio); 1443fec89c10SKirill A. Shutemov 1444369ea824SJérôme Glisse /* 1445017b1660SMike Kravetz * For THP, we have to assume the worse case ie pmd for invalidation. 1446017b1660SMike Kravetz * For hugetlb, it could be much worse if we need to do pud 1447017b1660SMike Kravetz * invalidation in the case of pmd sharing. 1448017b1660SMike Kravetz * 1449869f7ee6SMatthew Wilcox (Oracle) * Note that the folio can not be freed in this function as call of 1450869f7ee6SMatthew Wilcox (Oracle) * try_to_unmap() must hold a reference on the folio. 1451369ea824SJérôme Glisse */ 14522aff7a47SMatthew Wilcox (Oracle) range.end = vma_address_end(&pvmw); 14537269f999SJérôme Glisse mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, 1454494334e4SHugh Dickins address, range.end); 1455869f7ee6SMatthew Wilcox (Oracle) if (folio_test_hugetlb(folio)) { 1456017b1660SMike Kravetz /* 1457017b1660SMike Kravetz * If sharing is possible, start and end will be adjusted 1458017b1660SMike Kravetz * accordingly. 1459017b1660SMike Kravetz */ 1460ac46d4f3SJérôme Glisse adjust_range_if_pmd_sharing_possible(vma, &range.start, 1461ac46d4f3SJérôme Glisse &range.end); 1462017b1660SMike Kravetz } 1463ac46d4f3SJérôme Glisse mmu_notifier_invalidate_range_start(&range); 1464369ea824SJérôme Glisse 1465c7ab0d2fSKirill A. Shutemov while (page_vma_mapped_walk(&pvmw)) { 1466cea86fe2SHugh Dickins /* Unexpected PMD-mapped THP? */ 1467869f7ee6SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(!pvmw.pte, folio); 1468cea86fe2SHugh Dickins 14691da177e4SLinus Torvalds /* 1470869f7ee6SMatthew Wilcox (Oracle) * If the folio is in an mlock()d vma, we must not swap it out. 14711da177e4SLinus Torvalds */ 1472efdb6720SHugh Dickins if (!(flags & TTU_IGNORE_MLOCK) && 1473efdb6720SHugh Dickins (vma->vm_flags & VM_LOCKED)) { 1474cea86fe2SHugh Dickins /* Restore the mlock which got missed */ 1475869f7ee6SMatthew Wilcox (Oracle) mlock_vma_folio(folio, vma, false); 1476c7ab0d2fSKirill A. Shutemov page_vma_mapped_walk_done(&pvmw); 1477efdb6720SHugh Dickins ret = false; 1478c7ab0d2fSKirill A. Shutemov break; 1479b87537d9SHugh Dickins } 1480c7ab0d2fSKirill A. Shutemov 1481869f7ee6SMatthew Wilcox (Oracle) subpage = folio_page(folio, 1482869f7ee6SMatthew Wilcox (Oracle) pte_pfn(*pvmw.pte) - folio_pfn(folio)); 1483785373b4SLinus Torvalds address = pvmw.address; 1484785373b4SLinus Torvalds 1485869f7ee6SMatthew Wilcox (Oracle) if (folio_test_hugetlb(folio) && !folio_test_anon(folio)) { 1486c0d0381aSMike Kravetz /* 1487c0d0381aSMike Kravetz * To call huge_pmd_unshare, i_mmap_rwsem must be 1488c0d0381aSMike Kravetz * held in write mode. Caller needs to explicitly 1489c0d0381aSMike Kravetz * do this outside rmap routines. 1490c0d0381aSMike Kravetz */ 1491c0d0381aSMike Kravetz VM_BUG_ON(!(flags & TTU_RMAP_LOCKED)); 149234ae204fSMike Kravetz if (huge_pmd_unshare(mm, vma, &address, pvmw.pte)) { 1493017b1660SMike Kravetz /* 1494017b1660SMike Kravetz * huge_pmd_unshare unmapped an entire PMD 1495017b1660SMike Kravetz * page. There is no way of knowing exactly 1496017b1660SMike Kravetz * which PMDs may be cached for this mm, so 1497017b1660SMike Kravetz * we must flush them all. start/end were 1498017b1660SMike Kravetz * already adjusted above to cover this range. 1499017b1660SMike Kravetz */ 1500ac46d4f3SJérôme Glisse flush_cache_range(vma, range.start, range.end); 1501ac46d4f3SJérôme Glisse flush_tlb_range(vma, range.start, range.end); 1502ac46d4f3SJérôme Glisse mmu_notifier_invalidate_range(mm, range.start, 1503ac46d4f3SJérôme Glisse range.end); 1504017b1660SMike Kravetz 1505017b1660SMike Kravetz /* 1506017b1660SMike Kravetz * The ref count of the PMD page was dropped 1507017b1660SMike Kravetz * which is part of the way map counting 1508017b1660SMike Kravetz * is done for shared PMDs. Return 'true' 1509017b1660SMike Kravetz * here. When there is no other sharing, 1510017b1660SMike Kravetz * huge_pmd_unshare returns false and we will 1511017b1660SMike Kravetz * unmap the actual page and drop map count 1512017b1660SMike Kravetz * to zero. 1513017b1660SMike Kravetz */ 1514017b1660SMike Kravetz page_vma_mapped_walk_done(&pvmw); 1515017b1660SMike Kravetz break; 1516017b1660SMike Kravetz } 1517017b1660SMike Kravetz } 15188346242aSKirill A. Shutemov 15191da177e4SLinus Torvalds /* Nuke the page table entry. */ 1520785373b4SLinus Torvalds flush_cache_page(vma, address, pte_pfn(*pvmw.pte)); 152172b252aeSMel Gorman if (should_defer_flush(mm, flags)) { 152272b252aeSMel Gorman /* 1523c7ab0d2fSKirill A. Shutemov * We clear the PTE but do not flush so potentially 1524869f7ee6SMatthew Wilcox (Oracle) * a remote CPU could still be writing to the folio. 1525c7ab0d2fSKirill A. Shutemov * If the entry was previously clean then the 1526c7ab0d2fSKirill A. Shutemov * architecture must guarantee that a clear->dirty 1527c7ab0d2fSKirill A. Shutemov * transition on a cached TLB entry is written through 1528c7ab0d2fSKirill A. Shutemov * and traps if the PTE is unmapped. 152972b252aeSMel Gorman */ 1530785373b4SLinus Torvalds pteval = ptep_get_and_clear(mm, address, pvmw.pte); 153172b252aeSMel Gorman 1532c7ab0d2fSKirill A. Shutemov set_tlb_ubc_flush_pending(mm, pte_dirty(pteval)); 153372b252aeSMel Gorman } else { 1534785373b4SLinus Torvalds pteval = ptep_clear_flush(vma, address, pvmw.pte); 153572b252aeSMel Gorman } 15361da177e4SLinus Torvalds 1537869f7ee6SMatthew Wilcox (Oracle) /* Set the dirty flag on the folio now the pte is gone. */ 15381da177e4SLinus Torvalds if (pte_dirty(pteval)) 1539869f7ee6SMatthew Wilcox (Oracle) folio_mark_dirty(folio); 15401da177e4SLinus Torvalds 1541365e9c87SHugh Dickins /* Update high watermark before we lower rss */ 1542365e9c87SHugh Dickins update_hiwater_rss(mm); 1543365e9c87SHugh Dickins 1544da358d5cSMatthew Wilcox (Oracle) if (PageHWPoison(subpage) && !(flags & TTU_IGNORE_HWPOISON)) { 15455fd27b8eSPunit Agrawal pteval = swp_entry_to_pte(make_hwpoison_entry(subpage)); 1546869f7ee6SMatthew Wilcox (Oracle) if (folio_test_hugetlb(folio)) { 1547869f7ee6SMatthew Wilcox (Oracle) hugetlb_count_sub(folio_nr_pages(folio), mm); 1548785373b4SLinus Torvalds set_huge_swap_pte_at(mm, address, 15495fd27b8eSPunit Agrawal pvmw.pte, pteval, 15505fd27b8eSPunit Agrawal vma_mmu_pagesize(vma)); 15515d317b2bSNaoya Horiguchi } else { 1552869f7ee6SMatthew Wilcox (Oracle) dec_mm_counter(mm, mm_counter(&folio->page)); 1553785373b4SLinus Torvalds set_pte_at(mm, address, pvmw.pte, pteval); 15545f24ae58SNaoya Horiguchi } 1555c7ab0d2fSKirill A. Shutemov 1556bce73e48SChristian Borntraeger } else if (pte_unused(pteval) && !userfaultfd_armed(vma)) { 155745961722SKonstantin Weitz /* 155845961722SKonstantin Weitz * The guest indicated that the page content is of no 155945961722SKonstantin Weitz * interest anymore. Simply discard the pte, vmscan 156045961722SKonstantin Weitz * will take care of the rest. 1561bce73e48SChristian Borntraeger * A future reference will then fault in a new zero 1562bce73e48SChristian Borntraeger * page. When userfaultfd is active, we must not drop 1563bce73e48SChristian Borntraeger * this page though, as its main user (postcopy 1564bce73e48SChristian Borntraeger * migration) will not expect userfaults on already 1565bce73e48SChristian Borntraeger * copied pages. 156645961722SKonstantin Weitz */ 1567869f7ee6SMatthew Wilcox (Oracle) dec_mm_counter(mm, mm_counter(&folio->page)); 15680f10851eSJérôme Glisse /* We have to invalidate as we cleared the pte */ 15690f10851eSJérôme Glisse mmu_notifier_invalidate_range(mm, address, 15700f10851eSJérôme Glisse address + PAGE_SIZE); 1571869f7ee6SMatthew Wilcox (Oracle) } else if (folio_test_anon(folio)) { 1572c7ab0d2fSKirill A. Shutemov swp_entry_t entry = { .val = page_private(subpage) }; 1573179ef71cSCyrill Gorcunov pte_t swp_pte; 15741da177e4SLinus Torvalds /* 15751da177e4SLinus Torvalds * Store the swap location in the pte. 15761da177e4SLinus Torvalds * See handle_pte_fault() ... 15771da177e4SLinus Torvalds */ 1578869f7ee6SMatthew Wilcox (Oracle) if (unlikely(folio_test_swapbacked(folio) != 1579869f7ee6SMatthew Wilcox (Oracle) folio_test_swapcache(folio))) { 1580eb94a878SMinchan Kim WARN_ON_ONCE(1); 158183612a94SMinchan Kim ret = false; 1582369ea824SJérôme Glisse /* We have to invalidate as we cleared the pte */ 15830f10851eSJérôme Glisse mmu_notifier_invalidate_range(mm, address, 15840f10851eSJérôme Glisse address + PAGE_SIZE); 1585eb94a878SMinchan Kim page_vma_mapped_walk_done(&pvmw); 1586eb94a878SMinchan Kim break; 1587eb94a878SMinchan Kim } 1588854e9ed0SMinchan Kim 1589802a3a92SShaohua Li /* MADV_FREE page check */ 1590869f7ee6SMatthew Wilcox (Oracle) if (!folio_test_swapbacked(folio)) { 1591869f7ee6SMatthew Wilcox (Oracle) if (!folio_test_dirty(folio)) { 15920f10851eSJérôme Glisse /* Invalidate as we cleared the pte */ 15930f10851eSJérôme Glisse mmu_notifier_invalidate_range(mm, 15940f10851eSJérôme Glisse address, address + PAGE_SIZE); 1595854e9ed0SMinchan Kim dec_mm_counter(mm, MM_ANONPAGES); 1596854e9ed0SMinchan Kim goto discard; 1597854e9ed0SMinchan Kim } 1598854e9ed0SMinchan Kim 1599802a3a92SShaohua Li /* 1600869f7ee6SMatthew Wilcox (Oracle) * If the folio was redirtied, it cannot be 1601802a3a92SShaohua Li * discarded. Remap the page to page table. 1602802a3a92SShaohua Li */ 1603785373b4SLinus Torvalds set_pte_at(mm, address, pvmw.pte, pteval); 1604869f7ee6SMatthew Wilcox (Oracle) folio_set_swapbacked(folio); 1605e4b82222SMinchan Kim ret = false; 1606802a3a92SShaohua Li page_vma_mapped_walk_done(&pvmw); 1607802a3a92SShaohua Li break; 1608802a3a92SShaohua Li } 1609802a3a92SShaohua Li 1610570a335bSHugh Dickins if (swap_duplicate(entry) < 0) { 1611785373b4SLinus Torvalds set_pte_at(mm, address, pvmw.pte, pteval); 1612e4b82222SMinchan Kim ret = false; 1613c7ab0d2fSKirill A. Shutemov page_vma_mapped_walk_done(&pvmw); 1614c7ab0d2fSKirill A. Shutemov break; 1615570a335bSHugh Dickins } 1616ca827d55SKhalid Aziz if (arch_unmap_one(mm, vma, address, pteval) < 0) { 1617ca827d55SKhalid Aziz set_pte_at(mm, address, pvmw.pte, pteval); 1618ca827d55SKhalid Aziz ret = false; 1619ca827d55SKhalid Aziz page_vma_mapped_walk_done(&pvmw); 1620ca827d55SKhalid Aziz break; 1621ca827d55SKhalid Aziz } 16221da177e4SLinus Torvalds if (list_empty(&mm->mmlist)) { 16231da177e4SLinus Torvalds spin_lock(&mmlist_lock); 1624f412ac08SHugh Dickins if (list_empty(&mm->mmlist)) 16251da177e4SLinus Torvalds list_add(&mm->mmlist, &init_mm.mmlist); 16261da177e4SLinus Torvalds spin_unlock(&mmlist_lock); 16271da177e4SLinus Torvalds } 1628d559db08SKAMEZAWA Hiroyuki dec_mm_counter(mm, MM_ANONPAGES); 1629b084d435SKAMEZAWA Hiroyuki inc_mm_counter(mm, MM_SWAPENTS); 1630179ef71cSCyrill Gorcunov swp_pte = swp_entry_to_pte(entry); 1631179ef71cSCyrill Gorcunov if (pte_soft_dirty(pteval)) 1632179ef71cSCyrill Gorcunov swp_pte = pte_swp_mksoft_dirty(swp_pte); 1633f45ec5ffSPeter Xu if (pte_uffd_wp(pteval)) 1634f45ec5ffSPeter Xu swp_pte = pte_swp_mkuffd_wp(swp_pte); 1635785373b4SLinus Torvalds set_pte_at(mm, address, pvmw.pte, swp_pte); 16360f10851eSJérôme Glisse /* Invalidate as we cleared the pte */ 1637369ea824SJérôme Glisse mmu_notifier_invalidate_range(mm, address, 1638369ea824SJérôme Glisse address + PAGE_SIZE); 16390f10851eSJérôme Glisse } else { 16400f10851eSJérôme Glisse /* 1641869f7ee6SMatthew Wilcox (Oracle) * This is a locked file-backed folio, 1642869f7ee6SMatthew Wilcox (Oracle) * so it cannot be removed from the page 1643869f7ee6SMatthew Wilcox (Oracle) * cache and replaced by a new folio before 1644869f7ee6SMatthew Wilcox (Oracle) * mmu_notifier_invalidate_range_end, so no 1645869f7ee6SMatthew Wilcox (Oracle) * concurrent thread might update its page table 1646869f7ee6SMatthew Wilcox (Oracle) * to point at a new folio while a device is 1647869f7ee6SMatthew Wilcox (Oracle) * still using this folio. 16480f10851eSJérôme Glisse * 1649ad56b738SMike Rapoport * See Documentation/vm/mmu_notifier.rst 16500f10851eSJérôme Glisse */ 1651869f7ee6SMatthew Wilcox (Oracle) dec_mm_counter(mm, mm_counter_file(&folio->page)); 16520f10851eSJérôme Glisse } 16530f10851eSJérôme Glisse discard: 16540f10851eSJérôme Glisse /* 16550f10851eSJérôme Glisse * No need to call mmu_notifier_invalidate_range() it has be 16560f10851eSJérôme Glisse * done above for all cases requiring it to happen under page 16570f10851eSJérôme Glisse * table lock before mmu_notifier_invalidate_range_end() 16580f10851eSJérôme Glisse * 1659ad56b738SMike Rapoport * See Documentation/vm/mmu_notifier.rst 16600f10851eSJérôme Glisse */ 1661869f7ee6SMatthew Wilcox (Oracle) page_remove_rmap(subpage, vma, folio_test_hugetlb(folio)); 1662b7435507SHugh Dickins if (vma->vm_flags & VM_LOCKED) 1663b7435507SHugh Dickins mlock_page_drain(smp_processor_id()); 1664869f7ee6SMatthew Wilcox (Oracle) folio_put(folio); 1665c7ab0d2fSKirill A. Shutemov } 1666369ea824SJérôme Glisse 1667ac46d4f3SJérôme Glisse mmu_notifier_invalidate_range_end(&range); 1668369ea824SJérôme Glisse 1669caed0f48SKOSAKI Motohiro return ret; 16701da177e4SLinus Torvalds } 16711da177e4SLinus Torvalds 167252629506SJoonsoo Kim static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg) 167352629506SJoonsoo Kim { 1674222100eeSAnshuman Khandual return vma_is_temporary_stack(vma); 167552629506SJoonsoo Kim } 167652629506SJoonsoo Kim 16772f031c6fSMatthew Wilcox (Oracle) static int page_not_mapped(struct folio *folio) 167852629506SJoonsoo Kim { 16792f031c6fSMatthew Wilcox (Oracle) return !folio_mapped(folio); 16802a52bcbcSKirill A. Shutemov } 168152629506SJoonsoo Kim 16821da177e4SLinus Torvalds /** 1683869f7ee6SMatthew Wilcox (Oracle) * try_to_unmap - Try to remove all page table mappings to a folio. 1684869f7ee6SMatthew Wilcox (Oracle) * @folio: The folio to unmap. 168514fa31b8SAndi Kleen * @flags: action and flags 16861da177e4SLinus Torvalds * 16871da177e4SLinus Torvalds * Tries to remove all the page table entries which are mapping this 1688869f7ee6SMatthew Wilcox (Oracle) * folio. It is the caller's responsibility to check if the folio is 1689869f7ee6SMatthew Wilcox (Oracle) * still mapped if needed (use TTU_SYNC to prevent accounting races). 16901da177e4SLinus Torvalds * 1691869f7ee6SMatthew Wilcox (Oracle) * Context: Caller must hold the folio lock. 16921da177e4SLinus Torvalds */ 1693869f7ee6SMatthew Wilcox (Oracle) void try_to_unmap(struct folio *folio, enum ttu_flags flags) 16941da177e4SLinus Torvalds { 169552629506SJoonsoo Kim struct rmap_walk_control rwc = { 169652629506SJoonsoo Kim .rmap_one = try_to_unmap_one, 1697802a3a92SShaohua Li .arg = (void *)flags, 1698b7e188ecSMiaohe Lin .done = page_not_mapped, 16992f031c6fSMatthew Wilcox (Oracle) .anon_lock = folio_lock_anon_vma_read, 170052629506SJoonsoo Kim }; 17011da177e4SLinus Torvalds 1702a98a2f0cSAlistair Popple if (flags & TTU_RMAP_LOCKED) 17032f031c6fSMatthew Wilcox (Oracle) rmap_walk_locked(folio, &rwc); 1704a98a2f0cSAlistair Popple else 17052f031c6fSMatthew Wilcox (Oracle) rmap_walk(folio, &rwc); 1706a98a2f0cSAlistair Popple } 1707a98a2f0cSAlistair Popple 1708a98a2f0cSAlistair Popple /* 1709a98a2f0cSAlistair Popple * @arg: enum ttu_flags will be passed to this argument. 1710a98a2f0cSAlistair Popple * 1711a98a2f0cSAlistair Popple * If TTU_SPLIT_HUGE_PMD is specified any PMD mappings will be split into PTEs 171264b586d1SHugh Dickins * containing migration entries. 1713a98a2f0cSAlistair Popple */ 17142f031c6fSMatthew Wilcox (Oracle) static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, 1715a98a2f0cSAlistair Popple unsigned long address, void *arg) 1716a98a2f0cSAlistair Popple { 1717a98a2f0cSAlistair Popple struct mm_struct *mm = vma->vm_mm; 17184b8554c5SMatthew Wilcox (Oracle) DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); 1719a98a2f0cSAlistair Popple pte_t pteval; 1720a98a2f0cSAlistair Popple struct page *subpage; 1721a98a2f0cSAlistair Popple bool ret = true; 1722a98a2f0cSAlistair Popple struct mmu_notifier_range range; 1723a98a2f0cSAlistair Popple enum ttu_flags flags = (enum ttu_flags)(long)arg; 1724a98a2f0cSAlistair Popple 1725a98a2f0cSAlistair Popple /* 1726a98a2f0cSAlistair Popple * When racing against e.g. zap_pte_range() on another cpu, 1727a98a2f0cSAlistair Popple * in between its ptep_get_and_clear_full() and page_remove_rmap(), 1728a98a2f0cSAlistair Popple * try_to_migrate() may return before page_mapped() has become false, 1729a98a2f0cSAlistair Popple * if page table locking is skipped: use TTU_SYNC to wait for that. 1730a98a2f0cSAlistair Popple */ 1731a98a2f0cSAlistair Popple if (flags & TTU_SYNC) 1732a98a2f0cSAlistair Popple pvmw.flags = PVMW_SYNC; 1733a98a2f0cSAlistair Popple 1734a98a2f0cSAlistair Popple /* 1735a98a2f0cSAlistair Popple * unmap_page() in mm/huge_memory.c is the only user of migration with 1736a98a2f0cSAlistair Popple * TTU_SPLIT_HUGE_PMD and it wants to freeze. 1737a98a2f0cSAlistair Popple */ 1738a98a2f0cSAlistair Popple if (flags & TTU_SPLIT_HUGE_PMD) 1739af28a988SMatthew Wilcox (Oracle) split_huge_pmd_address(vma, address, true, folio); 1740a98a2f0cSAlistair Popple 1741a98a2f0cSAlistair Popple /* 1742a98a2f0cSAlistair Popple * For THP, we have to assume the worse case ie pmd for invalidation. 1743a98a2f0cSAlistair Popple * For hugetlb, it could be much worse if we need to do pud 1744a98a2f0cSAlistair Popple * invalidation in the case of pmd sharing. 1745a98a2f0cSAlistair Popple * 1746a98a2f0cSAlistair Popple * Note that the page can not be free in this function as call of 1747a98a2f0cSAlistair Popple * try_to_unmap() must hold a reference on the page. 1748a98a2f0cSAlistair Popple */ 17492aff7a47SMatthew Wilcox (Oracle) range.end = vma_address_end(&pvmw); 1750a98a2f0cSAlistair Popple mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, 1751a98a2f0cSAlistair Popple address, range.end); 17524b8554c5SMatthew Wilcox (Oracle) if (folio_test_hugetlb(folio)) { 1753a98a2f0cSAlistair Popple /* 1754a98a2f0cSAlistair Popple * If sharing is possible, start and end will be adjusted 1755a98a2f0cSAlistair Popple * accordingly. 1756a98a2f0cSAlistair Popple */ 1757a98a2f0cSAlistair Popple adjust_range_if_pmd_sharing_possible(vma, &range.start, 1758a98a2f0cSAlistair Popple &range.end); 1759a98a2f0cSAlistair Popple } 1760a98a2f0cSAlistair Popple mmu_notifier_invalidate_range_start(&range); 1761a98a2f0cSAlistair Popple 1762a98a2f0cSAlistair Popple while (page_vma_mapped_walk(&pvmw)) { 1763a98a2f0cSAlistair Popple #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 1764a98a2f0cSAlistair Popple /* PMD-mapped THP migration entry */ 1765a98a2f0cSAlistair Popple if (!pvmw.pte) { 17664b8554c5SMatthew Wilcox (Oracle) subpage = folio_page(folio, 17674b8554c5SMatthew Wilcox (Oracle) pmd_pfn(*pvmw.pmd) - folio_pfn(folio)); 17684b8554c5SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) || 17694b8554c5SMatthew Wilcox (Oracle) !folio_test_pmd_mappable(folio), folio); 1770a98a2f0cSAlistair Popple 17714b8554c5SMatthew Wilcox (Oracle) set_pmd_migration_entry(&pvmw, subpage); 1772a98a2f0cSAlistair Popple continue; 1773a98a2f0cSAlistair Popple } 1774a98a2f0cSAlistair Popple #endif 1775a98a2f0cSAlistair Popple 1776a98a2f0cSAlistair Popple /* Unexpected PMD-mapped THP? */ 17774b8554c5SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(!pvmw.pte, folio); 1778a98a2f0cSAlistair Popple 17794b8554c5SMatthew Wilcox (Oracle) subpage = folio_page(folio, 17804b8554c5SMatthew Wilcox (Oracle) pte_pfn(*pvmw.pte) - folio_pfn(folio)); 1781a98a2f0cSAlistair Popple address = pvmw.address; 1782a98a2f0cSAlistair Popple 17834b8554c5SMatthew Wilcox (Oracle) if (folio_test_hugetlb(folio) && !folio_test_anon(folio)) { 1784a98a2f0cSAlistair Popple /* 1785a98a2f0cSAlistair Popple * To call huge_pmd_unshare, i_mmap_rwsem must be 1786a98a2f0cSAlistair Popple * held in write mode. Caller needs to explicitly 1787a98a2f0cSAlistair Popple * do this outside rmap routines. 1788a98a2f0cSAlistair Popple */ 1789a98a2f0cSAlistair Popple VM_BUG_ON(!(flags & TTU_RMAP_LOCKED)); 1790a98a2f0cSAlistair Popple if (huge_pmd_unshare(mm, vma, &address, pvmw.pte)) { 1791a98a2f0cSAlistair Popple /* 1792a98a2f0cSAlistair Popple * huge_pmd_unshare unmapped an entire PMD 1793a98a2f0cSAlistair Popple * page. There is no way of knowing exactly 1794a98a2f0cSAlistair Popple * which PMDs may be cached for this mm, so 1795a98a2f0cSAlistair Popple * we must flush them all. start/end were 1796a98a2f0cSAlistair Popple * already adjusted above to cover this range. 1797a98a2f0cSAlistair Popple */ 1798a98a2f0cSAlistair Popple flush_cache_range(vma, range.start, range.end); 1799a98a2f0cSAlistair Popple flush_tlb_range(vma, range.start, range.end); 1800a98a2f0cSAlistair Popple mmu_notifier_invalidate_range(mm, range.start, 1801a98a2f0cSAlistair Popple range.end); 1802a98a2f0cSAlistair Popple 1803a98a2f0cSAlistair Popple /* 1804a98a2f0cSAlistair Popple * The ref count of the PMD page was dropped 1805a98a2f0cSAlistair Popple * which is part of the way map counting 1806a98a2f0cSAlistair Popple * is done for shared PMDs. Return 'true' 1807a98a2f0cSAlistair Popple * here. When there is no other sharing, 1808a98a2f0cSAlistair Popple * huge_pmd_unshare returns false and we will 1809a98a2f0cSAlistair Popple * unmap the actual page and drop map count 1810a98a2f0cSAlistair Popple * to zero. 1811a98a2f0cSAlistair Popple */ 1812a98a2f0cSAlistair Popple page_vma_mapped_walk_done(&pvmw); 1813a98a2f0cSAlistair Popple break; 1814a98a2f0cSAlistair Popple } 1815a98a2f0cSAlistair Popple } 1816a98a2f0cSAlistair Popple 1817a98a2f0cSAlistair Popple /* Nuke the page table entry. */ 1818a98a2f0cSAlistair Popple flush_cache_page(vma, address, pte_pfn(*pvmw.pte)); 1819a98a2f0cSAlistair Popple pteval = ptep_clear_flush(vma, address, pvmw.pte); 1820a98a2f0cSAlistair Popple 18214b8554c5SMatthew Wilcox (Oracle) /* Set the dirty flag on the folio now the pte is gone. */ 1822a98a2f0cSAlistair Popple if (pte_dirty(pteval)) 18234b8554c5SMatthew Wilcox (Oracle) folio_mark_dirty(folio); 1824a98a2f0cSAlistair Popple 1825a98a2f0cSAlistair Popple /* Update high watermark before we lower rss */ 1826a98a2f0cSAlistair Popple update_hiwater_rss(mm); 1827a98a2f0cSAlistair Popple 18284b8554c5SMatthew Wilcox (Oracle) if (folio_is_zone_device(folio)) { 18294b8554c5SMatthew Wilcox (Oracle) unsigned long pfn = folio_pfn(folio); 1830a98a2f0cSAlistair Popple swp_entry_t entry; 1831a98a2f0cSAlistair Popple pte_t swp_pte; 1832a98a2f0cSAlistair Popple 1833a98a2f0cSAlistair Popple /* 1834a98a2f0cSAlistair Popple * Store the pfn of the page in a special migration 1835a98a2f0cSAlistair Popple * pte. do_swap_page() will wait until the migration 1836a98a2f0cSAlistair Popple * pte is removed and then restart fault handling. 1837a98a2f0cSAlistair Popple */ 18383d88705cSAlistair Popple entry = pte_to_swp_entry(pteval); 18393d88705cSAlistair Popple if (is_writable_device_private_entry(entry)) 18403d88705cSAlistair Popple entry = make_writable_migration_entry(pfn); 18413d88705cSAlistair Popple else 18423d88705cSAlistair Popple entry = make_readable_migration_entry(pfn); 1843a98a2f0cSAlistair Popple swp_pte = swp_entry_to_pte(entry); 1844a98a2f0cSAlistair Popple 1845a98a2f0cSAlistair Popple /* 1846a98a2f0cSAlistair Popple * pteval maps a zone device page and is therefore 1847a98a2f0cSAlistair Popple * a swap pte. 1848a98a2f0cSAlistair Popple */ 1849a98a2f0cSAlistair Popple if (pte_swp_soft_dirty(pteval)) 1850a98a2f0cSAlistair Popple swp_pte = pte_swp_mksoft_dirty(swp_pte); 1851a98a2f0cSAlistair Popple if (pte_swp_uffd_wp(pteval)) 1852a98a2f0cSAlistair Popple swp_pte = pte_swp_mkuffd_wp(swp_pte); 1853a98a2f0cSAlistair Popple set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte); 1854*4cc79b33SAnshuman Khandual trace_set_migration_pte(pvmw.address, pte_val(swp_pte), 1855*4cc79b33SAnshuman Khandual compound_order(&folio->page)); 1856a98a2f0cSAlistair Popple /* 1857a98a2f0cSAlistair Popple * No need to invalidate here it will synchronize on 1858a98a2f0cSAlistair Popple * against the special swap migration pte. 1859a98a2f0cSAlistair Popple * 1860a98a2f0cSAlistair Popple * The assignment to subpage above was computed from a 1861a98a2f0cSAlistair Popple * swap PTE which results in an invalid pointer. 1862a98a2f0cSAlistair Popple * Since only PAGE_SIZE pages can currently be 1863a98a2f0cSAlistair Popple * migrated, just set it to page. This will need to be 1864a98a2f0cSAlistair Popple * changed when hugepage migrations to device private 1865a98a2f0cSAlistair Popple * memory are supported. 1866a98a2f0cSAlistair Popple */ 18674b8554c5SMatthew Wilcox (Oracle) subpage = &folio->page; 1868da358d5cSMatthew Wilcox (Oracle) } else if (PageHWPoison(subpage)) { 1869a98a2f0cSAlistair Popple pteval = swp_entry_to_pte(make_hwpoison_entry(subpage)); 18704b8554c5SMatthew Wilcox (Oracle) if (folio_test_hugetlb(folio)) { 18714b8554c5SMatthew Wilcox (Oracle) hugetlb_count_sub(folio_nr_pages(folio), mm); 1872a98a2f0cSAlistair Popple set_huge_swap_pte_at(mm, address, 1873a98a2f0cSAlistair Popple pvmw.pte, pteval, 1874a98a2f0cSAlistair Popple vma_mmu_pagesize(vma)); 1875a98a2f0cSAlistair Popple } else { 18764b8554c5SMatthew Wilcox (Oracle) dec_mm_counter(mm, mm_counter(&folio->page)); 1877a98a2f0cSAlistair Popple set_pte_at(mm, address, pvmw.pte, pteval); 1878a98a2f0cSAlistair Popple } 1879a98a2f0cSAlistair Popple 1880a98a2f0cSAlistair Popple } else if (pte_unused(pteval) && !userfaultfd_armed(vma)) { 1881a98a2f0cSAlistair Popple /* 1882a98a2f0cSAlistair Popple * The guest indicated that the page content is of no 1883a98a2f0cSAlistair Popple * interest anymore. Simply discard the pte, vmscan 1884a98a2f0cSAlistair Popple * will take care of the rest. 1885a98a2f0cSAlistair Popple * A future reference will then fault in a new zero 1886a98a2f0cSAlistair Popple * page. When userfaultfd is active, we must not drop 1887a98a2f0cSAlistair Popple * this page though, as its main user (postcopy 1888a98a2f0cSAlistair Popple * migration) will not expect userfaults on already 1889a98a2f0cSAlistair Popple * copied pages. 1890a98a2f0cSAlistair Popple */ 18914b8554c5SMatthew Wilcox (Oracle) dec_mm_counter(mm, mm_counter(&folio->page)); 1892a98a2f0cSAlistair Popple /* We have to invalidate as we cleared the pte */ 1893a98a2f0cSAlistair Popple mmu_notifier_invalidate_range(mm, address, 1894a98a2f0cSAlistair Popple address + PAGE_SIZE); 1895a98a2f0cSAlistair Popple } else { 1896a98a2f0cSAlistair Popple swp_entry_t entry; 1897a98a2f0cSAlistair Popple pte_t swp_pte; 1898a98a2f0cSAlistair Popple 1899a98a2f0cSAlistair Popple if (arch_unmap_one(mm, vma, address, pteval) < 0) { 1900a98a2f0cSAlistair Popple set_pte_at(mm, address, pvmw.pte, pteval); 1901a98a2f0cSAlistair Popple ret = false; 1902a98a2f0cSAlistair Popple page_vma_mapped_walk_done(&pvmw); 1903a98a2f0cSAlistair Popple break; 1904a98a2f0cSAlistair Popple } 1905a98a2f0cSAlistair Popple 1906a98a2f0cSAlistair Popple /* 1907a98a2f0cSAlistair Popple * Store the pfn of the page in a special migration 1908a98a2f0cSAlistair Popple * pte. do_swap_page() will wait until the migration 1909a98a2f0cSAlistair Popple * pte is removed and then restart fault handling. 1910a98a2f0cSAlistair Popple */ 1911a98a2f0cSAlistair Popple if (pte_write(pteval)) 1912a98a2f0cSAlistair Popple entry = make_writable_migration_entry( 1913a98a2f0cSAlistair Popple page_to_pfn(subpage)); 1914a98a2f0cSAlistair Popple else 1915a98a2f0cSAlistair Popple entry = make_readable_migration_entry( 1916a98a2f0cSAlistair Popple page_to_pfn(subpage)); 1917a98a2f0cSAlistair Popple 1918a98a2f0cSAlistair Popple swp_pte = swp_entry_to_pte(entry); 1919a98a2f0cSAlistair Popple if (pte_soft_dirty(pteval)) 1920a98a2f0cSAlistair Popple swp_pte = pte_swp_mksoft_dirty(swp_pte); 1921a98a2f0cSAlistair Popple if (pte_uffd_wp(pteval)) 1922a98a2f0cSAlistair Popple swp_pte = pte_swp_mkuffd_wp(swp_pte); 1923a98a2f0cSAlistair Popple set_pte_at(mm, address, pvmw.pte, swp_pte); 1924*4cc79b33SAnshuman Khandual trace_set_migration_pte(address, pte_val(swp_pte), 1925*4cc79b33SAnshuman Khandual compound_order(&folio->page)); 1926a98a2f0cSAlistair Popple /* 1927a98a2f0cSAlistair Popple * No need to invalidate here it will synchronize on 1928a98a2f0cSAlistair Popple * against the special swap migration pte. 1929a98a2f0cSAlistair Popple */ 1930a98a2f0cSAlistair Popple } 1931a98a2f0cSAlistair Popple 1932a98a2f0cSAlistair Popple /* 1933a98a2f0cSAlistair Popple * No need to call mmu_notifier_invalidate_range() it has be 1934a98a2f0cSAlistair Popple * done above for all cases requiring it to happen under page 1935a98a2f0cSAlistair Popple * table lock before mmu_notifier_invalidate_range_end() 1936a98a2f0cSAlistair Popple * 1937a98a2f0cSAlistair Popple * See Documentation/vm/mmu_notifier.rst 1938a98a2f0cSAlistair Popple */ 19394b8554c5SMatthew Wilcox (Oracle) page_remove_rmap(subpage, vma, folio_test_hugetlb(folio)); 1940b7435507SHugh Dickins if (vma->vm_flags & VM_LOCKED) 1941b7435507SHugh Dickins mlock_page_drain(smp_processor_id()); 19424b8554c5SMatthew Wilcox (Oracle) folio_put(folio); 1943a98a2f0cSAlistair Popple } 1944a98a2f0cSAlistair Popple 1945a98a2f0cSAlistair Popple mmu_notifier_invalidate_range_end(&range); 1946a98a2f0cSAlistair Popple 1947a98a2f0cSAlistair Popple return ret; 1948a98a2f0cSAlistair Popple } 1949a98a2f0cSAlistair Popple 1950a98a2f0cSAlistair Popple /** 1951a98a2f0cSAlistair Popple * try_to_migrate - try to replace all page table mappings with swap entries 19524b8554c5SMatthew Wilcox (Oracle) * @folio: the folio to replace page table entries for 1953a98a2f0cSAlistair Popple * @flags: action and flags 1954a98a2f0cSAlistair Popple * 19554b8554c5SMatthew Wilcox (Oracle) * Tries to remove all the page table entries which are mapping this folio and 19564b8554c5SMatthew Wilcox (Oracle) * replace them with special swap entries. Caller must hold the folio lock. 1957a98a2f0cSAlistair Popple */ 19584b8554c5SMatthew Wilcox (Oracle) void try_to_migrate(struct folio *folio, enum ttu_flags flags) 1959a98a2f0cSAlistair Popple { 1960a98a2f0cSAlistair Popple struct rmap_walk_control rwc = { 1961a98a2f0cSAlistair Popple .rmap_one = try_to_migrate_one, 1962a98a2f0cSAlistair Popple .arg = (void *)flags, 1963a98a2f0cSAlistair Popple .done = page_not_mapped, 19642f031c6fSMatthew Wilcox (Oracle) .anon_lock = folio_lock_anon_vma_read, 1965a98a2f0cSAlistair Popple }; 1966a98a2f0cSAlistair Popple 1967a98a2f0cSAlistair Popple /* 1968a98a2f0cSAlistair Popple * Migration always ignores mlock and only supports TTU_RMAP_LOCKED and 1969a98a2f0cSAlistair Popple * TTU_SPLIT_HUGE_PMD and TTU_SYNC flags. 1970a98a2f0cSAlistair Popple */ 1971a98a2f0cSAlistair Popple if (WARN_ON_ONCE(flags & ~(TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD | 1972a98a2f0cSAlistair Popple TTU_SYNC))) 1973a98a2f0cSAlistair Popple return; 1974a98a2f0cSAlistair Popple 19754b8554c5SMatthew Wilcox (Oracle) if (folio_is_zone_device(folio) && !folio_is_device_private(folio)) 19766c855fceSHugh Dickins return; 19776c855fceSHugh Dickins 197852629506SJoonsoo Kim /* 197952629506SJoonsoo Kim * During exec, a temporary VMA is setup and later moved. 198052629506SJoonsoo Kim * The VMA is moved under the anon_vma lock but not the 198152629506SJoonsoo Kim * page tables leading to a race where migration cannot 198252629506SJoonsoo Kim * find the migration ptes. Rather than increasing the 198352629506SJoonsoo Kim * locking requirements of exec(), migration skips 198452629506SJoonsoo Kim * temporary VMAs until after exec() completes. 198552629506SJoonsoo Kim */ 19864b8554c5SMatthew Wilcox (Oracle) if (!folio_test_ksm(folio) && folio_test_anon(folio)) 198752629506SJoonsoo Kim rwc.invalid_vma = invalid_migration_vma; 198852629506SJoonsoo Kim 19892a52bcbcSKirill A. Shutemov if (flags & TTU_RMAP_LOCKED) 19902f031c6fSMatthew Wilcox (Oracle) rmap_walk_locked(folio, &rwc); 19912a52bcbcSKirill A. Shutemov else 19922f031c6fSMatthew Wilcox (Oracle) rmap_walk(folio, &rwc); 1993b291f000SNick Piggin } 1994e9995ef9SHugh Dickins 1995b756a3b5SAlistair Popple #ifdef CONFIG_DEVICE_PRIVATE 1996b756a3b5SAlistair Popple struct make_exclusive_args { 1997b756a3b5SAlistair Popple struct mm_struct *mm; 1998b756a3b5SAlistair Popple unsigned long address; 1999b756a3b5SAlistair Popple void *owner; 2000b756a3b5SAlistair Popple bool valid; 2001b756a3b5SAlistair Popple }; 2002b756a3b5SAlistair Popple 20032f031c6fSMatthew Wilcox (Oracle) static bool page_make_device_exclusive_one(struct folio *folio, 2004b756a3b5SAlistair Popple struct vm_area_struct *vma, unsigned long address, void *priv) 2005b756a3b5SAlistair Popple { 2006b756a3b5SAlistair Popple struct mm_struct *mm = vma->vm_mm; 20070d251485SMatthew Wilcox (Oracle) DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); 2008b756a3b5SAlistair Popple struct make_exclusive_args *args = priv; 2009b756a3b5SAlistair Popple pte_t pteval; 2010b756a3b5SAlistair Popple struct page *subpage; 2011b756a3b5SAlistair Popple bool ret = true; 2012b756a3b5SAlistair Popple struct mmu_notifier_range range; 2013b756a3b5SAlistair Popple swp_entry_t entry; 2014b756a3b5SAlistair Popple pte_t swp_pte; 2015b756a3b5SAlistair Popple 2016b756a3b5SAlistair Popple mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0, vma, 2017b756a3b5SAlistair Popple vma->vm_mm, address, min(vma->vm_end, 20180d251485SMatthew Wilcox (Oracle) address + folio_size(folio)), 20190d251485SMatthew Wilcox (Oracle) args->owner); 2020b756a3b5SAlistair Popple mmu_notifier_invalidate_range_start(&range); 2021b756a3b5SAlistair Popple 2022b756a3b5SAlistair Popple while (page_vma_mapped_walk(&pvmw)) { 2023b756a3b5SAlistair Popple /* Unexpected PMD-mapped THP? */ 20240d251485SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(!pvmw.pte, folio); 2025b756a3b5SAlistair Popple 2026b756a3b5SAlistair Popple if (!pte_present(*pvmw.pte)) { 2027b756a3b5SAlistair Popple ret = false; 2028b756a3b5SAlistair Popple page_vma_mapped_walk_done(&pvmw); 2029b756a3b5SAlistair Popple break; 2030b756a3b5SAlistair Popple } 2031b756a3b5SAlistair Popple 20320d251485SMatthew Wilcox (Oracle) subpage = folio_page(folio, 20330d251485SMatthew Wilcox (Oracle) pte_pfn(*pvmw.pte) - folio_pfn(folio)); 2034b756a3b5SAlistair Popple address = pvmw.address; 2035b756a3b5SAlistair Popple 2036b756a3b5SAlistair Popple /* Nuke the page table entry. */ 2037b756a3b5SAlistair Popple flush_cache_page(vma, address, pte_pfn(*pvmw.pte)); 2038b756a3b5SAlistair Popple pteval = ptep_clear_flush(vma, address, pvmw.pte); 2039b756a3b5SAlistair Popple 20400d251485SMatthew Wilcox (Oracle) /* Set the dirty flag on the folio now the pte is gone. */ 2041b756a3b5SAlistair Popple if (pte_dirty(pteval)) 20420d251485SMatthew Wilcox (Oracle) folio_mark_dirty(folio); 2043b756a3b5SAlistair Popple 2044b756a3b5SAlistair Popple /* 2045b756a3b5SAlistair Popple * Check that our target page is still mapped at the expected 2046b756a3b5SAlistair Popple * address. 2047b756a3b5SAlistair Popple */ 2048b756a3b5SAlistair Popple if (args->mm == mm && args->address == address && 2049b756a3b5SAlistair Popple pte_write(pteval)) 2050b756a3b5SAlistair Popple args->valid = true; 2051b756a3b5SAlistair Popple 2052b756a3b5SAlistair Popple /* 2053b756a3b5SAlistair Popple * Store the pfn of the page in a special migration 2054b756a3b5SAlistair Popple * pte. do_swap_page() will wait until the migration 2055b756a3b5SAlistair Popple * pte is removed and then restart fault handling. 2056b756a3b5SAlistair Popple */ 2057b756a3b5SAlistair Popple if (pte_write(pteval)) 2058b756a3b5SAlistair Popple entry = make_writable_device_exclusive_entry( 2059b756a3b5SAlistair Popple page_to_pfn(subpage)); 2060b756a3b5SAlistair Popple else 2061b756a3b5SAlistair Popple entry = make_readable_device_exclusive_entry( 2062b756a3b5SAlistair Popple page_to_pfn(subpage)); 2063b756a3b5SAlistair Popple swp_pte = swp_entry_to_pte(entry); 2064b756a3b5SAlistair Popple if (pte_soft_dirty(pteval)) 2065b756a3b5SAlistair Popple swp_pte = pte_swp_mksoft_dirty(swp_pte); 2066b756a3b5SAlistair Popple if (pte_uffd_wp(pteval)) 2067b756a3b5SAlistair Popple swp_pte = pte_swp_mkuffd_wp(swp_pte); 2068b756a3b5SAlistair Popple 2069b756a3b5SAlistair Popple set_pte_at(mm, address, pvmw.pte, swp_pte); 2070b756a3b5SAlistair Popple 2071b756a3b5SAlistair Popple /* 2072b756a3b5SAlistair Popple * There is a reference on the page for the swap entry which has 2073b756a3b5SAlistair Popple * been removed, so shouldn't take another. 2074b756a3b5SAlistair Popple */ 2075cea86fe2SHugh Dickins page_remove_rmap(subpage, vma, false); 2076b756a3b5SAlistair Popple } 2077b756a3b5SAlistair Popple 2078b756a3b5SAlistair Popple mmu_notifier_invalidate_range_end(&range); 2079b756a3b5SAlistair Popple 2080b756a3b5SAlistair Popple return ret; 2081b756a3b5SAlistair Popple } 2082b756a3b5SAlistair Popple 2083b756a3b5SAlistair Popple /** 20840d251485SMatthew Wilcox (Oracle) * folio_make_device_exclusive - Mark the folio exclusively owned by a device. 20850d251485SMatthew Wilcox (Oracle) * @folio: The folio to replace page table entries for. 20860d251485SMatthew Wilcox (Oracle) * @mm: The mm_struct where the folio is expected to be mapped. 20870d251485SMatthew Wilcox (Oracle) * @address: Address where the folio is expected to be mapped. 2088b756a3b5SAlistair Popple * @owner: passed to MMU_NOTIFY_EXCLUSIVE range notifier callbacks 2089b756a3b5SAlistair Popple * 20900d251485SMatthew Wilcox (Oracle) * Tries to remove all the page table entries which are mapping this 20910d251485SMatthew Wilcox (Oracle) * folio and replace them with special device exclusive swap entries to 20920d251485SMatthew Wilcox (Oracle) * grant a device exclusive access to the folio. 2093b756a3b5SAlistair Popple * 20940d251485SMatthew Wilcox (Oracle) * Context: Caller must hold the folio lock. 20950d251485SMatthew Wilcox (Oracle) * Return: false if the page is still mapped, or if it could not be unmapped 2096b756a3b5SAlistair Popple * from the expected address. Otherwise returns true (success). 2097b756a3b5SAlistair Popple */ 20980d251485SMatthew Wilcox (Oracle) static bool folio_make_device_exclusive(struct folio *folio, 20990d251485SMatthew Wilcox (Oracle) struct mm_struct *mm, unsigned long address, void *owner) 2100b756a3b5SAlistair Popple { 2101b756a3b5SAlistair Popple struct make_exclusive_args args = { 2102b756a3b5SAlistair Popple .mm = mm, 2103b756a3b5SAlistair Popple .address = address, 2104b756a3b5SAlistair Popple .owner = owner, 2105b756a3b5SAlistair Popple .valid = false, 2106b756a3b5SAlistair Popple }; 2107b756a3b5SAlistair Popple struct rmap_walk_control rwc = { 2108b756a3b5SAlistair Popple .rmap_one = page_make_device_exclusive_one, 2109b756a3b5SAlistair Popple .done = page_not_mapped, 21102f031c6fSMatthew Wilcox (Oracle) .anon_lock = folio_lock_anon_vma_read, 2111b756a3b5SAlistair Popple .arg = &args, 2112b756a3b5SAlistair Popple }; 2113b756a3b5SAlistair Popple 2114b756a3b5SAlistair Popple /* 21150d251485SMatthew Wilcox (Oracle) * Restrict to anonymous folios for now to avoid potential writeback 21160d251485SMatthew Wilcox (Oracle) * issues. 2117b756a3b5SAlistair Popple */ 21180d251485SMatthew Wilcox (Oracle) if (!folio_test_anon(folio)) 2119b756a3b5SAlistair Popple return false; 2120b756a3b5SAlistair Popple 21212f031c6fSMatthew Wilcox (Oracle) rmap_walk(folio, &rwc); 2122b756a3b5SAlistair Popple 21230d251485SMatthew Wilcox (Oracle) return args.valid && !folio_mapcount(folio); 2124b756a3b5SAlistair Popple } 2125b756a3b5SAlistair Popple 2126b756a3b5SAlistair Popple /** 2127b756a3b5SAlistair Popple * make_device_exclusive_range() - Mark a range for exclusive use by a device 2128b756a3b5SAlistair Popple * @mm: mm_struct of assoicated target process 2129b756a3b5SAlistair Popple * @start: start of the region to mark for exclusive device access 2130b756a3b5SAlistair Popple * @end: end address of region 2131b756a3b5SAlistair Popple * @pages: returns the pages which were successfully marked for exclusive access 2132b756a3b5SAlistair Popple * @owner: passed to MMU_NOTIFY_EXCLUSIVE range notifier to allow filtering 2133b756a3b5SAlistair Popple * 2134b756a3b5SAlistair Popple * Returns: number of pages found in the range by GUP. A page is marked for 2135b756a3b5SAlistair Popple * exclusive access only if the page pointer is non-NULL. 2136b756a3b5SAlistair Popple * 2137b756a3b5SAlistair Popple * This function finds ptes mapping page(s) to the given address range, locks 2138b756a3b5SAlistair Popple * them and replaces mappings with special swap entries preventing userspace CPU 2139b756a3b5SAlistair Popple * access. On fault these entries are replaced with the original mapping after 2140b756a3b5SAlistair Popple * calling MMU notifiers. 2141b756a3b5SAlistair Popple * 2142b756a3b5SAlistair Popple * A driver using this to program access from a device must use a mmu notifier 2143b756a3b5SAlistair Popple * critical section to hold a device specific lock during programming. Once 2144b756a3b5SAlistair Popple * programming is complete it should drop the page lock and reference after 2145b756a3b5SAlistair Popple * which point CPU access to the page will revoke the exclusive access. 2146b756a3b5SAlistair Popple */ 2147b756a3b5SAlistair Popple int make_device_exclusive_range(struct mm_struct *mm, unsigned long start, 2148b756a3b5SAlistair Popple unsigned long end, struct page **pages, 2149b756a3b5SAlistair Popple void *owner) 2150b756a3b5SAlistair Popple { 2151b756a3b5SAlistair Popple long npages = (end - start) >> PAGE_SHIFT; 2152b756a3b5SAlistair Popple long i; 2153b756a3b5SAlistair Popple 2154b756a3b5SAlistair Popple npages = get_user_pages_remote(mm, start, npages, 2155b756a3b5SAlistair Popple FOLL_GET | FOLL_WRITE | FOLL_SPLIT_PMD, 2156b756a3b5SAlistair Popple pages, NULL, NULL); 2157b756a3b5SAlistair Popple if (npages < 0) 2158b756a3b5SAlistair Popple return npages; 2159b756a3b5SAlistair Popple 2160b756a3b5SAlistair Popple for (i = 0; i < npages; i++, start += PAGE_SIZE) { 21610d251485SMatthew Wilcox (Oracle) struct folio *folio = page_folio(pages[i]); 21620d251485SMatthew Wilcox (Oracle) if (PageTail(pages[i]) || !folio_trylock(folio)) { 21630d251485SMatthew Wilcox (Oracle) folio_put(folio); 2164b756a3b5SAlistair Popple pages[i] = NULL; 2165b756a3b5SAlistair Popple continue; 2166b756a3b5SAlistair Popple } 2167b756a3b5SAlistair Popple 21680d251485SMatthew Wilcox (Oracle) if (!folio_make_device_exclusive(folio, mm, start, owner)) { 21690d251485SMatthew Wilcox (Oracle) folio_unlock(folio); 21700d251485SMatthew Wilcox (Oracle) folio_put(folio); 2171b756a3b5SAlistair Popple pages[i] = NULL; 2172b756a3b5SAlistair Popple } 2173b756a3b5SAlistair Popple } 2174b756a3b5SAlistair Popple 2175b756a3b5SAlistair Popple return npages; 2176b756a3b5SAlistair Popple } 2177b756a3b5SAlistair Popple EXPORT_SYMBOL_GPL(make_device_exclusive_range); 2178b756a3b5SAlistair Popple #endif 2179b756a3b5SAlistair Popple 218001d8b20dSPeter Zijlstra void __put_anon_vma(struct anon_vma *anon_vma) 218176545066SRik van Riel { 218276545066SRik van Riel struct anon_vma *root = anon_vma->root; 218376545066SRik van Riel 2184624483f3SAndrey Ryabinin anon_vma_free(anon_vma); 218501d8b20dSPeter Zijlstra if (root != anon_vma && atomic_dec_and_test(&root->refcount)) 218676545066SRik van Riel anon_vma_free(root); 218776545066SRik van Riel } 218876545066SRik van Riel 21892f031c6fSMatthew Wilcox (Oracle) static struct anon_vma *rmap_walk_anon_lock(struct folio *folio, 219084fbbe21SMatthew Wilcox (Oracle) const struct rmap_walk_control *rwc) 2191faecd8ddSJoonsoo Kim { 2192faecd8ddSJoonsoo Kim struct anon_vma *anon_vma; 2193faecd8ddSJoonsoo Kim 21940dd1c7bbSJoonsoo Kim if (rwc->anon_lock) 21952f031c6fSMatthew Wilcox (Oracle) return rwc->anon_lock(folio); 21960dd1c7bbSJoonsoo Kim 2197faecd8ddSJoonsoo Kim /* 21982f031c6fSMatthew Wilcox (Oracle) * Note: remove_migration_ptes() cannot use folio_lock_anon_vma_read() 2199faecd8ddSJoonsoo Kim * because that depends on page_mapped(); but not all its usages 2200c1e8d7c6SMichel Lespinasse * are holding mmap_lock. Users without mmap_lock are required to 2201faecd8ddSJoonsoo Kim * take a reference count to prevent the anon_vma disappearing 2202faecd8ddSJoonsoo Kim */ 2203e05b3453SMatthew Wilcox (Oracle) anon_vma = folio_anon_vma(folio); 2204faecd8ddSJoonsoo Kim if (!anon_vma) 2205faecd8ddSJoonsoo Kim return NULL; 2206faecd8ddSJoonsoo Kim 2207faecd8ddSJoonsoo Kim anon_vma_lock_read(anon_vma); 2208faecd8ddSJoonsoo Kim return anon_vma; 2209faecd8ddSJoonsoo Kim } 2210faecd8ddSJoonsoo Kim 2211e9995ef9SHugh Dickins /* 2212e8351ac9SJoonsoo Kim * rmap_walk_anon - do something to anonymous page using the object-based 2213e8351ac9SJoonsoo Kim * rmap method 2214e8351ac9SJoonsoo Kim * @page: the page to be handled 2215e8351ac9SJoonsoo Kim * @rwc: control variable according to each walk type 2216e8351ac9SJoonsoo Kim * 2217e8351ac9SJoonsoo Kim * Find all the mappings of a page using the mapping pointer and the vma chains 2218e8351ac9SJoonsoo Kim * contained in the anon_vma struct it points to. 2219e9995ef9SHugh Dickins */ 222084fbbe21SMatthew Wilcox (Oracle) static void rmap_walk_anon(struct folio *folio, 222184fbbe21SMatthew Wilcox (Oracle) const struct rmap_walk_control *rwc, bool locked) 2222e9995ef9SHugh Dickins { 2223e9995ef9SHugh Dickins struct anon_vma *anon_vma; 2224a8fa41adSKirill A. Shutemov pgoff_t pgoff_start, pgoff_end; 22255beb4930SRik van Riel struct anon_vma_chain *avc; 2226e9995ef9SHugh Dickins 2227b9773199SKirill A. Shutemov if (locked) { 2228e05b3453SMatthew Wilcox (Oracle) anon_vma = folio_anon_vma(folio); 2229b9773199SKirill A. Shutemov /* anon_vma disappear under us? */ 2230e05b3453SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(!anon_vma, folio); 2231b9773199SKirill A. Shutemov } else { 22322f031c6fSMatthew Wilcox (Oracle) anon_vma = rmap_walk_anon_lock(folio, rwc); 2233b9773199SKirill A. Shutemov } 2234e9995ef9SHugh Dickins if (!anon_vma) 22351df631aeSMinchan Kim return; 2236faecd8ddSJoonsoo Kim 22372f031c6fSMatthew Wilcox (Oracle) pgoff_start = folio_pgoff(folio); 22382f031c6fSMatthew Wilcox (Oracle) pgoff_end = pgoff_start + folio_nr_pages(folio) - 1; 2239a8fa41adSKirill A. Shutemov anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, 2240a8fa41adSKirill A. Shutemov pgoff_start, pgoff_end) { 22415beb4930SRik van Riel struct vm_area_struct *vma = avc->vma; 22422f031c6fSMatthew Wilcox (Oracle) unsigned long address = vma_address(&folio->page, vma); 22430dd1c7bbSJoonsoo Kim 2244494334e4SHugh Dickins VM_BUG_ON_VMA(address == -EFAULT, vma); 2245ad12695fSAndrea Arcangeli cond_resched(); 2246ad12695fSAndrea Arcangeli 22470dd1c7bbSJoonsoo Kim if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) 22480dd1c7bbSJoonsoo Kim continue; 22490dd1c7bbSJoonsoo Kim 22502f031c6fSMatthew Wilcox (Oracle) if (!rwc->rmap_one(folio, vma, address, rwc->arg)) 2251e9995ef9SHugh Dickins break; 22522f031c6fSMatthew Wilcox (Oracle) if (rwc->done && rwc->done(folio)) 22530dd1c7bbSJoonsoo Kim break; 2254e9995ef9SHugh Dickins } 2255b9773199SKirill A. Shutemov 2256b9773199SKirill A. Shutemov if (!locked) 22574fc3f1d6SIngo Molnar anon_vma_unlock_read(anon_vma); 2258e9995ef9SHugh Dickins } 2259e9995ef9SHugh Dickins 2260e8351ac9SJoonsoo Kim /* 2261e8351ac9SJoonsoo Kim * rmap_walk_file - do something to file page using the object-based rmap method 2262e8351ac9SJoonsoo Kim * @page: the page to be handled 2263e8351ac9SJoonsoo Kim * @rwc: control variable according to each walk type 2264e8351ac9SJoonsoo Kim * 2265e8351ac9SJoonsoo Kim * Find all the mappings of a page using the mapping pointer and the vma chains 2266e8351ac9SJoonsoo Kim * contained in the address_space struct it points to. 2267e8351ac9SJoonsoo Kim */ 226884fbbe21SMatthew Wilcox (Oracle) static void rmap_walk_file(struct folio *folio, 226984fbbe21SMatthew Wilcox (Oracle) const struct rmap_walk_control *rwc, bool locked) 2270e9995ef9SHugh Dickins { 22712f031c6fSMatthew Wilcox (Oracle) struct address_space *mapping = folio_mapping(folio); 2272a8fa41adSKirill A. Shutemov pgoff_t pgoff_start, pgoff_end; 2273e9995ef9SHugh Dickins struct vm_area_struct *vma; 2274e9995ef9SHugh Dickins 22759f32624bSJoonsoo Kim /* 22769f32624bSJoonsoo Kim * The page lock not only makes sure that page->mapping cannot 22779f32624bSJoonsoo Kim * suddenly be NULLified by truncation, it makes sure that the 22789f32624bSJoonsoo Kim * structure at mapping cannot be freed and reused yet, 2279c8c06efaSDavidlohr Bueso * so we can safely take mapping->i_mmap_rwsem. 22809f32624bSJoonsoo Kim */ 22812f031c6fSMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 22829f32624bSJoonsoo Kim 2283e9995ef9SHugh Dickins if (!mapping) 22841df631aeSMinchan Kim return; 22853dec0ba0SDavidlohr Bueso 22862f031c6fSMatthew Wilcox (Oracle) pgoff_start = folio_pgoff(folio); 22872f031c6fSMatthew Wilcox (Oracle) pgoff_end = pgoff_start + folio_nr_pages(folio) - 1; 2288b9773199SKirill A. Shutemov if (!locked) 22893dec0ba0SDavidlohr Bueso i_mmap_lock_read(mapping); 2290a8fa41adSKirill A. Shutemov vma_interval_tree_foreach(vma, &mapping->i_mmap, 2291a8fa41adSKirill A. Shutemov pgoff_start, pgoff_end) { 22922f031c6fSMatthew Wilcox (Oracle) unsigned long address = vma_address(&folio->page, vma); 22930dd1c7bbSJoonsoo Kim 2294494334e4SHugh Dickins VM_BUG_ON_VMA(address == -EFAULT, vma); 2295ad12695fSAndrea Arcangeli cond_resched(); 2296ad12695fSAndrea Arcangeli 22970dd1c7bbSJoonsoo Kim if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) 22980dd1c7bbSJoonsoo Kim continue; 22990dd1c7bbSJoonsoo Kim 23002f031c6fSMatthew Wilcox (Oracle) if (!rwc->rmap_one(folio, vma, address, rwc->arg)) 23010dd1c7bbSJoonsoo Kim goto done; 23022f031c6fSMatthew Wilcox (Oracle) if (rwc->done && rwc->done(folio)) 23030dd1c7bbSJoonsoo Kim goto done; 2304e9995ef9SHugh Dickins } 23050dd1c7bbSJoonsoo Kim 23060dd1c7bbSJoonsoo Kim done: 2307b9773199SKirill A. Shutemov if (!locked) 23083dec0ba0SDavidlohr Bueso i_mmap_unlock_read(mapping); 2309e9995ef9SHugh Dickins } 2310e9995ef9SHugh Dickins 231184fbbe21SMatthew Wilcox (Oracle) void rmap_walk(struct folio *folio, const struct rmap_walk_control *rwc) 2312e9995ef9SHugh Dickins { 23132f031c6fSMatthew Wilcox (Oracle) if (unlikely(folio_test_ksm(folio))) 23142f031c6fSMatthew Wilcox (Oracle) rmap_walk_ksm(folio, rwc); 23152f031c6fSMatthew Wilcox (Oracle) else if (folio_test_anon(folio)) 23162f031c6fSMatthew Wilcox (Oracle) rmap_walk_anon(folio, rwc, false); 2317e9995ef9SHugh Dickins else 23182f031c6fSMatthew Wilcox (Oracle) rmap_walk_file(folio, rwc, false); 2319b9773199SKirill A. Shutemov } 2320b9773199SKirill A. Shutemov 2321b9773199SKirill A. Shutemov /* Like rmap_walk, but caller holds relevant rmap lock */ 232284fbbe21SMatthew Wilcox (Oracle) void rmap_walk_locked(struct folio *folio, const struct rmap_walk_control *rwc) 2323b9773199SKirill A. Shutemov { 2324b9773199SKirill A. Shutemov /* no ksm support for now */ 23252f031c6fSMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(folio_test_ksm(folio), folio); 23262f031c6fSMatthew Wilcox (Oracle) if (folio_test_anon(folio)) 23272f031c6fSMatthew Wilcox (Oracle) rmap_walk_anon(folio, rwc, true); 2328b9773199SKirill A. Shutemov else 23292f031c6fSMatthew Wilcox (Oracle) rmap_walk_file(folio, rwc, true); 2330e9995ef9SHugh Dickins } 23310fe6e20bSNaoya Horiguchi 2332e3390f67SNaoya Horiguchi #ifdef CONFIG_HUGETLB_PAGE 23330fe6e20bSNaoya Horiguchi /* 2334451b9514SKirill Tkhai * The following two functions are for anonymous (private mapped) hugepages. 23350fe6e20bSNaoya Horiguchi * Unlike common anonymous pages, anonymous hugepages have no accounting code 23360fe6e20bSNaoya Horiguchi * and no lru code, because we handle hugepages differently from common pages. 23370fe6e20bSNaoya Horiguchi */ 23380fe6e20bSNaoya Horiguchi void hugepage_add_anon_rmap(struct page *page, 23390fe6e20bSNaoya Horiguchi struct vm_area_struct *vma, unsigned long address) 23400fe6e20bSNaoya Horiguchi { 23410fe6e20bSNaoya Horiguchi struct anon_vma *anon_vma = vma->anon_vma; 23420fe6e20bSNaoya Horiguchi int first; 2343a850ea30SNaoya Horiguchi 2344a850ea30SNaoya Horiguchi BUG_ON(!PageLocked(page)); 23450fe6e20bSNaoya Horiguchi BUG_ON(!anon_vma); 23465dbe0af4SHugh Dickins /* address might be in next vma when migration races vma_adjust */ 234753f9263bSKirill A. Shutemov first = atomic_inc_and_test(compound_mapcount_ptr(page)); 23480fe6e20bSNaoya Horiguchi if (first) 2349451b9514SKirill Tkhai __page_set_anon_rmap(page, vma, address, 0); 23500fe6e20bSNaoya Horiguchi } 23510fe6e20bSNaoya Horiguchi 23520fe6e20bSNaoya Horiguchi void hugepage_add_new_anon_rmap(struct page *page, 23530fe6e20bSNaoya Horiguchi struct vm_area_struct *vma, unsigned long address) 23540fe6e20bSNaoya Horiguchi { 23550fe6e20bSNaoya Horiguchi BUG_ON(address < vma->vm_start || address >= vma->vm_end); 235653f9263bSKirill A. Shutemov atomic_set(compound_mapcount_ptr(page), 0); 235747e29d32SJohn Hubbard atomic_set(compound_pincount_ptr(page), 0); 235847e29d32SJohn Hubbard 2359451b9514SKirill Tkhai __page_set_anon_rmap(page, vma, address, 1); 23600fe6e20bSNaoya Horiguchi } 2361e3390f67SNaoya Horiguchi #endif /* CONFIG_HUGETLB_PAGE */ 2362