11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * mm/rmap.c - physical to virtual reverse mappings 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright 2001, Rik van Riel <riel@conectiva.com.br> 51da177e4SLinus Torvalds * Released under the General Public License (GPL). 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * Simple, low overhead reverse mapping scheme. 81da177e4SLinus Torvalds * Please try to keep this thing as modular as possible. 91da177e4SLinus Torvalds * 101da177e4SLinus Torvalds * Provides methods for unmapping each kind of mapped page: 111da177e4SLinus Torvalds * the anon methods track anonymous pages, and 121da177e4SLinus Torvalds * the file methods track pages belonging to an inode. 131da177e4SLinus Torvalds * 141da177e4SLinus Torvalds * Original design by Rik van Riel <riel@conectiva.com.br> 2001 151da177e4SLinus Torvalds * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004 161da177e4SLinus Torvalds * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004 1798f32602SHugh Dickins * Contributions by Hugh Dickins 2003, 2004 181da177e4SLinus Torvalds */ 191da177e4SLinus Torvalds 201da177e4SLinus Torvalds /* 211da177e4SLinus Torvalds * Lock ordering in mm: 221da177e4SLinus Torvalds * 231b1dcc1bSJes Sorensen * inode->i_mutex (while writing or truncating, not reading or faulting) 241da177e4SLinus Torvalds * mm->mmap_sem 251da177e4SLinus Torvalds * page->flags PG_locked (lock_page) 263d48ae45SPeter Zijlstra * mapping->i_mmap_mutex 275a505085SIngo Molnar * anon_vma->rwsem 28b8072f09SHugh Dickins * mm->page_table_lock or pte_lock 29053837fcSNick Piggin * zone->lru_lock (in mark_page_accessed, isolate_lru_page) 305d337b91SHugh Dickins * swap_lock (in swap_duplicate, swap_info_get) 311da177e4SLinus Torvalds * mmlist_lock (in mmput, drain_mmlist and others) 321da177e4SLinus Torvalds * mapping->private_lock (in __set_page_dirty_buffers) 33250df6edSDave Chinner * inode->i_lock (in set_page_dirty's __mark_inode_dirty) 34f758eeabSChristoph Hellwig * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty) 351da177e4SLinus Torvalds * sb_lock (within inode_lock in fs/fs-writeback.c) 361da177e4SLinus Torvalds * mapping->tree_lock (widely used, in set_page_dirty, 371da177e4SLinus Torvalds * in arch-dependent flush_dcache_mmap_lock, 38f758eeabSChristoph Hellwig * within bdi.wb->list_lock in __sync_single_inode) 396a46079cSAndi Kleen * 405a505085SIngo Molnar * anon_vma->rwsem,mapping->i_mutex (memory_failure, collect_procs_anon) 416a46079cSAndi Kleen * ->tasklist_lock 426a46079cSAndi Kleen * pte map lock 431da177e4SLinus Torvalds */ 441da177e4SLinus Torvalds 451da177e4SLinus Torvalds #include <linux/mm.h> 461da177e4SLinus Torvalds #include <linux/pagemap.h> 471da177e4SLinus Torvalds #include <linux/swap.h> 481da177e4SLinus Torvalds #include <linux/swapops.h> 491da177e4SLinus Torvalds #include <linux/slab.h> 501da177e4SLinus Torvalds #include <linux/init.h> 515ad64688SHugh Dickins #include <linux/ksm.h> 521da177e4SLinus Torvalds #include <linux/rmap.h> 531da177e4SLinus Torvalds #include <linux/rcupdate.h> 54b95f1b31SPaul Gortmaker #include <linux/export.h> 558a9f3ccdSBalbir Singh #include <linux/memcontrol.h> 56cddb8a5cSAndrea Arcangeli #include <linux/mmu_notifier.h> 5764cdd548SKOSAKI Motohiro #include <linux/migrate.h> 580fe6e20bSNaoya Horiguchi #include <linux/hugetlb.h> 59ef5d437fSJan Kara #include <linux/backing-dev.h> 601da177e4SLinus Torvalds 611da177e4SLinus Torvalds #include <asm/tlbflush.h> 621da177e4SLinus Torvalds 63b291f000SNick Piggin #include "internal.h" 64b291f000SNick Piggin 65fdd2e5f8SAdrian Bunk static struct kmem_cache *anon_vma_cachep; 665beb4930SRik van Riel static struct kmem_cache *anon_vma_chain_cachep; 67fdd2e5f8SAdrian Bunk 68fdd2e5f8SAdrian Bunk static inline struct anon_vma *anon_vma_alloc(void) 69fdd2e5f8SAdrian Bunk { 7001d8b20dSPeter Zijlstra struct anon_vma *anon_vma; 7101d8b20dSPeter Zijlstra 7201d8b20dSPeter Zijlstra anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL); 7301d8b20dSPeter Zijlstra if (anon_vma) { 7401d8b20dSPeter Zijlstra atomic_set(&anon_vma->refcount, 1); 7501d8b20dSPeter Zijlstra /* 7601d8b20dSPeter Zijlstra * Initialise the anon_vma root to point to itself. If called 7701d8b20dSPeter Zijlstra * from fork, the root will be reset to the parents anon_vma. 7801d8b20dSPeter Zijlstra */ 7901d8b20dSPeter Zijlstra anon_vma->root = anon_vma; 80fdd2e5f8SAdrian Bunk } 81fdd2e5f8SAdrian Bunk 8201d8b20dSPeter Zijlstra return anon_vma; 8301d8b20dSPeter Zijlstra } 8401d8b20dSPeter Zijlstra 8501d8b20dSPeter Zijlstra static inline void anon_vma_free(struct anon_vma *anon_vma) 86fdd2e5f8SAdrian Bunk { 8701d8b20dSPeter Zijlstra VM_BUG_ON(atomic_read(&anon_vma->refcount)); 8888c22088SPeter Zijlstra 8988c22088SPeter Zijlstra /* 904fc3f1d6SIngo Molnar * Synchronize against page_lock_anon_vma_read() such that 9188c22088SPeter Zijlstra * we can safely hold the lock without the anon_vma getting 9288c22088SPeter Zijlstra * freed. 9388c22088SPeter Zijlstra * 9488c22088SPeter Zijlstra * Relies on the full mb implied by the atomic_dec_and_test() from 9588c22088SPeter Zijlstra * put_anon_vma() against the acquire barrier implied by 964fc3f1d6SIngo Molnar * down_read_trylock() from page_lock_anon_vma_read(). This orders: 9788c22088SPeter Zijlstra * 984fc3f1d6SIngo Molnar * page_lock_anon_vma_read() VS put_anon_vma() 994fc3f1d6SIngo Molnar * down_read_trylock() atomic_dec_and_test() 10088c22088SPeter Zijlstra * LOCK MB 1014fc3f1d6SIngo Molnar * atomic_read() rwsem_is_locked() 10288c22088SPeter Zijlstra * 10388c22088SPeter Zijlstra * LOCK should suffice since the actual taking of the lock must 10488c22088SPeter Zijlstra * happen _before_ what follows. 10588c22088SPeter Zijlstra */ 1065a505085SIngo Molnar if (rwsem_is_locked(&anon_vma->root->rwsem)) { 1074fc3f1d6SIngo Molnar anon_vma_lock_write(anon_vma); 10808b52706SKonstantin Khlebnikov anon_vma_unlock_write(anon_vma); 10988c22088SPeter Zijlstra } 11088c22088SPeter Zijlstra 111fdd2e5f8SAdrian Bunk kmem_cache_free(anon_vma_cachep, anon_vma); 112fdd2e5f8SAdrian Bunk } 1131da177e4SLinus Torvalds 114dd34739cSLinus Torvalds static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp) 1155beb4930SRik van Riel { 116dd34739cSLinus Torvalds return kmem_cache_alloc(anon_vma_chain_cachep, gfp); 1175beb4930SRik van Riel } 1185beb4930SRik van Riel 119e574b5fdSNamhyung Kim static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain) 1205beb4930SRik van Riel { 1215beb4930SRik van Riel kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain); 1225beb4930SRik van Riel } 1235beb4930SRik van Riel 1246583a843SKautuk Consul static void anon_vma_chain_link(struct vm_area_struct *vma, 1256583a843SKautuk Consul struct anon_vma_chain *avc, 1266583a843SKautuk Consul struct anon_vma *anon_vma) 1276583a843SKautuk Consul { 1286583a843SKautuk Consul avc->vma = vma; 1296583a843SKautuk Consul avc->anon_vma = anon_vma; 1306583a843SKautuk Consul list_add(&avc->same_vma, &vma->anon_vma_chain); 131bf181b9fSMichel Lespinasse anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); 1326583a843SKautuk Consul } 1336583a843SKautuk Consul 134d9d332e0SLinus Torvalds /** 135d9d332e0SLinus Torvalds * anon_vma_prepare - attach an anon_vma to a memory region 136d9d332e0SLinus Torvalds * @vma: the memory region in question 137d9d332e0SLinus Torvalds * 138d9d332e0SLinus Torvalds * This makes sure the memory mapping described by 'vma' has 139d9d332e0SLinus Torvalds * an 'anon_vma' attached to it, so that we can associate the 140d9d332e0SLinus Torvalds * anonymous pages mapped into it with that anon_vma. 141d9d332e0SLinus Torvalds * 142d9d332e0SLinus Torvalds * The common case will be that we already have one, but if 14323a0790aSFigo.zhang * not we either need to find an adjacent mapping that we 144d9d332e0SLinus Torvalds * can re-use the anon_vma from (very common when the only 145d9d332e0SLinus Torvalds * reason for splitting a vma has been mprotect()), or we 146d9d332e0SLinus Torvalds * allocate a new one. 147d9d332e0SLinus Torvalds * 148d9d332e0SLinus Torvalds * Anon-vma allocations are very subtle, because we may have 1494fc3f1d6SIngo Molnar * optimistically looked up an anon_vma in page_lock_anon_vma_read() 150d9d332e0SLinus Torvalds * and that may actually touch the spinlock even in the newly 151d9d332e0SLinus Torvalds * allocated vma (it depends on RCU to make sure that the 152d9d332e0SLinus Torvalds * anon_vma isn't actually destroyed). 153d9d332e0SLinus Torvalds * 154d9d332e0SLinus Torvalds * As a result, we need to do proper anon_vma locking even 155d9d332e0SLinus Torvalds * for the new allocation. At the same time, we do not want 156d9d332e0SLinus Torvalds * to do any locking for the common case of already having 157d9d332e0SLinus Torvalds * an anon_vma. 158d9d332e0SLinus Torvalds * 159d9d332e0SLinus Torvalds * This must be called with the mmap_sem held for reading. 160d9d332e0SLinus Torvalds */ 1611da177e4SLinus Torvalds int anon_vma_prepare(struct vm_area_struct *vma) 1621da177e4SLinus Torvalds { 1631da177e4SLinus Torvalds struct anon_vma *anon_vma = vma->anon_vma; 1645beb4930SRik van Riel struct anon_vma_chain *avc; 1651da177e4SLinus Torvalds 1661da177e4SLinus Torvalds might_sleep(); 1671da177e4SLinus Torvalds if (unlikely(!anon_vma)) { 1681da177e4SLinus Torvalds struct mm_struct *mm = vma->vm_mm; 169d9d332e0SLinus Torvalds struct anon_vma *allocated; 1701da177e4SLinus Torvalds 171dd34739cSLinus Torvalds avc = anon_vma_chain_alloc(GFP_KERNEL); 1725beb4930SRik van Riel if (!avc) 1735beb4930SRik van Riel goto out_enomem; 1745beb4930SRik van Riel 1751da177e4SLinus Torvalds anon_vma = find_mergeable_anon_vma(vma); 1761da177e4SLinus Torvalds allocated = NULL; 177d9d332e0SLinus Torvalds if (!anon_vma) { 1781da177e4SLinus Torvalds anon_vma = anon_vma_alloc(); 1791da177e4SLinus Torvalds if (unlikely(!anon_vma)) 1805beb4930SRik van Riel goto out_enomem_free_avc; 1811da177e4SLinus Torvalds allocated = anon_vma; 1821da177e4SLinus Torvalds } 1831da177e4SLinus Torvalds 1844fc3f1d6SIngo Molnar anon_vma_lock_write(anon_vma); 1851da177e4SLinus Torvalds /* page_table_lock to protect against threads */ 1861da177e4SLinus Torvalds spin_lock(&mm->page_table_lock); 1871da177e4SLinus Torvalds if (likely(!vma->anon_vma)) { 1881da177e4SLinus Torvalds vma->anon_vma = anon_vma; 1896583a843SKautuk Consul anon_vma_chain_link(vma, avc, anon_vma); 1901da177e4SLinus Torvalds allocated = NULL; 19131f2b0ebSOleg Nesterov avc = NULL; 1921da177e4SLinus Torvalds } 1931da177e4SLinus Torvalds spin_unlock(&mm->page_table_lock); 19408b52706SKonstantin Khlebnikov anon_vma_unlock_write(anon_vma); 19531f2b0ebSOleg Nesterov 19631f2b0ebSOleg Nesterov if (unlikely(allocated)) 19701d8b20dSPeter Zijlstra put_anon_vma(allocated); 19831f2b0ebSOleg Nesterov if (unlikely(avc)) 1995beb4930SRik van Riel anon_vma_chain_free(avc); 2005beb4930SRik van Riel } 2011da177e4SLinus Torvalds return 0; 2025beb4930SRik van Riel 2035beb4930SRik van Riel out_enomem_free_avc: 2045beb4930SRik van Riel anon_vma_chain_free(avc); 2055beb4930SRik van Riel out_enomem: 2065beb4930SRik van Riel return -ENOMEM; 2071da177e4SLinus Torvalds } 2081da177e4SLinus Torvalds 209bb4aa396SLinus Torvalds /* 210bb4aa396SLinus Torvalds * This is a useful helper function for locking the anon_vma root as 211bb4aa396SLinus Torvalds * we traverse the vma->anon_vma_chain, looping over anon_vma's that 212bb4aa396SLinus Torvalds * have the same vma. 213bb4aa396SLinus Torvalds * 214bb4aa396SLinus Torvalds * Such anon_vma's should have the same root, so you'd expect to see 215bb4aa396SLinus Torvalds * just a single mutex_lock for the whole traversal. 216bb4aa396SLinus Torvalds */ 217bb4aa396SLinus Torvalds static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma) 218bb4aa396SLinus Torvalds { 219bb4aa396SLinus Torvalds struct anon_vma *new_root = anon_vma->root; 220bb4aa396SLinus Torvalds if (new_root != root) { 221bb4aa396SLinus Torvalds if (WARN_ON_ONCE(root)) 2225a505085SIngo Molnar up_write(&root->rwsem); 223bb4aa396SLinus Torvalds root = new_root; 2245a505085SIngo Molnar down_write(&root->rwsem); 225bb4aa396SLinus Torvalds } 226bb4aa396SLinus Torvalds return root; 227bb4aa396SLinus Torvalds } 228bb4aa396SLinus Torvalds 229bb4aa396SLinus Torvalds static inline void unlock_anon_vma_root(struct anon_vma *root) 230bb4aa396SLinus Torvalds { 231bb4aa396SLinus Torvalds if (root) 2325a505085SIngo Molnar up_write(&root->rwsem); 233bb4aa396SLinus Torvalds } 234bb4aa396SLinus Torvalds 2355beb4930SRik van Riel /* 2365beb4930SRik van Riel * Attach the anon_vmas from src to dst. 2375beb4930SRik van Riel * Returns 0 on success, -ENOMEM on failure. 2385beb4930SRik van Riel */ 2395beb4930SRik van Riel int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src) 2405beb4930SRik van Riel { 2415beb4930SRik van Riel struct anon_vma_chain *avc, *pavc; 242bb4aa396SLinus Torvalds struct anon_vma *root = NULL; 2435beb4930SRik van Riel 244646d87b4SLinus Torvalds list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) { 245bb4aa396SLinus Torvalds struct anon_vma *anon_vma; 246bb4aa396SLinus Torvalds 247dd34739cSLinus Torvalds avc = anon_vma_chain_alloc(GFP_NOWAIT | __GFP_NOWARN); 248dd34739cSLinus Torvalds if (unlikely(!avc)) { 249dd34739cSLinus Torvalds unlock_anon_vma_root(root); 250dd34739cSLinus Torvalds root = NULL; 251dd34739cSLinus Torvalds avc = anon_vma_chain_alloc(GFP_KERNEL); 2525beb4930SRik van Riel if (!avc) 2535beb4930SRik van Riel goto enomem_failure; 254dd34739cSLinus Torvalds } 255bb4aa396SLinus Torvalds anon_vma = pavc->anon_vma; 256bb4aa396SLinus Torvalds root = lock_anon_vma_root(root, anon_vma); 257bb4aa396SLinus Torvalds anon_vma_chain_link(dst, avc, anon_vma); 2585beb4930SRik van Riel } 259bb4aa396SLinus Torvalds unlock_anon_vma_root(root); 2605beb4930SRik van Riel return 0; 2615beb4930SRik van Riel 2625beb4930SRik van Riel enomem_failure: 2635beb4930SRik van Riel unlink_anon_vmas(dst); 2645beb4930SRik van Riel return -ENOMEM; 2651da177e4SLinus Torvalds } 2661da177e4SLinus Torvalds 2675beb4930SRik van Riel /* 2685beb4930SRik van Riel * Attach vma to its own anon_vma, as well as to the anon_vmas that 2695beb4930SRik van Riel * the corresponding VMA in the parent process is attached to. 2705beb4930SRik van Riel * Returns 0 on success, non-zero on failure. 2715beb4930SRik van Riel */ 2725beb4930SRik van Riel int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) 2731da177e4SLinus Torvalds { 2745beb4930SRik van Riel struct anon_vma_chain *avc; 2755beb4930SRik van Riel struct anon_vma *anon_vma; 2765beb4930SRik van Riel 2775beb4930SRik van Riel /* Don't bother if the parent process has no anon_vma here. */ 2785beb4930SRik van Riel if (!pvma->anon_vma) 2795beb4930SRik van Riel return 0; 2805beb4930SRik van Riel 2815beb4930SRik van Riel /* 2825beb4930SRik van Riel * First, attach the new VMA to the parent VMA's anon_vmas, 2835beb4930SRik van Riel * so rmap can find non-COWed pages in child processes. 2845beb4930SRik van Riel */ 2855beb4930SRik van Riel if (anon_vma_clone(vma, pvma)) 2865beb4930SRik van Riel return -ENOMEM; 2875beb4930SRik van Riel 2885beb4930SRik van Riel /* Then add our own anon_vma. */ 2895beb4930SRik van Riel anon_vma = anon_vma_alloc(); 2905beb4930SRik van Riel if (!anon_vma) 2915beb4930SRik van Riel goto out_error; 292dd34739cSLinus Torvalds avc = anon_vma_chain_alloc(GFP_KERNEL); 2935beb4930SRik van Riel if (!avc) 2945beb4930SRik van Riel goto out_error_free_anon_vma; 2955c341ee1SRik van Riel 2965c341ee1SRik van Riel /* 2975c341ee1SRik van Riel * The root anon_vma's spinlock is the lock actually used when we 2985c341ee1SRik van Riel * lock any of the anon_vmas in this anon_vma tree. 2995c341ee1SRik van Riel */ 3005c341ee1SRik van Riel anon_vma->root = pvma->anon_vma->root; 30176545066SRik van Riel /* 30201d8b20dSPeter Zijlstra * With refcounts, an anon_vma can stay around longer than the 30301d8b20dSPeter Zijlstra * process it belongs to. The root anon_vma needs to be pinned until 30401d8b20dSPeter Zijlstra * this anon_vma is freed, because the lock lives in the root. 30576545066SRik van Riel */ 30676545066SRik van Riel get_anon_vma(anon_vma->root); 3075beb4930SRik van Riel /* Mark this anon_vma as the one where our new (COWed) pages go. */ 3085beb4930SRik van Riel vma->anon_vma = anon_vma; 3094fc3f1d6SIngo Molnar anon_vma_lock_write(anon_vma); 3105c341ee1SRik van Riel anon_vma_chain_link(vma, avc, anon_vma); 31108b52706SKonstantin Khlebnikov anon_vma_unlock_write(anon_vma); 3125beb4930SRik van Riel 3135beb4930SRik van Riel return 0; 3145beb4930SRik van Riel 3155beb4930SRik van Riel out_error_free_anon_vma: 31601d8b20dSPeter Zijlstra put_anon_vma(anon_vma); 3175beb4930SRik van Riel out_error: 3184946d54cSRik van Riel unlink_anon_vmas(vma); 3195beb4930SRik van Riel return -ENOMEM; 3205beb4930SRik van Riel } 3215beb4930SRik van Riel 3225beb4930SRik van Riel void unlink_anon_vmas(struct vm_area_struct *vma) 3235beb4930SRik van Riel { 3245beb4930SRik van Riel struct anon_vma_chain *avc, *next; 325eee2acbaSPeter Zijlstra struct anon_vma *root = NULL; 3265beb4930SRik van Riel 3275c341ee1SRik van Riel /* 3285c341ee1SRik van Riel * Unlink each anon_vma chained to the VMA. This list is ordered 3295c341ee1SRik van Riel * from newest to oldest, ensuring the root anon_vma gets freed last. 3305c341ee1SRik van Riel */ 3315beb4930SRik van Riel list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { 332eee2acbaSPeter Zijlstra struct anon_vma *anon_vma = avc->anon_vma; 333eee2acbaSPeter Zijlstra 334eee2acbaSPeter Zijlstra root = lock_anon_vma_root(root, anon_vma); 335bf181b9fSMichel Lespinasse anon_vma_interval_tree_remove(avc, &anon_vma->rb_root); 336eee2acbaSPeter Zijlstra 337eee2acbaSPeter Zijlstra /* 338eee2acbaSPeter Zijlstra * Leave empty anon_vmas on the list - we'll need 339eee2acbaSPeter Zijlstra * to free them outside the lock. 340eee2acbaSPeter Zijlstra */ 341bf181b9fSMichel Lespinasse if (RB_EMPTY_ROOT(&anon_vma->rb_root)) 342eee2acbaSPeter Zijlstra continue; 343eee2acbaSPeter Zijlstra 344eee2acbaSPeter Zijlstra list_del(&avc->same_vma); 345eee2acbaSPeter Zijlstra anon_vma_chain_free(avc); 346eee2acbaSPeter Zijlstra } 347eee2acbaSPeter Zijlstra unlock_anon_vma_root(root); 348eee2acbaSPeter Zijlstra 349eee2acbaSPeter Zijlstra /* 350eee2acbaSPeter Zijlstra * Iterate the list once more, it now only contains empty and unlinked 351eee2acbaSPeter Zijlstra * anon_vmas, destroy them. Could not do before due to __put_anon_vma() 3525a505085SIngo Molnar * needing to write-acquire the anon_vma->root->rwsem. 353eee2acbaSPeter Zijlstra */ 354eee2acbaSPeter Zijlstra list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { 355eee2acbaSPeter Zijlstra struct anon_vma *anon_vma = avc->anon_vma; 356eee2acbaSPeter Zijlstra 357eee2acbaSPeter Zijlstra put_anon_vma(anon_vma); 358eee2acbaSPeter Zijlstra 3595beb4930SRik van Riel list_del(&avc->same_vma); 3605beb4930SRik van Riel anon_vma_chain_free(avc); 3615beb4930SRik van Riel } 3625beb4930SRik van Riel } 3635beb4930SRik van Riel 36451cc5068SAlexey Dobriyan static void anon_vma_ctor(void *data) 3651da177e4SLinus Torvalds { 3661da177e4SLinus Torvalds struct anon_vma *anon_vma = data; 3671da177e4SLinus Torvalds 3685a505085SIngo Molnar init_rwsem(&anon_vma->rwsem); 36983813267SPeter Zijlstra atomic_set(&anon_vma->refcount, 0); 370bf181b9fSMichel Lespinasse anon_vma->rb_root = RB_ROOT; 3711da177e4SLinus Torvalds } 3721da177e4SLinus Torvalds 3731da177e4SLinus Torvalds void __init anon_vma_init(void) 3741da177e4SLinus Torvalds { 3751da177e4SLinus Torvalds anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma), 37620c2df83SPaul Mundt 0, SLAB_DESTROY_BY_RCU|SLAB_PANIC, anon_vma_ctor); 3775beb4930SRik van Riel anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain, SLAB_PANIC); 3781da177e4SLinus Torvalds } 3791da177e4SLinus Torvalds 3801da177e4SLinus Torvalds /* 3816111e4caSPeter Zijlstra * Getting a lock on a stable anon_vma from a page off the LRU is tricky! 3826111e4caSPeter Zijlstra * 3836111e4caSPeter Zijlstra * Since there is no serialization what so ever against page_remove_rmap() 3846111e4caSPeter Zijlstra * the best this function can do is return a locked anon_vma that might 3856111e4caSPeter Zijlstra * have been relevant to this page. 3866111e4caSPeter Zijlstra * 3876111e4caSPeter Zijlstra * The page might have been remapped to a different anon_vma or the anon_vma 3886111e4caSPeter Zijlstra * returned may already be freed (and even reused). 3896111e4caSPeter Zijlstra * 390bc658c96SPeter Zijlstra * In case it was remapped to a different anon_vma, the new anon_vma will be a 391bc658c96SPeter Zijlstra * child of the old anon_vma, and the anon_vma lifetime rules will therefore 392bc658c96SPeter Zijlstra * ensure that any anon_vma obtained from the page will still be valid for as 393bc658c96SPeter Zijlstra * long as we observe page_mapped() [ hence all those page_mapped() tests ]. 394bc658c96SPeter Zijlstra * 3956111e4caSPeter Zijlstra * All users of this function must be very careful when walking the anon_vma 3966111e4caSPeter Zijlstra * chain and verify that the page in question is indeed mapped in it 3976111e4caSPeter Zijlstra * [ something equivalent to page_mapped_in_vma() ]. 3986111e4caSPeter Zijlstra * 3996111e4caSPeter Zijlstra * Since anon_vma's slab is DESTROY_BY_RCU and we know from page_remove_rmap() 4006111e4caSPeter Zijlstra * that the anon_vma pointer from page->mapping is valid if there is a 4016111e4caSPeter Zijlstra * mapcount, we can dereference the anon_vma after observing those. 4021da177e4SLinus Torvalds */ 403746b18d4SPeter Zijlstra struct anon_vma *page_get_anon_vma(struct page *page) 4041da177e4SLinus Torvalds { 405746b18d4SPeter Zijlstra struct anon_vma *anon_vma = NULL; 4061da177e4SLinus Torvalds unsigned long anon_mapping; 4071da177e4SLinus Torvalds 4081da177e4SLinus Torvalds rcu_read_lock(); 40980e14822SHugh Dickins anon_mapping = (unsigned long) ACCESS_ONCE(page->mapping); 4103ca7b3c5SHugh Dickins if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) 4111da177e4SLinus Torvalds goto out; 4121da177e4SLinus Torvalds if (!page_mapped(page)) 4131da177e4SLinus Torvalds goto out; 4141da177e4SLinus Torvalds 4151da177e4SLinus Torvalds anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); 416746b18d4SPeter Zijlstra if (!atomic_inc_not_zero(&anon_vma->refcount)) { 417746b18d4SPeter Zijlstra anon_vma = NULL; 418746b18d4SPeter Zijlstra goto out; 419746b18d4SPeter Zijlstra } 420f1819427SHugh Dickins 421f1819427SHugh Dickins /* 422f1819427SHugh Dickins * If this page is still mapped, then its anon_vma cannot have been 423746b18d4SPeter Zijlstra * freed. But if it has been unmapped, we have no security against the 424746b18d4SPeter Zijlstra * anon_vma structure being freed and reused (for another anon_vma: 425746b18d4SPeter Zijlstra * SLAB_DESTROY_BY_RCU guarantees that - so the atomic_inc_not_zero() 426746b18d4SPeter Zijlstra * above cannot corrupt). 427f1819427SHugh Dickins */ 428746b18d4SPeter Zijlstra if (!page_mapped(page)) { 429746b18d4SPeter Zijlstra put_anon_vma(anon_vma); 430746b18d4SPeter Zijlstra anon_vma = NULL; 431746b18d4SPeter Zijlstra } 4321da177e4SLinus Torvalds out: 4331da177e4SLinus Torvalds rcu_read_unlock(); 434746b18d4SPeter Zijlstra 435746b18d4SPeter Zijlstra return anon_vma; 436746b18d4SPeter Zijlstra } 437746b18d4SPeter Zijlstra 43888c22088SPeter Zijlstra /* 43988c22088SPeter Zijlstra * Similar to page_get_anon_vma() except it locks the anon_vma. 44088c22088SPeter Zijlstra * 44188c22088SPeter Zijlstra * Its a little more complex as it tries to keep the fast path to a single 44288c22088SPeter Zijlstra * atomic op -- the trylock. If we fail the trylock, we fall back to getting a 44388c22088SPeter Zijlstra * reference like with page_get_anon_vma() and then block on the mutex. 44488c22088SPeter Zijlstra */ 4454fc3f1d6SIngo Molnar struct anon_vma *page_lock_anon_vma_read(struct page *page) 446746b18d4SPeter Zijlstra { 44788c22088SPeter Zijlstra struct anon_vma *anon_vma = NULL; 448eee0f252SHugh Dickins struct anon_vma *root_anon_vma; 44988c22088SPeter Zijlstra unsigned long anon_mapping; 450746b18d4SPeter Zijlstra 45188c22088SPeter Zijlstra rcu_read_lock(); 45288c22088SPeter Zijlstra anon_mapping = (unsigned long) ACCESS_ONCE(page->mapping); 45388c22088SPeter Zijlstra if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) 45488c22088SPeter Zijlstra goto out; 45588c22088SPeter Zijlstra if (!page_mapped(page)) 45688c22088SPeter Zijlstra goto out; 45788c22088SPeter Zijlstra 45888c22088SPeter Zijlstra anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); 459eee0f252SHugh Dickins root_anon_vma = ACCESS_ONCE(anon_vma->root); 4604fc3f1d6SIngo Molnar if (down_read_trylock(&root_anon_vma->rwsem)) { 46188c22088SPeter Zijlstra /* 462eee0f252SHugh Dickins * If the page is still mapped, then this anon_vma is still 463eee0f252SHugh Dickins * its anon_vma, and holding the mutex ensures that it will 464bc658c96SPeter Zijlstra * not go away, see anon_vma_free(). 46588c22088SPeter Zijlstra */ 466eee0f252SHugh Dickins if (!page_mapped(page)) { 4674fc3f1d6SIngo Molnar up_read(&root_anon_vma->rwsem); 46888c22088SPeter Zijlstra anon_vma = NULL; 46988c22088SPeter Zijlstra } 47088c22088SPeter Zijlstra goto out; 47188c22088SPeter Zijlstra } 47288c22088SPeter Zijlstra 47388c22088SPeter Zijlstra /* trylock failed, we got to sleep */ 47488c22088SPeter Zijlstra if (!atomic_inc_not_zero(&anon_vma->refcount)) { 47588c22088SPeter Zijlstra anon_vma = NULL; 47688c22088SPeter Zijlstra goto out; 47788c22088SPeter Zijlstra } 47888c22088SPeter Zijlstra 47988c22088SPeter Zijlstra if (!page_mapped(page)) { 48088c22088SPeter Zijlstra put_anon_vma(anon_vma); 48188c22088SPeter Zijlstra anon_vma = NULL; 48288c22088SPeter Zijlstra goto out; 48388c22088SPeter Zijlstra } 48488c22088SPeter Zijlstra 48588c22088SPeter Zijlstra /* we pinned the anon_vma, its safe to sleep */ 48688c22088SPeter Zijlstra rcu_read_unlock(); 4874fc3f1d6SIngo Molnar anon_vma_lock_read(anon_vma); 488746b18d4SPeter Zijlstra 48988c22088SPeter Zijlstra if (atomic_dec_and_test(&anon_vma->refcount)) { 49088c22088SPeter Zijlstra /* 49188c22088SPeter Zijlstra * Oops, we held the last refcount, release the lock 49288c22088SPeter Zijlstra * and bail -- can't simply use put_anon_vma() because 4934fc3f1d6SIngo Molnar * we'll deadlock on the anon_vma_lock_write() recursion. 49488c22088SPeter Zijlstra */ 4954fc3f1d6SIngo Molnar anon_vma_unlock_read(anon_vma); 49688c22088SPeter Zijlstra __put_anon_vma(anon_vma); 49788c22088SPeter Zijlstra anon_vma = NULL; 49888c22088SPeter Zijlstra } 49988c22088SPeter Zijlstra 50088c22088SPeter Zijlstra return anon_vma; 50188c22088SPeter Zijlstra 50288c22088SPeter Zijlstra out: 50388c22088SPeter Zijlstra rcu_read_unlock(); 504746b18d4SPeter Zijlstra return anon_vma; 50534bbd704SOleg Nesterov } 50634bbd704SOleg Nesterov 5074fc3f1d6SIngo Molnar void page_unlock_anon_vma_read(struct anon_vma *anon_vma) 50834bbd704SOleg Nesterov { 5094fc3f1d6SIngo Molnar anon_vma_unlock_read(anon_vma); 5101da177e4SLinus Torvalds } 5111da177e4SLinus Torvalds 5121da177e4SLinus Torvalds /* 5133ad33b24SLee Schermerhorn * At what user virtual address is page expected in @vma? 5141da177e4SLinus Torvalds */ 51586c2ad19SMichel Lespinasse static inline unsigned long 51686c2ad19SMichel Lespinasse __vma_address(struct page *page, struct vm_area_struct *vma) 5171da177e4SLinus Torvalds { 5181da177e4SLinus Torvalds pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); 5191da177e4SLinus Torvalds 5200fe6e20bSNaoya Horiguchi if (unlikely(is_vm_hugetlb_page(vma))) 5210fe6e20bSNaoya Horiguchi pgoff = page->index << huge_page_order(page_hstate(page)); 52286c2ad19SMichel Lespinasse 52386c2ad19SMichel Lespinasse return vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); 5241da177e4SLinus Torvalds } 52586c2ad19SMichel Lespinasse 52686c2ad19SMichel Lespinasse inline unsigned long 52786c2ad19SMichel Lespinasse vma_address(struct page *page, struct vm_area_struct *vma) 52886c2ad19SMichel Lespinasse { 52986c2ad19SMichel Lespinasse unsigned long address = __vma_address(page, vma); 53086c2ad19SMichel Lespinasse 53186c2ad19SMichel Lespinasse /* page should be within @vma mapping range */ 53286c2ad19SMichel Lespinasse VM_BUG_ON(address < vma->vm_start || address >= vma->vm_end); 53386c2ad19SMichel Lespinasse 5341da177e4SLinus Torvalds return address; 5351da177e4SLinus Torvalds } 5361da177e4SLinus Torvalds 5371da177e4SLinus Torvalds /* 538bf89c8c8SHuang Shijie * At what user virtual address is page expected in vma? 539ab941e0fSNaoya Horiguchi * Caller should check the page is actually part of the vma. 5401da177e4SLinus Torvalds */ 5411da177e4SLinus Torvalds unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) 5421da177e4SLinus Torvalds { 54386c2ad19SMichel Lespinasse unsigned long address; 54421d0d443SAndrea Arcangeli if (PageAnon(page)) { 5454829b906SHugh Dickins struct anon_vma *page__anon_vma = page_anon_vma(page); 5464829b906SHugh Dickins /* 5474829b906SHugh Dickins * Note: swapoff's unuse_vma() is more efficient with this 5484829b906SHugh Dickins * check, and needs it to match anon_vma when KSM is active. 5494829b906SHugh Dickins */ 5504829b906SHugh Dickins if (!vma->anon_vma || !page__anon_vma || 5514829b906SHugh Dickins vma->anon_vma->root != page__anon_vma->root) 55221d0d443SAndrea Arcangeli return -EFAULT; 55321d0d443SAndrea Arcangeli } else if (page->mapping && !(vma->vm_flags & VM_NONLINEAR)) { 554ee498ed7SHugh Dickins if (!vma->vm_file || 555ee498ed7SHugh Dickins vma->vm_file->f_mapping != page->mapping) 5561da177e4SLinus Torvalds return -EFAULT; 5571da177e4SLinus Torvalds } else 5581da177e4SLinus Torvalds return -EFAULT; 55986c2ad19SMichel Lespinasse address = __vma_address(page, vma); 56086c2ad19SMichel Lespinasse if (unlikely(address < vma->vm_start || address >= vma->vm_end)) 56186c2ad19SMichel Lespinasse return -EFAULT; 56286c2ad19SMichel Lespinasse return address; 5631da177e4SLinus Torvalds } 5641da177e4SLinus Torvalds 5656219049aSBob Liu pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) 5666219049aSBob Liu { 5676219049aSBob Liu pgd_t *pgd; 5686219049aSBob Liu pud_t *pud; 5696219049aSBob Liu pmd_t *pmd = NULL; 5706219049aSBob Liu 5716219049aSBob Liu pgd = pgd_offset(mm, address); 5726219049aSBob Liu if (!pgd_present(*pgd)) 5736219049aSBob Liu goto out; 5746219049aSBob Liu 5756219049aSBob Liu pud = pud_offset(pgd, address); 5766219049aSBob Liu if (!pud_present(*pud)) 5776219049aSBob Liu goto out; 5786219049aSBob Liu 5796219049aSBob Liu pmd = pmd_offset(pud, address); 5806219049aSBob Liu if (!pmd_present(*pmd)) 5816219049aSBob Liu pmd = NULL; 5826219049aSBob Liu out: 5836219049aSBob Liu return pmd; 5846219049aSBob Liu } 5856219049aSBob Liu 5861da177e4SLinus Torvalds /* 58781b4082dSNikita Danilov * Check that @page is mapped at @address into @mm. 58881b4082dSNikita Danilov * 589479db0bfSNick Piggin * If @sync is false, page_check_address may perform a racy check to avoid 590479db0bfSNick Piggin * the page table lock when the pte is not present (helpful when reclaiming 591479db0bfSNick Piggin * highly shared pages). 592479db0bfSNick Piggin * 593b8072f09SHugh Dickins * On success returns with pte mapped and locked. 59481b4082dSNikita Danilov */ 595e9a81a82SNamhyung Kim pte_t *__page_check_address(struct page *page, struct mm_struct *mm, 596479db0bfSNick Piggin unsigned long address, spinlock_t **ptlp, int sync) 59781b4082dSNikita Danilov { 59881b4082dSNikita Danilov pmd_t *pmd; 59981b4082dSNikita Danilov pte_t *pte; 600c0718806SHugh Dickins spinlock_t *ptl; 60181b4082dSNikita Danilov 6020fe6e20bSNaoya Horiguchi if (unlikely(PageHuge(page))) { 60398398c32SJianguo Wu /* when pud is not present, pte will be NULL */ 6040fe6e20bSNaoya Horiguchi pte = huge_pte_offset(mm, address); 60598398c32SJianguo Wu if (!pte) 60698398c32SJianguo Wu return NULL; 60798398c32SJianguo Wu 608cb900f41SKirill A. Shutemov ptl = huge_pte_lockptr(page_hstate(page), mm, pte); 6090fe6e20bSNaoya Horiguchi goto check; 6100fe6e20bSNaoya Horiguchi } 6110fe6e20bSNaoya Horiguchi 6126219049aSBob Liu pmd = mm_find_pmd(mm, address); 6136219049aSBob Liu if (!pmd) 614c0718806SHugh Dickins return NULL; 615c0718806SHugh Dickins 61671e3aac0SAndrea Arcangeli if (pmd_trans_huge(*pmd)) 61771e3aac0SAndrea Arcangeli return NULL; 618c0718806SHugh Dickins 61981b4082dSNikita Danilov pte = pte_offset_map(pmd, address); 620c0718806SHugh Dickins /* Make a quick check before getting the lock */ 621479db0bfSNick Piggin if (!sync && !pte_present(*pte)) { 62281b4082dSNikita Danilov pte_unmap(pte); 623c0718806SHugh Dickins return NULL; 62481b4082dSNikita Danilov } 625c0718806SHugh Dickins 6264c21e2f2SHugh Dickins ptl = pte_lockptr(mm, pmd); 6270fe6e20bSNaoya Horiguchi check: 628c0718806SHugh Dickins spin_lock(ptl); 629c0718806SHugh Dickins if (pte_present(*pte) && page_to_pfn(page) == pte_pfn(*pte)) { 630c0718806SHugh Dickins *ptlp = ptl; 631c0718806SHugh Dickins return pte; 63281b4082dSNikita Danilov } 633c0718806SHugh Dickins pte_unmap_unlock(pte, ptl); 634c0718806SHugh Dickins return NULL; 63581b4082dSNikita Danilov } 63681b4082dSNikita Danilov 637b291f000SNick Piggin /** 638b291f000SNick Piggin * page_mapped_in_vma - check whether a page is really mapped in a VMA 639b291f000SNick Piggin * @page: the page to test 640b291f000SNick Piggin * @vma: the VMA to test 641b291f000SNick Piggin * 642b291f000SNick Piggin * Returns 1 if the page is mapped into the page tables of the VMA, 0 643b291f000SNick Piggin * if the page is not mapped into the page tables of this VMA. Only 644b291f000SNick Piggin * valid for normal file or anonymous VMAs. 645b291f000SNick Piggin */ 6466a46079cSAndi Kleen int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma) 647b291f000SNick Piggin { 648b291f000SNick Piggin unsigned long address; 649b291f000SNick Piggin pte_t *pte; 650b291f000SNick Piggin spinlock_t *ptl; 651b291f000SNick Piggin 65286c2ad19SMichel Lespinasse address = __vma_address(page, vma); 65386c2ad19SMichel Lespinasse if (unlikely(address < vma->vm_start || address >= vma->vm_end)) 654b291f000SNick Piggin return 0; 655b291f000SNick Piggin pte = page_check_address(page, vma->vm_mm, address, &ptl, 1); 656b291f000SNick Piggin if (!pte) /* the page is not in this mm */ 657b291f000SNick Piggin return 0; 658b291f000SNick Piggin pte_unmap_unlock(pte, ptl); 659b291f000SNick Piggin 660b291f000SNick Piggin return 1; 661b291f000SNick Piggin } 662b291f000SNick Piggin 6639f32624bSJoonsoo Kim struct page_referenced_arg { 6649f32624bSJoonsoo Kim int mapcount; 6659f32624bSJoonsoo Kim int referenced; 6669f32624bSJoonsoo Kim unsigned long vm_flags; 6679f32624bSJoonsoo Kim struct mem_cgroup *memcg; 6689f32624bSJoonsoo Kim }; 66981b4082dSNikita Danilov /* 6709f32624bSJoonsoo Kim * arg: page_referenced_arg will be passed 6711da177e4SLinus Torvalds */ 6725ad64688SHugh Dickins int page_referenced_one(struct page *page, struct vm_area_struct *vma, 6739f32624bSJoonsoo Kim unsigned long address, void *arg) 6741da177e4SLinus Torvalds { 6751da177e4SLinus Torvalds struct mm_struct *mm = vma->vm_mm; 676117b0791SKirill A. Shutemov spinlock_t *ptl; 6771da177e4SLinus Torvalds int referenced = 0; 6789f32624bSJoonsoo Kim struct page_referenced_arg *pra = arg; 6791da177e4SLinus Torvalds 6802da28bfdSAndrea Arcangeli if (unlikely(PageTransHuge(page))) { 6812da28bfdSAndrea Arcangeli pmd_t *pmd; 6822da28bfdSAndrea Arcangeli 683b291f000SNick Piggin /* 6842da28bfdSAndrea Arcangeli * rmap might return false positives; we must filter 6852da28bfdSAndrea Arcangeli * these out using page_check_address_pmd(). 686b291f000SNick Piggin */ 6872da28bfdSAndrea Arcangeli pmd = page_check_address_pmd(page, mm, address, 688117b0791SKirill A. Shutemov PAGE_CHECK_ADDRESS_PMD_FLAG, &ptl); 689117b0791SKirill A. Shutemov if (!pmd) 6909f32624bSJoonsoo Kim return SWAP_AGAIN; 6912da28bfdSAndrea Arcangeli 6925a9bbdcdSHugh Dickins if (vma->vm_flags & VM_LOCKED) { 693117b0791SKirill A. Shutemov spin_unlock(ptl); 6949f32624bSJoonsoo Kim pra->vm_flags |= VM_LOCKED; 6959f32624bSJoonsoo Kim return SWAP_FAIL; /* To break the loop */ 696b291f000SNick Piggin } 697b291f000SNick Piggin 6982da28bfdSAndrea Arcangeli /* go ahead even if the pmd is pmd_trans_splitting() */ 6992da28bfdSAndrea Arcangeli if (pmdp_clear_flush_young_notify(vma, address, pmd)) 70071e3aac0SAndrea Arcangeli referenced++; 701117b0791SKirill A. Shutemov spin_unlock(ptl); 70271e3aac0SAndrea Arcangeli } else { 70371e3aac0SAndrea Arcangeli pte_t *pte; 70471e3aac0SAndrea Arcangeli 7052da28bfdSAndrea Arcangeli /* 7062da28bfdSAndrea Arcangeli * rmap might return false positives; we must filter 7072da28bfdSAndrea Arcangeli * these out using page_check_address(). 7082da28bfdSAndrea Arcangeli */ 70971e3aac0SAndrea Arcangeli pte = page_check_address(page, mm, address, &ptl, 0); 71071e3aac0SAndrea Arcangeli if (!pte) 7119f32624bSJoonsoo Kim return SWAP_AGAIN; 71271e3aac0SAndrea Arcangeli 7132da28bfdSAndrea Arcangeli if (vma->vm_flags & VM_LOCKED) { 7142da28bfdSAndrea Arcangeli pte_unmap_unlock(pte, ptl); 7159f32624bSJoonsoo Kim pra->vm_flags |= VM_LOCKED; 7169f32624bSJoonsoo Kim return SWAP_FAIL; /* To break the loop */ 7172da28bfdSAndrea Arcangeli } 7182da28bfdSAndrea Arcangeli 7194917e5d0SJohannes Weiner if (ptep_clear_flush_young_notify(vma, address, pte)) { 7204917e5d0SJohannes Weiner /* 7214917e5d0SJohannes Weiner * Don't treat a reference through a sequentially read 7224917e5d0SJohannes Weiner * mapping as such. If the page has been used in 7234917e5d0SJohannes Weiner * another mapping, we will catch it; if this other 7244917e5d0SJohannes Weiner * mapping is already gone, the unmap path will have 7254917e5d0SJohannes Weiner * set PG_referenced or activated the page. 7264917e5d0SJohannes Weiner */ 72764363aadSJoe Perches if (likely(!(vma->vm_flags & VM_SEQ_READ))) 7281da177e4SLinus Torvalds referenced++; 7294917e5d0SJohannes Weiner } 730c0718806SHugh Dickins pte_unmap_unlock(pte, ptl); 73171e3aac0SAndrea Arcangeli } 73271e3aac0SAndrea Arcangeli 7339f32624bSJoonsoo Kim if (referenced) { 7349f32624bSJoonsoo Kim pra->referenced++; 7359f32624bSJoonsoo Kim pra->vm_flags |= vma->vm_flags; 7361da177e4SLinus Torvalds } 7371da177e4SLinus Torvalds 7389f32624bSJoonsoo Kim pra->mapcount--; 7399f32624bSJoonsoo Kim if (!pra->mapcount) 7409f32624bSJoonsoo Kim return SWAP_SUCCESS; /* To break the loop */ 7419f32624bSJoonsoo Kim 7429f32624bSJoonsoo Kim return SWAP_AGAIN; 7439f32624bSJoonsoo Kim } 7449f32624bSJoonsoo Kim 7459f32624bSJoonsoo Kim static bool invalid_page_referenced_vma(struct vm_area_struct *vma, void *arg) 7461da177e4SLinus Torvalds { 7479f32624bSJoonsoo Kim struct page_referenced_arg *pra = arg; 7489f32624bSJoonsoo Kim struct mem_cgroup *memcg = pra->memcg; 7491da177e4SLinus Torvalds 7509f32624bSJoonsoo Kim if (!mm_match_cgroup(vma->vm_mm, memcg)) 7519f32624bSJoonsoo Kim return true; 7521da177e4SLinus Torvalds 7539f32624bSJoonsoo Kim return false; 7541da177e4SLinus Torvalds } 7551da177e4SLinus Torvalds 7561da177e4SLinus Torvalds /** 7571da177e4SLinus Torvalds * page_referenced - test if the page was referenced 7581da177e4SLinus Torvalds * @page: the page to test 7591da177e4SLinus Torvalds * @is_locked: caller holds lock on the page 76072835c86SJohannes Weiner * @memcg: target memory cgroup 7616fe6b7e3SWu Fengguang * @vm_flags: collect encountered vma->vm_flags who actually referenced the page 7621da177e4SLinus Torvalds * 7631da177e4SLinus Torvalds * Quick test_and_clear_referenced for all mappings to a page, 7641da177e4SLinus Torvalds * returns the number of ptes which referenced the page. 7651da177e4SLinus Torvalds */ 7666fe6b7e3SWu Fengguang int page_referenced(struct page *page, 7676fe6b7e3SWu Fengguang int is_locked, 76872835c86SJohannes Weiner struct mem_cgroup *memcg, 7696fe6b7e3SWu Fengguang unsigned long *vm_flags) 7701da177e4SLinus Torvalds { 7719f32624bSJoonsoo Kim int ret; 7725ad64688SHugh Dickins int we_locked = 0; 7739f32624bSJoonsoo Kim struct page_referenced_arg pra = { 7749f32624bSJoonsoo Kim .mapcount = page_mapcount(page), 7759f32624bSJoonsoo Kim .memcg = memcg, 7769f32624bSJoonsoo Kim }; 7779f32624bSJoonsoo Kim struct rmap_walk_control rwc = { 7789f32624bSJoonsoo Kim .rmap_one = page_referenced_one, 7799f32624bSJoonsoo Kim .arg = (void *)&pra, 7809f32624bSJoonsoo Kim .anon_lock = page_lock_anon_vma_read, 7819f32624bSJoonsoo Kim }; 7821da177e4SLinus Torvalds 7836fe6b7e3SWu Fengguang *vm_flags = 0; 7849f32624bSJoonsoo Kim if (!page_mapped(page)) 7859f32624bSJoonsoo Kim return 0; 7869f32624bSJoonsoo Kim 7879f32624bSJoonsoo Kim if (!page_rmapping(page)) 7889f32624bSJoonsoo Kim return 0; 7899f32624bSJoonsoo Kim 7905ad64688SHugh Dickins if (!is_locked && (!PageAnon(page) || PageKsm(page))) { 7915ad64688SHugh Dickins we_locked = trylock_page(page); 7929f32624bSJoonsoo Kim if (!we_locked) 7939f32624bSJoonsoo Kim return 1; 7945ad64688SHugh Dickins } 7959f32624bSJoonsoo Kim 7969f32624bSJoonsoo Kim /* 7979f32624bSJoonsoo Kim * If we are reclaiming on behalf of a cgroup, skip 7989f32624bSJoonsoo Kim * counting on behalf of references from different 7999f32624bSJoonsoo Kim * cgroups 8009f32624bSJoonsoo Kim */ 8019f32624bSJoonsoo Kim if (memcg) { 8029f32624bSJoonsoo Kim rwc.invalid_vma = invalid_page_referenced_vma; 8035ad64688SHugh Dickins } 8049f32624bSJoonsoo Kim 8059f32624bSJoonsoo Kim ret = rmap_walk(page, &rwc); 8069f32624bSJoonsoo Kim *vm_flags = pra.vm_flags; 8079f32624bSJoonsoo Kim 8085ad64688SHugh Dickins if (we_locked) 8091da177e4SLinus Torvalds unlock_page(page); 8109f32624bSJoonsoo Kim 8119f32624bSJoonsoo Kim return pra.referenced; 8121da177e4SLinus Torvalds } 8131da177e4SLinus Torvalds 8141cb1729bSHugh Dickins static int page_mkclean_one(struct page *page, struct vm_area_struct *vma, 815*9853a407SJoonsoo Kim unsigned long address, void *arg) 816d08b3851SPeter Zijlstra { 817d08b3851SPeter Zijlstra struct mm_struct *mm = vma->vm_mm; 818c2fda5feSPeter Zijlstra pte_t *pte; 819d08b3851SPeter Zijlstra spinlock_t *ptl; 820d08b3851SPeter Zijlstra int ret = 0; 821*9853a407SJoonsoo Kim int *cleaned = arg; 822d08b3851SPeter Zijlstra 823479db0bfSNick Piggin pte = page_check_address(page, mm, address, &ptl, 1); 824d08b3851SPeter Zijlstra if (!pte) 825d08b3851SPeter Zijlstra goto out; 826d08b3851SPeter Zijlstra 827c2fda5feSPeter Zijlstra if (pte_dirty(*pte) || pte_write(*pte)) { 828c2fda5feSPeter Zijlstra pte_t entry; 829d08b3851SPeter Zijlstra 830c2fda5feSPeter Zijlstra flush_cache_page(vma, address, pte_pfn(*pte)); 8312ec74c3eSSagi Grimberg entry = ptep_clear_flush(vma, address, pte); 832d08b3851SPeter Zijlstra entry = pte_wrprotect(entry); 833c2fda5feSPeter Zijlstra entry = pte_mkclean(entry); 834d6e88e67SAl Viro set_pte_at(mm, address, pte, entry); 835d08b3851SPeter Zijlstra ret = 1; 836c2fda5feSPeter Zijlstra } 837d08b3851SPeter Zijlstra 838d08b3851SPeter Zijlstra pte_unmap_unlock(pte, ptl); 8392ec74c3eSSagi Grimberg 840*9853a407SJoonsoo Kim if (ret) { 8412ec74c3eSSagi Grimberg mmu_notifier_invalidate_page(mm, address); 842*9853a407SJoonsoo Kim (*cleaned)++; 843*9853a407SJoonsoo Kim } 844d08b3851SPeter Zijlstra out: 845*9853a407SJoonsoo Kim return SWAP_AGAIN; 846d08b3851SPeter Zijlstra } 847d08b3851SPeter Zijlstra 848*9853a407SJoonsoo Kim static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg) 849d08b3851SPeter Zijlstra { 850*9853a407SJoonsoo Kim if (vma->vm_flags & VM_SHARED) 851*9853a407SJoonsoo Kim return 0; 852d08b3851SPeter Zijlstra 853*9853a407SJoonsoo Kim return 1; 854d08b3851SPeter Zijlstra } 855d08b3851SPeter Zijlstra 856d08b3851SPeter Zijlstra int page_mkclean(struct page *page) 857d08b3851SPeter Zijlstra { 858*9853a407SJoonsoo Kim int cleaned = 0; 859*9853a407SJoonsoo Kim struct address_space *mapping; 860*9853a407SJoonsoo Kim struct rmap_walk_control rwc = { 861*9853a407SJoonsoo Kim .arg = (void *)&cleaned, 862*9853a407SJoonsoo Kim .rmap_one = page_mkclean_one, 863*9853a407SJoonsoo Kim .invalid_vma = invalid_mkclean_vma, 864*9853a407SJoonsoo Kim }; 865d08b3851SPeter Zijlstra 866d08b3851SPeter Zijlstra BUG_ON(!PageLocked(page)); 867d08b3851SPeter Zijlstra 868*9853a407SJoonsoo Kim if (!page_mapped(page)) 869*9853a407SJoonsoo Kim return 0; 870d08b3851SPeter Zijlstra 871*9853a407SJoonsoo Kim mapping = page_mapping(page); 872*9853a407SJoonsoo Kim if (!mapping) 873*9853a407SJoonsoo Kim return 0; 874*9853a407SJoonsoo Kim 875*9853a407SJoonsoo Kim rmap_walk(page, &rwc); 876*9853a407SJoonsoo Kim 877*9853a407SJoonsoo Kim return cleaned; 878d08b3851SPeter Zijlstra } 87960b59beaSJaya Kumar EXPORT_SYMBOL_GPL(page_mkclean); 880d08b3851SPeter Zijlstra 8811da177e4SLinus Torvalds /** 882c44b6743SRik van Riel * page_move_anon_rmap - move a page to our anon_vma 883c44b6743SRik van Riel * @page: the page to move to our anon_vma 884c44b6743SRik van Riel * @vma: the vma the page belongs to 885c44b6743SRik van Riel * @address: the user virtual address mapped 886c44b6743SRik van Riel * 887c44b6743SRik van Riel * When a page belongs exclusively to one process after a COW event, 888c44b6743SRik van Riel * that page can be moved into the anon_vma that belongs to just that 889c44b6743SRik van Riel * process, so the rmap code will not search the parent or sibling 890c44b6743SRik van Riel * processes. 891c44b6743SRik van Riel */ 892c44b6743SRik van Riel void page_move_anon_rmap(struct page *page, 893c44b6743SRik van Riel struct vm_area_struct *vma, unsigned long address) 894c44b6743SRik van Riel { 895c44b6743SRik van Riel struct anon_vma *anon_vma = vma->anon_vma; 896c44b6743SRik van Riel 897c44b6743SRik van Riel VM_BUG_ON(!PageLocked(page)); 898c44b6743SRik van Riel VM_BUG_ON(!anon_vma); 899c44b6743SRik van Riel VM_BUG_ON(page->index != linear_page_index(vma, address)); 900c44b6743SRik van Riel 901c44b6743SRik van Riel anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 902c44b6743SRik van Riel page->mapping = (struct address_space *) anon_vma; 903c44b6743SRik van Riel } 904c44b6743SRik van Riel 905c44b6743SRik van Riel /** 90643d8eac4SRandy Dunlap * __page_set_anon_rmap - set up new anonymous rmap 9074e1c1975SAndi Kleen * @page: Page to add to rmap 9084e1c1975SAndi Kleen * @vma: VM area to add page to. 9094e1c1975SAndi Kleen * @address: User virtual address of the mapping 910e8a03febSRik van Riel * @exclusive: the page is exclusively owned by the current process 9111da177e4SLinus Torvalds */ 9129617d95eSNick Piggin static void __page_set_anon_rmap(struct page *page, 913e8a03febSRik van Riel struct vm_area_struct *vma, unsigned long address, int exclusive) 9141da177e4SLinus Torvalds { 915e8a03febSRik van Riel struct anon_vma *anon_vma = vma->anon_vma; 9162822c1aaSNick Piggin 917e8a03febSRik van Riel BUG_ON(!anon_vma); 918ea90002bSLinus Torvalds 9194e1c1975SAndi Kleen if (PageAnon(page)) 9204e1c1975SAndi Kleen return; 9214e1c1975SAndi Kleen 922ea90002bSLinus Torvalds /* 923e8a03febSRik van Riel * If the page isn't exclusively mapped into this vma, 924e8a03febSRik van Riel * we must use the _oldest_ possible anon_vma for the 925e8a03febSRik van Riel * page mapping! 926ea90002bSLinus Torvalds */ 9274e1c1975SAndi Kleen if (!exclusive) 928288468c3SAndrea Arcangeli anon_vma = anon_vma->root; 929ea90002bSLinus Torvalds 9301da177e4SLinus Torvalds anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 9312822c1aaSNick Piggin page->mapping = (struct address_space *) anon_vma; 9324d7670e0SNick Piggin page->index = linear_page_index(vma, address); 9331da177e4SLinus Torvalds } 9349617d95eSNick Piggin 9359617d95eSNick Piggin /** 93643d8eac4SRandy Dunlap * __page_check_anon_rmap - sanity check anonymous rmap addition 937c97a9e10SNick Piggin * @page: the page to add the mapping to 938c97a9e10SNick Piggin * @vma: the vm area in which the mapping is added 939c97a9e10SNick Piggin * @address: the user virtual address mapped 940c97a9e10SNick Piggin */ 941c97a9e10SNick Piggin static void __page_check_anon_rmap(struct page *page, 942c97a9e10SNick Piggin struct vm_area_struct *vma, unsigned long address) 943c97a9e10SNick Piggin { 944c97a9e10SNick Piggin #ifdef CONFIG_DEBUG_VM 945c97a9e10SNick Piggin /* 946c97a9e10SNick Piggin * The page's anon-rmap details (mapping and index) are guaranteed to 947c97a9e10SNick Piggin * be set up correctly at this point. 948c97a9e10SNick Piggin * 949c97a9e10SNick Piggin * We have exclusion against page_add_anon_rmap because the caller 950c97a9e10SNick Piggin * always holds the page locked, except if called from page_dup_rmap, 951c97a9e10SNick Piggin * in which case the page is already known to be setup. 952c97a9e10SNick Piggin * 953c97a9e10SNick Piggin * We have exclusion against page_add_new_anon_rmap because those pages 954c97a9e10SNick Piggin * are initially only visible via the pagetables, and the pte is locked 955c97a9e10SNick Piggin * over the call to page_add_new_anon_rmap. 956c97a9e10SNick Piggin */ 95744ab57a0SAndrea Arcangeli BUG_ON(page_anon_vma(page)->root != vma->anon_vma->root); 958c97a9e10SNick Piggin BUG_ON(page->index != linear_page_index(vma, address)); 959c97a9e10SNick Piggin #endif 960c97a9e10SNick Piggin } 961c97a9e10SNick Piggin 962c97a9e10SNick Piggin /** 9639617d95eSNick Piggin * page_add_anon_rmap - add pte mapping to an anonymous page 9649617d95eSNick Piggin * @page: the page to add the mapping to 9659617d95eSNick Piggin * @vma: the vm area in which the mapping is added 9669617d95eSNick Piggin * @address: the user virtual address mapped 9679617d95eSNick Piggin * 9685ad64688SHugh Dickins * The caller needs to hold the pte lock, and the page must be locked in 96980e14822SHugh Dickins * the anon_vma case: to serialize mapping,index checking after setting, 97080e14822SHugh Dickins * and to ensure that PageAnon is not being upgraded racily to PageKsm 97180e14822SHugh Dickins * (but PageKsm is never downgraded to PageAnon). 9729617d95eSNick Piggin */ 9739617d95eSNick Piggin void page_add_anon_rmap(struct page *page, 9749617d95eSNick Piggin struct vm_area_struct *vma, unsigned long address) 9759617d95eSNick Piggin { 976ad8c2ee8SRik van Riel do_page_add_anon_rmap(page, vma, address, 0); 977ad8c2ee8SRik van Riel } 978ad8c2ee8SRik van Riel 979ad8c2ee8SRik van Riel /* 980ad8c2ee8SRik van Riel * Special version of the above for do_swap_page, which often runs 981ad8c2ee8SRik van Riel * into pages that are exclusively owned by the current process. 982ad8c2ee8SRik van Riel * Everybody else should continue to use page_add_anon_rmap above. 983ad8c2ee8SRik van Riel */ 984ad8c2ee8SRik van Riel void do_page_add_anon_rmap(struct page *page, 985ad8c2ee8SRik van Riel struct vm_area_struct *vma, unsigned long address, int exclusive) 986ad8c2ee8SRik van Riel { 9875ad64688SHugh Dickins int first = atomic_inc_and_test(&page->_mapcount); 98879134171SAndrea Arcangeli if (first) { 9893cd14fcdSKirill A. Shutemov if (PageTransHuge(page)) 99079134171SAndrea Arcangeli __inc_zone_page_state(page, 99179134171SAndrea Arcangeli NR_ANON_TRANSPARENT_HUGEPAGES); 9923cd14fcdSKirill A. Shutemov __mod_zone_page_state(page_zone(page), NR_ANON_PAGES, 9933cd14fcdSKirill A. Shutemov hpage_nr_pages(page)); 99479134171SAndrea Arcangeli } 9955ad64688SHugh Dickins if (unlikely(PageKsm(page))) 9965ad64688SHugh Dickins return; 9975ad64688SHugh Dickins 998c97a9e10SNick Piggin VM_BUG_ON(!PageLocked(page)); 9995dbe0af4SHugh Dickins /* address might be in next vma when migration races vma_adjust */ 10005ad64688SHugh Dickins if (first) 1001ad8c2ee8SRik van Riel __page_set_anon_rmap(page, vma, address, exclusive); 100269029cd5SKAMEZAWA Hiroyuki else 1003c97a9e10SNick Piggin __page_check_anon_rmap(page, vma, address); 10041da177e4SLinus Torvalds } 10051da177e4SLinus Torvalds 100643d8eac4SRandy Dunlap /** 10079617d95eSNick Piggin * page_add_new_anon_rmap - add pte mapping to a new anonymous page 10089617d95eSNick Piggin * @page: the page to add the mapping to 10099617d95eSNick Piggin * @vma: the vm area in which the mapping is added 10109617d95eSNick Piggin * @address: the user virtual address mapped 10119617d95eSNick Piggin * 10129617d95eSNick Piggin * Same as page_add_anon_rmap but must only be called on *new* pages. 10139617d95eSNick Piggin * This means the inc-and-test can be bypassed. 1014c97a9e10SNick Piggin * Page does not have to be locked. 10159617d95eSNick Piggin */ 10169617d95eSNick Piggin void page_add_new_anon_rmap(struct page *page, 10179617d95eSNick Piggin struct vm_area_struct *vma, unsigned long address) 10189617d95eSNick Piggin { 1019b5934c53SHugh Dickins VM_BUG_ON(address < vma->vm_start || address >= vma->vm_end); 1020cbf84b7aSHugh Dickins SetPageSwapBacked(page); 1021cbf84b7aSHugh Dickins atomic_set(&page->_mapcount, 0); /* increment count (starts at -1) */ 10223cd14fcdSKirill A. Shutemov if (PageTransHuge(page)) 102379134171SAndrea Arcangeli __inc_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES); 10243cd14fcdSKirill A. Shutemov __mod_zone_page_state(page_zone(page), NR_ANON_PAGES, 10253cd14fcdSKirill A. Shutemov hpage_nr_pages(page)); 1026e8a03febSRik van Riel __page_set_anon_rmap(page, vma, address, 1); 1027c53954a0SMel Gorman if (!mlocked_vma_newpage(vma, page)) { 1028c53954a0SMel Gorman SetPageActive(page); 1029c53954a0SMel Gorman lru_cache_add(page); 1030c53954a0SMel Gorman } else 1031b5934c53SHugh Dickins add_page_to_unevictable_list(page); 10329617d95eSNick Piggin } 10339617d95eSNick Piggin 10341da177e4SLinus Torvalds /** 10351da177e4SLinus Torvalds * page_add_file_rmap - add pte mapping to a file page 10361da177e4SLinus Torvalds * @page: the page to add the mapping to 10371da177e4SLinus Torvalds * 1038b8072f09SHugh Dickins * The caller needs to hold the pte lock. 10391da177e4SLinus Torvalds */ 10401da177e4SLinus Torvalds void page_add_file_rmap(struct page *page) 10411da177e4SLinus Torvalds { 104289c06bd5SKAMEZAWA Hiroyuki bool locked; 104389c06bd5SKAMEZAWA Hiroyuki unsigned long flags; 104489c06bd5SKAMEZAWA Hiroyuki 104589c06bd5SKAMEZAWA Hiroyuki mem_cgroup_begin_update_page_stat(page, &locked, &flags); 1046d69b042fSBalbir Singh if (atomic_inc_and_test(&page->_mapcount)) { 104765ba55f5SChristoph Lameter __inc_zone_page_state(page, NR_FILE_MAPPED); 104868b4876dSSha Zhengju mem_cgroup_inc_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED); 1049d69b042fSBalbir Singh } 105089c06bd5SKAMEZAWA Hiroyuki mem_cgroup_end_update_page_stat(page, &locked, &flags); 10511da177e4SLinus Torvalds } 10521da177e4SLinus Torvalds 10531da177e4SLinus Torvalds /** 10541da177e4SLinus Torvalds * page_remove_rmap - take down pte mapping from a page 10551da177e4SLinus Torvalds * @page: page to remove mapping from 10561da177e4SLinus Torvalds * 1057b8072f09SHugh Dickins * The caller needs to hold the pte lock. 10581da177e4SLinus Torvalds */ 1059edc315fdSHugh Dickins void page_remove_rmap(struct page *page) 10601da177e4SLinus Torvalds { 106189c06bd5SKAMEZAWA Hiroyuki bool anon = PageAnon(page); 106289c06bd5SKAMEZAWA Hiroyuki bool locked; 106389c06bd5SKAMEZAWA Hiroyuki unsigned long flags; 106489c06bd5SKAMEZAWA Hiroyuki 106589c06bd5SKAMEZAWA Hiroyuki /* 106689c06bd5SKAMEZAWA Hiroyuki * The anon case has no mem_cgroup page_stat to update; but may 106789c06bd5SKAMEZAWA Hiroyuki * uncharge_page() below, where the lock ordering can deadlock if 106889c06bd5SKAMEZAWA Hiroyuki * we hold the lock against page_stat move: so avoid it on anon. 106989c06bd5SKAMEZAWA Hiroyuki */ 107089c06bd5SKAMEZAWA Hiroyuki if (!anon) 107189c06bd5SKAMEZAWA Hiroyuki mem_cgroup_begin_update_page_stat(page, &locked, &flags); 107289c06bd5SKAMEZAWA Hiroyuki 1073b904dcfeSKOSAKI Motohiro /* page still mapped by someone else? */ 1074b904dcfeSKOSAKI Motohiro if (!atomic_add_negative(-1, &page->_mapcount)) 107589c06bd5SKAMEZAWA Hiroyuki goto out; 1076b904dcfeSKOSAKI Motohiro 10771da177e4SLinus Torvalds /* 10780fe6e20bSNaoya Horiguchi * Hugepages are not counted in NR_ANON_PAGES nor NR_FILE_MAPPED 10790fe6e20bSNaoya Horiguchi * and not charged by memcg for now. 10800fe6e20bSNaoya Horiguchi */ 10810fe6e20bSNaoya Horiguchi if (unlikely(PageHuge(page))) 108289c06bd5SKAMEZAWA Hiroyuki goto out; 108389c06bd5SKAMEZAWA Hiroyuki if (anon) { 108416f8c5b2SHugh Dickins mem_cgroup_uncharge_page(page); 10853cd14fcdSKirill A. Shutemov if (PageTransHuge(page)) 108679134171SAndrea Arcangeli __dec_zone_page_state(page, 108779134171SAndrea Arcangeli NR_ANON_TRANSPARENT_HUGEPAGES); 10883cd14fcdSKirill A. Shutemov __mod_zone_page_state(page_zone(page), NR_ANON_PAGES, 10893cd14fcdSKirill A. Shutemov -hpage_nr_pages(page)); 1090b904dcfeSKOSAKI Motohiro } else { 1091b904dcfeSKOSAKI Motohiro __dec_zone_page_state(page, NR_FILE_MAPPED); 109268b4876dSSha Zhengju mem_cgroup_dec_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED); 1093e6c509f8SHugh Dickins mem_cgroup_end_update_page_stat(page, &locked, &flags); 1094b904dcfeSKOSAKI Motohiro } 1095e6c509f8SHugh Dickins if (unlikely(PageMlocked(page))) 1096e6c509f8SHugh Dickins clear_page_mlock(page); 109716f8c5b2SHugh Dickins /* 10981da177e4SLinus Torvalds * It would be tidy to reset the PageAnon mapping here, 10991da177e4SLinus Torvalds * but that might overwrite a racing page_add_anon_rmap 11001da177e4SLinus Torvalds * which increments mapcount after us but sets mapping 11011da177e4SLinus Torvalds * before us: so leave the reset to free_hot_cold_page, 11021da177e4SLinus Torvalds * and remember that it's only reliable while mapped. 11031da177e4SLinus Torvalds * Leaving it set also helps swapoff to reinstate ptes 11041da177e4SLinus Torvalds * faster for those pages still in swapcache. 11051da177e4SLinus Torvalds */ 1106e6c509f8SHugh Dickins return; 110789c06bd5SKAMEZAWA Hiroyuki out: 110889c06bd5SKAMEZAWA Hiroyuki if (!anon) 110989c06bd5SKAMEZAWA Hiroyuki mem_cgroup_end_update_page_stat(page, &locked, &flags); 11101da177e4SLinus Torvalds } 11111da177e4SLinus Torvalds 11121da177e4SLinus Torvalds /* 111352629506SJoonsoo Kim * @arg: enum ttu_flags will be passed to this argument 11141da177e4SLinus Torvalds */ 11155ad64688SHugh Dickins int try_to_unmap_one(struct page *page, struct vm_area_struct *vma, 111652629506SJoonsoo Kim unsigned long address, void *arg) 11171da177e4SLinus Torvalds { 11181da177e4SLinus Torvalds struct mm_struct *mm = vma->vm_mm; 11191da177e4SLinus Torvalds pte_t *pte; 11201da177e4SLinus Torvalds pte_t pteval; 1121c0718806SHugh Dickins spinlock_t *ptl; 11221da177e4SLinus Torvalds int ret = SWAP_AGAIN; 112352629506SJoonsoo Kim enum ttu_flags flags = (enum ttu_flags)arg; 11241da177e4SLinus Torvalds 1125479db0bfSNick Piggin pte = page_check_address(page, mm, address, &ptl, 0); 1126c0718806SHugh Dickins if (!pte) 112781b4082dSNikita Danilov goto out; 11281da177e4SLinus Torvalds 11291da177e4SLinus Torvalds /* 11301da177e4SLinus Torvalds * If the page is mlock()d, we cannot swap it out. 11311da177e4SLinus Torvalds * If it's recently referenced (perhaps page_referenced 11321da177e4SLinus Torvalds * skipped over this mm) then we should reactivate it. 11331da177e4SLinus Torvalds */ 113414fa31b8SAndi Kleen if (!(flags & TTU_IGNORE_MLOCK)) { 1135caed0f48SKOSAKI Motohiro if (vma->vm_flags & VM_LOCKED) 1136caed0f48SKOSAKI Motohiro goto out_mlock; 1137caed0f48SKOSAKI Motohiro 1138af8e3354SHugh Dickins if (TTU_ACTION(flags) == TTU_MUNLOCK) 113953f79acbSHugh Dickins goto out_unmap; 114014fa31b8SAndi Kleen } 114114fa31b8SAndi Kleen if (!(flags & TTU_IGNORE_ACCESS)) { 1142b291f000SNick Piggin if (ptep_clear_flush_young_notify(vma, address, pte)) { 11431da177e4SLinus Torvalds ret = SWAP_FAIL; 11441da177e4SLinus Torvalds goto out_unmap; 11451da177e4SLinus Torvalds } 1146b291f000SNick Piggin } 11471da177e4SLinus Torvalds 11481da177e4SLinus Torvalds /* Nuke the page table entry. */ 11491da177e4SLinus Torvalds flush_cache_page(vma, address, page_to_pfn(page)); 11502ec74c3eSSagi Grimberg pteval = ptep_clear_flush(vma, address, pte); 11511da177e4SLinus Torvalds 11521da177e4SLinus Torvalds /* Move the dirty bit to the physical page now the pte is gone. */ 11531da177e4SLinus Torvalds if (pte_dirty(pteval)) 11541da177e4SLinus Torvalds set_page_dirty(page); 11551da177e4SLinus Torvalds 1156365e9c87SHugh Dickins /* Update high watermark before we lower rss */ 1157365e9c87SHugh Dickins update_hiwater_rss(mm); 1158365e9c87SHugh Dickins 1159888b9f7cSAndi Kleen if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) { 11605f24ae58SNaoya Horiguchi if (!PageHuge(page)) { 1161888b9f7cSAndi Kleen if (PageAnon(page)) 1162d559db08SKAMEZAWA Hiroyuki dec_mm_counter(mm, MM_ANONPAGES); 1163888b9f7cSAndi Kleen else 1164d559db08SKAMEZAWA Hiroyuki dec_mm_counter(mm, MM_FILEPAGES); 11655f24ae58SNaoya Horiguchi } 1166888b9f7cSAndi Kleen set_pte_at(mm, address, pte, 1167888b9f7cSAndi Kleen swp_entry_to_pte(make_hwpoison_entry(page))); 1168888b9f7cSAndi Kleen } else if (PageAnon(page)) { 11694c21e2f2SHugh Dickins swp_entry_t entry = { .val = page_private(page) }; 1170179ef71cSCyrill Gorcunov pte_t swp_pte; 11710697212aSChristoph Lameter 11720697212aSChristoph Lameter if (PageSwapCache(page)) { 11731da177e4SLinus Torvalds /* 11741da177e4SLinus Torvalds * Store the swap location in the pte. 11751da177e4SLinus Torvalds * See handle_pte_fault() ... 11761da177e4SLinus Torvalds */ 1177570a335bSHugh Dickins if (swap_duplicate(entry) < 0) { 1178570a335bSHugh Dickins set_pte_at(mm, address, pte, pteval); 1179570a335bSHugh Dickins ret = SWAP_FAIL; 1180570a335bSHugh Dickins goto out_unmap; 1181570a335bSHugh Dickins } 11821da177e4SLinus Torvalds if (list_empty(&mm->mmlist)) { 11831da177e4SLinus Torvalds spin_lock(&mmlist_lock); 1184f412ac08SHugh Dickins if (list_empty(&mm->mmlist)) 11851da177e4SLinus Torvalds list_add(&mm->mmlist, &init_mm.mmlist); 11861da177e4SLinus Torvalds spin_unlock(&mmlist_lock); 11871da177e4SLinus Torvalds } 1188d559db08SKAMEZAWA Hiroyuki dec_mm_counter(mm, MM_ANONPAGES); 1189b084d435SKAMEZAWA Hiroyuki inc_mm_counter(mm, MM_SWAPENTS); 1190ce1744f4SKonstantin Khlebnikov } else if (IS_ENABLED(CONFIG_MIGRATION)) { 11910697212aSChristoph Lameter /* 11920697212aSChristoph Lameter * Store the pfn of the page in a special migration 11930697212aSChristoph Lameter * pte. do_swap_page() will wait until the migration 11940697212aSChristoph Lameter * pte is removed and then restart fault handling. 11950697212aSChristoph Lameter */ 119614fa31b8SAndi Kleen BUG_ON(TTU_ACTION(flags) != TTU_MIGRATION); 11970697212aSChristoph Lameter entry = make_migration_entry(page, pte_write(pteval)); 11980697212aSChristoph Lameter } 1199179ef71cSCyrill Gorcunov swp_pte = swp_entry_to_pte(entry); 1200179ef71cSCyrill Gorcunov if (pte_soft_dirty(pteval)) 1201179ef71cSCyrill Gorcunov swp_pte = pte_swp_mksoft_dirty(swp_pte); 1202179ef71cSCyrill Gorcunov set_pte_at(mm, address, pte, swp_pte); 12031da177e4SLinus Torvalds BUG_ON(pte_file(*pte)); 1204ce1744f4SKonstantin Khlebnikov } else if (IS_ENABLED(CONFIG_MIGRATION) && 1205ce1744f4SKonstantin Khlebnikov (TTU_ACTION(flags) == TTU_MIGRATION)) { 120604e62a29SChristoph Lameter /* Establish migration entry for a file page */ 120704e62a29SChristoph Lameter swp_entry_t entry; 120804e62a29SChristoph Lameter entry = make_migration_entry(page, pte_write(pteval)); 120904e62a29SChristoph Lameter set_pte_at(mm, address, pte, swp_entry_to_pte(entry)); 121004e62a29SChristoph Lameter } else 1211d559db08SKAMEZAWA Hiroyuki dec_mm_counter(mm, MM_FILEPAGES); 12121da177e4SLinus Torvalds 1213edc315fdSHugh Dickins page_remove_rmap(page); 12141da177e4SLinus Torvalds page_cache_release(page); 12151da177e4SLinus Torvalds 12161da177e4SLinus Torvalds out_unmap: 1217c0718806SHugh Dickins pte_unmap_unlock(pte, ptl); 12182ec74c3eSSagi Grimberg if (ret != SWAP_FAIL) 12192ec74c3eSSagi Grimberg mmu_notifier_invalidate_page(mm, address); 1220caed0f48SKOSAKI Motohiro out: 1221caed0f48SKOSAKI Motohiro return ret; 122253f79acbSHugh Dickins 1223caed0f48SKOSAKI Motohiro out_mlock: 1224caed0f48SKOSAKI Motohiro pte_unmap_unlock(pte, ptl); 1225caed0f48SKOSAKI Motohiro 1226caed0f48SKOSAKI Motohiro 1227caed0f48SKOSAKI Motohiro /* 1228caed0f48SKOSAKI Motohiro * We need mmap_sem locking, Otherwise VM_LOCKED check makes 1229caed0f48SKOSAKI Motohiro * unstable result and race. Plus, We can't wait here because 12305a505085SIngo Molnar * we now hold anon_vma->rwsem or mapping->i_mmap_mutex. 1231caed0f48SKOSAKI Motohiro * if trylock failed, the page remain in evictable lru and later 1232caed0f48SKOSAKI Motohiro * vmscan could retry to move the page to unevictable lru if the 1233caed0f48SKOSAKI Motohiro * page is actually mlocked. 1234caed0f48SKOSAKI Motohiro */ 123553f79acbSHugh Dickins if (down_read_trylock(&vma->vm_mm->mmap_sem)) { 123653f79acbSHugh Dickins if (vma->vm_flags & VM_LOCKED) { 123753f79acbSHugh Dickins mlock_vma_page(page); 123853f79acbSHugh Dickins ret = SWAP_MLOCK; 123953f79acbSHugh Dickins } 124053f79acbSHugh Dickins up_read(&vma->vm_mm->mmap_sem); 124153f79acbSHugh Dickins } 12421da177e4SLinus Torvalds return ret; 12431da177e4SLinus Torvalds } 12441da177e4SLinus Torvalds 12451da177e4SLinus Torvalds /* 12461da177e4SLinus Torvalds * objrmap doesn't work for nonlinear VMAs because the assumption that 12471da177e4SLinus Torvalds * offset-into-file correlates with offset-into-virtual-addresses does not hold. 12481da177e4SLinus Torvalds * Consequently, given a particular page and its ->index, we cannot locate the 12491da177e4SLinus Torvalds * ptes which are mapping that page without an exhaustive linear search. 12501da177e4SLinus Torvalds * 12511da177e4SLinus Torvalds * So what this code does is a mini "virtual scan" of each nonlinear VMA which 12521da177e4SLinus Torvalds * maps the file to which the target page belongs. The ->vm_private_data field 12531da177e4SLinus Torvalds * holds the current cursor into that scan. Successive searches will circulate 12541da177e4SLinus Torvalds * around the vma's virtual address space. 12551da177e4SLinus Torvalds * 12561da177e4SLinus Torvalds * So as more replacement pressure is applied to the pages in a nonlinear VMA, 12571da177e4SLinus Torvalds * more scanning pressure is placed against them as well. Eventually pages 12581da177e4SLinus Torvalds * will become fully unmapped and are eligible for eviction. 12591da177e4SLinus Torvalds * 12601da177e4SLinus Torvalds * For very sparsely populated VMAs this is a little inefficient - chances are 12611da177e4SLinus Torvalds * there there won't be many ptes located within the scan cluster. In this case 12621da177e4SLinus Torvalds * maybe we could scan further - to the end of the pte page, perhaps. 1263b291f000SNick Piggin * 1264b291f000SNick Piggin * Mlocked pages: check VM_LOCKED under mmap_sem held for read, if we can 1265b291f000SNick Piggin * acquire it without blocking. If vma locked, mlock the pages in the cluster, 1266b291f000SNick Piggin * rather than unmapping them. If we encounter the "check_page" that vmscan is 1267b291f000SNick Piggin * trying to unmap, return SWAP_MLOCK, else default SWAP_AGAIN. 12681da177e4SLinus Torvalds */ 12691da177e4SLinus Torvalds #define CLUSTER_SIZE min(32*PAGE_SIZE, PMD_SIZE) 12701da177e4SLinus Torvalds #define CLUSTER_MASK (~(CLUSTER_SIZE - 1)) 12711da177e4SLinus Torvalds 1272b291f000SNick Piggin static int try_to_unmap_cluster(unsigned long cursor, unsigned int *mapcount, 1273b291f000SNick Piggin struct vm_area_struct *vma, struct page *check_page) 12741da177e4SLinus Torvalds { 12751da177e4SLinus Torvalds struct mm_struct *mm = vma->vm_mm; 12761da177e4SLinus Torvalds pmd_t *pmd; 1277c0718806SHugh Dickins pte_t *pte; 12781da177e4SLinus Torvalds pte_t pteval; 1279c0718806SHugh Dickins spinlock_t *ptl; 12801da177e4SLinus Torvalds struct page *page; 12811da177e4SLinus Torvalds unsigned long address; 12822ec74c3eSSagi Grimberg unsigned long mmun_start; /* For mmu_notifiers */ 12832ec74c3eSSagi Grimberg unsigned long mmun_end; /* For mmu_notifiers */ 12841da177e4SLinus Torvalds unsigned long end; 1285b291f000SNick Piggin int ret = SWAP_AGAIN; 1286b291f000SNick Piggin int locked_vma = 0; 12871da177e4SLinus Torvalds 12881da177e4SLinus Torvalds address = (vma->vm_start + cursor) & CLUSTER_MASK; 12891da177e4SLinus Torvalds end = address + CLUSTER_SIZE; 12901da177e4SLinus Torvalds if (address < vma->vm_start) 12911da177e4SLinus Torvalds address = vma->vm_start; 12921da177e4SLinus Torvalds if (end > vma->vm_end) 12931da177e4SLinus Torvalds end = vma->vm_end; 12941da177e4SLinus Torvalds 12956219049aSBob Liu pmd = mm_find_pmd(mm, address); 12966219049aSBob Liu if (!pmd) 1297b291f000SNick Piggin return ret; 1298b291f000SNick Piggin 12992ec74c3eSSagi Grimberg mmun_start = address; 13002ec74c3eSSagi Grimberg mmun_end = end; 13012ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 13022ec74c3eSSagi Grimberg 1303b291f000SNick Piggin /* 1304af8e3354SHugh Dickins * If we can acquire the mmap_sem for read, and vma is VM_LOCKED, 1305b291f000SNick Piggin * keep the sem while scanning the cluster for mlocking pages. 1306b291f000SNick Piggin */ 1307af8e3354SHugh Dickins if (down_read_trylock(&vma->vm_mm->mmap_sem)) { 1308b291f000SNick Piggin locked_vma = (vma->vm_flags & VM_LOCKED); 1309b291f000SNick Piggin if (!locked_vma) 1310b291f000SNick Piggin up_read(&vma->vm_mm->mmap_sem); /* don't need it */ 1311b291f000SNick Piggin } 1312c0718806SHugh Dickins 1313c0718806SHugh Dickins pte = pte_offset_map_lock(mm, pmd, address, &ptl); 13141da177e4SLinus Torvalds 1315365e9c87SHugh Dickins /* Update high watermark before we lower rss */ 1316365e9c87SHugh Dickins update_hiwater_rss(mm); 1317365e9c87SHugh Dickins 1318c0718806SHugh Dickins for (; address < end; pte++, address += PAGE_SIZE) { 13191da177e4SLinus Torvalds if (!pte_present(*pte)) 13201da177e4SLinus Torvalds continue; 13216aab341eSLinus Torvalds page = vm_normal_page(vma, address, *pte); 13226aab341eSLinus Torvalds BUG_ON(!page || PageAnon(page)); 13231da177e4SLinus Torvalds 1324b291f000SNick Piggin if (locked_vma) { 1325b291f000SNick Piggin mlock_vma_page(page); /* no-op if already mlocked */ 1326b291f000SNick Piggin if (page == check_page) 1327b291f000SNick Piggin ret = SWAP_MLOCK; 1328b291f000SNick Piggin continue; /* don't unmap */ 1329b291f000SNick Piggin } 1330b291f000SNick Piggin 1331cddb8a5cSAndrea Arcangeli if (ptep_clear_flush_young_notify(vma, address, pte)) 13321da177e4SLinus Torvalds continue; 13331da177e4SLinus Torvalds 13341da177e4SLinus Torvalds /* Nuke the page table entry. */ 1335eca35133SBen Collins flush_cache_page(vma, address, pte_pfn(*pte)); 13362ec74c3eSSagi Grimberg pteval = ptep_clear_flush(vma, address, pte); 13371da177e4SLinus Torvalds 13381da177e4SLinus Torvalds /* If nonlinear, store the file page offset in the pte. */ 133941bb3476SCyrill Gorcunov if (page->index != linear_page_index(vma, address)) { 134041bb3476SCyrill Gorcunov pte_t ptfile = pgoff_to_pte(page->index); 134141bb3476SCyrill Gorcunov if (pte_soft_dirty(pteval)) 134241bb3476SCyrill Gorcunov pte_file_mksoft_dirty(ptfile); 134341bb3476SCyrill Gorcunov set_pte_at(mm, address, pte, ptfile); 134441bb3476SCyrill Gorcunov } 13451da177e4SLinus Torvalds 13461da177e4SLinus Torvalds /* Move the dirty bit to the physical page now the pte is gone. */ 13471da177e4SLinus Torvalds if (pte_dirty(pteval)) 13481da177e4SLinus Torvalds set_page_dirty(page); 13491da177e4SLinus Torvalds 1350edc315fdSHugh Dickins page_remove_rmap(page); 13511da177e4SLinus Torvalds page_cache_release(page); 1352d559db08SKAMEZAWA Hiroyuki dec_mm_counter(mm, MM_FILEPAGES); 13531da177e4SLinus Torvalds (*mapcount)--; 13541da177e4SLinus Torvalds } 1355c0718806SHugh Dickins pte_unmap_unlock(pte - 1, ptl); 13562ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 1357b291f000SNick Piggin if (locked_vma) 1358b291f000SNick Piggin up_read(&vma->vm_mm->mmap_sem); 1359b291f000SNick Piggin return ret; 13601da177e4SLinus Torvalds } 13611da177e4SLinus Torvalds 13620f843c6aSJoonsoo Kim static int try_to_unmap_nonlinear(struct page *page, 13630f843c6aSJoonsoo Kim struct address_space *mapping, struct vm_area_struct *vma) 13640f843c6aSJoonsoo Kim { 13650f843c6aSJoonsoo Kim int ret = SWAP_AGAIN; 13660f843c6aSJoonsoo Kim unsigned long cursor; 13670f843c6aSJoonsoo Kim unsigned long max_nl_cursor = 0; 13680f843c6aSJoonsoo Kim unsigned long max_nl_size = 0; 13690f843c6aSJoonsoo Kim unsigned int mapcount; 13700f843c6aSJoonsoo Kim 13710f843c6aSJoonsoo Kim list_for_each_entry(vma, 13720f843c6aSJoonsoo Kim &mapping->i_mmap_nonlinear, shared.nonlinear) { 13730f843c6aSJoonsoo Kim 13740f843c6aSJoonsoo Kim cursor = (unsigned long) vma->vm_private_data; 13750f843c6aSJoonsoo Kim if (cursor > max_nl_cursor) 13760f843c6aSJoonsoo Kim max_nl_cursor = cursor; 13770f843c6aSJoonsoo Kim cursor = vma->vm_end - vma->vm_start; 13780f843c6aSJoonsoo Kim if (cursor > max_nl_size) 13790f843c6aSJoonsoo Kim max_nl_size = cursor; 13800f843c6aSJoonsoo Kim } 13810f843c6aSJoonsoo Kim 13820f843c6aSJoonsoo Kim if (max_nl_size == 0) { /* all nonlinears locked or reserved ? */ 13830f843c6aSJoonsoo Kim return SWAP_FAIL; 13840f843c6aSJoonsoo Kim } 13850f843c6aSJoonsoo Kim 13860f843c6aSJoonsoo Kim /* 13870f843c6aSJoonsoo Kim * We don't try to search for this page in the nonlinear vmas, 13880f843c6aSJoonsoo Kim * and page_referenced wouldn't have found it anyway. Instead 13890f843c6aSJoonsoo Kim * just walk the nonlinear vmas trying to age and unmap some. 13900f843c6aSJoonsoo Kim * The mapcount of the page we came in with is irrelevant, 13910f843c6aSJoonsoo Kim * but even so use it as a guide to how hard we should try? 13920f843c6aSJoonsoo Kim */ 13930f843c6aSJoonsoo Kim mapcount = page_mapcount(page); 13940f843c6aSJoonsoo Kim if (!mapcount) 13950f843c6aSJoonsoo Kim return ret; 13960f843c6aSJoonsoo Kim 13970f843c6aSJoonsoo Kim cond_resched(); 13980f843c6aSJoonsoo Kim 13990f843c6aSJoonsoo Kim max_nl_size = (max_nl_size + CLUSTER_SIZE - 1) & CLUSTER_MASK; 14000f843c6aSJoonsoo Kim if (max_nl_cursor == 0) 14010f843c6aSJoonsoo Kim max_nl_cursor = CLUSTER_SIZE; 14020f843c6aSJoonsoo Kim 14030f843c6aSJoonsoo Kim do { 14040f843c6aSJoonsoo Kim list_for_each_entry(vma, 14050f843c6aSJoonsoo Kim &mapping->i_mmap_nonlinear, shared.nonlinear) { 14060f843c6aSJoonsoo Kim 14070f843c6aSJoonsoo Kim cursor = (unsigned long) vma->vm_private_data; 14080f843c6aSJoonsoo Kim while (cursor < max_nl_cursor && 14090f843c6aSJoonsoo Kim cursor < vma->vm_end - vma->vm_start) { 14100f843c6aSJoonsoo Kim if (try_to_unmap_cluster(cursor, &mapcount, 14110f843c6aSJoonsoo Kim vma, page) == SWAP_MLOCK) 14120f843c6aSJoonsoo Kim ret = SWAP_MLOCK; 14130f843c6aSJoonsoo Kim cursor += CLUSTER_SIZE; 14140f843c6aSJoonsoo Kim vma->vm_private_data = (void *) cursor; 14150f843c6aSJoonsoo Kim if ((int)mapcount <= 0) 14160f843c6aSJoonsoo Kim return ret; 14170f843c6aSJoonsoo Kim } 14180f843c6aSJoonsoo Kim vma->vm_private_data = (void *) max_nl_cursor; 14190f843c6aSJoonsoo Kim } 14200f843c6aSJoonsoo Kim cond_resched(); 14210f843c6aSJoonsoo Kim max_nl_cursor += CLUSTER_SIZE; 14220f843c6aSJoonsoo Kim } while (max_nl_cursor <= max_nl_size); 14230f843c6aSJoonsoo Kim 14240f843c6aSJoonsoo Kim /* 14250f843c6aSJoonsoo Kim * Don't loop forever (perhaps all the remaining pages are 14260f843c6aSJoonsoo Kim * in locked vmas). Reset cursor on all unreserved nonlinear 14270f843c6aSJoonsoo Kim * vmas, now forgetting on which ones it had fallen behind. 14280f843c6aSJoonsoo Kim */ 14290f843c6aSJoonsoo Kim list_for_each_entry(vma, &mapping->i_mmap_nonlinear, shared.nonlinear) 14300f843c6aSJoonsoo Kim vma->vm_private_data = NULL; 14310f843c6aSJoonsoo Kim 14320f843c6aSJoonsoo Kim return ret; 14330f843c6aSJoonsoo Kim } 14340f843c6aSJoonsoo Kim 143571e3aac0SAndrea Arcangeli bool is_vma_temporary_stack(struct vm_area_struct *vma) 1436a8bef8ffSMel Gorman { 1437a8bef8ffSMel Gorman int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP); 1438a8bef8ffSMel Gorman 1439a8bef8ffSMel Gorman if (!maybe_stack) 1440a8bef8ffSMel Gorman return false; 1441a8bef8ffSMel Gorman 1442a8bef8ffSMel Gorman if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == 1443a8bef8ffSMel Gorman VM_STACK_INCOMPLETE_SETUP) 1444a8bef8ffSMel Gorman return true; 1445a8bef8ffSMel Gorman 1446a8bef8ffSMel Gorman return false; 1447a8bef8ffSMel Gorman } 1448a8bef8ffSMel Gorman 144952629506SJoonsoo Kim static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg) 145052629506SJoonsoo Kim { 145152629506SJoonsoo Kim return is_vma_temporary_stack(vma); 145252629506SJoonsoo Kim } 145352629506SJoonsoo Kim 145452629506SJoonsoo Kim static int page_not_mapped(struct page *page) 145552629506SJoonsoo Kim { 145652629506SJoonsoo Kim return !page_mapped(page); 145752629506SJoonsoo Kim }; 145852629506SJoonsoo Kim 14591da177e4SLinus Torvalds /** 14601da177e4SLinus Torvalds * try_to_unmap - try to remove all page table mappings to a page 14611da177e4SLinus Torvalds * @page: the page to get unmapped 146214fa31b8SAndi Kleen * @flags: action and flags 14631da177e4SLinus Torvalds * 14641da177e4SLinus Torvalds * Tries to remove all the page table entries which are mapping this 14651da177e4SLinus Torvalds * page, used in the pageout path. Caller must hold the page lock. 14661da177e4SLinus Torvalds * Return values are: 14671da177e4SLinus Torvalds * 14681da177e4SLinus Torvalds * SWAP_SUCCESS - we succeeded in removing all mappings 14691da177e4SLinus Torvalds * SWAP_AGAIN - we missed a mapping, try again later 14701da177e4SLinus Torvalds * SWAP_FAIL - the page is unswappable 1471b291f000SNick Piggin * SWAP_MLOCK - page is mlocked. 14721da177e4SLinus Torvalds */ 147314fa31b8SAndi Kleen int try_to_unmap(struct page *page, enum ttu_flags flags) 14741da177e4SLinus Torvalds { 14751da177e4SLinus Torvalds int ret; 147652629506SJoonsoo Kim struct rmap_walk_control rwc = { 147752629506SJoonsoo Kim .rmap_one = try_to_unmap_one, 147852629506SJoonsoo Kim .arg = (void *)flags, 147952629506SJoonsoo Kim .done = page_not_mapped, 148052629506SJoonsoo Kim .file_nonlinear = try_to_unmap_nonlinear, 148152629506SJoonsoo Kim .anon_lock = page_lock_anon_vma_read, 148252629506SJoonsoo Kim }; 14831da177e4SLinus Torvalds 148491600e9eSAndrea Arcangeli VM_BUG_ON(!PageHuge(page) && PageTransHuge(page)); 14851da177e4SLinus Torvalds 148652629506SJoonsoo Kim /* 148752629506SJoonsoo Kim * During exec, a temporary VMA is setup and later moved. 148852629506SJoonsoo Kim * The VMA is moved under the anon_vma lock but not the 148952629506SJoonsoo Kim * page tables leading to a race where migration cannot 149052629506SJoonsoo Kim * find the migration ptes. Rather than increasing the 149152629506SJoonsoo Kim * locking requirements of exec(), migration skips 149252629506SJoonsoo Kim * temporary VMAs until after exec() completes. 149352629506SJoonsoo Kim */ 149452629506SJoonsoo Kim if (flags & TTU_MIGRATION && !PageKsm(page) && PageAnon(page)) 149552629506SJoonsoo Kim rwc.invalid_vma = invalid_migration_vma; 149652629506SJoonsoo Kim 149752629506SJoonsoo Kim ret = rmap_walk(page, &rwc); 149852629506SJoonsoo Kim 1499b291f000SNick Piggin if (ret != SWAP_MLOCK && !page_mapped(page)) 15001da177e4SLinus Torvalds ret = SWAP_SUCCESS; 15011da177e4SLinus Torvalds return ret; 15021da177e4SLinus Torvalds } 150381b4082dSNikita Danilov 1504b291f000SNick Piggin /** 1505b291f000SNick Piggin * try_to_munlock - try to munlock a page 1506b291f000SNick Piggin * @page: the page to be munlocked 1507b291f000SNick Piggin * 1508b291f000SNick Piggin * Called from munlock code. Checks all of the VMAs mapping the page 1509b291f000SNick Piggin * to make sure nobody else has this page mlocked. The page will be 1510b291f000SNick Piggin * returned with PG_mlocked cleared if no other vmas have it mlocked. 1511b291f000SNick Piggin * 1512b291f000SNick Piggin * Return values are: 1513b291f000SNick Piggin * 151453f79acbSHugh Dickins * SWAP_AGAIN - no vma is holding page mlocked, or, 1515b291f000SNick Piggin * SWAP_AGAIN - page mapped in mlocked vma -- couldn't acquire mmap sem 15165ad64688SHugh Dickins * SWAP_FAIL - page cannot be located at present 1517b291f000SNick Piggin * SWAP_MLOCK - page is now mlocked. 1518b291f000SNick Piggin */ 1519b291f000SNick Piggin int try_to_munlock(struct page *page) 1520b291f000SNick Piggin { 1521e8351ac9SJoonsoo Kim int ret; 1522e8351ac9SJoonsoo Kim struct rmap_walk_control rwc = { 1523e8351ac9SJoonsoo Kim .rmap_one = try_to_unmap_one, 1524e8351ac9SJoonsoo Kim .arg = (void *)TTU_MUNLOCK, 1525e8351ac9SJoonsoo Kim .done = page_not_mapped, 1526e8351ac9SJoonsoo Kim /* 1527e8351ac9SJoonsoo Kim * We don't bother to try to find the munlocked page in 1528e8351ac9SJoonsoo Kim * nonlinears. It's costly. Instead, later, page reclaim logic 1529e8351ac9SJoonsoo Kim * may call try_to_unmap() and recover PG_mlocked lazily. 1530e8351ac9SJoonsoo Kim */ 1531e8351ac9SJoonsoo Kim .file_nonlinear = NULL, 1532e8351ac9SJoonsoo Kim .anon_lock = page_lock_anon_vma_read, 1533e8351ac9SJoonsoo Kim 1534e8351ac9SJoonsoo Kim }; 1535e8351ac9SJoonsoo Kim 1536b291f000SNick Piggin VM_BUG_ON(!PageLocked(page) || PageLRU(page)); 1537b291f000SNick Piggin 1538e8351ac9SJoonsoo Kim ret = rmap_walk(page, &rwc); 1539e8351ac9SJoonsoo Kim return ret; 1540b291f000SNick Piggin } 1541e9995ef9SHugh Dickins 154201d8b20dSPeter Zijlstra void __put_anon_vma(struct anon_vma *anon_vma) 154376545066SRik van Riel { 154476545066SRik van Riel struct anon_vma *root = anon_vma->root; 154576545066SRik van Riel 154601d8b20dSPeter Zijlstra if (root != anon_vma && atomic_dec_and_test(&root->refcount)) 154776545066SRik van Riel anon_vma_free(root); 154801d8b20dSPeter Zijlstra 154901d8b20dSPeter Zijlstra anon_vma_free(anon_vma); 155076545066SRik van Riel } 155176545066SRik van Riel 15520dd1c7bbSJoonsoo Kim static struct anon_vma *rmap_walk_anon_lock(struct page *page, 15530dd1c7bbSJoonsoo Kim struct rmap_walk_control *rwc) 1554faecd8ddSJoonsoo Kim { 1555faecd8ddSJoonsoo Kim struct anon_vma *anon_vma; 1556faecd8ddSJoonsoo Kim 15570dd1c7bbSJoonsoo Kim if (rwc->anon_lock) 15580dd1c7bbSJoonsoo Kim return rwc->anon_lock(page); 15590dd1c7bbSJoonsoo Kim 1560faecd8ddSJoonsoo Kim /* 1561faecd8ddSJoonsoo Kim * Note: remove_migration_ptes() cannot use page_lock_anon_vma_read() 1562faecd8ddSJoonsoo Kim * because that depends on page_mapped(); but not all its usages 1563faecd8ddSJoonsoo Kim * are holding mmap_sem. Users without mmap_sem are required to 1564faecd8ddSJoonsoo Kim * take a reference count to prevent the anon_vma disappearing 1565faecd8ddSJoonsoo Kim */ 1566faecd8ddSJoonsoo Kim anon_vma = page_anon_vma(page); 1567faecd8ddSJoonsoo Kim if (!anon_vma) 1568faecd8ddSJoonsoo Kim return NULL; 1569faecd8ddSJoonsoo Kim 1570faecd8ddSJoonsoo Kim anon_vma_lock_read(anon_vma); 1571faecd8ddSJoonsoo Kim return anon_vma; 1572faecd8ddSJoonsoo Kim } 1573faecd8ddSJoonsoo Kim 1574e9995ef9SHugh Dickins /* 1575e8351ac9SJoonsoo Kim * rmap_walk_anon - do something to anonymous page using the object-based 1576e8351ac9SJoonsoo Kim * rmap method 1577e8351ac9SJoonsoo Kim * @page: the page to be handled 1578e8351ac9SJoonsoo Kim * @rwc: control variable according to each walk type 1579e8351ac9SJoonsoo Kim * 1580e8351ac9SJoonsoo Kim * Find all the mappings of a page using the mapping pointer and the vma chains 1581e8351ac9SJoonsoo Kim * contained in the anon_vma struct it points to. 1582e8351ac9SJoonsoo Kim * 1583e8351ac9SJoonsoo Kim * When called from try_to_munlock(), the mmap_sem of the mm containing the vma 1584e8351ac9SJoonsoo Kim * where the page was found will be held for write. So, we won't recheck 1585e8351ac9SJoonsoo Kim * vm_flags for that VMA. That should be OK, because that vma shouldn't be 1586e8351ac9SJoonsoo Kim * LOCKED. 1587e9995ef9SHugh Dickins */ 1588051ac83aSJoonsoo Kim static int rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc) 1589e9995ef9SHugh Dickins { 1590e9995ef9SHugh Dickins struct anon_vma *anon_vma; 1591bf181b9fSMichel Lespinasse pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); 15925beb4930SRik van Riel struct anon_vma_chain *avc; 1593e9995ef9SHugh Dickins int ret = SWAP_AGAIN; 1594e9995ef9SHugh Dickins 15950dd1c7bbSJoonsoo Kim anon_vma = rmap_walk_anon_lock(page, rwc); 1596e9995ef9SHugh Dickins if (!anon_vma) 1597e9995ef9SHugh Dickins return ret; 1598faecd8ddSJoonsoo Kim 1599bf181b9fSMichel Lespinasse anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, pgoff) { 16005beb4930SRik van Riel struct vm_area_struct *vma = avc->vma; 1601e9995ef9SHugh Dickins unsigned long address = vma_address(page, vma); 16020dd1c7bbSJoonsoo Kim 16030dd1c7bbSJoonsoo Kim if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) 16040dd1c7bbSJoonsoo Kim continue; 16050dd1c7bbSJoonsoo Kim 1606051ac83aSJoonsoo Kim ret = rwc->rmap_one(page, vma, address, rwc->arg); 1607e9995ef9SHugh Dickins if (ret != SWAP_AGAIN) 1608e9995ef9SHugh Dickins break; 16090dd1c7bbSJoonsoo Kim if (rwc->done && rwc->done(page)) 16100dd1c7bbSJoonsoo Kim break; 1611e9995ef9SHugh Dickins } 16124fc3f1d6SIngo Molnar anon_vma_unlock_read(anon_vma); 1613e9995ef9SHugh Dickins return ret; 1614e9995ef9SHugh Dickins } 1615e9995ef9SHugh Dickins 1616e8351ac9SJoonsoo Kim /* 1617e8351ac9SJoonsoo Kim * rmap_walk_file - do something to file page using the object-based rmap method 1618e8351ac9SJoonsoo Kim * @page: the page to be handled 1619e8351ac9SJoonsoo Kim * @rwc: control variable according to each walk type 1620e8351ac9SJoonsoo Kim * 1621e8351ac9SJoonsoo Kim * Find all the mappings of a page using the mapping pointer and the vma chains 1622e8351ac9SJoonsoo Kim * contained in the address_space struct it points to. 1623e8351ac9SJoonsoo Kim * 1624e8351ac9SJoonsoo Kim * When called from try_to_munlock(), the mmap_sem of the mm containing the vma 1625e8351ac9SJoonsoo Kim * where the page was found will be held for write. So, we won't recheck 1626e8351ac9SJoonsoo Kim * vm_flags for that VMA. That should be OK, because that vma shouldn't be 1627e8351ac9SJoonsoo Kim * LOCKED. 1628e8351ac9SJoonsoo Kim */ 1629051ac83aSJoonsoo Kim static int rmap_walk_file(struct page *page, struct rmap_walk_control *rwc) 1630e9995ef9SHugh Dickins { 1631e9995ef9SHugh Dickins struct address_space *mapping = page->mapping; 1632b854f711SJoonsoo Kim pgoff_t pgoff = page->index << compound_order(page); 1633e9995ef9SHugh Dickins struct vm_area_struct *vma; 1634e9995ef9SHugh Dickins int ret = SWAP_AGAIN; 1635e9995ef9SHugh Dickins 16369f32624bSJoonsoo Kim /* 16379f32624bSJoonsoo Kim * The page lock not only makes sure that page->mapping cannot 16389f32624bSJoonsoo Kim * suddenly be NULLified by truncation, it makes sure that the 16399f32624bSJoonsoo Kim * structure at mapping cannot be freed and reused yet, 16409f32624bSJoonsoo Kim * so we can safely take mapping->i_mmap_mutex. 16419f32624bSJoonsoo Kim */ 16429f32624bSJoonsoo Kim VM_BUG_ON(!PageLocked(page)); 16439f32624bSJoonsoo Kim 1644e9995ef9SHugh Dickins if (!mapping) 1645e9995ef9SHugh Dickins return ret; 16463d48ae45SPeter Zijlstra mutex_lock(&mapping->i_mmap_mutex); 16476b2dbba8SMichel Lespinasse vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { 1648e9995ef9SHugh Dickins unsigned long address = vma_address(page, vma); 16490dd1c7bbSJoonsoo Kim 16500dd1c7bbSJoonsoo Kim if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) 16510dd1c7bbSJoonsoo Kim continue; 16520dd1c7bbSJoonsoo Kim 1653051ac83aSJoonsoo Kim ret = rwc->rmap_one(page, vma, address, rwc->arg); 1654e9995ef9SHugh Dickins if (ret != SWAP_AGAIN) 16550dd1c7bbSJoonsoo Kim goto done; 16560dd1c7bbSJoonsoo Kim if (rwc->done && rwc->done(page)) 16570dd1c7bbSJoonsoo Kim goto done; 1658e9995ef9SHugh Dickins } 16590dd1c7bbSJoonsoo Kim 16600dd1c7bbSJoonsoo Kim if (!rwc->file_nonlinear) 16610dd1c7bbSJoonsoo Kim goto done; 16620dd1c7bbSJoonsoo Kim 16630dd1c7bbSJoonsoo Kim if (list_empty(&mapping->i_mmap_nonlinear)) 16640dd1c7bbSJoonsoo Kim goto done; 16650dd1c7bbSJoonsoo Kim 16660dd1c7bbSJoonsoo Kim ret = rwc->file_nonlinear(page, mapping, vma); 16670dd1c7bbSJoonsoo Kim 16680dd1c7bbSJoonsoo Kim done: 16693d48ae45SPeter Zijlstra mutex_unlock(&mapping->i_mmap_mutex); 1670e9995ef9SHugh Dickins return ret; 1671e9995ef9SHugh Dickins } 1672e9995ef9SHugh Dickins 1673051ac83aSJoonsoo Kim int rmap_walk(struct page *page, struct rmap_walk_control *rwc) 1674e9995ef9SHugh Dickins { 1675e9995ef9SHugh Dickins if (unlikely(PageKsm(page))) 1676051ac83aSJoonsoo Kim return rmap_walk_ksm(page, rwc); 1677e9995ef9SHugh Dickins else if (PageAnon(page)) 1678051ac83aSJoonsoo Kim return rmap_walk_anon(page, rwc); 1679e9995ef9SHugh Dickins else 1680051ac83aSJoonsoo Kim return rmap_walk_file(page, rwc); 1681e9995ef9SHugh Dickins } 16820fe6e20bSNaoya Horiguchi 1683e3390f67SNaoya Horiguchi #ifdef CONFIG_HUGETLB_PAGE 16840fe6e20bSNaoya Horiguchi /* 16850fe6e20bSNaoya Horiguchi * The following three functions are for anonymous (private mapped) hugepages. 16860fe6e20bSNaoya Horiguchi * Unlike common anonymous pages, anonymous hugepages have no accounting code 16870fe6e20bSNaoya Horiguchi * and no lru code, because we handle hugepages differently from common pages. 16880fe6e20bSNaoya Horiguchi */ 16890fe6e20bSNaoya Horiguchi static void __hugepage_set_anon_rmap(struct page *page, 16900fe6e20bSNaoya Horiguchi struct vm_area_struct *vma, unsigned long address, int exclusive) 16910fe6e20bSNaoya Horiguchi { 16920fe6e20bSNaoya Horiguchi struct anon_vma *anon_vma = vma->anon_vma; 1693433abed6SNaoya Horiguchi 16940fe6e20bSNaoya Horiguchi BUG_ON(!anon_vma); 1695433abed6SNaoya Horiguchi 1696433abed6SNaoya Horiguchi if (PageAnon(page)) 1697433abed6SNaoya Horiguchi return; 1698433abed6SNaoya Horiguchi if (!exclusive) 1699433abed6SNaoya Horiguchi anon_vma = anon_vma->root; 1700433abed6SNaoya Horiguchi 17010fe6e20bSNaoya Horiguchi anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 17020fe6e20bSNaoya Horiguchi page->mapping = (struct address_space *) anon_vma; 17030fe6e20bSNaoya Horiguchi page->index = linear_page_index(vma, address); 17040fe6e20bSNaoya Horiguchi } 17050fe6e20bSNaoya Horiguchi 17060fe6e20bSNaoya Horiguchi void hugepage_add_anon_rmap(struct page *page, 17070fe6e20bSNaoya Horiguchi struct vm_area_struct *vma, unsigned long address) 17080fe6e20bSNaoya Horiguchi { 17090fe6e20bSNaoya Horiguchi struct anon_vma *anon_vma = vma->anon_vma; 17100fe6e20bSNaoya Horiguchi int first; 1711a850ea30SNaoya Horiguchi 1712a850ea30SNaoya Horiguchi BUG_ON(!PageLocked(page)); 17130fe6e20bSNaoya Horiguchi BUG_ON(!anon_vma); 17145dbe0af4SHugh Dickins /* address might be in next vma when migration races vma_adjust */ 17150fe6e20bSNaoya Horiguchi first = atomic_inc_and_test(&page->_mapcount); 17160fe6e20bSNaoya Horiguchi if (first) 17170fe6e20bSNaoya Horiguchi __hugepage_set_anon_rmap(page, vma, address, 0); 17180fe6e20bSNaoya Horiguchi } 17190fe6e20bSNaoya Horiguchi 17200fe6e20bSNaoya Horiguchi void hugepage_add_new_anon_rmap(struct page *page, 17210fe6e20bSNaoya Horiguchi struct vm_area_struct *vma, unsigned long address) 17220fe6e20bSNaoya Horiguchi { 17230fe6e20bSNaoya Horiguchi BUG_ON(address < vma->vm_start || address >= vma->vm_end); 17240fe6e20bSNaoya Horiguchi atomic_set(&page->_mapcount, 0); 17250fe6e20bSNaoya Horiguchi __hugepage_set_anon_rmap(page, vma, address, 1); 17260fe6e20bSNaoya Horiguchi } 1727e3390f67SNaoya Horiguchi #endif /* CONFIG_HUGETLB_PAGE */ 1728