1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * linux/mm/swap.c 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 61da177e4SLinus Torvalds */ 71da177e4SLinus Torvalds 81da177e4SLinus Torvalds /* 9183ff22bSSimon Arlott * This file contains the default values for the operation of the 101da177e4SLinus Torvalds * Linux VM subsystem. Fine-tuning documentation can be found in 1157043247SMauro Carvalho Chehab * Documentation/admin-guide/sysctl/vm.rst. 121da177e4SLinus Torvalds * Started 18.12.91 131da177e4SLinus Torvalds * Swap aging added 23.2.95, Stephen Tweedie. 141da177e4SLinus Torvalds * Buffermem limits added 12.3.98, Rik van Riel. 151da177e4SLinus Torvalds */ 161da177e4SLinus Torvalds 171da177e4SLinus Torvalds #include <linux/mm.h> 181da177e4SLinus Torvalds #include <linux/sched.h> 191da177e4SLinus Torvalds #include <linux/kernel_stat.h> 201da177e4SLinus Torvalds #include <linux/swap.h> 211da177e4SLinus Torvalds #include <linux/mman.h> 221da177e4SLinus Torvalds #include <linux/pagemap.h> 231da177e4SLinus Torvalds #include <linux/pagevec.h> 241da177e4SLinus Torvalds #include <linux/init.h> 25b95f1b31SPaul Gortmaker #include <linux/export.h> 261da177e4SLinus Torvalds #include <linux/mm_inline.h> 271da177e4SLinus Torvalds #include <linux/percpu_counter.h> 283565fce3SDan Williams #include <linux/memremap.h> 291da177e4SLinus Torvalds #include <linux/percpu.h> 301da177e4SLinus Torvalds #include <linux/cpu.h> 311da177e4SLinus Torvalds #include <linux/notifier.h> 32e0bf68ddSPeter Zijlstra #include <linux/backing-dev.h> 3366e1707bSBalbir Singh #include <linux/memcontrol.h> 345a0e3ad6STejun Heo #include <linux/gfp.h> 35a27bb332SKent Overstreet #include <linux/uio.h> 36822fc613SNaoya Horiguchi #include <linux/hugetlb.h> 3733c3fc71SVladimir Davydov #include <linux/page_idle.h> 381da177e4SLinus Torvalds 3964d6519dSLee Schermerhorn #include "internal.h" 4064d6519dSLee Schermerhorn 41c6286c98SMel Gorman #define CREATE_TRACE_POINTS 42c6286c98SMel Gorman #include <trace/events/pagemap.h> 43c6286c98SMel Gorman 441da177e4SLinus Torvalds /* How many pages do we try to swap or page in/out together? */ 451da177e4SLinus Torvalds int page_cluster; 461da177e4SLinus Torvalds 4713f7f789SMel Gorman static DEFINE_PER_CPU(struct pagevec, lru_add_pvec); 48f84f9504SVegard Nossum static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs); 49cc5993bdSMinchan Kim static DEFINE_PER_CPU(struct pagevec, lru_deactivate_file_pvecs); 509c276cc6SMinchan Kim static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs); 51f7ad2a6cSShaohua Li static DEFINE_PER_CPU(struct pagevec, lru_lazyfree_pvecs); 52a4a921aaSMing Li #ifdef CONFIG_SMP 53a4a921aaSMing Li static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs); 54a4a921aaSMing Li #endif 55902aaed0SHisashi Hifumi 56b221385bSAdrian Bunk /* 57b221385bSAdrian Bunk * This path almost never happens for VM activity - pages are normally 58b221385bSAdrian Bunk * freed via pagevecs. But it gets used by networking. 59b221385bSAdrian Bunk */ 60920c7a5dSHarvey Harrison static void __page_cache_release(struct page *page) 61b221385bSAdrian Bunk { 62b221385bSAdrian Bunk if (PageLRU(page)) { 63f4b7e272SAndrey Ryabinin pg_data_t *pgdat = page_pgdat(page); 64fa9add64SHugh Dickins struct lruvec *lruvec; 65fa9add64SHugh Dickins unsigned long flags; 66b221385bSAdrian Bunk 67f4b7e272SAndrey Ryabinin spin_lock_irqsave(&pgdat->lru_lock, flags); 68f4b7e272SAndrey Ryabinin lruvec = mem_cgroup_page_lruvec(page, pgdat); 69309381feSSasha Levin VM_BUG_ON_PAGE(!PageLRU(page), page); 70b221385bSAdrian Bunk __ClearPageLRU(page); 71fa9add64SHugh Dickins del_page_from_lru_list(page, lruvec, page_off_lru(page)); 72f4b7e272SAndrey Ryabinin spin_unlock_irqrestore(&pgdat->lru_lock, flags); 73b221385bSAdrian Bunk } 7462906027SNicholas Piggin __ClearPageWaiters(page); 7591807063SAndrea Arcangeli } 7691807063SAndrea Arcangeli 7791807063SAndrea Arcangeli static void __put_single_page(struct page *page) 7891807063SAndrea Arcangeli { 7991807063SAndrea Arcangeli __page_cache_release(page); 807ae88534SYang Shi mem_cgroup_uncharge(page); 812d4894b5SMel Gorman free_unref_page(page); 82b221385bSAdrian Bunk } 83b221385bSAdrian Bunk 8491807063SAndrea Arcangeli static void __put_compound_page(struct page *page) 8591807063SAndrea Arcangeli { 8691807063SAndrea Arcangeli compound_page_dtor *dtor; 8791807063SAndrea Arcangeli 88822fc613SNaoya Horiguchi /* 89822fc613SNaoya Horiguchi * __page_cache_release() is supposed to be called for thp, not for 90822fc613SNaoya Horiguchi * hugetlb. This is because hugetlb page does never have PageLRU set 91822fc613SNaoya Horiguchi * (it's never listed to any LRU lists) and no memcg routines should 92822fc613SNaoya Horiguchi * be called for hugetlb (it has a separate hugetlb_cgroup.) 93822fc613SNaoya Horiguchi */ 94822fc613SNaoya Horiguchi if (!PageHuge(page)) 9591807063SAndrea Arcangeli __page_cache_release(page); 9691807063SAndrea Arcangeli dtor = get_compound_page_dtor(page); 9791807063SAndrea Arcangeli (*dtor)(page); 9891807063SAndrea Arcangeli } 9991807063SAndrea Arcangeli 100ddc58f27SKirill A. Shutemov void __put_page(struct page *page) 101c747ce79SJianyu Zhan { 10271389703SDan Williams if (is_zone_device_page(page)) { 10371389703SDan Williams put_dev_pagemap(page->pgmap); 10471389703SDan Williams 10571389703SDan Williams /* 10671389703SDan Williams * The page belongs to the device that created pgmap. Do 10771389703SDan Williams * not return it to page allocator. 10871389703SDan Williams */ 10971389703SDan Williams return; 11071389703SDan Williams } 11171389703SDan Williams 112ddc58f27SKirill A. Shutemov if (unlikely(PageCompound(page))) 11326296ad2SAndrew Morton __put_compound_page(page); 11426296ad2SAndrew Morton else 11526296ad2SAndrew Morton __put_single_page(page); 11626296ad2SAndrew Morton } 117ddc58f27SKirill A. Shutemov EXPORT_SYMBOL(__put_page); 11870b50f94SAndrea Arcangeli 1191d7ea732SAlexander Zarochentsev /** 1207682486bSRandy Dunlap * put_pages_list() - release a list of pages 1217682486bSRandy Dunlap * @pages: list of pages threaded on page->lru 1221d7ea732SAlexander Zarochentsev * 1231d7ea732SAlexander Zarochentsev * Release a list of pages which are strung together on page.lru. Currently 1241d7ea732SAlexander Zarochentsev * used by read_cache_pages() and related error recovery code. 1251d7ea732SAlexander Zarochentsev */ 1261d7ea732SAlexander Zarochentsev void put_pages_list(struct list_head *pages) 1271d7ea732SAlexander Zarochentsev { 1281d7ea732SAlexander Zarochentsev while (!list_empty(pages)) { 1291d7ea732SAlexander Zarochentsev struct page *victim; 1301d7ea732SAlexander Zarochentsev 131f86196eaSNikolay Borisov victim = lru_to_page(pages); 1321d7ea732SAlexander Zarochentsev list_del(&victim->lru); 13309cbfeafSKirill A. Shutemov put_page(victim); 1341d7ea732SAlexander Zarochentsev } 1351d7ea732SAlexander Zarochentsev } 1361d7ea732SAlexander Zarochentsev EXPORT_SYMBOL(put_pages_list); 1371d7ea732SAlexander Zarochentsev 13818022c5dSMel Gorman /* 13918022c5dSMel Gorman * get_kernel_pages() - pin kernel pages in memory 14018022c5dSMel Gorman * @kiov: An array of struct kvec structures 14118022c5dSMel Gorman * @nr_segs: number of segments to pin 14218022c5dSMel Gorman * @write: pinning for read/write, currently ignored 14318022c5dSMel Gorman * @pages: array that receives pointers to the pages pinned. 14418022c5dSMel Gorman * Should be at least nr_segs long. 14518022c5dSMel Gorman * 14618022c5dSMel Gorman * Returns number of pages pinned. This may be fewer than the number 14718022c5dSMel Gorman * requested. If nr_pages is 0 or negative, returns 0. If no pages 14818022c5dSMel Gorman * were pinned, returns -errno. Each page returned must be released 14918022c5dSMel Gorman * with a put_page() call when it is finished with. 15018022c5dSMel Gorman */ 15118022c5dSMel Gorman int get_kernel_pages(const struct kvec *kiov, int nr_segs, int write, 15218022c5dSMel Gorman struct page **pages) 15318022c5dSMel Gorman { 15418022c5dSMel Gorman int seg; 15518022c5dSMel Gorman 15618022c5dSMel Gorman for (seg = 0; seg < nr_segs; seg++) { 15718022c5dSMel Gorman if (WARN_ON(kiov[seg].iov_len != PAGE_SIZE)) 15818022c5dSMel Gorman return seg; 15918022c5dSMel Gorman 1605a178119SMel Gorman pages[seg] = kmap_to_page(kiov[seg].iov_base); 16109cbfeafSKirill A. Shutemov get_page(pages[seg]); 16218022c5dSMel Gorman } 16318022c5dSMel Gorman 16418022c5dSMel Gorman return seg; 16518022c5dSMel Gorman } 16618022c5dSMel Gorman EXPORT_SYMBOL_GPL(get_kernel_pages); 16718022c5dSMel Gorman 16818022c5dSMel Gorman /* 16918022c5dSMel Gorman * get_kernel_page() - pin a kernel page in memory 17018022c5dSMel Gorman * @start: starting kernel address 17118022c5dSMel Gorman * @write: pinning for read/write, currently ignored 17218022c5dSMel Gorman * @pages: array that receives pointer to the page pinned. 17318022c5dSMel Gorman * Must be at least nr_segs long. 17418022c5dSMel Gorman * 17518022c5dSMel Gorman * Returns 1 if page is pinned. If the page was not pinned, returns 17618022c5dSMel Gorman * -errno. The page returned must be released with a put_page() call 17718022c5dSMel Gorman * when it is finished with. 17818022c5dSMel Gorman */ 17918022c5dSMel Gorman int get_kernel_page(unsigned long start, int write, struct page **pages) 18018022c5dSMel Gorman { 18118022c5dSMel Gorman const struct kvec kiov = { 18218022c5dSMel Gorman .iov_base = (void *)start, 18318022c5dSMel Gorman .iov_len = PAGE_SIZE 18418022c5dSMel Gorman }; 18518022c5dSMel Gorman 18618022c5dSMel Gorman return get_kernel_pages(&kiov, 1, write, pages); 18718022c5dSMel Gorman } 18818022c5dSMel Gorman EXPORT_SYMBOL_GPL(get_kernel_page); 18918022c5dSMel Gorman 1903dd7ae8eSShaohua Li static void pagevec_lru_move_fn(struct pagevec *pvec, 191fa9add64SHugh Dickins void (*move_fn)(struct page *page, struct lruvec *lruvec, void *arg), 1923dd7ae8eSShaohua Li void *arg) 193902aaed0SHisashi Hifumi { 194902aaed0SHisashi Hifumi int i; 19568eb0731SMel Gorman struct pglist_data *pgdat = NULL; 196fa9add64SHugh Dickins struct lruvec *lruvec; 1973dd7ae8eSShaohua Li unsigned long flags = 0; 198902aaed0SHisashi Hifumi 199902aaed0SHisashi Hifumi for (i = 0; i < pagevec_count(pvec); i++) { 200902aaed0SHisashi Hifumi struct page *page = pvec->pages[i]; 20168eb0731SMel Gorman struct pglist_data *pagepgdat = page_pgdat(page); 202902aaed0SHisashi Hifumi 20368eb0731SMel Gorman if (pagepgdat != pgdat) { 20468eb0731SMel Gorman if (pgdat) 20568eb0731SMel Gorman spin_unlock_irqrestore(&pgdat->lru_lock, flags); 20668eb0731SMel Gorman pgdat = pagepgdat; 20768eb0731SMel Gorman spin_lock_irqsave(&pgdat->lru_lock, flags); 208902aaed0SHisashi Hifumi } 2093dd7ae8eSShaohua Li 21068eb0731SMel Gorman lruvec = mem_cgroup_page_lruvec(page, pgdat); 211fa9add64SHugh Dickins (*move_fn)(page, lruvec, arg); 2123dd7ae8eSShaohua Li } 21368eb0731SMel Gorman if (pgdat) 21468eb0731SMel Gorman spin_unlock_irqrestore(&pgdat->lru_lock, flags); 215c6f92f9fSMel Gorman release_pages(pvec->pages, pvec->nr); 2163dd7ae8eSShaohua Li pagevec_reinit(pvec); 2173dd7ae8eSShaohua Li } 2183dd7ae8eSShaohua Li 219fa9add64SHugh Dickins static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec, 220fa9add64SHugh Dickins void *arg) 2213dd7ae8eSShaohua Li { 2223dd7ae8eSShaohua Li int *pgmoved = arg; 2233dd7ae8eSShaohua Li 224c55e8d03SJohannes Weiner if (PageLRU(page) && !PageUnevictable(page)) { 225c55e8d03SJohannes Weiner del_page_from_lru_list(page, lruvec, page_lru(page)); 226c55e8d03SJohannes Weiner ClearPageActive(page); 227c55e8d03SJohannes Weiner add_page_to_lru_list_tail(page, lruvec, page_lru(page)); 2283dd7ae8eSShaohua Li (*pgmoved)++; 229902aaed0SHisashi Hifumi } 230902aaed0SHisashi Hifumi } 2313dd7ae8eSShaohua Li 2323dd7ae8eSShaohua Li /* 2333dd7ae8eSShaohua Li * pagevec_move_tail() must be called with IRQ disabled. 2343dd7ae8eSShaohua Li * Otherwise this may cause nasty races. 2353dd7ae8eSShaohua Li */ 2363dd7ae8eSShaohua Li static void pagevec_move_tail(struct pagevec *pvec) 2373dd7ae8eSShaohua Li { 2383dd7ae8eSShaohua Li int pgmoved = 0; 2393dd7ae8eSShaohua Li 2403dd7ae8eSShaohua Li pagevec_lru_move_fn(pvec, pagevec_move_tail_fn, &pgmoved); 241902aaed0SHisashi Hifumi __count_vm_events(PGROTATED, pgmoved); 242902aaed0SHisashi Hifumi } 243902aaed0SHisashi Hifumi 244902aaed0SHisashi Hifumi /* 2451da177e4SLinus Torvalds * Writeback is about to end against a page which has been marked for immediate 2461da177e4SLinus Torvalds * reclaim. If it still appears to be reclaimable, move it to the tail of the 247902aaed0SHisashi Hifumi * inactive list. 2481da177e4SLinus Torvalds */ 249ac6aadb2SMiklos Szeredi void rotate_reclaimable_page(struct page *page) 2501da177e4SLinus Torvalds { 251c55e8d03SJohannes Weiner if (!PageLocked(page) && !PageDirty(page) && 252894bc310SLee Schermerhorn !PageUnevictable(page) && PageLRU(page)) { 253902aaed0SHisashi Hifumi struct pagevec *pvec; 2541da177e4SLinus Torvalds unsigned long flags; 2551da177e4SLinus Torvalds 25609cbfeafSKirill A. Shutemov get_page(page); 257902aaed0SHisashi Hifumi local_irq_save(flags); 2587c8e0181SChristoph Lameter pvec = this_cpu_ptr(&lru_rotate_pvecs); 2598f182270SLukasz Odzioba if (!pagevec_add(pvec, page) || PageCompound(page)) 260902aaed0SHisashi Hifumi pagevec_move_tail(pvec); 261902aaed0SHisashi Hifumi local_irq_restore(flags); 262ac6aadb2SMiklos Szeredi } 2631da177e4SLinus Torvalds } 2641da177e4SLinus Torvalds 265fa9add64SHugh Dickins static void update_page_reclaim_stat(struct lruvec *lruvec, 2663e2f41f1SKOSAKI Motohiro int file, int rotated) 2673e2f41f1SKOSAKI Motohiro { 268fa9add64SHugh Dickins struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat; 2693e2f41f1SKOSAKI Motohiro 2703e2f41f1SKOSAKI Motohiro reclaim_stat->recent_scanned[file]++; 2713e2f41f1SKOSAKI Motohiro if (rotated) 2723e2f41f1SKOSAKI Motohiro reclaim_stat->recent_rotated[file]++; 2733e2f41f1SKOSAKI Motohiro } 2743e2f41f1SKOSAKI Motohiro 275fa9add64SHugh Dickins static void __activate_page(struct page *page, struct lruvec *lruvec, 276fa9add64SHugh Dickins void *arg) 277744ed144SShaohua Li { 2787a608572SLinus Torvalds if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { 279744ed144SShaohua Li int file = page_is_file_cache(page); 280744ed144SShaohua Li int lru = page_lru_base_type(page); 281744ed144SShaohua Li 282fa9add64SHugh Dickins del_page_from_lru_list(page, lruvec, lru); 283744ed144SShaohua Li SetPageActive(page); 284744ed144SShaohua Li lru += LRU_ACTIVE; 285fa9add64SHugh Dickins add_page_to_lru_list(page, lruvec, lru); 28624b7e581SMel Gorman trace_mm_lru_activate(page); 2877a608572SLinus Torvalds 288fa9add64SHugh Dickins __count_vm_event(PGACTIVATE); 289fa9add64SHugh Dickins update_page_reclaim_stat(lruvec, file, 1); 290744ed144SShaohua Li } 291eb709b0dSShaohua Li } 292eb709b0dSShaohua Li 293eb709b0dSShaohua Li #ifdef CONFIG_SMP 294eb709b0dSShaohua Li static void activate_page_drain(int cpu) 295eb709b0dSShaohua Li { 296eb709b0dSShaohua Li struct pagevec *pvec = &per_cpu(activate_page_pvecs, cpu); 297eb709b0dSShaohua Li 298eb709b0dSShaohua Li if (pagevec_count(pvec)) 299eb709b0dSShaohua Li pagevec_lru_move_fn(pvec, __activate_page, NULL); 300eb709b0dSShaohua Li } 301eb709b0dSShaohua Li 3025fbc4616SChris Metcalf static bool need_activate_page_drain(int cpu) 3035fbc4616SChris Metcalf { 3045fbc4616SChris Metcalf return pagevec_count(&per_cpu(activate_page_pvecs, cpu)) != 0; 3055fbc4616SChris Metcalf } 3065fbc4616SChris Metcalf 307eb709b0dSShaohua Li void activate_page(struct page *page) 308eb709b0dSShaohua Li { 309800d8c63SKirill A. Shutemov page = compound_head(page); 310eb709b0dSShaohua Li if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { 311eb709b0dSShaohua Li struct pagevec *pvec = &get_cpu_var(activate_page_pvecs); 312eb709b0dSShaohua Li 31309cbfeafSKirill A. Shutemov get_page(page); 3148f182270SLukasz Odzioba if (!pagevec_add(pvec, page) || PageCompound(page)) 315eb709b0dSShaohua Li pagevec_lru_move_fn(pvec, __activate_page, NULL); 316eb709b0dSShaohua Li put_cpu_var(activate_page_pvecs); 317eb709b0dSShaohua Li } 318eb709b0dSShaohua Li } 319eb709b0dSShaohua Li 320eb709b0dSShaohua Li #else 321eb709b0dSShaohua Li static inline void activate_page_drain(int cpu) 322eb709b0dSShaohua Li { 323eb709b0dSShaohua Li } 324eb709b0dSShaohua Li 325eb709b0dSShaohua Li void activate_page(struct page *page) 326eb709b0dSShaohua Li { 327f4b7e272SAndrey Ryabinin pg_data_t *pgdat = page_pgdat(page); 328eb709b0dSShaohua Li 329800d8c63SKirill A. Shutemov page = compound_head(page); 330f4b7e272SAndrey Ryabinin spin_lock_irq(&pgdat->lru_lock); 331f4b7e272SAndrey Ryabinin __activate_page(page, mem_cgroup_page_lruvec(page, pgdat), NULL); 332f4b7e272SAndrey Ryabinin spin_unlock_irq(&pgdat->lru_lock); 3331da177e4SLinus Torvalds } 334eb709b0dSShaohua Li #endif 3351da177e4SLinus Torvalds 336059285a2SMel Gorman static void __lru_cache_activate_page(struct page *page) 337059285a2SMel Gorman { 338059285a2SMel Gorman struct pagevec *pvec = &get_cpu_var(lru_add_pvec); 339059285a2SMel Gorman int i; 340059285a2SMel Gorman 341059285a2SMel Gorman /* 342059285a2SMel Gorman * Search backwards on the optimistic assumption that the page being 343059285a2SMel Gorman * activated has just been added to this pagevec. Note that only 344059285a2SMel Gorman * the local pagevec is examined as a !PageLRU page could be in the 345059285a2SMel Gorman * process of being released, reclaimed, migrated or on a remote 346059285a2SMel Gorman * pagevec that is currently being drained. Furthermore, marking 347059285a2SMel Gorman * a remote pagevec's page PageActive potentially hits a race where 348059285a2SMel Gorman * a page is marked PageActive just after it is added to the inactive 349059285a2SMel Gorman * list causing accounting errors and BUG_ON checks to trigger. 350059285a2SMel Gorman */ 351059285a2SMel Gorman for (i = pagevec_count(pvec) - 1; i >= 0; i--) { 352059285a2SMel Gorman struct page *pagevec_page = pvec->pages[i]; 353059285a2SMel Gorman 354059285a2SMel Gorman if (pagevec_page == page) { 355059285a2SMel Gorman SetPageActive(page); 356059285a2SMel Gorman break; 357059285a2SMel Gorman } 358059285a2SMel Gorman } 359059285a2SMel Gorman 360059285a2SMel Gorman put_cpu_var(lru_add_pvec); 361059285a2SMel Gorman } 362059285a2SMel Gorman 3631da177e4SLinus Torvalds /* 3641da177e4SLinus Torvalds * Mark a page as having seen activity. 3651da177e4SLinus Torvalds * 3661da177e4SLinus Torvalds * inactive,unreferenced -> inactive,referenced 3671da177e4SLinus Torvalds * inactive,referenced -> active,unreferenced 3681da177e4SLinus Torvalds * active,unreferenced -> active,referenced 369eb39d618SHugh Dickins * 370eb39d618SHugh Dickins * When a newly allocated page is not yet visible, so safe for non-atomic ops, 371eb39d618SHugh Dickins * __SetPageReferenced(page) may be substituted for mark_page_accessed(page). 3721da177e4SLinus Torvalds */ 373920c7a5dSHarvey Harrison void mark_page_accessed(struct page *page) 3741da177e4SLinus Torvalds { 375e90309c9SKirill A. Shutemov page = compound_head(page); 376059285a2SMel Gorman 377a1100a74SFengguang Wu if (!PageReferenced(page)) { 378a1100a74SFengguang Wu SetPageReferenced(page); 379a1100a74SFengguang Wu } else if (PageUnevictable(page)) { 380a1100a74SFengguang Wu /* 381a1100a74SFengguang Wu * Unevictable pages are on the "LRU_UNEVICTABLE" list. But, 382a1100a74SFengguang Wu * this list is never rotated or maintained, so marking an 383a1100a74SFengguang Wu * evictable page accessed has no effect. 384a1100a74SFengguang Wu */ 385a1100a74SFengguang Wu } else if (!PageActive(page)) { 386059285a2SMel Gorman /* 387059285a2SMel Gorman * If the page is on the LRU, queue it for activation via 388059285a2SMel Gorman * activate_page_pvecs. Otherwise, assume the page is on a 389059285a2SMel Gorman * pagevec, mark it active and it'll be moved to the active 390059285a2SMel Gorman * LRU on the next drain. 391059285a2SMel Gorman */ 392059285a2SMel Gorman if (PageLRU(page)) 3931da177e4SLinus Torvalds activate_page(page); 394059285a2SMel Gorman else 395059285a2SMel Gorman __lru_cache_activate_page(page); 3961da177e4SLinus Torvalds ClearPageReferenced(page); 397a528910eSJohannes Weiner if (page_is_file_cache(page)) 398a528910eSJohannes Weiner workingset_activation(page); 3991da177e4SLinus Torvalds } 40033c3fc71SVladimir Davydov if (page_is_idle(page)) 40133c3fc71SVladimir Davydov clear_page_idle(page); 4021da177e4SLinus Torvalds } 4031da177e4SLinus Torvalds EXPORT_SYMBOL(mark_page_accessed); 4041da177e4SLinus Torvalds 4052329d375SJianyu Zhan static void __lru_cache_add(struct page *page) 4061da177e4SLinus Torvalds { 40713f7f789SMel Gorman struct pagevec *pvec = &get_cpu_var(lru_add_pvec); 40813f7f789SMel Gorman 40909cbfeafSKirill A. Shutemov get_page(page); 4108f182270SLukasz Odzioba if (!pagevec_add(pvec, page) || PageCompound(page)) 411a0b8cab3SMel Gorman __pagevec_lru_add(pvec); 41213f7f789SMel Gorman put_cpu_var(lru_add_pvec); 4131da177e4SLinus Torvalds } 4142329d375SJianyu Zhan 4152329d375SJianyu Zhan /** 416e02a9f04SRandy Dunlap * lru_cache_add_anon - add a page to the page lists 4172329d375SJianyu Zhan * @page: the page to add 4182329d375SJianyu Zhan */ 4192329d375SJianyu Zhan void lru_cache_add_anon(struct page *page) 4202329d375SJianyu Zhan { 4216fb81a17SMel Gorman if (PageActive(page)) 4222329d375SJianyu Zhan ClearPageActive(page); 4232329d375SJianyu Zhan __lru_cache_add(page); 4242329d375SJianyu Zhan } 4252329d375SJianyu Zhan 4262329d375SJianyu Zhan void lru_cache_add_file(struct page *page) 4272329d375SJianyu Zhan { 4286fb81a17SMel Gorman if (PageActive(page)) 4292329d375SJianyu Zhan ClearPageActive(page); 4302329d375SJianyu Zhan __lru_cache_add(page); 4312329d375SJianyu Zhan } 4322329d375SJianyu Zhan EXPORT_SYMBOL(lru_cache_add_file); 4331da177e4SLinus Torvalds 434f04e9ebbSKOSAKI Motohiro /** 435c53954a0SMel Gorman * lru_cache_add - add a page to a page list 436f04e9ebbSKOSAKI Motohiro * @page: the page to be added to the LRU. 4372329d375SJianyu Zhan * 4382329d375SJianyu Zhan * Queue the page for addition to the LRU via pagevec. The decision on whether 4392329d375SJianyu Zhan * to add the page to the [in]active [file|anon] list is deferred until the 4402329d375SJianyu Zhan * pagevec is drained. This gives a chance for the caller of lru_cache_add() 4412329d375SJianyu Zhan * have the page added to the active list using mark_page_accessed(). 442f04e9ebbSKOSAKI Motohiro */ 443c53954a0SMel Gorman void lru_cache_add(struct page *page) 4441da177e4SLinus Torvalds { 445309381feSSasha Levin VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page); 446309381feSSasha Levin VM_BUG_ON_PAGE(PageLRU(page), page); 447c53954a0SMel Gorman __lru_cache_add(page); 4481da177e4SLinus Torvalds } 4491da177e4SLinus Torvalds 450894bc310SLee Schermerhorn /** 45100501b53SJohannes Weiner * lru_cache_add_active_or_unevictable 45200501b53SJohannes Weiner * @page: the page to be added to LRU 45300501b53SJohannes Weiner * @vma: vma in which page is mapped for determining reclaimability 45400501b53SJohannes Weiner * 45500501b53SJohannes Weiner * Place @page on the active or unevictable LRU list, depending on its 45600501b53SJohannes Weiner * evictability. Note that if the page is not evictable, it goes 45700501b53SJohannes Weiner * directly back onto it's zone's unevictable list, it does NOT use a 45800501b53SJohannes Weiner * per cpu pagevec. 45900501b53SJohannes Weiner */ 46000501b53SJohannes Weiner void lru_cache_add_active_or_unevictable(struct page *page, 46100501b53SJohannes Weiner struct vm_area_struct *vma) 46200501b53SJohannes Weiner { 46300501b53SJohannes Weiner VM_BUG_ON_PAGE(PageLRU(page), page); 46400501b53SJohannes Weiner 4659c4e6b1aSShakeel Butt if (likely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) != VM_LOCKED)) 46600501b53SJohannes Weiner SetPageActive(page); 4679c4e6b1aSShakeel Butt else if (!TestSetPageMlocked(page)) { 46800501b53SJohannes Weiner /* 46900501b53SJohannes Weiner * We use the irq-unsafe __mod_zone_page_stat because this 47000501b53SJohannes Weiner * counter is not modified from interrupt context, and the pte 47100501b53SJohannes Weiner * lock is held(spinlock), which implies preemption disabled. 47200501b53SJohannes Weiner */ 47300501b53SJohannes Weiner __mod_zone_page_state(page_zone(page), NR_MLOCK, 47400501b53SJohannes Weiner hpage_nr_pages(page)); 47500501b53SJohannes Weiner count_vm_event(UNEVICTABLE_PGMLOCKED); 47600501b53SJohannes Weiner } 4779c4e6b1aSShakeel Butt lru_cache_add(page); 47800501b53SJohannes Weiner } 47900501b53SJohannes Weiner 480902aaed0SHisashi Hifumi /* 48131560180SMinchan Kim * If the page can not be invalidated, it is moved to the 48231560180SMinchan Kim * inactive list to speed up its reclaim. It is moved to the 48331560180SMinchan Kim * head of the list, rather than the tail, to give the flusher 48431560180SMinchan Kim * threads some time to write it out, as this is much more 48531560180SMinchan Kim * effective than the single-page writeout from reclaim. 486278df9f4SMinchan Kim * 487278df9f4SMinchan Kim * If the page isn't page_mapped and dirty/writeback, the page 488278df9f4SMinchan Kim * could reclaim asap using PG_reclaim. 489278df9f4SMinchan Kim * 490278df9f4SMinchan Kim * 1. active, mapped page -> none 491278df9f4SMinchan Kim * 2. active, dirty/writeback page -> inactive, head, PG_reclaim 492278df9f4SMinchan Kim * 3. inactive, mapped page -> none 493278df9f4SMinchan Kim * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim 494278df9f4SMinchan Kim * 5. inactive, clean -> inactive, tail 495278df9f4SMinchan Kim * 6. Others -> none 496278df9f4SMinchan Kim * 497278df9f4SMinchan Kim * In 4, why it moves inactive's head, the VM expects the page would 498278df9f4SMinchan Kim * be write it out by flusher threads as this is much more effective 499278df9f4SMinchan Kim * than the single-page writeout from reclaim. 50031560180SMinchan Kim */ 501cc5993bdSMinchan Kim static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec, 502fa9add64SHugh Dickins void *arg) 50331560180SMinchan Kim { 50431560180SMinchan Kim int lru, file; 505278df9f4SMinchan Kim bool active; 50631560180SMinchan Kim 507278df9f4SMinchan Kim if (!PageLRU(page)) 50831560180SMinchan Kim return; 50931560180SMinchan Kim 510bad49d9cSMinchan Kim if (PageUnevictable(page)) 511bad49d9cSMinchan Kim return; 512bad49d9cSMinchan Kim 51331560180SMinchan Kim /* Some processes are using the page */ 51431560180SMinchan Kim if (page_mapped(page)) 51531560180SMinchan Kim return; 51631560180SMinchan Kim 517278df9f4SMinchan Kim active = PageActive(page); 51831560180SMinchan Kim file = page_is_file_cache(page); 51931560180SMinchan Kim lru = page_lru_base_type(page); 520fa9add64SHugh Dickins 521fa9add64SHugh Dickins del_page_from_lru_list(page, lruvec, lru + active); 52231560180SMinchan Kim ClearPageActive(page); 52331560180SMinchan Kim ClearPageReferenced(page); 52431560180SMinchan Kim 525278df9f4SMinchan Kim if (PageWriteback(page) || PageDirty(page)) { 526278df9f4SMinchan Kim /* 527278df9f4SMinchan Kim * PG_reclaim could be raced with end_page_writeback 528278df9f4SMinchan Kim * It can make readahead confusing. But race window 529278df9f4SMinchan Kim * is _really_ small and it's non-critical problem. 530278df9f4SMinchan Kim */ 531e7a1aaf2SYu Zhao add_page_to_lru_list(page, lruvec, lru); 532278df9f4SMinchan Kim SetPageReclaim(page); 533278df9f4SMinchan Kim } else { 534278df9f4SMinchan Kim /* 535278df9f4SMinchan Kim * The page's writeback ends up during pagevec 536278df9f4SMinchan Kim * We moves tha page into tail of inactive. 537278df9f4SMinchan Kim */ 538e7a1aaf2SYu Zhao add_page_to_lru_list_tail(page, lruvec, lru); 539278df9f4SMinchan Kim __count_vm_event(PGROTATED); 540278df9f4SMinchan Kim } 541278df9f4SMinchan Kim 542278df9f4SMinchan Kim if (active) 543278df9f4SMinchan Kim __count_vm_event(PGDEACTIVATE); 544fa9add64SHugh Dickins update_page_reclaim_stat(lruvec, file, 0); 54531560180SMinchan Kim } 54631560180SMinchan Kim 5479c276cc6SMinchan Kim static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec, 5489c276cc6SMinchan Kim void *arg) 5499c276cc6SMinchan Kim { 5509c276cc6SMinchan Kim if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) { 5519c276cc6SMinchan Kim int file = page_is_file_cache(page); 5529c276cc6SMinchan Kim int lru = page_lru_base_type(page); 5539c276cc6SMinchan Kim 5549c276cc6SMinchan Kim del_page_from_lru_list(page, lruvec, lru + LRU_ACTIVE); 5559c276cc6SMinchan Kim ClearPageActive(page); 5569c276cc6SMinchan Kim ClearPageReferenced(page); 5579c276cc6SMinchan Kim add_page_to_lru_list(page, lruvec, lru); 5589c276cc6SMinchan Kim 5599c276cc6SMinchan Kim __count_vm_events(PGDEACTIVATE, hpage_nr_pages(page)); 5609c276cc6SMinchan Kim update_page_reclaim_stat(lruvec, file, 0); 5619c276cc6SMinchan Kim } 5629c276cc6SMinchan Kim } 56310853a03SMinchan Kim 564f7ad2a6cSShaohua Li static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec, 56510853a03SMinchan Kim void *arg) 56610853a03SMinchan Kim { 567f7ad2a6cSShaohua Li if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) && 56824c92eb7SShaohua Li !PageSwapCache(page) && !PageUnevictable(page)) { 569f7ad2a6cSShaohua Li bool active = PageActive(page); 57010853a03SMinchan Kim 571f7ad2a6cSShaohua Li del_page_from_lru_list(page, lruvec, 572f7ad2a6cSShaohua Li LRU_INACTIVE_ANON + active); 57310853a03SMinchan Kim ClearPageActive(page); 57410853a03SMinchan Kim ClearPageReferenced(page); 575f7ad2a6cSShaohua Li /* 576f7ad2a6cSShaohua Li * lazyfree pages are clean anonymous pages. They have 577f7ad2a6cSShaohua Li * SwapBacked flag cleared to distinguish normal anonymous 578f7ad2a6cSShaohua Li * pages 579f7ad2a6cSShaohua Li */ 580f7ad2a6cSShaohua Li ClearPageSwapBacked(page); 581f7ad2a6cSShaohua Li add_page_to_lru_list(page, lruvec, LRU_INACTIVE_FILE); 58210853a03SMinchan Kim 583f7ad2a6cSShaohua Li __count_vm_events(PGLAZYFREE, hpage_nr_pages(page)); 5842262185cSRoman Gushchin count_memcg_page_event(page, PGLAZYFREE); 585f7ad2a6cSShaohua Li update_page_reclaim_stat(lruvec, 1, 0); 58610853a03SMinchan Kim } 58710853a03SMinchan Kim } 58810853a03SMinchan Kim 58931560180SMinchan Kim /* 590902aaed0SHisashi Hifumi * Drain pages out of the cpu's pagevecs. 591902aaed0SHisashi Hifumi * Either "cpu" is the current CPU, and preemption has already been 592902aaed0SHisashi Hifumi * disabled; or "cpu" is being hot-unplugged, and is already dead. 593902aaed0SHisashi Hifumi */ 594f0cb3c76SKonstantin Khlebnikov void lru_add_drain_cpu(int cpu) 5951da177e4SLinus Torvalds { 59613f7f789SMel Gorman struct pagevec *pvec = &per_cpu(lru_add_pvec, cpu); 5971da177e4SLinus Torvalds 5981da177e4SLinus Torvalds if (pagevec_count(pvec)) 599a0b8cab3SMel Gorman __pagevec_lru_add(pvec); 600902aaed0SHisashi Hifumi 601902aaed0SHisashi Hifumi pvec = &per_cpu(lru_rotate_pvecs, cpu); 602902aaed0SHisashi Hifumi if (pagevec_count(pvec)) { 603902aaed0SHisashi Hifumi unsigned long flags; 604902aaed0SHisashi Hifumi 605902aaed0SHisashi Hifumi /* No harm done if a racing interrupt already did this */ 606902aaed0SHisashi Hifumi local_irq_save(flags); 607902aaed0SHisashi Hifumi pagevec_move_tail(pvec); 608902aaed0SHisashi Hifumi local_irq_restore(flags); 609902aaed0SHisashi Hifumi } 61031560180SMinchan Kim 611cc5993bdSMinchan Kim pvec = &per_cpu(lru_deactivate_file_pvecs, cpu); 61231560180SMinchan Kim if (pagevec_count(pvec)) 613cc5993bdSMinchan Kim pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL); 614eb709b0dSShaohua Li 6159c276cc6SMinchan Kim pvec = &per_cpu(lru_deactivate_pvecs, cpu); 6169c276cc6SMinchan Kim if (pagevec_count(pvec)) 6179c276cc6SMinchan Kim pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL); 6189c276cc6SMinchan Kim 619f7ad2a6cSShaohua Li pvec = &per_cpu(lru_lazyfree_pvecs, cpu); 62010853a03SMinchan Kim if (pagevec_count(pvec)) 621f7ad2a6cSShaohua Li pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL); 62210853a03SMinchan Kim 623eb709b0dSShaohua Li activate_page_drain(cpu); 62431560180SMinchan Kim } 62531560180SMinchan Kim 62631560180SMinchan Kim /** 627cc5993bdSMinchan Kim * deactivate_file_page - forcefully deactivate a file page 62831560180SMinchan Kim * @page: page to deactivate 62931560180SMinchan Kim * 63031560180SMinchan Kim * This function hints the VM that @page is a good reclaim candidate, 63131560180SMinchan Kim * for example if its invalidation fails due to the page being dirty 63231560180SMinchan Kim * or under writeback. 63331560180SMinchan Kim */ 634cc5993bdSMinchan Kim void deactivate_file_page(struct page *page) 63531560180SMinchan Kim { 636821ed6bbSMinchan Kim /* 637cc5993bdSMinchan Kim * In a workload with many unevictable page such as mprotect, 638cc5993bdSMinchan Kim * unevictable page deactivation for accelerating reclaim is pointless. 639821ed6bbSMinchan Kim */ 640821ed6bbSMinchan Kim if (PageUnevictable(page)) 641821ed6bbSMinchan Kim return; 642821ed6bbSMinchan Kim 64331560180SMinchan Kim if (likely(get_page_unless_zero(page))) { 644cc5993bdSMinchan Kim struct pagevec *pvec = &get_cpu_var(lru_deactivate_file_pvecs); 64531560180SMinchan Kim 6468f182270SLukasz Odzioba if (!pagevec_add(pvec, page) || PageCompound(page)) 647cc5993bdSMinchan Kim pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL); 648cc5993bdSMinchan Kim put_cpu_var(lru_deactivate_file_pvecs); 64931560180SMinchan Kim } 65080bfed90SAndrew Morton } 65180bfed90SAndrew Morton 6529c276cc6SMinchan Kim /* 6539c276cc6SMinchan Kim * deactivate_page - deactivate a page 6549c276cc6SMinchan Kim * @page: page to deactivate 6559c276cc6SMinchan Kim * 6569c276cc6SMinchan Kim * deactivate_page() moves @page to the inactive list if @page was on the active 6579c276cc6SMinchan Kim * list and was not an unevictable page. This is done to accelerate the reclaim 6589c276cc6SMinchan Kim * of @page. 6599c276cc6SMinchan Kim */ 6609c276cc6SMinchan Kim void deactivate_page(struct page *page) 6619c276cc6SMinchan Kim { 6629c276cc6SMinchan Kim if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) { 6639c276cc6SMinchan Kim struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs); 6649c276cc6SMinchan Kim 6659c276cc6SMinchan Kim get_page(page); 6669c276cc6SMinchan Kim if (!pagevec_add(pvec, page) || PageCompound(page)) 6679c276cc6SMinchan Kim pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL); 6689c276cc6SMinchan Kim put_cpu_var(lru_deactivate_pvecs); 6699c276cc6SMinchan Kim } 6709c276cc6SMinchan Kim } 6719c276cc6SMinchan Kim 67210853a03SMinchan Kim /** 673f7ad2a6cSShaohua Li * mark_page_lazyfree - make an anon page lazyfree 67410853a03SMinchan Kim * @page: page to deactivate 67510853a03SMinchan Kim * 676f7ad2a6cSShaohua Li * mark_page_lazyfree() moves @page to the inactive file list. 677f7ad2a6cSShaohua Li * This is done to accelerate the reclaim of @page. 67810853a03SMinchan Kim */ 679f7ad2a6cSShaohua Li void mark_page_lazyfree(struct page *page) 68010853a03SMinchan Kim { 681f7ad2a6cSShaohua Li if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) && 68224c92eb7SShaohua Li !PageSwapCache(page) && !PageUnevictable(page)) { 683f7ad2a6cSShaohua Li struct pagevec *pvec = &get_cpu_var(lru_lazyfree_pvecs); 68410853a03SMinchan Kim 68509cbfeafSKirill A. Shutemov get_page(page); 6868f182270SLukasz Odzioba if (!pagevec_add(pvec, page) || PageCompound(page)) 687f7ad2a6cSShaohua Li pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL); 688f7ad2a6cSShaohua Li put_cpu_var(lru_lazyfree_pvecs); 68910853a03SMinchan Kim } 69010853a03SMinchan Kim } 69110853a03SMinchan Kim 69280bfed90SAndrew Morton void lru_add_drain(void) 69380bfed90SAndrew Morton { 694f0cb3c76SKonstantin Khlebnikov lru_add_drain_cpu(get_cpu()); 69580bfed90SAndrew Morton put_cpu(); 6961da177e4SLinus Torvalds } 6971da177e4SLinus Torvalds 6986ea183d6SMichal Hocko #ifdef CONFIG_SMP 6996ea183d6SMichal Hocko 7006ea183d6SMichal Hocko static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work); 7016ea183d6SMichal Hocko 702c4028958SDavid Howells static void lru_add_drain_per_cpu(struct work_struct *dummy) 703053837fcSNick Piggin { 704053837fcSNick Piggin lru_add_drain(); 705053837fcSNick Piggin } 706053837fcSNick Piggin 7079852a721SMichal Hocko /* 7089852a721SMichal Hocko * Doesn't need any cpu hotplug locking because we do rely on per-cpu 7099852a721SMichal Hocko * kworkers being shut down before our page_alloc_cpu_dead callback is 7109852a721SMichal Hocko * executed on the offlined cpu. 7119852a721SMichal Hocko * Calling this function with cpu hotplug locks held can actually lead 7129852a721SMichal Hocko * to obscure indirect dependencies via WQ context. 7139852a721SMichal Hocko */ 7149852a721SMichal Hocko void lru_add_drain_all(void) 715053837fcSNick Piggin { 716eef1a429SKonstantin Khlebnikov static seqcount_t seqcount = SEQCNT_ZERO(seqcount); 7175fbc4616SChris Metcalf static DEFINE_MUTEX(lock); 7185fbc4616SChris Metcalf static struct cpumask has_work; 719eef1a429SKonstantin Khlebnikov int cpu, seq; 7205fbc4616SChris Metcalf 721ce612879SMichal Hocko /* 722ce612879SMichal Hocko * Make sure nobody triggers this path before mm_percpu_wq is fully 723ce612879SMichal Hocko * initialized. 724ce612879SMichal Hocko */ 725ce612879SMichal Hocko if (WARN_ON(!mm_percpu_wq)) 726ce612879SMichal Hocko return; 727ce612879SMichal Hocko 728eef1a429SKonstantin Khlebnikov seq = raw_read_seqcount_latch(&seqcount); 729eef1a429SKonstantin Khlebnikov 7305fbc4616SChris Metcalf mutex_lock(&lock); 731eef1a429SKonstantin Khlebnikov 732eef1a429SKonstantin Khlebnikov /* 733eef1a429SKonstantin Khlebnikov * Piggyback on drain started and finished while we waited for lock: 734eef1a429SKonstantin Khlebnikov * all pages pended at the time of our enter were drained from vectors. 735eef1a429SKonstantin Khlebnikov */ 736eef1a429SKonstantin Khlebnikov if (__read_seqcount_retry(&seqcount, seq)) 737eef1a429SKonstantin Khlebnikov goto done; 738eef1a429SKonstantin Khlebnikov 739eef1a429SKonstantin Khlebnikov raw_write_seqcount_latch(&seqcount); 740eef1a429SKonstantin Khlebnikov 7415fbc4616SChris Metcalf cpumask_clear(&has_work); 7425fbc4616SChris Metcalf 7435fbc4616SChris Metcalf for_each_online_cpu(cpu) { 7445fbc4616SChris Metcalf struct work_struct *work = &per_cpu(lru_add_drain_work, cpu); 7455fbc4616SChris Metcalf 7465fbc4616SChris Metcalf if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) || 7475fbc4616SChris Metcalf pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) || 748cc5993bdSMinchan Kim pagevec_count(&per_cpu(lru_deactivate_file_pvecs, cpu)) || 7499c276cc6SMinchan Kim pagevec_count(&per_cpu(lru_deactivate_pvecs, cpu)) || 750f7ad2a6cSShaohua Li pagevec_count(&per_cpu(lru_lazyfree_pvecs, cpu)) || 7515fbc4616SChris Metcalf need_activate_page_drain(cpu)) { 7525fbc4616SChris Metcalf INIT_WORK(work, lru_add_drain_per_cpu); 753ce612879SMichal Hocko queue_work_on(cpu, mm_percpu_wq, work); 7545fbc4616SChris Metcalf cpumask_set_cpu(cpu, &has_work); 7555fbc4616SChris Metcalf } 7565fbc4616SChris Metcalf } 7575fbc4616SChris Metcalf 7585fbc4616SChris Metcalf for_each_cpu(cpu, &has_work) 7595fbc4616SChris Metcalf flush_work(&per_cpu(lru_add_drain_work, cpu)); 7605fbc4616SChris Metcalf 761eef1a429SKonstantin Khlebnikov done: 7625fbc4616SChris Metcalf mutex_unlock(&lock); 763053837fcSNick Piggin } 7646ea183d6SMichal Hocko #else 7656ea183d6SMichal Hocko void lru_add_drain_all(void) 7666ea183d6SMichal Hocko { 7676ea183d6SMichal Hocko lru_add_drain(); 7686ea183d6SMichal Hocko } 7696ea183d6SMichal Hocko #endif 770053837fcSNick Piggin 771aabfb572SMichal Hocko /** 772ea1754a0SKirill A. Shutemov * release_pages - batched put_page() 773aabfb572SMichal Hocko * @pages: array of pages to release 774aabfb572SMichal Hocko * @nr: number of pages 7751da177e4SLinus Torvalds * 776aabfb572SMichal Hocko * Decrement the reference count on all the pages in @pages. If it 777aabfb572SMichal Hocko * fell to zero, remove the page from the LRU and free it. 7781da177e4SLinus Torvalds */ 779c6f92f9fSMel Gorman void release_pages(struct page **pages, int nr) 7801da177e4SLinus Torvalds { 7811da177e4SLinus Torvalds int i; 782cc59850eSKonstantin Khlebnikov LIST_HEAD(pages_to_free); 783599d0c95SMel Gorman struct pglist_data *locked_pgdat = NULL; 784fa9add64SHugh Dickins struct lruvec *lruvec; 785902aaed0SHisashi Hifumi unsigned long uninitialized_var(flags); 786aabfb572SMichal Hocko unsigned int uninitialized_var(lock_batch); 7871da177e4SLinus Torvalds 7881da177e4SLinus Torvalds for (i = 0; i < nr; i++) { 7891da177e4SLinus Torvalds struct page *page = pages[i]; 7901da177e4SLinus Torvalds 791aabfb572SMichal Hocko /* 792aabfb572SMichal Hocko * Make sure the IRQ-safe lock-holding time does not get 793aabfb572SMichal Hocko * excessive with a continuous string of pages from the 794599d0c95SMel Gorman * same pgdat. The lock is held only if pgdat != NULL. 795aabfb572SMichal Hocko */ 796599d0c95SMel Gorman if (locked_pgdat && ++lock_batch == SWAP_CLUSTER_MAX) { 797599d0c95SMel Gorman spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); 798599d0c95SMel Gorman locked_pgdat = NULL; 799aabfb572SMichal Hocko } 800aabfb572SMichal Hocko 8016fcb52a5SAaron Lu if (is_huge_zero_page(page)) 802aa88b68cSKirill A. Shutemov continue; 803aa88b68cSKirill A. Shutemov 804c5d6c45eSIra Weiny if (is_zone_device_page(page)) { 805df6ad698SJérôme Glisse if (locked_pgdat) { 806df6ad698SJérôme Glisse spin_unlock_irqrestore(&locked_pgdat->lru_lock, 807df6ad698SJérôme Glisse flags); 808df6ad698SJérôme Glisse locked_pgdat = NULL; 809df6ad698SJérôme Glisse } 810c5d6c45eSIra Weiny /* 811c5d6c45eSIra Weiny * ZONE_DEVICE pages that return 'false' from 812c5d6c45eSIra Weiny * put_devmap_managed_page() do not require special 813c5d6c45eSIra Weiny * processing, and instead, expect a call to 814c5d6c45eSIra Weiny * put_page_testzero(). 815c5d6c45eSIra Weiny */ 816*07d80269SJohn Hubbard if (page_is_devmap_managed(page)) { 817*07d80269SJohn Hubbard put_devmap_managed_page(page); 818df6ad698SJérôme Glisse continue; 819df6ad698SJérôme Glisse } 820*07d80269SJohn Hubbard } 821df6ad698SJérôme Glisse 822ddc58f27SKirill A. Shutemov page = compound_head(page); 823b5810039SNick Piggin if (!put_page_testzero(page)) 8241da177e4SLinus Torvalds continue; 8251da177e4SLinus Torvalds 826ddc58f27SKirill A. Shutemov if (PageCompound(page)) { 827599d0c95SMel Gorman if (locked_pgdat) { 828599d0c95SMel Gorman spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); 829599d0c95SMel Gorman locked_pgdat = NULL; 830ddc58f27SKirill A. Shutemov } 831ddc58f27SKirill A. Shutemov __put_compound_page(page); 832ddc58f27SKirill A. Shutemov continue; 833ddc58f27SKirill A. Shutemov } 834ddc58f27SKirill A. Shutemov 83546453a6eSNick Piggin if (PageLRU(page)) { 836599d0c95SMel Gorman struct pglist_data *pgdat = page_pgdat(page); 837894bc310SLee Schermerhorn 838599d0c95SMel Gorman if (pgdat != locked_pgdat) { 839599d0c95SMel Gorman if (locked_pgdat) 840599d0c95SMel Gorman spin_unlock_irqrestore(&locked_pgdat->lru_lock, 841902aaed0SHisashi Hifumi flags); 842aabfb572SMichal Hocko lock_batch = 0; 843599d0c95SMel Gorman locked_pgdat = pgdat; 844599d0c95SMel Gorman spin_lock_irqsave(&locked_pgdat->lru_lock, flags); 8451da177e4SLinus Torvalds } 846fa9add64SHugh Dickins 847599d0c95SMel Gorman lruvec = mem_cgroup_page_lruvec(page, locked_pgdat); 848309381feSSasha Levin VM_BUG_ON_PAGE(!PageLRU(page), page); 84967453911SNick Piggin __ClearPageLRU(page); 850fa9add64SHugh Dickins del_page_from_lru_list(page, lruvec, page_off_lru(page)); 85146453a6eSNick Piggin } 85246453a6eSNick Piggin 853c53954a0SMel Gorman /* Clear Active bit in case of parallel mark_page_accessed */ 854e3741b50SMel Gorman __ClearPageActive(page); 85562906027SNicholas Piggin __ClearPageWaiters(page); 856c53954a0SMel Gorman 857cc59850eSKonstantin Khlebnikov list_add(&page->lru, &pages_to_free); 8581da177e4SLinus Torvalds } 859599d0c95SMel Gorman if (locked_pgdat) 860599d0c95SMel Gorman spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); 8611da177e4SLinus Torvalds 862747db954SJohannes Weiner mem_cgroup_uncharge_list(&pages_to_free); 8632d4894b5SMel Gorman free_unref_page_list(&pages_to_free); 8641da177e4SLinus Torvalds } 8650be8557bSMiklos Szeredi EXPORT_SYMBOL(release_pages); 8661da177e4SLinus Torvalds 8671da177e4SLinus Torvalds /* 8681da177e4SLinus Torvalds * The pages which we're about to release may be in the deferred lru-addition 8691da177e4SLinus Torvalds * queues. That would prevent them from really being freed right now. That's 8701da177e4SLinus Torvalds * OK from a correctness point of view but is inefficient - those pages may be 8711da177e4SLinus Torvalds * cache-warm and we want to give them back to the page allocator ASAP. 8721da177e4SLinus Torvalds * 8731da177e4SLinus Torvalds * So __pagevec_release() will drain those queues here. __pagevec_lru_add() 8741da177e4SLinus Torvalds * and __pagevec_lru_add_active() call release_pages() directly to avoid 8751da177e4SLinus Torvalds * mutual recursion. 8761da177e4SLinus Torvalds */ 8771da177e4SLinus Torvalds void __pagevec_release(struct pagevec *pvec) 8781da177e4SLinus Torvalds { 8797f0b5fb9SMel Gorman if (!pvec->percpu_pvec_drained) { 8801da177e4SLinus Torvalds lru_add_drain(); 8817f0b5fb9SMel Gorman pvec->percpu_pvec_drained = true; 882d9ed0d08SMel Gorman } 883c6f92f9fSMel Gorman release_pages(pvec->pages, pagevec_count(pvec)); 8841da177e4SLinus Torvalds pagevec_reinit(pvec); 8851da177e4SLinus Torvalds } 8867f285701SSteve French EXPORT_SYMBOL(__pagevec_release); 8877f285701SSteve French 88812d27107SHugh Dickins #ifdef CONFIG_TRANSPARENT_HUGEPAGE 88971e3aac0SAndrea Arcangeli /* used by __split_huge_page_refcount() */ 890fa9add64SHugh Dickins void lru_add_page_tail(struct page *page, struct page *page_tail, 8915bc7b8acSShaohua Li struct lruvec *lruvec, struct list_head *list) 89271e3aac0SAndrea Arcangeli { 89371e3aac0SAndrea Arcangeli const int file = 0; 89471e3aac0SAndrea Arcangeli 895309381feSSasha Levin VM_BUG_ON_PAGE(!PageHead(page), page); 896309381feSSasha Levin VM_BUG_ON_PAGE(PageCompound(page_tail), page); 897309381feSSasha Levin VM_BUG_ON_PAGE(PageLRU(page_tail), page); 89835f3aa39SLance Roy lockdep_assert_held(&lruvec_pgdat(lruvec)->lru_lock); 89971e3aac0SAndrea Arcangeli 9005bc7b8acSShaohua Li if (!list) 90171e3aac0SAndrea Arcangeli SetPageLRU(page_tail); 90271e3aac0SAndrea Arcangeli 90312d27107SHugh Dickins if (likely(PageLRU(page))) 90412d27107SHugh Dickins list_add_tail(&page_tail->lru, &page->lru); 9055bc7b8acSShaohua Li else if (list) { 9065bc7b8acSShaohua Li /* page reclaim is reclaiming a huge page */ 9075bc7b8acSShaohua Li get_page(page_tail); 9085bc7b8acSShaohua Li list_add_tail(&page_tail->lru, list); 9095bc7b8acSShaohua Li } else { 91012d27107SHugh Dickins /* 91112d27107SHugh Dickins * Head page has not yet been counted, as an hpage, 91212d27107SHugh Dickins * so we must account for each subpage individually. 91312d27107SHugh Dickins * 914e7a1aaf2SYu Zhao * Put page_tail on the list at the correct position 915e7a1aaf2SYu Zhao * so they all end up in order. 91612d27107SHugh Dickins */ 917e7a1aaf2SYu Zhao add_page_to_lru_list_tail(page_tail, lruvec, 918e7a1aaf2SYu Zhao page_lru(page_tail)); 91971e3aac0SAndrea Arcangeli } 9207512102cSHugh Dickins 9217512102cSHugh Dickins if (!PageUnevictable(page)) 922e180cf80SKirill A. Shutemov update_page_reclaim_stat(lruvec, file, PageActive(page_tail)); 92371e3aac0SAndrea Arcangeli } 92412d27107SHugh Dickins #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 92571e3aac0SAndrea Arcangeli 926fa9add64SHugh Dickins static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec, 927fa9add64SHugh Dickins void *arg) 9283dd7ae8eSShaohua Li { 9299c4e6b1aSShakeel Butt enum lru_list lru; 9309c4e6b1aSShakeel Butt int was_unevictable = TestClearPageUnevictable(page); 9313dd7ae8eSShaohua Li 932309381feSSasha Levin VM_BUG_ON_PAGE(PageLRU(page), page); 9333dd7ae8eSShaohua Li 9343dd7ae8eSShaohua Li SetPageLRU(page); 9359c4e6b1aSShakeel Butt /* 9369c4e6b1aSShakeel Butt * Page becomes evictable in two ways: 937dae966dcSPeng Fan * 1) Within LRU lock [munlock_vma_page() and __munlock_pagevec()]. 9389c4e6b1aSShakeel Butt * 2) Before acquiring LRU lock to put the page to correct LRU and then 9399c4e6b1aSShakeel Butt * a) do PageLRU check with lock [check_move_unevictable_pages] 9409c4e6b1aSShakeel Butt * b) do PageLRU check before lock [clear_page_mlock] 9419c4e6b1aSShakeel Butt * 9429c4e6b1aSShakeel Butt * (1) & (2a) are ok as LRU lock will serialize them. For (2b), we need 9439c4e6b1aSShakeel Butt * following strict ordering: 9449c4e6b1aSShakeel Butt * 9459c4e6b1aSShakeel Butt * #0: __pagevec_lru_add_fn #1: clear_page_mlock 9469c4e6b1aSShakeel Butt * 9479c4e6b1aSShakeel Butt * SetPageLRU() TestClearPageMlocked() 9489c4e6b1aSShakeel Butt * smp_mb() // explicit ordering // above provides strict 9499c4e6b1aSShakeel Butt * // ordering 9509c4e6b1aSShakeel Butt * PageMlocked() PageLRU() 9519c4e6b1aSShakeel Butt * 9529c4e6b1aSShakeel Butt * 9539c4e6b1aSShakeel Butt * if '#1' does not observe setting of PG_lru by '#0' and fails 9549c4e6b1aSShakeel Butt * isolation, the explicit barrier will make sure that page_evictable 9559c4e6b1aSShakeel Butt * check will put the page in correct LRU. Without smp_mb(), SetPageLRU 9569c4e6b1aSShakeel Butt * can be reordered after PageMlocked check and can make '#1' to fail 9579c4e6b1aSShakeel Butt * the isolation of the page whose Mlocked bit is cleared (#0 is also 9589c4e6b1aSShakeel Butt * looking at the same page) and the evictable page will be stranded 9599c4e6b1aSShakeel Butt * in an unevictable LRU. 9609c4e6b1aSShakeel Butt */ 9619c4e6b1aSShakeel Butt smp_mb(); 9629c4e6b1aSShakeel Butt 9639c4e6b1aSShakeel Butt if (page_evictable(page)) { 9649c4e6b1aSShakeel Butt lru = page_lru(page); 9659c4e6b1aSShakeel Butt update_page_reclaim_stat(lruvec, page_is_file_cache(page), 9669c4e6b1aSShakeel Butt PageActive(page)); 9679c4e6b1aSShakeel Butt if (was_unevictable) 9689c4e6b1aSShakeel Butt count_vm_event(UNEVICTABLE_PGRESCUED); 9699c4e6b1aSShakeel Butt } else { 9709c4e6b1aSShakeel Butt lru = LRU_UNEVICTABLE; 9719c4e6b1aSShakeel Butt ClearPageActive(page); 9729c4e6b1aSShakeel Butt SetPageUnevictable(page); 9739c4e6b1aSShakeel Butt if (!was_unevictable) 9749c4e6b1aSShakeel Butt count_vm_event(UNEVICTABLE_PGCULLED); 9759c4e6b1aSShakeel Butt } 9769c4e6b1aSShakeel Butt 977fa9add64SHugh Dickins add_page_to_lru_list(page, lruvec, lru); 97824b7e581SMel Gorman trace_mm_lru_insertion(page, lru); 9793dd7ae8eSShaohua Li } 9803dd7ae8eSShaohua Li 9811da177e4SLinus Torvalds /* 9821da177e4SLinus Torvalds * Add the passed pages to the LRU, then drop the caller's refcount 9831da177e4SLinus Torvalds * on them. Reinitialises the caller's pagevec. 9841da177e4SLinus Torvalds */ 985a0b8cab3SMel Gorman void __pagevec_lru_add(struct pagevec *pvec) 9861da177e4SLinus Torvalds { 987a0b8cab3SMel Gorman pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, NULL); 9881da177e4SLinus Torvalds } 9895095ae83SHugh Dickins EXPORT_SYMBOL(__pagevec_lru_add); 990f04e9ebbSKOSAKI Motohiro 9911da177e4SLinus Torvalds /** 9920cd6144aSJohannes Weiner * pagevec_lookup_entries - gang pagecache lookup 9930cd6144aSJohannes Weiner * @pvec: Where the resulting entries are placed 9940cd6144aSJohannes Weiner * @mapping: The address_space to search 9950cd6144aSJohannes Weiner * @start: The starting entry index 996cb6f0f34SMike Rapoport * @nr_entries: The maximum number of pages 9970cd6144aSJohannes Weiner * @indices: The cache indices corresponding to the entries in @pvec 9980cd6144aSJohannes Weiner * 9990cd6144aSJohannes Weiner * pagevec_lookup_entries() will search for and return a group of up 1000f144c390SMike Rapoport * to @nr_pages pages and shadow entries in the mapping. All 10010cd6144aSJohannes Weiner * entries are placed in @pvec. pagevec_lookup_entries() takes a 10020cd6144aSJohannes Weiner * reference against actual pages in @pvec. 10030cd6144aSJohannes Weiner * 10040cd6144aSJohannes Weiner * The search returns a group of mapping-contiguous entries with 10050cd6144aSJohannes Weiner * ascending indexes. There may be holes in the indices due to 10060cd6144aSJohannes Weiner * not-present entries. 10070cd6144aSJohannes Weiner * 10080cd6144aSJohannes Weiner * pagevec_lookup_entries() returns the number of entries which were 10090cd6144aSJohannes Weiner * found. 10100cd6144aSJohannes Weiner */ 10110cd6144aSJohannes Weiner unsigned pagevec_lookup_entries(struct pagevec *pvec, 10120cd6144aSJohannes Weiner struct address_space *mapping, 1013e02a9f04SRandy Dunlap pgoff_t start, unsigned nr_entries, 10140cd6144aSJohannes Weiner pgoff_t *indices) 10150cd6144aSJohannes Weiner { 1016e02a9f04SRandy Dunlap pvec->nr = find_get_entries(mapping, start, nr_entries, 10170cd6144aSJohannes Weiner pvec->pages, indices); 10180cd6144aSJohannes Weiner return pagevec_count(pvec); 10190cd6144aSJohannes Weiner } 10200cd6144aSJohannes Weiner 10210cd6144aSJohannes Weiner /** 10220cd6144aSJohannes Weiner * pagevec_remove_exceptionals - pagevec exceptionals pruning 10230cd6144aSJohannes Weiner * @pvec: The pagevec to prune 10240cd6144aSJohannes Weiner * 10250cd6144aSJohannes Weiner * pagevec_lookup_entries() fills both pages and exceptional radix 10260cd6144aSJohannes Weiner * tree entries into the pagevec. This function prunes all 10270cd6144aSJohannes Weiner * exceptionals from @pvec without leaving holes, so that it can be 10280cd6144aSJohannes Weiner * passed on to page-only pagevec operations. 10290cd6144aSJohannes Weiner */ 10300cd6144aSJohannes Weiner void pagevec_remove_exceptionals(struct pagevec *pvec) 10310cd6144aSJohannes Weiner { 10320cd6144aSJohannes Weiner int i, j; 10330cd6144aSJohannes Weiner 10340cd6144aSJohannes Weiner for (i = 0, j = 0; i < pagevec_count(pvec); i++) { 10350cd6144aSJohannes Weiner struct page *page = pvec->pages[i]; 10363159f943SMatthew Wilcox if (!xa_is_value(page)) 10370cd6144aSJohannes Weiner pvec->pages[j++] = page; 10380cd6144aSJohannes Weiner } 10390cd6144aSJohannes Weiner pvec->nr = j; 10400cd6144aSJohannes Weiner } 10410cd6144aSJohannes Weiner 10420cd6144aSJohannes Weiner /** 1043b947cee4SJan Kara * pagevec_lookup_range - gang pagecache lookup 10441da177e4SLinus Torvalds * @pvec: Where the resulting pages are placed 10451da177e4SLinus Torvalds * @mapping: The address_space to search 10461da177e4SLinus Torvalds * @start: The starting page index 1047b947cee4SJan Kara * @end: The final page index 10481da177e4SLinus Torvalds * 1049e02a9f04SRandy Dunlap * pagevec_lookup_range() will search for & return a group of up to PAGEVEC_SIZE 1050b947cee4SJan Kara * pages in the mapping starting from index @start and upto index @end 1051b947cee4SJan Kara * (inclusive). The pages are placed in @pvec. pagevec_lookup() takes a 10521da177e4SLinus Torvalds * reference against the pages in @pvec. 10531da177e4SLinus Torvalds * 10541da177e4SLinus Torvalds * The search returns a group of mapping-contiguous pages with ascending 1055d72dc8a2SJan Kara * indexes. There may be holes in the indices due to not-present pages. We 1056d72dc8a2SJan Kara * also update @start to index the next page for the traversal. 10571da177e4SLinus Torvalds * 1058b947cee4SJan Kara * pagevec_lookup_range() returns the number of pages which were found. If this 1059e02a9f04SRandy Dunlap * number is smaller than PAGEVEC_SIZE, the end of specified range has been 1060b947cee4SJan Kara * reached. 10611da177e4SLinus Torvalds */ 1062b947cee4SJan Kara unsigned pagevec_lookup_range(struct pagevec *pvec, 1063397162ffSJan Kara struct address_space *mapping, pgoff_t *start, pgoff_t end) 10641da177e4SLinus Torvalds { 1065397162ffSJan Kara pvec->nr = find_get_pages_range(mapping, start, end, PAGEVEC_SIZE, 1066b947cee4SJan Kara pvec->pages); 10671da177e4SLinus Torvalds return pagevec_count(pvec); 10681da177e4SLinus Torvalds } 1069b947cee4SJan Kara EXPORT_SYMBOL(pagevec_lookup_range); 107078539fdfSChristoph Hellwig 107172b045aeSJan Kara unsigned pagevec_lookup_range_tag(struct pagevec *pvec, 107272b045aeSJan Kara struct address_space *mapping, pgoff_t *index, pgoff_t end, 107310bbd235SMatthew Wilcox xa_mark_t tag) 10741da177e4SLinus Torvalds { 107572b045aeSJan Kara pvec->nr = find_get_pages_range_tag(mapping, index, end, tag, 107667fd707fSJan Kara PAGEVEC_SIZE, pvec->pages); 10771da177e4SLinus Torvalds return pagevec_count(pvec); 10781da177e4SLinus Torvalds } 107972b045aeSJan Kara EXPORT_SYMBOL(pagevec_lookup_range_tag); 10801da177e4SLinus Torvalds 108193d3b714SJan Kara unsigned pagevec_lookup_range_nr_tag(struct pagevec *pvec, 108293d3b714SJan Kara struct address_space *mapping, pgoff_t *index, pgoff_t end, 108310bbd235SMatthew Wilcox xa_mark_t tag, unsigned max_pages) 108493d3b714SJan Kara { 108593d3b714SJan Kara pvec->nr = find_get_pages_range_tag(mapping, index, end, tag, 108693d3b714SJan Kara min_t(unsigned int, max_pages, PAGEVEC_SIZE), pvec->pages); 108793d3b714SJan Kara return pagevec_count(pvec); 108893d3b714SJan Kara } 108993d3b714SJan Kara EXPORT_SYMBOL(pagevec_lookup_range_nr_tag); 10901da177e4SLinus Torvalds /* 10911da177e4SLinus Torvalds * Perform any setup for the swap system 10921da177e4SLinus Torvalds */ 10931da177e4SLinus Torvalds void __init swap_setup(void) 10941da177e4SLinus Torvalds { 1095ca79b0c2SArun KS unsigned long megs = totalram_pages() >> (20 - PAGE_SHIFT); 1096e0bf68ddSPeter Zijlstra 10971da177e4SLinus Torvalds /* Use a smaller cluster for small-memory machines */ 10981da177e4SLinus Torvalds if (megs < 16) 10991da177e4SLinus Torvalds page_cluster = 2; 11001da177e4SLinus Torvalds else 11011da177e4SLinus Torvalds page_cluster = 3; 11021da177e4SLinus Torvalds /* 11031da177e4SLinus Torvalds * Right now other parts of the system means that we 11041da177e4SLinus Torvalds * _really_ don't want to cluster much more 11051da177e4SLinus Torvalds */ 11061da177e4SLinus Torvalds } 1107*07d80269SJohn Hubbard 1108*07d80269SJohn Hubbard #ifdef CONFIG_DEV_PAGEMAP_OPS 1109*07d80269SJohn Hubbard void put_devmap_managed_page(struct page *page) 1110*07d80269SJohn Hubbard { 1111*07d80269SJohn Hubbard int count; 1112*07d80269SJohn Hubbard 1113*07d80269SJohn Hubbard if (WARN_ON_ONCE(!page_is_devmap_managed(page))) 1114*07d80269SJohn Hubbard return; 1115*07d80269SJohn Hubbard 1116*07d80269SJohn Hubbard count = page_ref_dec_return(page); 1117*07d80269SJohn Hubbard 1118*07d80269SJohn Hubbard /* 1119*07d80269SJohn Hubbard * devmap page refcounts are 1-based, rather than 0-based: if 1120*07d80269SJohn Hubbard * refcount is 1, then the page is free and the refcount is 1121*07d80269SJohn Hubbard * stable because nobody holds a reference on the page. 1122*07d80269SJohn Hubbard */ 1123*07d80269SJohn Hubbard if (count == 1) 1124*07d80269SJohn Hubbard free_devmap_managed_page(page); 1125*07d80269SJohn Hubbard else if (!count) 1126*07d80269SJohn Hubbard __put_page(page); 1127*07d80269SJohn Hubbard } 1128*07d80269SJohn Hubbard EXPORT_SYMBOL(put_devmap_managed_page); 1129*07d80269SJohn Hubbard #endif 1130