1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * linux/mm/swap.c 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 61da177e4SLinus Torvalds */ 71da177e4SLinus Torvalds 81da177e4SLinus Torvalds /* 9183ff22bSSimon Arlott * This file contains the default values for the operation of the 101da177e4SLinus Torvalds * Linux VM subsystem. Fine-tuning documentation can be found in 1157043247SMauro Carvalho Chehab * Documentation/admin-guide/sysctl/vm.rst. 121da177e4SLinus Torvalds * Started 18.12.91 131da177e4SLinus Torvalds * Swap aging added 23.2.95, Stephen Tweedie. 141da177e4SLinus Torvalds * Buffermem limits added 12.3.98, Rik van Riel. 151da177e4SLinus Torvalds */ 161da177e4SLinus Torvalds 171da177e4SLinus Torvalds #include <linux/mm.h> 181da177e4SLinus Torvalds #include <linux/sched.h> 191da177e4SLinus Torvalds #include <linux/kernel_stat.h> 201da177e4SLinus Torvalds #include <linux/swap.h> 211da177e4SLinus Torvalds #include <linux/mman.h> 221da177e4SLinus Torvalds #include <linux/pagemap.h> 231da177e4SLinus Torvalds #include <linux/pagevec.h> 241da177e4SLinus Torvalds #include <linux/init.h> 25b95f1b31SPaul Gortmaker #include <linux/export.h> 261da177e4SLinus Torvalds #include <linux/mm_inline.h> 271da177e4SLinus Torvalds #include <linux/percpu_counter.h> 283565fce3SDan Williams #include <linux/memremap.h> 291da177e4SLinus Torvalds #include <linux/percpu.h> 301da177e4SLinus Torvalds #include <linux/cpu.h> 311da177e4SLinus Torvalds #include <linux/notifier.h> 32e0bf68ddSPeter Zijlstra #include <linux/backing-dev.h> 3366e1707bSBalbir Singh #include <linux/memcontrol.h> 345a0e3ad6STejun Heo #include <linux/gfp.h> 35a27bb332SKent Overstreet #include <linux/uio.h> 36822fc613SNaoya Horiguchi #include <linux/hugetlb.h> 3733c3fc71SVladimir Davydov #include <linux/page_idle.h> 38b01b2141SIngo Molnar #include <linux/local_lock.h> 391da177e4SLinus Torvalds 4064d6519dSLee Schermerhorn #include "internal.h" 4164d6519dSLee Schermerhorn 42c6286c98SMel Gorman #define CREATE_TRACE_POINTS 43c6286c98SMel Gorman #include <trace/events/pagemap.h> 44c6286c98SMel Gorman 451da177e4SLinus Torvalds /* How many pages do we try to swap or page in/out together? */ 461da177e4SLinus Torvalds int page_cluster; 471da177e4SLinus Torvalds 48b01b2141SIngo Molnar /* Protecting only lru_rotate.pvec which requires disabling interrupts */ 49b01b2141SIngo Molnar struct lru_rotate { 50b01b2141SIngo Molnar local_lock_t lock; 51b01b2141SIngo Molnar struct pagevec pvec; 52b01b2141SIngo Molnar }; 53b01b2141SIngo Molnar static DEFINE_PER_CPU(struct lru_rotate, lru_rotate) = { 54b01b2141SIngo Molnar .lock = INIT_LOCAL_LOCK(lock), 55b01b2141SIngo Molnar }; 56b01b2141SIngo Molnar 57b01b2141SIngo Molnar /* 58b01b2141SIngo Molnar * The following struct pagevec are grouped together because they are protected 59b01b2141SIngo Molnar * by disabling preemption (and interrupts remain enabled). 60b01b2141SIngo Molnar */ 61b01b2141SIngo Molnar struct lru_pvecs { 62b01b2141SIngo Molnar local_lock_t lock; 63b01b2141SIngo Molnar struct pagevec lru_add; 64b01b2141SIngo Molnar struct pagevec lru_deactivate_file; 65b01b2141SIngo Molnar struct pagevec lru_deactivate; 66b01b2141SIngo Molnar struct pagevec lru_lazyfree; 67a4a921aaSMing Li #ifdef CONFIG_SMP 68b01b2141SIngo Molnar struct pagevec activate_page; 69a4a921aaSMing Li #endif 70b01b2141SIngo Molnar }; 71b01b2141SIngo Molnar static DEFINE_PER_CPU(struct lru_pvecs, lru_pvecs) = { 72b01b2141SIngo Molnar .lock = INIT_LOCAL_LOCK(lock), 73b01b2141SIngo Molnar }; 74902aaed0SHisashi Hifumi 75b221385bSAdrian Bunk /* 76b221385bSAdrian Bunk * This path almost never happens for VM activity - pages are normally 77b221385bSAdrian Bunk * freed via pagevecs. But it gets used by networking. 78b221385bSAdrian Bunk */ 79920c7a5dSHarvey Harrison static void __page_cache_release(struct page *page) 80b221385bSAdrian Bunk { 81b221385bSAdrian Bunk if (PageLRU(page)) { 82f4b7e272SAndrey Ryabinin pg_data_t *pgdat = page_pgdat(page); 83fa9add64SHugh Dickins struct lruvec *lruvec; 84fa9add64SHugh Dickins unsigned long flags; 85b221385bSAdrian Bunk 86f4b7e272SAndrey Ryabinin spin_lock_irqsave(&pgdat->lru_lock, flags); 87f4b7e272SAndrey Ryabinin lruvec = mem_cgroup_page_lruvec(page, pgdat); 88309381feSSasha Levin VM_BUG_ON_PAGE(!PageLRU(page), page); 89b221385bSAdrian Bunk __ClearPageLRU(page); 90fa9add64SHugh Dickins del_page_from_lru_list(page, lruvec, page_off_lru(page)); 91f4b7e272SAndrey Ryabinin spin_unlock_irqrestore(&pgdat->lru_lock, flags); 92b221385bSAdrian Bunk } 9362906027SNicholas Piggin __ClearPageWaiters(page); 9491807063SAndrea Arcangeli } 9591807063SAndrea Arcangeli 9691807063SAndrea Arcangeli static void __put_single_page(struct page *page) 9791807063SAndrea Arcangeli { 9891807063SAndrea Arcangeli __page_cache_release(page); 997ae88534SYang Shi mem_cgroup_uncharge(page); 1002d4894b5SMel Gorman free_unref_page(page); 101b221385bSAdrian Bunk } 102b221385bSAdrian Bunk 10391807063SAndrea Arcangeli static void __put_compound_page(struct page *page) 10491807063SAndrea Arcangeli { 105822fc613SNaoya Horiguchi /* 106822fc613SNaoya Horiguchi * __page_cache_release() is supposed to be called for thp, not for 107822fc613SNaoya Horiguchi * hugetlb. This is because hugetlb page does never have PageLRU set 108822fc613SNaoya Horiguchi * (it's never listed to any LRU lists) and no memcg routines should 109822fc613SNaoya Horiguchi * be called for hugetlb (it has a separate hugetlb_cgroup.) 110822fc613SNaoya Horiguchi */ 111822fc613SNaoya Horiguchi if (!PageHuge(page)) 11291807063SAndrea Arcangeli __page_cache_release(page); 113ff45fc3cSMatthew Wilcox (Oracle) destroy_compound_page(page); 11491807063SAndrea Arcangeli } 11591807063SAndrea Arcangeli 116ddc58f27SKirill A. Shutemov void __put_page(struct page *page) 117c747ce79SJianyu Zhan { 11871389703SDan Williams if (is_zone_device_page(page)) { 11971389703SDan Williams put_dev_pagemap(page->pgmap); 12071389703SDan Williams 12171389703SDan Williams /* 12271389703SDan Williams * The page belongs to the device that created pgmap. Do 12371389703SDan Williams * not return it to page allocator. 12471389703SDan Williams */ 12571389703SDan Williams return; 12671389703SDan Williams } 12771389703SDan Williams 128ddc58f27SKirill A. Shutemov if (unlikely(PageCompound(page))) 12926296ad2SAndrew Morton __put_compound_page(page); 13026296ad2SAndrew Morton else 13126296ad2SAndrew Morton __put_single_page(page); 13226296ad2SAndrew Morton } 133ddc58f27SKirill A. Shutemov EXPORT_SYMBOL(__put_page); 13470b50f94SAndrea Arcangeli 1351d7ea732SAlexander Zarochentsev /** 1367682486bSRandy Dunlap * put_pages_list() - release a list of pages 1377682486bSRandy Dunlap * @pages: list of pages threaded on page->lru 1381d7ea732SAlexander Zarochentsev * 1391d7ea732SAlexander Zarochentsev * Release a list of pages which are strung together on page.lru. Currently 1401d7ea732SAlexander Zarochentsev * used by read_cache_pages() and related error recovery code. 1411d7ea732SAlexander Zarochentsev */ 1421d7ea732SAlexander Zarochentsev void put_pages_list(struct list_head *pages) 1431d7ea732SAlexander Zarochentsev { 1441d7ea732SAlexander Zarochentsev while (!list_empty(pages)) { 1451d7ea732SAlexander Zarochentsev struct page *victim; 1461d7ea732SAlexander Zarochentsev 147f86196eaSNikolay Borisov victim = lru_to_page(pages); 1481d7ea732SAlexander Zarochentsev list_del(&victim->lru); 14909cbfeafSKirill A. Shutemov put_page(victim); 1501d7ea732SAlexander Zarochentsev } 1511d7ea732SAlexander Zarochentsev } 1521d7ea732SAlexander Zarochentsev EXPORT_SYMBOL(put_pages_list); 1531d7ea732SAlexander Zarochentsev 15418022c5dSMel Gorman /* 15518022c5dSMel Gorman * get_kernel_pages() - pin kernel pages in memory 15618022c5dSMel Gorman * @kiov: An array of struct kvec structures 15718022c5dSMel Gorman * @nr_segs: number of segments to pin 15818022c5dSMel Gorman * @write: pinning for read/write, currently ignored 15918022c5dSMel Gorman * @pages: array that receives pointers to the pages pinned. 16018022c5dSMel Gorman * Should be at least nr_segs long. 16118022c5dSMel Gorman * 16218022c5dSMel Gorman * Returns number of pages pinned. This may be fewer than the number 16318022c5dSMel Gorman * requested. If nr_pages is 0 or negative, returns 0. If no pages 16418022c5dSMel Gorman * were pinned, returns -errno. Each page returned must be released 16518022c5dSMel Gorman * with a put_page() call when it is finished with. 16618022c5dSMel Gorman */ 16718022c5dSMel Gorman int get_kernel_pages(const struct kvec *kiov, int nr_segs, int write, 16818022c5dSMel Gorman struct page **pages) 16918022c5dSMel Gorman { 17018022c5dSMel Gorman int seg; 17118022c5dSMel Gorman 17218022c5dSMel Gorman for (seg = 0; seg < nr_segs; seg++) { 17318022c5dSMel Gorman if (WARN_ON(kiov[seg].iov_len != PAGE_SIZE)) 17418022c5dSMel Gorman return seg; 17518022c5dSMel Gorman 1765a178119SMel Gorman pages[seg] = kmap_to_page(kiov[seg].iov_base); 17709cbfeafSKirill A. Shutemov get_page(pages[seg]); 17818022c5dSMel Gorman } 17918022c5dSMel Gorman 18018022c5dSMel Gorman return seg; 18118022c5dSMel Gorman } 18218022c5dSMel Gorman EXPORT_SYMBOL_GPL(get_kernel_pages); 18318022c5dSMel Gorman 18418022c5dSMel Gorman /* 18518022c5dSMel Gorman * get_kernel_page() - pin a kernel page in memory 18618022c5dSMel Gorman * @start: starting kernel address 18718022c5dSMel Gorman * @write: pinning for read/write, currently ignored 18818022c5dSMel Gorman * @pages: array that receives pointer to the page pinned. 18918022c5dSMel Gorman * Must be at least nr_segs long. 19018022c5dSMel Gorman * 19118022c5dSMel Gorman * Returns 1 if page is pinned. If the page was not pinned, returns 19218022c5dSMel Gorman * -errno. The page returned must be released with a put_page() call 19318022c5dSMel Gorman * when it is finished with. 19418022c5dSMel Gorman */ 19518022c5dSMel Gorman int get_kernel_page(unsigned long start, int write, struct page **pages) 19618022c5dSMel Gorman { 19718022c5dSMel Gorman const struct kvec kiov = { 19818022c5dSMel Gorman .iov_base = (void *)start, 19918022c5dSMel Gorman .iov_len = PAGE_SIZE 20018022c5dSMel Gorman }; 20118022c5dSMel Gorman 20218022c5dSMel Gorman return get_kernel_pages(&kiov, 1, write, pages); 20318022c5dSMel Gorman } 20418022c5dSMel Gorman EXPORT_SYMBOL_GPL(get_kernel_page); 20518022c5dSMel Gorman 2063dd7ae8eSShaohua Li static void pagevec_lru_move_fn(struct pagevec *pvec, 207fa9add64SHugh Dickins void (*move_fn)(struct page *page, struct lruvec *lruvec, void *arg), 2083dd7ae8eSShaohua Li void *arg) 209902aaed0SHisashi Hifumi { 210902aaed0SHisashi Hifumi int i; 21168eb0731SMel Gorman struct pglist_data *pgdat = NULL; 212fa9add64SHugh Dickins struct lruvec *lruvec; 2133dd7ae8eSShaohua Li unsigned long flags = 0; 214902aaed0SHisashi Hifumi 215902aaed0SHisashi Hifumi for (i = 0; i < pagevec_count(pvec); i++) { 216902aaed0SHisashi Hifumi struct page *page = pvec->pages[i]; 21768eb0731SMel Gorman struct pglist_data *pagepgdat = page_pgdat(page); 218902aaed0SHisashi Hifumi 21968eb0731SMel Gorman if (pagepgdat != pgdat) { 22068eb0731SMel Gorman if (pgdat) 22168eb0731SMel Gorman spin_unlock_irqrestore(&pgdat->lru_lock, flags); 22268eb0731SMel Gorman pgdat = pagepgdat; 22368eb0731SMel Gorman spin_lock_irqsave(&pgdat->lru_lock, flags); 224902aaed0SHisashi Hifumi } 2253dd7ae8eSShaohua Li 22668eb0731SMel Gorman lruvec = mem_cgroup_page_lruvec(page, pgdat); 227fa9add64SHugh Dickins (*move_fn)(page, lruvec, arg); 2283dd7ae8eSShaohua Li } 22968eb0731SMel Gorman if (pgdat) 23068eb0731SMel Gorman spin_unlock_irqrestore(&pgdat->lru_lock, flags); 231c6f92f9fSMel Gorman release_pages(pvec->pages, pvec->nr); 2323dd7ae8eSShaohua Li pagevec_reinit(pvec); 2333dd7ae8eSShaohua Li } 2343dd7ae8eSShaohua Li 235fa9add64SHugh Dickins static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec, 236fa9add64SHugh Dickins void *arg) 2373dd7ae8eSShaohua Li { 2383dd7ae8eSShaohua Li int *pgmoved = arg; 2393dd7ae8eSShaohua Li 240c55e8d03SJohannes Weiner if (PageLRU(page) && !PageUnevictable(page)) { 241c55e8d03SJohannes Weiner del_page_from_lru_list(page, lruvec, page_lru(page)); 242c55e8d03SJohannes Weiner ClearPageActive(page); 243c55e8d03SJohannes Weiner add_page_to_lru_list_tail(page, lruvec, page_lru(page)); 2443dd7ae8eSShaohua Li (*pgmoved)++; 245902aaed0SHisashi Hifumi } 246902aaed0SHisashi Hifumi } 2473dd7ae8eSShaohua Li 2483dd7ae8eSShaohua Li /* 2493dd7ae8eSShaohua Li * pagevec_move_tail() must be called with IRQ disabled. 2503dd7ae8eSShaohua Li * Otherwise this may cause nasty races. 2513dd7ae8eSShaohua Li */ 2523dd7ae8eSShaohua Li static void pagevec_move_tail(struct pagevec *pvec) 2533dd7ae8eSShaohua Li { 2543dd7ae8eSShaohua Li int pgmoved = 0; 2553dd7ae8eSShaohua Li 2563dd7ae8eSShaohua Li pagevec_lru_move_fn(pvec, pagevec_move_tail_fn, &pgmoved); 257902aaed0SHisashi Hifumi __count_vm_events(PGROTATED, pgmoved); 258902aaed0SHisashi Hifumi } 259902aaed0SHisashi Hifumi 260902aaed0SHisashi Hifumi /* 2611da177e4SLinus Torvalds * Writeback is about to end against a page which has been marked for immediate 2621da177e4SLinus Torvalds * reclaim. If it still appears to be reclaimable, move it to the tail of the 263902aaed0SHisashi Hifumi * inactive list. 2641da177e4SLinus Torvalds */ 265ac6aadb2SMiklos Szeredi void rotate_reclaimable_page(struct page *page) 2661da177e4SLinus Torvalds { 267c55e8d03SJohannes Weiner if (!PageLocked(page) && !PageDirty(page) && 268894bc310SLee Schermerhorn !PageUnevictable(page) && PageLRU(page)) { 269902aaed0SHisashi Hifumi struct pagevec *pvec; 2701da177e4SLinus Torvalds unsigned long flags; 2711da177e4SLinus Torvalds 27209cbfeafSKirill A. Shutemov get_page(page); 273b01b2141SIngo Molnar local_lock_irqsave(&lru_rotate.lock, flags); 274b01b2141SIngo Molnar pvec = this_cpu_ptr(&lru_rotate.pvec); 2758f182270SLukasz Odzioba if (!pagevec_add(pvec, page) || PageCompound(page)) 276902aaed0SHisashi Hifumi pagevec_move_tail(pvec); 277b01b2141SIngo Molnar local_unlock_irqrestore(&lru_rotate.lock, flags); 278ac6aadb2SMiklos Szeredi } 2791da177e4SLinus Torvalds } 2801da177e4SLinus Torvalds 281314b57fbSJohannes Weiner void lru_note_cost(struct page *page) 2823e2f41f1SKOSAKI Motohiro { 283314b57fbSJohannes Weiner struct lruvec *lruvec = mem_cgroup_page_lruvec(page, page_pgdat(page)); 284314b57fbSJohannes Weiner 285*7cf111bcSJohannes Weiner do { 286*7cf111bcSJohannes Weiner unsigned long lrusize; 287*7cf111bcSJohannes Weiner 288*7cf111bcSJohannes Weiner /* Record cost event */ 289314b57fbSJohannes Weiner if (page_is_file_lru(page)) 290314b57fbSJohannes Weiner lruvec->file_cost++; 2911431d4d1SJohannes Weiner else 292314b57fbSJohannes Weiner lruvec->anon_cost++; 293*7cf111bcSJohannes Weiner 294*7cf111bcSJohannes Weiner /* 295*7cf111bcSJohannes Weiner * Decay previous events 296*7cf111bcSJohannes Weiner * 297*7cf111bcSJohannes Weiner * Because workloads change over time (and to avoid 298*7cf111bcSJohannes Weiner * overflow) we keep these statistics as a floating 299*7cf111bcSJohannes Weiner * average, which ends up weighing recent refaults 300*7cf111bcSJohannes Weiner * more than old ones. 301*7cf111bcSJohannes Weiner */ 302*7cf111bcSJohannes Weiner lrusize = lruvec_page_state(lruvec, NR_INACTIVE_ANON) + 303*7cf111bcSJohannes Weiner lruvec_page_state(lruvec, NR_ACTIVE_ANON) + 304*7cf111bcSJohannes Weiner lruvec_page_state(lruvec, NR_INACTIVE_FILE) + 305*7cf111bcSJohannes Weiner lruvec_page_state(lruvec, NR_ACTIVE_FILE); 306*7cf111bcSJohannes Weiner 307*7cf111bcSJohannes Weiner if (lruvec->file_cost + lruvec->anon_cost > lrusize / 4) { 308*7cf111bcSJohannes Weiner lruvec->file_cost /= 2; 309*7cf111bcSJohannes Weiner lruvec->anon_cost /= 2; 310*7cf111bcSJohannes Weiner } 311*7cf111bcSJohannes Weiner } while ((lruvec = parent_lruvec(lruvec))); 3123e2f41f1SKOSAKI Motohiro } 3133e2f41f1SKOSAKI Motohiro 314fa9add64SHugh Dickins static void __activate_page(struct page *page, struct lruvec *lruvec, 315fa9add64SHugh Dickins void *arg) 316744ed144SShaohua Li { 3177a608572SLinus Torvalds if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { 318744ed144SShaohua Li int lru = page_lru_base_type(page); 319744ed144SShaohua Li 320fa9add64SHugh Dickins del_page_from_lru_list(page, lruvec, lru); 321744ed144SShaohua Li SetPageActive(page); 322744ed144SShaohua Li lru += LRU_ACTIVE; 323fa9add64SHugh Dickins add_page_to_lru_list(page, lruvec, lru); 32424b7e581SMel Gorman trace_mm_lru_activate(page); 3257a608572SLinus Torvalds 326fa9add64SHugh Dickins __count_vm_event(PGACTIVATE); 327744ed144SShaohua Li } 328eb709b0dSShaohua Li } 329eb709b0dSShaohua Li 330eb709b0dSShaohua Li #ifdef CONFIG_SMP 331eb709b0dSShaohua Li static void activate_page_drain(int cpu) 332eb709b0dSShaohua Li { 333b01b2141SIngo Molnar struct pagevec *pvec = &per_cpu(lru_pvecs.activate_page, cpu); 334eb709b0dSShaohua Li 335eb709b0dSShaohua Li if (pagevec_count(pvec)) 336eb709b0dSShaohua Li pagevec_lru_move_fn(pvec, __activate_page, NULL); 337eb709b0dSShaohua Li } 338eb709b0dSShaohua Li 3395fbc4616SChris Metcalf static bool need_activate_page_drain(int cpu) 3405fbc4616SChris Metcalf { 341b01b2141SIngo Molnar return pagevec_count(&per_cpu(lru_pvecs.activate_page, cpu)) != 0; 3425fbc4616SChris Metcalf } 3435fbc4616SChris Metcalf 344eb709b0dSShaohua Li void activate_page(struct page *page) 345eb709b0dSShaohua Li { 346800d8c63SKirill A. Shutemov page = compound_head(page); 347eb709b0dSShaohua Li if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { 348b01b2141SIngo Molnar struct pagevec *pvec; 349eb709b0dSShaohua Li 350b01b2141SIngo Molnar local_lock(&lru_pvecs.lock); 351b01b2141SIngo Molnar pvec = this_cpu_ptr(&lru_pvecs.activate_page); 35209cbfeafSKirill A. Shutemov get_page(page); 3538f182270SLukasz Odzioba if (!pagevec_add(pvec, page) || PageCompound(page)) 354eb709b0dSShaohua Li pagevec_lru_move_fn(pvec, __activate_page, NULL); 355b01b2141SIngo Molnar local_unlock(&lru_pvecs.lock); 356eb709b0dSShaohua Li } 357eb709b0dSShaohua Li } 358eb709b0dSShaohua Li 359eb709b0dSShaohua Li #else 360eb709b0dSShaohua Li static inline void activate_page_drain(int cpu) 361eb709b0dSShaohua Li { 362eb709b0dSShaohua Li } 363eb709b0dSShaohua Li 364eb709b0dSShaohua Li void activate_page(struct page *page) 365eb709b0dSShaohua Li { 366f4b7e272SAndrey Ryabinin pg_data_t *pgdat = page_pgdat(page); 367eb709b0dSShaohua Li 368800d8c63SKirill A. Shutemov page = compound_head(page); 369f4b7e272SAndrey Ryabinin spin_lock_irq(&pgdat->lru_lock); 370f4b7e272SAndrey Ryabinin __activate_page(page, mem_cgroup_page_lruvec(page, pgdat), NULL); 371f4b7e272SAndrey Ryabinin spin_unlock_irq(&pgdat->lru_lock); 3721da177e4SLinus Torvalds } 373eb709b0dSShaohua Li #endif 3741da177e4SLinus Torvalds 375059285a2SMel Gorman static void __lru_cache_activate_page(struct page *page) 376059285a2SMel Gorman { 377b01b2141SIngo Molnar struct pagevec *pvec; 378059285a2SMel Gorman int i; 379059285a2SMel Gorman 380b01b2141SIngo Molnar local_lock(&lru_pvecs.lock); 381b01b2141SIngo Molnar pvec = this_cpu_ptr(&lru_pvecs.lru_add); 382b01b2141SIngo Molnar 383059285a2SMel Gorman /* 384059285a2SMel Gorman * Search backwards on the optimistic assumption that the page being 385059285a2SMel Gorman * activated has just been added to this pagevec. Note that only 386059285a2SMel Gorman * the local pagevec is examined as a !PageLRU page could be in the 387059285a2SMel Gorman * process of being released, reclaimed, migrated or on a remote 388059285a2SMel Gorman * pagevec that is currently being drained. Furthermore, marking 389059285a2SMel Gorman * a remote pagevec's page PageActive potentially hits a race where 390059285a2SMel Gorman * a page is marked PageActive just after it is added to the inactive 391059285a2SMel Gorman * list causing accounting errors and BUG_ON checks to trigger. 392059285a2SMel Gorman */ 393059285a2SMel Gorman for (i = pagevec_count(pvec) - 1; i >= 0; i--) { 394059285a2SMel Gorman struct page *pagevec_page = pvec->pages[i]; 395059285a2SMel Gorman 396059285a2SMel Gorman if (pagevec_page == page) { 397059285a2SMel Gorman SetPageActive(page); 398059285a2SMel Gorman break; 399059285a2SMel Gorman } 400059285a2SMel Gorman } 401059285a2SMel Gorman 402b01b2141SIngo Molnar local_unlock(&lru_pvecs.lock); 403059285a2SMel Gorman } 404059285a2SMel Gorman 4051da177e4SLinus Torvalds /* 4061da177e4SLinus Torvalds * Mark a page as having seen activity. 4071da177e4SLinus Torvalds * 4081da177e4SLinus Torvalds * inactive,unreferenced -> inactive,referenced 4091da177e4SLinus Torvalds * inactive,referenced -> active,unreferenced 4101da177e4SLinus Torvalds * active,unreferenced -> active,referenced 411eb39d618SHugh Dickins * 412eb39d618SHugh Dickins * When a newly allocated page is not yet visible, so safe for non-atomic ops, 413eb39d618SHugh Dickins * __SetPageReferenced(page) may be substituted for mark_page_accessed(page). 4141da177e4SLinus Torvalds */ 415920c7a5dSHarvey Harrison void mark_page_accessed(struct page *page) 4161da177e4SLinus Torvalds { 417e90309c9SKirill A. Shutemov page = compound_head(page); 418059285a2SMel Gorman 419a1100a74SFengguang Wu if (!PageReferenced(page)) { 420a1100a74SFengguang Wu SetPageReferenced(page); 421a1100a74SFengguang Wu } else if (PageUnevictable(page)) { 422a1100a74SFengguang Wu /* 423a1100a74SFengguang Wu * Unevictable pages are on the "LRU_UNEVICTABLE" list. But, 424a1100a74SFengguang Wu * this list is never rotated or maintained, so marking an 425a1100a74SFengguang Wu * evictable page accessed has no effect. 426a1100a74SFengguang Wu */ 427a1100a74SFengguang Wu } else if (!PageActive(page)) { 428059285a2SMel Gorman /* 429059285a2SMel Gorman * If the page is on the LRU, queue it for activation via 430b01b2141SIngo Molnar * lru_pvecs.activate_page. Otherwise, assume the page is on a 431059285a2SMel Gorman * pagevec, mark it active and it'll be moved to the active 432059285a2SMel Gorman * LRU on the next drain. 433059285a2SMel Gorman */ 434059285a2SMel Gorman if (PageLRU(page)) 4351da177e4SLinus Torvalds activate_page(page); 436059285a2SMel Gorman else 437059285a2SMel Gorman __lru_cache_activate_page(page); 4381da177e4SLinus Torvalds ClearPageReferenced(page); 4399de4f22aSHuang Ying if (page_is_file_lru(page)) 440a528910eSJohannes Weiner workingset_activation(page); 4411da177e4SLinus Torvalds } 44233c3fc71SVladimir Davydov if (page_is_idle(page)) 44333c3fc71SVladimir Davydov clear_page_idle(page); 4441da177e4SLinus Torvalds } 4451da177e4SLinus Torvalds EXPORT_SYMBOL(mark_page_accessed); 4461da177e4SLinus Torvalds 447f04e9ebbSKOSAKI Motohiro /** 448c53954a0SMel Gorman * lru_cache_add - add a page to a page list 449f04e9ebbSKOSAKI Motohiro * @page: the page to be added to the LRU. 4502329d375SJianyu Zhan * 4512329d375SJianyu Zhan * Queue the page for addition to the LRU via pagevec. The decision on whether 4522329d375SJianyu Zhan * to add the page to the [in]active [file|anon] list is deferred until the 4532329d375SJianyu Zhan * pagevec is drained. This gives a chance for the caller of lru_cache_add() 4542329d375SJianyu Zhan * have the page added to the active list using mark_page_accessed(). 455f04e9ebbSKOSAKI Motohiro */ 456c53954a0SMel Gorman void lru_cache_add(struct page *page) 4571da177e4SLinus Torvalds { 4586058eaecSJohannes Weiner struct pagevec *pvec; 4596058eaecSJohannes Weiner 460309381feSSasha Levin VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page); 461309381feSSasha Levin VM_BUG_ON_PAGE(PageLRU(page), page); 4626058eaecSJohannes Weiner 4636058eaecSJohannes Weiner get_page(page); 4646058eaecSJohannes Weiner local_lock(&lru_pvecs.lock); 4656058eaecSJohannes Weiner pvec = this_cpu_ptr(&lru_pvecs.lru_add); 4666058eaecSJohannes Weiner if (!pagevec_add(pvec, page) || PageCompound(page)) 4676058eaecSJohannes Weiner __pagevec_lru_add(pvec); 4686058eaecSJohannes Weiner local_unlock(&lru_pvecs.lock); 4691da177e4SLinus Torvalds } 4706058eaecSJohannes Weiner EXPORT_SYMBOL(lru_cache_add); 4711da177e4SLinus Torvalds 472894bc310SLee Schermerhorn /** 47300501b53SJohannes Weiner * lru_cache_add_active_or_unevictable 47400501b53SJohannes Weiner * @page: the page to be added to LRU 47500501b53SJohannes Weiner * @vma: vma in which page is mapped for determining reclaimability 47600501b53SJohannes Weiner * 47700501b53SJohannes Weiner * Place @page on the active or unevictable LRU list, depending on its 47800501b53SJohannes Weiner * evictability. Note that if the page is not evictable, it goes 47900501b53SJohannes Weiner * directly back onto it's zone's unevictable list, it does NOT use a 48000501b53SJohannes Weiner * per cpu pagevec. 48100501b53SJohannes Weiner */ 48200501b53SJohannes Weiner void lru_cache_add_active_or_unevictable(struct page *page, 48300501b53SJohannes Weiner struct vm_area_struct *vma) 48400501b53SJohannes Weiner { 48500501b53SJohannes Weiner VM_BUG_ON_PAGE(PageLRU(page), page); 48600501b53SJohannes Weiner 4879c4e6b1aSShakeel Butt if (likely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) != VM_LOCKED)) 48800501b53SJohannes Weiner SetPageActive(page); 4899c4e6b1aSShakeel Butt else if (!TestSetPageMlocked(page)) { 49000501b53SJohannes Weiner /* 49100501b53SJohannes Weiner * We use the irq-unsafe __mod_zone_page_stat because this 49200501b53SJohannes Weiner * counter is not modified from interrupt context, and the pte 49300501b53SJohannes Weiner * lock is held(spinlock), which implies preemption disabled. 49400501b53SJohannes Weiner */ 49500501b53SJohannes Weiner __mod_zone_page_state(page_zone(page), NR_MLOCK, 49600501b53SJohannes Weiner hpage_nr_pages(page)); 49700501b53SJohannes Weiner count_vm_event(UNEVICTABLE_PGMLOCKED); 49800501b53SJohannes Weiner } 4999c4e6b1aSShakeel Butt lru_cache_add(page); 50000501b53SJohannes Weiner } 50100501b53SJohannes Weiner 502902aaed0SHisashi Hifumi /* 50331560180SMinchan Kim * If the page can not be invalidated, it is moved to the 50431560180SMinchan Kim * inactive list to speed up its reclaim. It is moved to the 50531560180SMinchan Kim * head of the list, rather than the tail, to give the flusher 50631560180SMinchan Kim * threads some time to write it out, as this is much more 50731560180SMinchan Kim * effective than the single-page writeout from reclaim. 508278df9f4SMinchan Kim * 509278df9f4SMinchan Kim * If the page isn't page_mapped and dirty/writeback, the page 510278df9f4SMinchan Kim * could reclaim asap using PG_reclaim. 511278df9f4SMinchan Kim * 512278df9f4SMinchan Kim * 1. active, mapped page -> none 513278df9f4SMinchan Kim * 2. active, dirty/writeback page -> inactive, head, PG_reclaim 514278df9f4SMinchan Kim * 3. inactive, mapped page -> none 515278df9f4SMinchan Kim * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim 516278df9f4SMinchan Kim * 5. inactive, clean -> inactive, tail 517278df9f4SMinchan Kim * 6. Others -> none 518278df9f4SMinchan Kim * 519278df9f4SMinchan Kim * In 4, why it moves inactive's head, the VM expects the page would 520278df9f4SMinchan Kim * be write it out by flusher threads as this is much more effective 521278df9f4SMinchan Kim * than the single-page writeout from reclaim. 52231560180SMinchan Kim */ 523cc5993bdSMinchan Kim static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec, 524fa9add64SHugh Dickins void *arg) 52531560180SMinchan Kim { 526fbbb602eSJohannes Weiner int lru; 527278df9f4SMinchan Kim bool active; 52831560180SMinchan Kim 529278df9f4SMinchan Kim if (!PageLRU(page)) 53031560180SMinchan Kim return; 53131560180SMinchan Kim 532bad49d9cSMinchan Kim if (PageUnevictable(page)) 533bad49d9cSMinchan Kim return; 534bad49d9cSMinchan Kim 53531560180SMinchan Kim /* Some processes are using the page */ 53631560180SMinchan Kim if (page_mapped(page)) 53731560180SMinchan Kim return; 53831560180SMinchan Kim 539278df9f4SMinchan Kim active = PageActive(page); 54031560180SMinchan Kim lru = page_lru_base_type(page); 541fa9add64SHugh Dickins 542fa9add64SHugh Dickins del_page_from_lru_list(page, lruvec, lru + active); 54331560180SMinchan Kim ClearPageActive(page); 54431560180SMinchan Kim ClearPageReferenced(page); 54531560180SMinchan Kim 546278df9f4SMinchan Kim if (PageWriteback(page) || PageDirty(page)) { 547278df9f4SMinchan Kim /* 548278df9f4SMinchan Kim * PG_reclaim could be raced with end_page_writeback 549278df9f4SMinchan Kim * It can make readahead confusing. But race window 550278df9f4SMinchan Kim * is _really_ small and it's non-critical problem. 551278df9f4SMinchan Kim */ 552e7a1aaf2SYu Zhao add_page_to_lru_list(page, lruvec, lru); 553278df9f4SMinchan Kim SetPageReclaim(page); 554278df9f4SMinchan Kim } else { 555278df9f4SMinchan Kim /* 556278df9f4SMinchan Kim * The page's writeback ends up during pagevec 557278df9f4SMinchan Kim * We moves tha page into tail of inactive. 558278df9f4SMinchan Kim */ 559e7a1aaf2SYu Zhao add_page_to_lru_list_tail(page, lruvec, lru); 560278df9f4SMinchan Kim __count_vm_event(PGROTATED); 561278df9f4SMinchan Kim } 562278df9f4SMinchan Kim 563278df9f4SMinchan Kim if (active) 564278df9f4SMinchan Kim __count_vm_event(PGDEACTIVATE); 56531560180SMinchan Kim } 56631560180SMinchan Kim 5679c276cc6SMinchan Kim static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec, 5689c276cc6SMinchan Kim void *arg) 5699c276cc6SMinchan Kim { 5709c276cc6SMinchan Kim if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) { 5719c276cc6SMinchan Kim int lru = page_lru_base_type(page); 5729c276cc6SMinchan Kim 5739c276cc6SMinchan Kim del_page_from_lru_list(page, lruvec, lru + LRU_ACTIVE); 5749c276cc6SMinchan Kim ClearPageActive(page); 5759c276cc6SMinchan Kim ClearPageReferenced(page); 5769c276cc6SMinchan Kim add_page_to_lru_list(page, lruvec, lru); 5779c276cc6SMinchan Kim 5789c276cc6SMinchan Kim __count_vm_events(PGDEACTIVATE, hpage_nr_pages(page)); 5799c276cc6SMinchan Kim } 5809c276cc6SMinchan Kim } 58110853a03SMinchan Kim 582f7ad2a6cSShaohua Li static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec, 58310853a03SMinchan Kim void *arg) 58410853a03SMinchan Kim { 585f7ad2a6cSShaohua Li if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) && 58624c92eb7SShaohua Li !PageSwapCache(page) && !PageUnevictable(page)) { 587f7ad2a6cSShaohua Li bool active = PageActive(page); 58810853a03SMinchan Kim 589f7ad2a6cSShaohua Li del_page_from_lru_list(page, lruvec, 590f7ad2a6cSShaohua Li LRU_INACTIVE_ANON + active); 59110853a03SMinchan Kim ClearPageActive(page); 59210853a03SMinchan Kim ClearPageReferenced(page); 593f7ad2a6cSShaohua Li /* 5949de4f22aSHuang Ying * Lazyfree pages are clean anonymous pages. They have 5959de4f22aSHuang Ying * PG_swapbacked flag cleared, to distinguish them from normal 5969de4f22aSHuang Ying * anonymous pages 597f7ad2a6cSShaohua Li */ 598f7ad2a6cSShaohua Li ClearPageSwapBacked(page); 599f7ad2a6cSShaohua Li add_page_to_lru_list(page, lruvec, LRU_INACTIVE_FILE); 60010853a03SMinchan Kim 601f7ad2a6cSShaohua Li __count_vm_events(PGLAZYFREE, hpage_nr_pages(page)); 6022262185cSRoman Gushchin count_memcg_page_event(page, PGLAZYFREE); 60310853a03SMinchan Kim } 60410853a03SMinchan Kim } 60510853a03SMinchan Kim 60631560180SMinchan Kim /* 607902aaed0SHisashi Hifumi * Drain pages out of the cpu's pagevecs. 608902aaed0SHisashi Hifumi * Either "cpu" is the current CPU, and preemption has already been 609902aaed0SHisashi Hifumi * disabled; or "cpu" is being hot-unplugged, and is already dead. 610902aaed0SHisashi Hifumi */ 611f0cb3c76SKonstantin Khlebnikov void lru_add_drain_cpu(int cpu) 6121da177e4SLinus Torvalds { 613b01b2141SIngo Molnar struct pagevec *pvec = &per_cpu(lru_pvecs.lru_add, cpu); 6141da177e4SLinus Torvalds 6151da177e4SLinus Torvalds if (pagevec_count(pvec)) 616a0b8cab3SMel Gorman __pagevec_lru_add(pvec); 617902aaed0SHisashi Hifumi 618b01b2141SIngo Molnar pvec = &per_cpu(lru_rotate.pvec, cpu); 619902aaed0SHisashi Hifumi if (pagevec_count(pvec)) { 620902aaed0SHisashi Hifumi unsigned long flags; 621902aaed0SHisashi Hifumi 622902aaed0SHisashi Hifumi /* No harm done if a racing interrupt already did this */ 623b01b2141SIngo Molnar local_lock_irqsave(&lru_rotate.lock, flags); 624902aaed0SHisashi Hifumi pagevec_move_tail(pvec); 625b01b2141SIngo Molnar local_unlock_irqrestore(&lru_rotate.lock, flags); 626902aaed0SHisashi Hifumi } 62731560180SMinchan Kim 628b01b2141SIngo Molnar pvec = &per_cpu(lru_pvecs.lru_deactivate_file, cpu); 62931560180SMinchan Kim if (pagevec_count(pvec)) 630cc5993bdSMinchan Kim pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL); 631eb709b0dSShaohua Li 632b01b2141SIngo Molnar pvec = &per_cpu(lru_pvecs.lru_deactivate, cpu); 6339c276cc6SMinchan Kim if (pagevec_count(pvec)) 6349c276cc6SMinchan Kim pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL); 6359c276cc6SMinchan Kim 636b01b2141SIngo Molnar pvec = &per_cpu(lru_pvecs.lru_lazyfree, cpu); 63710853a03SMinchan Kim if (pagevec_count(pvec)) 638f7ad2a6cSShaohua Li pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL); 63910853a03SMinchan Kim 640eb709b0dSShaohua Li activate_page_drain(cpu); 64131560180SMinchan Kim } 64231560180SMinchan Kim 64331560180SMinchan Kim /** 644cc5993bdSMinchan Kim * deactivate_file_page - forcefully deactivate a file page 64531560180SMinchan Kim * @page: page to deactivate 64631560180SMinchan Kim * 64731560180SMinchan Kim * This function hints the VM that @page is a good reclaim candidate, 64831560180SMinchan Kim * for example if its invalidation fails due to the page being dirty 64931560180SMinchan Kim * or under writeback. 65031560180SMinchan Kim */ 651cc5993bdSMinchan Kim void deactivate_file_page(struct page *page) 65231560180SMinchan Kim { 653821ed6bbSMinchan Kim /* 654cc5993bdSMinchan Kim * In a workload with many unevictable page such as mprotect, 655cc5993bdSMinchan Kim * unevictable page deactivation for accelerating reclaim is pointless. 656821ed6bbSMinchan Kim */ 657821ed6bbSMinchan Kim if (PageUnevictable(page)) 658821ed6bbSMinchan Kim return; 659821ed6bbSMinchan Kim 66031560180SMinchan Kim if (likely(get_page_unless_zero(page))) { 661b01b2141SIngo Molnar struct pagevec *pvec; 662b01b2141SIngo Molnar 663b01b2141SIngo Molnar local_lock(&lru_pvecs.lock); 664b01b2141SIngo Molnar pvec = this_cpu_ptr(&lru_pvecs.lru_deactivate_file); 66531560180SMinchan Kim 6668f182270SLukasz Odzioba if (!pagevec_add(pvec, page) || PageCompound(page)) 667cc5993bdSMinchan Kim pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL); 668b01b2141SIngo Molnar local_unlock(&lru_pvecs.lock); 66931560180SMinchan Kim } 67080bfed90SAndrew Morton } 67180bfed90SAndrew Morton 6729c276cc6SMinchan Kim /* 6739c276cc6SMinchan Kim * deactivate_page - deactivate a page 6749c276cc6SMinchan Kim * @page: page to deactivate 6759c276cc6SMinchan Kim * 6769c276cc6SMinchan Kim * deactivate_page() moves @page to the inactive list if @page was on the active 6779c276cc6SMinchan Kim * list and was not an unevictable page. This is done to accelerate the reclaim 6789c276cc6SMinchan Kim * of @page. 6799c276cc6SMinchan Kim */ 6809c276cc6SMinchan Kim void deactivate_page(struct page *page) 6819c276cc6SMinchan Kim { 6829c276cc6SMinchan Kim if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) { 683b01b2141SIngo Molnar struct pagevec *pvec; 6849c276cc6SMinchan Kim 685b01b2141SIngo Molnar local_lock(&lru_pvecs.lock); 686b01b2141SIngo Molnar pvec = this_cpu_ptr(&lru_pvecs.lru_deactivate); 6879c276cc6SMinchan Kim get_page(page); 6889c276cc6SMinchan Kim if (!pagevec_add(pvec, page) || PageCompound(page)) 6899c276cc6SMinchan Kim pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL); 690b01b2141SIngo Molnar local_unlock(&lru_pvecs.lock); 6919c276cc6SMinchan Kim } 6929c276cc6SMinchan Kim } 6939c276cc6SMinchan Kim 69410853a03SMinchan Kim /** 695f7ad2a6cSShaohua Li * mark_page_lazyfree - make an anon page lazyfree 69610853a03SMinchan Kim * @page: page to deactivate 69710853a03SMinchan Kim * 698f7ad2a6cSShaohua Li * mark_page_lazyfree() moves @page to the inactive file list. 699f7ad2a6cSShaohua Li * This is done to accelerate the reclaim of @page. 70010853a03SMinchan Kim */ 701f7ad2a6cSShaohua Li void mark_page_lazyfree(struct page *page) 70210853a03SMinchan Kim { 703f7ad2a6cSShaohua Li if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) && 70424c92eb7SShaohua Li !PageSwapCache(page) && !PageUnevictable(page)) { 705b01b2141SIngo Molnar struct pagevec *pvec; 70610853a03SMinchan Kim 707b01b2141SIngo Molnar local_lock(&lru_pvecs.lock); 708b01b2141SIngo Molnar pvec = this_cpu_ptr(&lru_pvecs.lru_lazyfree); 70909cbfeafSKirill A. Shutemov get_page(page); 7108f182270SLukasz Odzioba if (!pagevec_add(pvec, page) || PageCompound(page)) 711f7ad2a6cSShaohua Li pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL); 712b01b2141SIngo Molnar local_unlock(&lru_pvecs.lock); 71310853a03SMinchan Kim } 71410853a03SMinchan Kim } 71510853a03SMinchan Kim 71680bfed90SAndrew Morton void lru_add_drain(void) 71780bfed90SAndrew Morton { 718b01b2141SIngo Molnar local_lock(&lru_pvecs.lock); 719b01b2141SIngo Molnar lru_add_drain_cpu(smp_processor_id()); 720b01b2141SIngo Molnar local_unlock(&lru_pvecs.lock); 721b01b2141SIngo Molnar } 722b01b2141SIngo Molnar 723b01b2141SIngo Molnar void lru_add_drain_cpu_zone(struct zone *zone) 724b01b2141SIngo Molnar { 725b01b2141SIngo Molnar local_lock(&lru_pvecs.lock); 726b01b2141SIngo Molnar lru_add_drain_cpu(smp_processor_id()); 727b01b2141SIngo Molnar drain_local_pages(zone); 728b01b2141SIngo Molnar local_unlock(&lru_pvecs.lock); 7291da177e4SLinus Torvalds } 7301da177e4SLinus Torvalds 7316ea183d6SMichal Hocko #ifdef CONFIG_SMP 7326ea183d6SMichal Hocko 7336ea183d6SMichal Hocko static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work); 7346ea183d6SMichal Hocko 735c4028958SDavid Howells static void lru_add_drain_per_cpu(struct work_struct *dummy) 736053837fcSNick Piggin { 737053837fcSNick Piggin lru_add_drain(); 738053837fcSNick Piggin } 739053837fcSNick Piggin 7409852a721SMichal Hocko /* 7419852a721SMichal Hocko * Doesn't need any cpu hotplug locking because we do rely on per-cpu 7429852a721SMichal Hocko * kworkers being shut down before our page_alloc_cpu_dead callback is 7439852a721SMichal Hocko * executed on the offlined cpu. 7449852a721SMichal Hocko * Calling this function with cpu hotplug locks held can actually lead 7459852a721SMichal Hocko * to obscure indirect dependencies via WQ context. 7469852a721SMichal Hocko */ 7479852a721SMichal Hocko void lru_add_drain_all(void) 748053837fcSNick Piggin { 749eef1a429SKonstantin Khlebnikov static seqcount_t seqcount = SEQCNT_ZERO(seqcount); 7505fbc4616SChris Metcalf static DEFINE_MUTEX(lock); 7515fbc4616SChris Metcalf static struct cpumask has_work; 752eef1a429SKonstantin Khlebnikov int cpu, seq; 7535fbc4616SChris Metcalf 754ce612879SMichal Hocko /* 755ce612879SMichal Hocko * Make sure nobody triggers this path before mm_percpu_wq is fully 756ce612879SMichal Hocko * initialized. 757ce612879SMichal Hocko */ 758ce612879SMichal Hocko if (WARN_ON(!mm_percpu_wq)) 759ce612879SMichal Hocko return; 760ce612879SMichal Hocko 761eef1a429SKonstantin Khlebnikov seq = raw_read_seqcount_latch(&seqcount); 762eef1a429SKonstantin Khlebnikov 7635fbc4616SChris Metcalf mutex_lock(&lock); 764eef1a429SKonstantin Khlebnikov 765eef1a429SKonstantin Khlebnikov /* 766eef1a429SKonstantin Khlebnikov * Piggyback on drain started and finished while we waited for lock: 767eef1a429SKonstantin Khlebnikov * all pages pended at the time of our enter were drained from vectors. 768eef1a429SKonstantin Khlebnikov */ 769eef1a429SKonstantin Khlebnikov if (__read_seqcount_retry(&seqcount, seq)) 770eef1a429SKonstantin Khlebnikov goto done; 771eef1a429SKonstantin Khlebnikov 772eef1a429SKonstantin Khlebnikov raw_write_seqcount_latch(&seqcount); 773eef1a429SKonstantin Khlebnikov 7745fbc4616SChris Metcalf cpumask_clear(&has_work); 7755fbc4616SChris Metcalf 7765fbc4616SChris Metcalf for_each_online_cpu(cpu) { 7775fbc4616SChris Metcalf struct work_struct *work = &per_cpu(lru_add_drain_work, cpu); 7785fbc4616SChris Metcalf 779b01b2141SIngo Molnar if (pagevec_count(&per_cpu(lru_pvecs.lru_add, cpu)) || 780b01b2141SIngo Molnar pagevec_count(&per_cpu(lru_rotate.pvec, cpu)) || 781b01b2141SIngo Molnar pagevec_count(&per_cpu(lru_pvecs.lru_deactivate_file, cpu)) || 782b01b2141SIngo Molnar pagevec_count(&per_cpu(lru_pvecs.lru_deactivate, cpu)) || 783b01b2141SIngo Molnar pagevec_count(&per_cpu(lru_pvecs.lru_lazyfree, cpu)) || 7845fbc4616SChris Metcalf need_activate_page_drain(cpu)) { 7855fbc4616SChris Metcalf INIT_WORK(work, lru_add_drain_per_cpu); 786ce612879SMichal Hocko queue_work_on(cpu, mm_percpu_wq, work); 7875fbc4616SChris Metcalf cpumask_set_cpu(cpu, &has_work); 7885fbc4616SChris Metcalf } 7895fbc4616SChris Metcalf } 7905fbc4616SChris Metcalf 7915fbc4616SChris Metcalf for_each_cpu(cpu, &has_work) 7925fbc4616SChris Metcalf flush_work(&per_cpu(lru_add_drain_work, cpu)); 7935fbc4616SChris Metcalf 794eef1a429SKonstantin Khlebnikov done: 7955fbc4616SChris Metcalf mutex_unlock(&lock); 796053837fcSNick Piggin } 7976ea183d6SMichal Hocko #else 7986ea183d6SMichal Hocko void lru_add_drain_all(void) 7996ea183d6SMichal Hocko { 8006ea183d6SMichal Hocko lru_add_drain(); 8016ea183d6SMichal Hocko } 8026ea183d6SMichal Hocko #endif 803053837fcSNick Piggin 804aabfb572SMichal Hocko /** 805ea1754a0SKirill A. Shutemov * release_pages - batched put_page() 806aabfb572SMichal Hocko * @pages: array of pages to release 807aabfb572SMichal Hocko * @nr: number of pages 8081da177e4SLinus Torvalds * 809aabfb572SMichal Hocko * Decrement the reference count on all the pages in @pages. If it 810aabfb572SMichal Hocko * fell to zero, remove the page from the LRU and free it. 8111da177e4SLinus Torvalds */ 812c6f92f9fSMel Gorman void release_pages(struct page **pages, int nr) 8131da177e4SLinus Torvalds { 8141da177e4SLinus Torvalds int i; 815cc59850eSKonstantin Khlebnikov LIST_HEAD(pages_to_free); 816599d0c95SMel Gorman struct pglist_data *locked_pgdat = NULL; 817fa9add64SHugh Dickins struct lruvec *lruvec; 818902aaed0SHisashi Hifumi unsigned long uninitialized_var(flags); 819aabfb572SMichal Hocko unsigned int uninitialized_var(lock_batch); 8201da177e4SLinus Torvalds 8211da177e4SLinus Torvalds for (i = 0; i < nr; i++) { 8221da177e4SLinus Torvalds struct page *page = pages[i]; 8231da177e4SLinus Torvalds 824aabfb572SMichal Hocko /* 825aabfb572SMichal Hocko * Make sure the IRQ-safe lock-holding time does not get 826aabfb572SMichal Hocko * excessive with a continuous string of pages from the 827599d0c95SMel Gorman * same pgdat. The lock is held only if pgdat != NULL. 828aabfb572SMichal Hocko */ 829599d0c95SMel Gorman if (locked_pgdat && ++lock_batch == SWAP_CLUSTER_MAX) { 830599d0c95SMel Gorman spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); 831599d0c95SMel Gorman locked_pgdat = NULL; 832aabfb572SMichal Hocko } 833aabfb572SMichal Hocko 8346fcb52a5SAaron Lu if (is_huge_zero_page(page)) 835aa88b68cSKirill A. Shutemov continue; 836aa88b68cSKirill A. Shutemov 837c5d6c45eSIra Weiny if (is_zone_device_page(page)) { 838df6ad698SJérôme Glisse if (locked_pgdat) { 839df6ad698SJérôme Glisse spin_unlock_irqrestore(&locked_pgdat->lru_lock, 840df6ad698SJérôme Glisse flags); 841df6ad698SJérôme Glisse locked_pgdat = NULL; 842df6ad698SJérôme Glisse } 843c5d6c45eSIra Weiny /* 844c5d6c45eSIra Weiny * ZONE_DEVICE pages that return 'false' from 845c5d6c45eSIra Weiny * put_devmap_managed_page() do not require special 846c5d6c45eSIra Weiny * processing, and instead, expect a call to 847c5d6c45eSIra Weiny * put_page_testzero(). 848c5d6c45eSIra Weiny */ 84907d80269SJohn Hubbard if (page_is_devmap_managed(page)) { 85007d80269SJohn Hubbard put_devmap_managed_page(page); 851df6ad698SJérôme Glisse continue; 852df6ad698SJérôme Glisse } 85307d80269SJohn Hubbard } 854df6ad698SJérôme Glisse 855ddc58f27SKirill A. Shutemov page = compound_head(page); 856b5810039SNick Piggin if (!put_page_testzero(page)) 8571da177e4SLinus Torvalds continue; 8581da177e4SLinus Torvalds 859ddc58f27SKirill A. Shutemov if (PageCompound(page)) { 860599d0c95SMel Gorman if (locked_pgdat) { 861599d0c95SMel Gorman spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); 862599d0c95SMel Gorman locked_pgdat = NULL; 863ddc58f27SKirill A. Shutemov } 864ddc58f27SKirill A. Shutemov __put_compound_page(page); 865ddc58f27SKirill A. Shutemov continue; 866ddc58f27SKirill A. Shutemov } 867ddc58f27SKirill A. Shutemov 86846453a6eSNick Piggin if (PageLRU(page)) { 869599d0c95SMel Gorman struct pglist_data *pgdat = page_pgdat(page); 870894bc310SLee Schermerhorn 871599d0c95SMel Gorman if (pgdat != locked_pgdat) { 872599d0c95SMel Gorman if (locked_pgdat) 873599d0c95SMel Gorman spin_unlock_irqrestore(&locked_pgdat->lru_lock, 874902aaed0SHisashi Hifumi flags); 875aabfb572SMichal Hocko lock_batch = 0; 876599d0c95SMel Gorman locked_pgdat = pgdat; 877599d0c95SMel Gorman spin_lock_irqsave(&locked_pgdat->lru_lock, flags); 8781da177e4SLinus Torvalds } 879fa9add64SHugh Dickins 880599d0c95SMel Gorman lruvec = mem_cgroup_page_lruvec(page, locked_pgdat); 881309381feSSasha Levin VM_BUG_ON_PAGE(!PageLRU(page), page); 88267453911SNick Piggin __ClearPageLRU(page); 883fa9add64SHugh Dickins del_page_from_lru_list(page, lruvec, page_off_lru(page)); 88446453a6eSNick Piggin } 88546453a6eSNick Piggin 886c53954a0SMel Gorman /* Clear Active bit in case of parallel mark_page_accessed */ 887e3741b50SMel Gorman __ClearPageActive(page); 88862906027SNicholas Piggin __ClearPageWaiters(page); 889c53954a0SMel Gorman 890cc59850eSKonstantin Khlebnikov list_add(&page->lru, &pages_to_free); 8911da177e4SLinus Torvalds } 892599d0c95SMel Gorman if (locked_pgdat) 893599d0c95SMel Gorman spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); 8941da177e4SLinus Torvalds 895747db954SJohannes Weiner mem_cgroup_uncharge_list(&pages_to_free); 8962d4894b5SMel Gorman free_unref_page_list(&pages_to_free); 8971da177e4SLinus Torvalds } 8980be8557bSMiklos Szeredi EXPORT_SYMBOL(release_pages); 8991da177e4SLinus Torvalds 9001da177e4SLinus Torvalds /* 9011da177e4SLinus Torvalds * The pages which we're about to release may be in the deferred lru-addition 9021da177e4SLinus Torvalds * queues. That would prevent them from really being freed right now. That's 9031da177e4SLinus Torvalds * OK from a correctness point of view but is inefficient - those pages may be 9041da177e4SLinus Torvalds * cache-warm and we want to give them back to the page allocator ASAP. 9051da177e4SLinus Torvalds * 9061da177e4SLinus Torvalds * So __pagevec_release() will drain those queues here. __pagevec_lru_add() 9071da177e4SLinus Torvalds * and __pagevec_lru_add_active() call release_pages() directly to avoid 9081da177e4SLinus Torvalds * mutual recursion. 9091da177e4SLinus Torvalds */ 9101da177e4SLinus Torvalds void __pagevec_release(struct pagevec *pvec) 9111da177e4SLinus Torvalds { 9127f0b5fb9SMel Gorman if (!pvec->percpu_pvec_drained) { 9131da177e4SLinus Torvalds lru_add_drain(); 9147f0b5fb9SMel Gorman pvec->percpu_pvec_drained = true; 915d9ed0d08SMel Gorman } 916c6f92f9fSMel Gorman release_pages(pvec->pages, pagevec_count(pvec)); 9171da177e4SLinus Torvalds pagevec_reinit(pvec); 9181da177e4SLinus Torvalds } 9197f285701SSteve French EXPORT_SYMBOL(__pagevec_release); 9207f285701SSteve French 92112d27107SHugh Dickins #ifdef CONFIG_TRANSPARENT_HUGEPAGE 92271e3aac0SAndrea Arcangeli /* used by __split_huge_page_refcount() */ 923fa9add64SHugh Dickins void lru_add_page_tail(struct page *page, struct page *page_tail, 9245bc7b8acSShaohua Li struct lruvec *lruvec, struct list_head *list) 92571e3aac0SAndrea Arcangeli { 926309381feSSasha Levin VM_BUG_ON_PAGE(!PageHead(page), page); 927309381feSSasha Levin VM_BUG_ON_PAGE(PageCompound(page_tail), page); 928309381feSSasha Levin VM_BUG_ON_PAGE(PageLRU(page_tail), page); 92935f3aa39SLance Roy lockdep_assert_held(&lruvec_pgdat(lruvec)->lru_lock); 93071e3aac0SAndrea Arcangeli 9315bc7b8acSShaohua Li if (!list) 93271e3aac0SAndrea Arcangeli SetPageLRU(page_tail); 93371e3aac0SAndrea Arcangeli 93412d27107SHugh Dickins if (likely(PageLRU(page))) 93512d27107SHugh Dickins list_add_tail(&page_tail->lru, &page->lru); 9365bc7b8acSShaohua Li else if (list) { 9375bc7b8acSShaohua Li /* page reclaim is reclaiming a huge page */ 9385bc7b8acSShaohua Li get_page(page_tail); 9395bc7b8acSShaohua Li list_add_tail(&page_tail->lru, list); 9405bc7b8acSShaohua Li } else { 94112d27107SHugh Dickins /* 94212d27107SHugh Dickins * Head page has not yet been counted, as an hpage, 94312d27107SHugh Dickins * so we must account for each subpage individually. 94412d27107SHugh Dickins * 945e7a1aaf2SYu Zhao * Put page_tail on the list at the correct position 946e7a1aaf2SYu Zhao * so they all end up in order. 94712d27107SHugh Dickins */ 948e7a1aaf2SYu Zhao add_page_to_lru_list_tail(page_tail, lruvec, 949e7a1aaf2SYu Zhao page_lru(page_tail)); 95071e3aac0SAndrea Arcangeli } 95171e3aac0SAndrea Arcangeli } 95212d27107SHugh Dickins #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 95371e3aac0SAndrea Arcangeli 954fa9add64SHugh Dickins static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec, 955fa9add64SHugh Dickins void *arg) 9563dd7ae8eSShaohua Li { 9579c4e6b1aSShakeel Butt enum lru_list lru; 9589c4e6b1aSShakeel Butt int was_unevictable = TestClearPageUnevictable(page); 9593dd7ae8eSShaohua Li 960309381feSSasha Levin VM_BUG_ON_PAGE(PageLRU(page), page); 9613dd7ae8eSShaohua Li 9629c4e6b1aSShakeel Butt /* 9639c4e6b1aSShakeel Butt * Page becomes evictable in two ways: 964dae966dcSPeng Fan * 1) Within LRU lock [munlock_vma_page() and __munlock_pagevec()]. 9659c4e6b1aSShakeel Butt * 2) Before acquiring LRU lock to put the page to correct LRU and then 9669c4e6b1aSShakeel Butt * a) do PageLRU check with lock [check_move_unevictable_pages] 9679c4e6b1aSShakeel Butt * b) do PageLRU check before lock [clear_page_mlock] 9689c4e6b1aSShakeel Butt * 9699c4e6b1aSShakeel Butt * (1) & (2a) are ok as LRU lock will serialize them. For (2b), we need 9709c4e6b1aSShakeel Butt * following strict ordering: 9719c4e6b1aSShakeel Butt * 9729c4e6b1aSShakeel Butt * #0: __pagevec_lru_add_fn #1: clear_page_mlock 9739c4e6b1aSShakeel Butt * 9749c4e6b1aSShakeel Butt * SetPageLRU() TestClearPageMlocked() 9759c4e6b1aSShakeel Butt * smp_mb() // explicit ordering // above provides strict 9769c4e6b1aSShakeel Butt * // ordering 9779c4e6b1aSShakeel Butt * PageMlocked() PageLRU() 9789c4e6b1aSShakeel Butt * 9799c4e6b1aSShakeel Butt * 9809c4e6b1aSShakeel Butt * if '#1' does not observe setting of PG_lru by '#0' and fails 9819c4e6b1aSShakeel Butt * isolation, the explicit barrier will make sure that page_evictable 9829c4e6b1aSShakeel Butt * check will put the page in correct LRU. Without smp_mb(), SetPageLRU 9839c4e6b1aSShakeel Butt * can be reordered after PageMlocked check and can make '#1' to fail 9849c4e6b1aSShakeel Butt * the isolation of the page whose Mlocked bit is cleared (#0 is also 9859c4e6b1aSShakeel Butt * looking at the same page) and the evictable page will be stranded 9869c4e6b1aSShakeel Butt * in an unevictable LRU. 9879c4e6b1aSShakeel Butt */ 9889a9b6cceSYang Shi SetPageLRU(page); 9899a9b6cceSYang Shi smp_mb__after_atomic(); 9909c4e6b1aSShakeel Butt 9919c4e6b1aSShakeel Butt if (page_evictable(page)) { 9929c4e6b1aSShakeel Butt lru = page_lru(page); 9939c4e6b1aSShakeel Butt if (was_unevictable) 9949c4e6b1aSShakeel Butt count_vm_event(UNEVICTABLE_PGRESCUED); 9959c4e6b1aSShakeel Butt } else { 9969c4e6b1aSShakeel Butt lru = LRU_UNEVICTABLE; 9979c4e6b1aSShakeel Butt ClearPageActive(page); 9989c4e6b1aSShakeel Butt SetPageUnevictable(page); 9999c4e6b1aSShakeel Butt if (!was_unevictable) 10009c4e6b1aSShakeel Butt count_vm_event(UNEVICTABLE_PGCULLED); 10019c4e6b1aSShakeel Butt } 10029c4e6b1aSShakeel Butt 1003fa9add64SHugh Dickins add_page_to_lru_list(page, lruvec, lru); 100424b7e581SMel Gorman trace_mm_lru_insertion(page, lru); 10053dd7ae8eSShaohua Li } 10063dd7ae8eSShaohua Li 10071da177e4SLinus Torvalds /* 10081da177e4SLinus Torvalds * Add the passed pages to the LRU, then drop the caller's refcount 10091da177e4SLinus Torvalds * on them. Reinitialises the caller's pagevec. 10101da177e4SLinus Torvalds */ 1011a0b8cab3SMel Gorman void __pagevec_lru_add(struct pagevec *pvec) 10121da177e4SLinus Torvalds { 1013a0b8cab3SMel Gorman pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, NULL); 10141da177e4SLinus Torvalds } 1015f04e9ebbSKOSAKI Motohiro 10161da177e4SLinus Torvalds /** 10170cd6144aSJohannes Weiner * pagevec_lookup_entries - gang pagecache lookup 10180cd6144aSJohannes Weiner * @pvec: Where the resulting entries are placed 10190cd6144aSJohannes Weiner * @mapping: The address_space to search 10200cd6144aSJohannes Weiner * @start: The starting entry index 1021cb6f0f34SMike Rapoport * @nr_entries: The maximum number of pages 10220cd6144aSJohannes Weiner * @indices: The cache indices corresponding to the entries in @pvec 10230cd6144aSJohannes Weiner * 10240cd6144aSJohannes Weiner * pagevec_lookup_entries() will search for and return a group of up 1025f144c390SMike Rapoport * to @nr_pages pages and shadow entries in the mapping. All 10260cd6144aSJohannes Weiner * entries are placed in @pvec. pagevec_lookup_entries() takes a 10270cd6144aSJohannes Weiner * reference against actual pages in @pvec. 10280cd6144aSJohannes Weiner * 10290cd6144aSJohannes Weiner * The search returns a group of mapping-contiguous entries with 10300cd6144aSJohannes Weiner * ascending indexes. There may be holes in the indices due to 10310cd6144aSJohannes Weiner * not-present entries. 10320cd6144aSJohannes Weiner * 103371725ed1SHugh Dickins * Only one subpage of a Transparent Huge Page is returned in one call: 103471725ed1SHugh Dickins * allowing truncate_inode_pages_range() to evict the whole THP without 103571725ed1SHugh Dickins * cycling through a pagevec of extra references. 103671725ed1SHugh Dickins * 10370cd6144aSJohannes Weiner * pagevec_lookup_entries() returns the number of entries which were 10380cd6144aSJohannes Weiner * found. 10390cd6144aSJohannes Weiner */ 10400cd6144aSJohannes Weiner unsigned pagevec_lookup_entries(struct pagevec *pvec, 10410cd6144aSJohannes Weiner struct address_space *mapping, 1042e02a9f04SRandy Dunlap pgoff_t start, unsigned nr_entries, 10430cd6144aSJohannes Weiner pgoff_t *indices) 10440cd6144aSJohannes Weiner { 1045e02a9f04SRandy Dunlap pvec->nr = find_get_entries(mapping, start, nr_entries, 10460cd6144aSJohannes Weiner pvec->pages, indices); 10470cd6144aSJohannes Weiner return pagevec_count(pvec); 10480cd6144aSJohannes Weiner } 10490cd6144aSJohannes Weiner 10500cd6144aSJohannes Weiner /** 10510cd6144aSJohannes Weiner * pagevec_remove_exceptionals - pagevec exceptionals pruning 10520cd6144aSJohannes Weiner * @pvec: The pagevec to prune 10530cd6144aSJohannes Weiner * 10540cd6144aSJohannes Weiner * pagevec_lookup_entries() fills both pages and exceptional radix 10550cd6144aSJohannes Weiner * tree entries into the pagevec. This function prunes all 10560cd6144aSJohannes Weiner * exceptionals from @pvec without leaving holes, so that it can be 10570cd6144aSJohannes Weiner * passed on to page-only pagevec operations. 10580cd6144aSJohannes Weiner */ 10590cd6144aSJohannes Weiner void pagevec_remove_exceptionals(struct pagevec *pvec) 10600cd6144aSJohannes Weiner { 10610cd6144aSJohannes Weiner int i, j; 10620cd6144aSJohannes Weiner 10630cd6144aSJohannes Weiner for (i = 0, j = 0; i < pagevec_count(pvec); i++) { 10640cd6144aSJohannes Weiner struct page *page = pvec->pages[i]; 10653159f943SMatthew Wilcox if (!xa_is_value(page)) 10660cd6144aSJohannes Weiner pvec->pages[j++] = page; 10670cd6144aSJohannes Weiner } 10680cd6144aSJohannes Weiner pvec->nr = j; 10690cd6144aSJohannes Weiner } 10700cd6144aSJohannes Weiner 10710cd6144aSJohannes Weiner /** 1072b947cee4SJan Kara * pagevec_lookup_range - gang pagecache lookup 10731da177e4SLinus Torvalds * @pvec: Where the resulting pages are placed 10741da177e4SLinus Torvalds * @mapping: The address_space to search 10751da177e4SLinus Torvalds * @start: The starting page index 1076b947cee4SJan Kara * @end: The final page index 10771da177e4SLinus Torvalds * 1078e02a9f04SRandy Dunlap * pagevec_lookup_range() will search for & return a group of up to PAGEVEC_SIZE 1079b947cee4SJan Kara * pages in the mapping starting from index @start and upto index @end 1080b947cee4SJan Kara * (inclusive). The pages are placed in @pvec. pagevec_lookup() takes a 10811da177e4SLinus Torvalds * reference against the pages in @pvec. 10821da177e4SLinus Torvalds * 10831da177e4SLinus Torvalds * The search returns a group of mapping-contiguous pages with ascending 1084d72dc8a2SJan Kara * indexes. There may be holes in the indices due to not-present pages. We 1085d72dc8a2SJan Kara * also update @start to index the next page for the traversal. 10861da177e4SLinus Torvalds * 1087b947cee4SJan Kara * pagevec_lookup_range() returns the number of pages which were found. If this 1088e02a9f04SRandy Dunlap * number is smaller than PAGEVEC_SIZE, the end of specified range has been 1089b947cee4SJan Kara * reached. 10901da177e4SLinus Torvalds */ 1091b947cee4SJan Kara unsigned pagevec_lookup_range(struct pagevec *pvec, 1092397162ffSJan Kara struct address_space *mapping, pgoff_t *start, pgoff_t end) 10931da177e4SLinus Torvalds { 1094397162ffSJan Kara pvec->nr = find_get_pages_range(mapping, start, end, PAGEVEC_SIZE, 1095b947cee4SJan Kara pvec->pages); 10961da177e4SLinus Torvalds return pagevec_count(pvec); 10971da177e4SLinus Torvalds } 1098b947cee4SJan Kara EXPORT_SYMBOL(pagevec_lookup_range); 109978539fdfSChristoph Hellwig 110072b045aeSJan Kara unsigned pagevec_lookup_range_tag(struct pagevec *pvec, 110172b045aeSJan Kara struct address_space *mapping, pgoff_t *index, pgoff_t end, 110210bbd235SMatthew Wilcox xa_mark_t tag) 11031da177e4SLinus Torvalds { 110472b045aeSJan Kara pvec->nr = find_get_pages_range_tag(mapping, index, end, tag, 110567fd707fSJan Kara PAGEVEC_SIZE, pvec->pages); 11061da177e4SLinus Torvalds return pagevec_count(pvec); 11071da177e4SLinus Torvalds } 110872b045aeSJan Kara EXPORT_SYMBOL(pagevec_lookup_range_tag); 11091da177e4SLinus Torvalds 111093d3b714SJan Kara unsigned pagevec_lookup_range_nr_tag(struct pagevec *pvec, 111193d3b714SJan Kara struct address_space *mapping, pgoff_t *index, pgoff_t end, 111210bbd235SMatthew Wilcox xa_mark_t tag, unsigned max_pages) 111393d3b714SJan Kara { 111493d3b714SJan Kara pvec->nr = find_get_pages_range_tag(mapping, index, end, tag, 111593d3b714SJan Kara min_t(unsigned int, max_pages, PAGEVEC_SIZE), pvec->pages); 111693d3b714SJan Kara return pagevec_count(pvec); 111793d3b714SJan Kara } 111893d3b714SJan Kara EXPORT_SYMBOL(pagevec_lookup_range_nr_tag); 11191da177e4SLinus Torvalds /* 11201da177e4SLinus Torvalds * Perform any setup for the swap system 11211da177e4SLinus Torvalds */ 11221da177e4SLinus Torvalds void __init swap_setup(void) 11231da177e4SLinus Torvalds { 1124ca79b0c2SArun KS unsigned long megs = totalram_pages() >> (20 - PAGE_SHIFT); 1125e0bf68ddSPeter Zijlstra 11261da177e4SLinus Torvalds /* Use a smaller cluster for small-memory machines */ 11271da177e4SLinus Torvalds if (megs < 16) 11281da177e4SLinus Torvalds page_cluster = 2; 11291da177e4SLinus Torvalds else 11301da177e4SLinus Torvalds page_cluster = 3; 11311da177e4SLinus Torvalds /* 11321da177e4SLinus Torvalds * Right now other parts of the system means that we 11331da177e4SLinus Torvalds * _really_ don't want to cluster much more 11341da177e4SLinus Torvalds */ 11351da177e4SLinus Torvalds } 113607d80269SJohn Hubbard 113707d80269SJohn Hubbard #ifdef CONFIG_DEV_PAGEMAP_OPS 113807d80269SJohn Hubbard void put_devmap_managed_page(struct page *page) 113907d80269SJohn Hubbard { 114007d80269SJohn Hubbard int count; 114107d80269SJohn Hubbard 114207d80269SJohn Hubbard if (WARN_ON_ONCE(!page_is_devmap_managed(page))) 114307d80269SJohn Hubbard return; 114407d80269SJohn Hubbard 114507d80269SJohn Hubbard count = page_ref_dec_return(page); 114607d80269SJohn Hubbard 114707d80269SJohn Hubbard /* 114807d80269SJohn Hubbard * devmap page refcounts are 1-based, rather than 0-based: if 114907d80269SJohn Hubbard * refcount is 1, then the page is free and the refcount is 115007d80269SJohn Hubbard * stable because nobody holds a reference on the page. 115107d80269SJohn Hubbard */ 115207d80269SJohn Hubbard if (count == 1) 115307d80269SJohn Hubbard free_devmap_managed_page(page); 115407d80269SJohn Hubbard else if (!count) 115507d80269SJohn Hubbard __put_page(page); 115607d80269SJohn Hubbard } 115707d80269SJohn Hubbard EXPORT_SYMBOL(put_devmap_managed_page); 115807d80269SJohn Hubbard #endif 1159