11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * linux/mm/swap.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 51da177e4SLinus Torvalds */ 61da177e4SLinus Torvalds 71da177e4SLinus Torvalds /* 8183ff22bSSimon Arlott * This file contains the default values for the operation of the 91da177e4SLinus Torvalds * Linux VM subsystem. Fine-tuning documentation can be found in 101da177e4SLinus Torvalds * Documentation/sysctl/vm.txt. 111da177e4SLinus Torvalds * Started 18.12.91 121da177e4SLinus Torvalds * Swap aging added 23.2.95, Stephen Tweedie. 131da177e4SLinus Torvalds * Buffermem limits added 12.3.98, Rik van Riel. 141da177e4SLinus Torvalds */ 151da177e4SLinus Torvalds 161da177e4SLinus Torvalds #include <linux/mm.h> 171da177e4SLinus Torvalds #include <linux/sched.h> 181da177e4SLinus Torvalds #include <linux/kernel_stat.h> 191da177e4SLinus Torvalds #include <linux/swap.h> 201da177e4SLinus Torvalds #include <linux/mman.h> 211da177e4SLinus Torvalds #include <linux/pagemap.h> 221da177e4SLinus Torvalds #include <linux/pagevec.h> 231da177e4SLinus Torvalds #include <linux/init.h> 24b95f1b31SPaul Gortmaker #include <linux/export.h> 251da177e4SLinus Torvalds #include <linux/mm_inline.h> 261da177e4SLinus Torvalds #include <linux/percpu_counter.h> 273565fce3SDan Williams #include <linux/memremap.h> 281da177e4SLinus Torvalds #include <linux/percpu.h> 291da177e4SLinus Torvalds #include <linux/cpu.h> 301da177e4SLinus Torvalds #include <linux/notifier.h> 31e0bf68ddSPeter Zijlstra #include <linux/backing-dev.h> 3266e1707bSBalbir Singh #include <linux/memcontrol.h> 335a0e3ad6STejun Heo #include <linux/gfp.h> 34a27bb332SKent Overstreet #include <linux/uio.h> 35822fc613SNaoya Horiguchi #include <linux/hugetlb.h> 3633c3fc71SVladimir Davydov #include <linux/page_idle.h> 371da177e4SLinus Torvalds 3864d6519dSLee Schermerhorn #include "internal.h" 3964d6519dSLee Schermerhorn 40c6286c98SMel Gorman #define CREATE_TRACE_POINTS 41c6286c98SMel Gorman #include <trace/events/pagemap.h> 42c6286c98SMel Gorman 431da177e4SLinus Torvalds /* How many pages do we try to swap or page in/out together? */ 441da177e4SLinus Torvalds int page_cluster; 451da177e4SLinus Torvalds 4613f7f789SMel Gorman static DEFINE_PER_CPU(struct pagevec, lru_add_pvec); 47f84f9504SVegard Nossum static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs); 48cc5993bdSMinchan Kim static DEFINE_PER_CPU(struct pagevec, lru_deactivate_file_pvecs); 49f7ad2a6cSShaohua Li static DEFINE_PER_CPU(struct pagevec, lru_lazyfree_pvecs); 50a4a921aaSMing Li #ifdef CONFIG_SMP 51a4a921aaSMing Li static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs); 52a4a921aaSMing Li #endif 53902aaed0SHisashi Hifumi 54b221385bSAdrian Bunk /* 55b221385bSAdrian Bunk * This path almost never happens for VM activity - pages are normally 56b221385bSAdrian Bunk * freed via pagevecs. But it gets used by networking. 57b221385bSAdrian Bunk */ 58920c7a5dSHarvey Harrison static void __page_cache_release(struct page *page) 59b221385bSAdrian Bunk { 60b221385bSAdrian Bunk if (PageLRU(page)) { 61b221385bSAdrian Bunk struct zone *zone = page_zone(page); 62fa9add64SHugh Dickins struct lruvec *lruvec; 63fa9add64SHugh Dickins unsigned long flags; 64b221385bSAdrian Bunk 65a52633d8SMel Gorman spin_lock_irqsave(zone_lru_lock(zone), flags); 66599d0c95SMel Gorman lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat); 67309381feSSasha Levin VM_BUG_ON_PAGE(!PageLRU(page), page); 68b221385bSAdrian Bunk __ClearPageLRU(page); 69fa9add64SHugh Dickins del_page_from_lru_list(page, lruvec, page_off_lru(page)); 70a52633d8SMel Gorman spin_unlock_irqrestore(zone_lru_lock(zone), flags); 71b221385bSAdrian Bunk } 7262906027SNicholas Piggin __ClearPageWaiters(page); 730a31bc97SJohannes Weiner mem_cgroup_uncharge(page); 7491807063SAndrea Arcangeli } 7591807063SAndrea Arcangeli 7691807063SAndrea Arcangeli static void __put_single_page(struct page *page) 7791807063SAndrea Arcangeli { 7891807063SAndrea Arcangeli __page_cache_release(page); 79b745bc85SMel Gorman free_hot_cold_page(page, false); 80b221385bSAdrian Bunk } 81b221385bSAdrian Bunk 8291807063SAndrea Arcangeli static void __put_compound_page(struct page *page) 8391807063SAndrea Arcangeli { 8491807063SAndrea Arcangeli compound_page_dtor *dtor; 8591807063SAndrea Arcangeli 86822fc613SNaoya Horiguchi /* 87822fc613SNaoya Horiguchi * __page_cache_release() is supposed to be called for thp, not for 88822fc613SNaoya Horiguchi * hugetlb. This is because hugetlb page does never have PageLRU set 89822fc613SNaoya Horiguchi * (it's never listed to any LRU lists) and no memcg routines should 90822fc613SNaoya Horiguchi * be called for hugetlb (it has a separate hugetlb_cgroup.) 91822fc613SNaoya Horiguchi */ 92822fc613SNaoya Horiguchi if (!PageHuge(page)) 9391807063SAndrea Arcangeli __page_cache_release(page); 9491807063SAndrea Arcangeli dtor = get_compound_page_dtor(page); 9591807063SAndrea Arcangeli (*dtor)(page); 9691807063SAndrea Arcangeli } 9791807063SAndrea Arcangeli 98ddc58f27SKirill A. Shutemov void __put_page(struct page *page) 99c747ce79SJianyu Zhan { 10071389703SDan Williams if (is_zone_device_page(page)) { 10171389703SDan Williams put_dev_pagemap(page->pgmap); 10271389703SDan Williams 10371389703SDan Williams /* 10471389703SDan Williams * The page belongs to the device that created pgmap. Do 10571389703SDan Williams * not return it to page allocator. 10671389703SDan Williams */ 10771389703SDan Williams return; 10871389703SDan Williams } 10971389703SDan Williams 110ddc58f27SKirill A. Shutemov if (unlikely(PageCompound(page))) 11126296ad2SAndrew Morton __put_compound_page(page); 11226296ad2SAndrew Morton else 11326296ad2SAndrew Morton __put_single_page(page); 11426296ad2SAndrew Morton } 115ddc58f27SKirill A. Shutemov EXPORT_SYMBOL(__put_page); 11670b50f94SAndrea Arcangeli 1171d7ea732SAlexander Zarochentsev /** 1187682486bSRandy Dunlap * put_pages_list() - release a list of pages 1197682486bSRandy Dunlap * @pages: list of pages threaded on page->lru 1201d7ea732SAlexander Zarochentsev * 1211d7ea732SAlexander Zarochentsev * Release a list of pages which are strung together on page.lru. Currently 1221d7ea732SAlexander Zarochentsev * used by read_cache_pages() and related error recovery code. 1231d7ea732SAlexander Zarochentsev */ 1241d7ea732SAlexander Zarochentsev void put_pages_list(struct list_head *pages) 1251d7ea732SAlexander Zarochentsev { 1261d7ea732SAlexander Zarochentsev while (!list_empty(pages)) { 1271d7ea732SAlexander Zarochentsev struct page *victim; 1281d7ea732SAlexander Zarochentsev 1291d7ea732SAlexander Zarochentsev victim = list_entry(pages->prev, struct page, lru); 1301d7ea732SAlexander Zarochentsev list_del(&victim->lru); 13109cbfeafSKirill A. Shutemov put_page(victim); 1321d7ea732SAlexander Zarochentsev } 1331d7ea732SAlexander Zarochentsev } 1341d7ea732SAlexander Zarochentsev EXPORT_SYMBOL(put_pages_list); 1351d7ea732SAlexander Zarochentsev 13618022c5dSMel Gorman /* 13718022c5dSMel Gorman * get_kernel_pages() - pin kernel pages in memory 13818022c5dSMel Gorman * @kiov: An array of struct kvec structures 13918022c5dSMel Gorman * @nr_segs: number of segments to pin 14018022c5dSMel Gorman * @write: pinning for read/write, currently ignored 14118022c5dSMel Gorman * @pages: array that receives pointers to the pages pinned. 14218022c5dSMel Gorman * Should be at least nr_segs long. 14318022c5dSMel Gorman * 14418022c5dSMel Gorman * Returns number of pages pinned. This may be fewer than the number 14518022c5dSMel Gorman * requested. If nr_pages is 0 or negative, returns 0. If no pages 14618022c5dSMel Gorman * were pinned, returns -errno. Each page returned must be released 14718022c5dSMel Gorman * with a put_page() call when it is finished with. 14818022c5dSMel Gorman */ 14918022c5dSMel Gorman int get_kernel_pages(const struct kvec *kiov, int nr_segs, int write, 15018022c5dSMel Gorman struct page **pages) 15118022c5dSMel Gorman { 15218022c5dSMel Gorman int seg; 15318022c5dSMel Gorman 15418022c5dSMel Gorman for (seg = 0; seg < nr_segs; seg++) { 15518022c5dSMel Gorman if (WARN_ON(kiov[seg].iov_len != PAGE_SIZE)) 15618022c5dSMel Gorman return seg; 15718022c5dSMel Gorman 1585a178119SMel Gorman pages[seg] = kmap_to_page(kiov[seg].iov_base); 15909cbfeafSKirill A. Shutemov get_page(pages[seg]); 16018022c5dSMel Gorman } 16118022c5dSMel Gorman 16218022c5dSMel Gorman return seg; 16318022c5dSMel Gorman } 16418022c5dSMel Gorman EXPORT_SYMBOL_GPL(get_kernel_pages); 16518022c5dSMel Gorman 16618022c5dSMel Gorman /* 16718022c5dSMel Gorman * get_kernel_page() - pin a kernel page in memory 16818022c5dSMel Gorman * @start: starting kernel address 16918022c5dSMel Gorman * @write: pinning for read/write, currently ignored 17018022c5dSMel Gorman * @pages: array that receives pointer to the page pinned. 17118022c5dSMel Gorman * Must be at least nr_segs long. 17218022c5dSMel Gorman * 17318022c5dSMel Gorman * Returns 1 if page is pinned. If the page was not pinned, returns 17418022c5dSMel Gorman * -errno. The page returned must be released with a put_page() call 17518022c5dSMel Gorman * when it is finished with. 17618022c5dSMel Gorman */ 17718022c5dSMel Gorman int get_kernel_page(unsigned long start, int write, struct page **pages) 17818022c5dSMel Gorman { 17918022c5dSMel Gorman const struct kvec kiov = { 18018022c5dSMel Gorman .iov_base = (void *)start, 18118022c5dSMel Gorman .iov_len = PAGE_SIZE 18218022c5dSMel Gorman }; 18318022c5dSMel Gorman 18418022c5dSMel Gorman return get_kernel_pages(&kiov, 1, write, pages); 18518022c5dSMel Gorman } 18618022c5dSMel Gorman EXPORT_SYMBOL_GPL(get_kernel_page); 18718022c5dSMel Gorman 1883dd7ae8eSShaohua Li static void pagevec_lru_move_fn(struct pagevec *pvec, 189fa9add64SHugh Dickins void (*move_fn)(struct page *page, struct lruvec *lruvec, void *arg), 1903dd7ae8eSShaohua Li void *arg) 191902aaed0SHisashi Hifumi { 192902aaed0SHisashi Hifumi int i; 19368eb0731SMel Gorman struct pglist_data *pgdat = NULL; 194fa9add64SHugh Dickins struct lruvec *lruvec; 1953dd7ae8eSShaohua Li unsigned long flags = 0; 196902aaed0SHisashi Hifumi 197902aaed0SHisashi Hifumi for (i = 0; i < pagevec_count(pvec); i++) { 198902aaed0SHisashi Hifumi struct page *page = pvec->pages[i]; 19968eb0731SMel Gorman struct pglist_data *pagepgdat = page_pgdat(page); 200902aaed0SHisashi Hifumi 20168eb0731SMel Gorman if (pagepgdat != pgdat) { 20268eb0731SMel Gorman if (pgdat) 20368eb0731SMel Gorman spin_unlock_irqrestore(&pgdat->lru_lock, flags); 20468eb0731SMel Gorman pgdat = pagepgdat; 20568eb0731SMel Gorman spin_lock_irqsave(&pgdat->lru_lock, flags); 206902aaed0SHisashi Hifumi } 2073dd7ae8eSShaohua Li 20868eb0731SMel Gorman lruvec = mem_cgroup_page_lruvec(page, pgdat); 209fa9add64SHugh Dickins (*move_fn)(page, lruvec, arg); 2103dd7ae8eSShaohua Li } 21168eb0731SMel Gorman if (pgdat) 21268eb0731SMel Gorman spin_unlock_irqrestore(&pgdat->lru_lock, flags); 2133dd7ae8eSShaohua Li release_pages(pvec->pages, pvec->nr, pvec->cold); 2143dd7ae8eSShaohua Li pagevec_reinit(pvec); 2153dd7ae8eSShaohua Li } 2163dd7ae8eSShaohua Li 217fa9add64SHugh Dickins static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec, 218fa9add64SHugh Dickins void *arg) 2193dd7ae8eSShaohua Li { 2203dd7ae8eSShaohua Li int *pgmoved = arg; 2213dd7ae8eSShaohua Li 222c55e8d03SJohannes Weiner if (PageLRU(page) && !PageUnevictable(page)) { 223c55e8d03SJohannes Weiner del_page_from_lru_list(page, lruvec, page_lru(page)); 224c55e8d03SJohannes Weiner ClearPageActive(page); 225c55e8d03SJohannes Weiner add_page_to_lru_list_tail(page, lruvec, page_lru(page)); 2263dd7ae8eSShaohua Li (*pgmoved)++; 227902aaed0SHisashi Hifumi } 228902aaed0SHisashi Hifumi } 2293dd7ae8eSShaohua Li 2303dd7ae8eSShaohua Li /* 2313dd7ae8eSShaohua Li * pagevec_move_tail() must be called with IRQ disabled. 2323dd7ae8eSShaohua Li * Otherwise this may cause nasty races. 2333dd7ae8eSShaohua Li */ 2343dd7ae8eSShaohua Li static void pagevec_move_tail(struct pagevec *pvec) 2353dd7ae8eSShaohua Li { 2363dd7ae8eSShaohua Li int pgmoved = 0; 2373dd7ae8eSShaohua Li 2383dd7ae8eSShaohua Li pagevec_lru_move_fn(pvec, pagevec_move_tail_fn, &pgmoved); 239902aaed0SHisashi Hifumi __count_vm_events(PGROTATED, pgmoved); 240902aaed0SHisashi Hifumi } 241902aaed0SHisashi Hifumi 242902aaed0SHisashi Hifumi /* 2431da177e4SLinus Torvalds * Writeback is about to end against a page which has been marked for immediate 2441da177e4SLinus Torvalds * reclaim. If it still appears to be reclaimable, move it to the tail of the 245902aaed0SHisashi Hifumi * inactive list. 2461da177e4SLinus Torvalds */ 247ac6aadb2SMiklos Szeredi void rotate_reclaimable_page(struct page *page) 2481da177e4SLinus Torvalds { 249c55e8d03SJohannes Weiner if (!PageLocked(page) && !PageDirty(page) && 250894bc310SLee Schermerhorn !PageUnevictable(page) && PageLRU(page)) { 251902aaed0SHisashi Hifumi struct pagevec *pvec; 2521da177e4SLinus Torvalds unsigned long flags; 2531da177e4SLinus Torvalds 25409cbfeafSKirill A. Shutemov get_page(page); 255902aaed0SHisashi Hifumi local_irq_save(flags); 2567c8e0181SChristoph Lameter pvec = this_cpu_ptr(&lru_rotate_pvecs); 2578f182270SLukasz Odzioba if (!pagevec_add(pvec, page) || PageCompound(page)) 258902aaed0SHisashi Hifumi pagevec_move_tail(pvec); 259902aaed0SHisashi Hifumi local_irq_restore(flags); 260ac6aadb2SMiklos Szeredi } 2611da177e4SLinus Torvalds } 2621da177e4SLinus Torvalds 263fa9add64SHugh Dickins static void update_page_reclaim_stat(struct lruvec *lruvec, 2643e2f41f1SKOSAKI Motohiro int file, int rotated) 2653e2f41f1SKOSAKI Motohiro { 266fa9add64SHugh Dickins struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat; 2673e2f41f1SKOSAKI Motohiro 2683e2f41f1SKOSAKI Motohiro reclaim_stat->recent_scanned[file]++; 2693e2f41f1SKOSAKI Motohiro if (rotated) 2703e2f41f1SKOSAKI Motohiro reclaim_stat->recent_rotated[file]++; 2713e2f41f1SKOSAKI Motohiro } 2723e2f41f1SKOSAKI Motohiro 273fa9add64SHugh Dickins static void __activate_page(struct page *page, struct lruvec *lruvec, 274fa9add64SHugh Dickins void *arg) 275744ed144SShaohua Li { 2767a608572SLinus Torvalds if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { 277744ed144SShaohua Li int file = page_is_file_cache(page); 278744ed144SShaohua Li int lru = page_lru_base_type(page); 279744ed144SShaohua Li 280fa9add64SHugh Dickins del_page_from_lru_list(page, lruvec, lru); 281744ed144SShaohua Li SetPageActive(page); 282744ed144SShaohua Li lru += LRU_ACTIVE; 283fa9add64SHugh Dickins add_page_to_lru_list(page, lruvec, lru); 28424b7e581SMel Gorman trace_mm_lru_activate(page); 2857a608572SLinus Torvalds 286fa9add64SHugh Dickins __count_vm_event(PGACTIVATE); 287fa9add64SHugh Dickins update_page_reclaim_stat(lruvec, file, 1); 288744ed144SShaohua Li } 289eb709b0dSShaohua Li } 290eb709b0dSShaohua Li 291eb709b0dSShaohua Li #ifdef CONFIG_SMP 292eb709b0dSShaohua Li static void activate_page_drain(int cpu) 293eb709b0dSShaohua Li { 294eb709b0dSShaohua Li struct pagevec *pvec = &per_cpu(activate_page_pvecs, cpu); 295eb709b0dSShaohua Li 296eb709b0dSShaohua Li if (pagevec_count(pvec)) 297eb709b0dSShaohua Li pagevec_lru_move_fn(pvec, __activate_page, NULL); 298eb709b0dSShaohua Li } 299eb709b0dSShaohua Li 3005fbc4616SChris Metcalf static bool need_activate_page_drain(int cpu) 3015fbc4616SChris Metcalf { 3025fbc4616SChris Metcalf return pagevec_count(&per_cpu(activate_page_pvecs, cpu)) != 0; 3035fbc4616SChris Metcalf } 3045fbc4616SChris Metcalf 305eb709b0dSShaohua Li void activate_page(struct page *page) 306eb709b0dSShaohua Li { 307800d8c63SKirill A. Shutemov page = compound_head(page); 308eb709b0dSShaohua Li if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { 309eb709b0dSShaohua Li struct pagevec *pvec = &get_cpu_var(activate_page_pvecs); 310eb709b0dSShaohua Li 31109cbfeafSKirill A. Shutemov get_page(page); 3128f182270SLukasz Odzioba if (!pagevec_add(pvec, page) || PageCompound(page)) 313eb709b0dSShaohua Li pagevec_lru_move_fn(pvec, __activate_page, NULL); 314eb709b0dSShaohua Li put_cpu_var(activate_page_pvecs); 315eb709b0dSShaohua Li } 316eb709b0dSShaohua Li } 317eb709b0dSShaohua Li 318eb709b0dSShaohua Li #else 319eb709b0dSShaohua Li static inline void activate_page_drain(int cpu) 320eb709b0dSShaohua Li { 321eb709b0dSShaohua Li } 322eb709b0dSShaohua Li 3235fbc4616SChris Metcalf static bool need_activate_page_drain(int cpu) 3245fbc4616SChris Metcalf { 3255fbc4616SChris Metcalf return false; 3265fbc4616SChris Metcalf } 3275fbc4616SChris Metcalf 328eb709b0dSShaohua Li void activate_page(struct page *page) 329eb709b0dSShaohua Li { 330eb709b0dSShaohua Li struct zone *zone = page_zone(page); 331eb709b0dSShaohua Li 332800d8c63SKirill A. Shutemov page = compound_head(page); 333a52633d8SMel Gorman spin_lock_irq(zone_lru_lock(zone)); 334599d0c95SMel Gorman __activate_page(page, mem_cgroup_page_lruvec(page, zone->zone_pgdat), NULL); 335a52633d8SMel Gorman spin_unlock_irq(zone_lru_lock(zone)); 3361da177e4SLinus Torvalds } 337eb709b0dSShaohua Li #endif 3381da177e4SLinus Torvalds 339059285a2SMel Gorman static void __lru_cache_activate_page(struct page *page) 340059285a2SMel Gorman { 341059285a2SMel Gorman struct pagevec *pvec = &get_cpu_var(lru_add_pvec); 342059285a2SMel Gorman int i; 343059285a2SMel Gorman 344059285a2SMel Gorman /* 345059285a2SMel Gorman * Search backwards on the optimistic assumption that the page being 346059285a2SMel Gorman * activated has just been added to this pagevec. Note that only 347059285a2SMel Gorman * the local pagevec is examined as a !PageLRU page could be in the 348059285a2SMel Gorman * process of being released, reclaimed, migrated or on a remote 349059285a2SMel Gorman * pagevec that is currently being drained. Furthermore, marking 350059285a2SMel Gorman * a remote pagevec's page PageActive potentially hits a race where 351059285a2SMel Gorman * a page is marked PageActive just after it is added to the inactive 352059285a2SMel Gorman * list causing accounting errors and BUG_ON checks to trigger. 353059285a2SMel Gorman */ 354059285a2SMel Gorman for (i = pagevec_count(pvec) - 1; i >= 0; i--) { 355059285a2SMel Gorman struct page *pagevec_page = pvec->pages[i]; 356059285a2SMel Gorman 357059285a2SMel Gorman if (pagevec_page == page) { 358059285a2SMel Gorman SetPageActive(page); 359059285a2SMel Gorman break; 360059285a2SMel Gorman } 361059285a2SMel Gorman } 362059285a2SMel Gorman 363059285a2SMel Gorman put_cpu_var(lru_add_pvec); 364059285a2SMel Gorman } 365059285a2SMel Gorman 3661da177e4SLinus Torvalds /* 3671da177e4SLinus Torvalds * Mark a page as having seen activity. 3681da177e4SLinus Torvalds * 3691da177e4SLinus Torvalds * inactive,unreferenced -> inactive,referenced 3701da177e4SLinus Torvalds * inactive,referenced -> active,unreferenced 3711da177e4SLinus Torvalds * active,unreferenced -> active,referenced 372eb39d618SHugh Dickins * 373eb39d618SHugh Dickins * When a newly allocated page is not yet visible, so safe for non-atomic ops, 374eb39d618SHugh Dickins * __SetPageReferenced(page) may be substituted for mark_page_accessed(page). 3751da177e4SLinus Torvalds */ 376920c7a5dSHarvey Harrison void mark_page_accessed(struct page *page) 3771da177e4SLinus Torvalds { 378e90309c9SKirill A. Shutemov page = compound_head(page); 379894bc310SLee Schermerhorn if (!PageActive(page) && !PageUnevictable(page) && 380059285a2SMel Gorman PageReferenced(page)) { 381059285a2SMel Gorman 382059285a2SMel Gorman /* 383059285a2SMel Gorman * If the page is on the LRU, queue it for activation via 384059285a2SMel Gorman * activate_page_pvecs. Otherwise, assume the page is on a 385059285a2SMel Gorman * pagevec, mark it active and it'll be moved to the active 386059285a2SMel Gorman * LRU on the next drain. 387059285a2SMel Gorman */ 388059285a2SMel Gorman if (PageLRU(page)) 3891da177e4SLinus Torvalds activate_page(page); 390059285a2SMel Gorman else 391059285a2SMel Gorman __lru_cache_activate_page(page); 3921da177e4SLinus Torvalds ClearPageReferenced(page); 393a528910eSJohannes Weiner if (page_is_file_cache(page)) 394a528910eSJohannes Weiner workingset_activation(page); 3951da177e4SLinus Torvalds } else if (!PageReferenced(page)) { 3961da177e4SLinus Torvalds SetPageReferenced(page); 3971da177e4SLinus Torvalds } 39833c3fc71SVladimir Davydov if (page_is_idle(page)) 39933c3fc71SVladimir Davydov clear_page_idle(page); 4001da177e4SLinus Torvalds } 4011da177e4SLinus Torvalds EXPORT_SYMBOL(mark_page_accessed); 4021da177e4SLinus Torvalds 4032329d375SJianyu Zhan static void __lru_cache_add(struct page *page) 4041da177e4SLinus Torvalds { 40513f7f789SMel Gorman struct pagevec *pvec = &get_cpu_var(lru_add_pvec); 40613f7f789SMel Gorman 40709cbfeafSKirill A. Shutemov get_page(page); 4088f182270SLukasz Odzioba if (!pagevec_add(pvec, page) || PageCompound(page)) 409a0b8cab3SMel Gorman __pagevec_lru_add(pvec); 41013f7f789SMel Gorman put_cpu_var(lru_add_pvec); 4111da177e4SLinus Torvalds } 4122329d375SJianyu Zhan 4132329d375SJianyu Zhan /** 4142329d375SJianyu Zhan * lru_cache_add: add a page to the page lists 4152329d375SJianyu Zhan * @page: the page to add 4162329d375SJianyu Zhan */ 4172329d375SJianyu Zhan void lru_cache_add_anon(struct page *page) 4182329d375SJianyu Zhan { 4196fb81a17SMel Gorman if (PageActive(page)) 4202329d375SJianyu Zhan ClearPageActive(page); 4212329d375SJianyu Zhan __lru_cache_add(page); 4222329d375SJianyu Zhan } 4232329d375SJianyu Zhan 4242329d375SJianyu Zhan void lru_cache_add_file(struct page *page) 4252329d375SJianyu Zhan { 4266fb81a17SMel Gorman if (PageActive(page)) 4272329d375SJianyu Zhan ClearPageActive(page); 4282329d375SJianyu Zhan __lru_cache_add(page); 4292329d375SJianyu Zhan } 4302329d375SJianyu Zhan EXPORT_SYMBOL(lru_cache_add_file); 4311da177e4SLinus Torvalds 432f04e9ebbSKOSAKI Motohiro /** 433c53954a0SMel Gorman * lru_cache_add - add a page to a page list 434f04e9ebbSKOSAKI Motohiro * @page: the page to be added to the LRU. 4352329d375SJianyu Zhan * 4362329d375SJianyu Zhan * Queue the page for addition to the LRU via pagevec. The decision on whether 4372329d375SJianyu Zhan * to add the page to the [in]active [file|anon] list is deferred until the 4382329d375SJianyu Zhan * pagevec is drained. This gives a chance for the caller of lru_cache_add() 4392329d375SJianyu Zhan * have the page added to the active list using mark_page_accessed(). 440f04e9ebbSKOSAKI Motohiro */ 441c53954a0SMel Gorman void lru_cache_add(struct page *page) 4421da177e4SLinus Torvalds { 443309381feSSasha Levin VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page); 444309381feSSasha Levin VM_BUG_ON_PAGE(PageLRU(page), page); 445c53954a0SMel Gorman __lru_cache_add(page); 4461da177e4SLinus Torvalds } 4471da177e4SLinus Torvalds 448894bc310SLee Schermerhorn /** 449894bc310SLee Schermerhorn * add_page_to_unevictable_list - add a page to the unevictable list 450894bc310SLee Schermerhorn * @page: the page to be added to the unevictable list 451894bc310SLee Schermerhorn * 452894bc310SLee Schermerhorn * Add page directly to its zone's unevictable list. To avoid races with 453894bc310SLee Schermerhorn * tasks that might be making the page evictable, through eg. munlock, 454894bc310SLee Schermerhorn * munmap or exit, while it's not on the lru, we want to add the page 455894bc310SLee Schermerhorn * while it's locked or otherwise "invisible" to other tasks. This is 456894bc310SLee Schermerhorn * difficult to do when using the pagevec cache, so bypass that. 457894bc310SLee Schermerhorn */ 458894bc310SLee Schermerhorn void add_page_to_unevictable_list(struct page *page) 459894bc310SLee Schermerhorn { 460599d0c95SMel Gorman struct pglist_data *pgdat = page_pgdat(page); 461fa9add64SHugh Dickins struct lruvec *lruvec; 462894bc310SLee Schermerhorn 463599d0c95SMel Gorman spin_lock_irq(&pgdat->lru_lock); 464599d0c95SMel Gorman lruvec = mem_cgroup_page_lruvec(page, pgdat); 465ef2a2cbdSNaoya Horiguchi ClearPageActive(page); 466894bc310SLee Schermerhorn SetPageUnevictable(page); 467894bc310SLee Schermerhorn SetPageLRU(page); 468fa9add64SHugh Dickins add_page_to_lru_list(page, lruvec, LRU_UNEVICTABLE); 469599d0c95SMel Gorman spin_unlock_irq(&pgdat->lru_lock); 470894bc310SLee Schermerhorn } 471894bc310SLee Schermerhorn 47200501b53SJohannes Weiner /** 47300501b53SJohannes Weiner * lru_cache_add_active_or_unevictable 47400501b53SJohannes Weiner * @page: the page to be added to LRU 47500501b53SJohannes Weiner * @vma: vma in which page is mapped for determining reclaimability 47600501b53SJohannes Weiner * 47700501b53SJohannes Weiner * Place @page on the active or unevictable LRU list, depending on its 47800501b53SJohannes Weiner * evictability. Note that if the page is not evictable, it goes 47900501b53SJohannes Weiner * directly back onto it's zone's unevictable list, it does NOT use a 48000501b53SJohannes Weiner * per cpu pagevec. 48100501b53SJohannes Weiner */ 48200501b53SJohannes Weiner void lru_cache_add_active_or_unevictable(struct page *page, 48300501b53SJohannes Weiner struct vm_area_struct *vma) 48400501b53SJohannes Weiner { 48500501b53SJohannes Weiner VM_BUG_ON_PAGE(PageLRU(page), page); 48600501b53SJohannes Weiner 48700501b53SJohannes Weiner if (likely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) != VM_LOCKED)) { 48800501b53SJohannes Weiner SetPageActive(page); 48900501b53SJohannes Weiner lru_cache_add(page); 49000501b53SJohannes Weiner return; 49100501b53SJohannes Weiner } 49200501b53SJohannes Weiner 49300501b53SJohannes Weiner if (!TestSetPageMlocked(page)) { 49400501b53SJohannes Weiner /* 49500501b53SJohannes Weiner * We use the irq-unsafe __mod_zone_page_stat because this 49600501b53SJohannes Weiner * counter is not modified from interrupt context, and the pte 49700501b53SJohannes Weiner * lock is held(spinlock), which implies preemption disabled. 49800501b53SJohannes Weiner */ 49900501b53SJohannes Weiner __mod_zone_page_state(page_zone(page), NR_MLOCK, 50000501b53SJohannes Weiner hpage_nr_pages(page)); 50100501b53SJohannes Weiner count_vm_event(UNEVICTABLE_PGMLOCKED); 50200501b53SJohannes Weiner } 50300501b53SJohannes Weiner add_page_to_unevictable_list(page); 50400501b53SJohannes Weiner } 50500501b53SJohannes Weiner 506902aaed0SHisashi Hifumi /* 50731560180SMinchan Kim * If the page can not be invalidated, it is moved to the 50831560180SMinchan Kim * inactive list to speed up its reclaim. It is moved to the 50931560180SMinchan Kim * head of the list, rather than the tail, to give the flusher 51031560180SMinchan Kim * threads some time to write it out, as this is much more 51131560180SMinchan Kim * effective than the single-page writeout from reclaim. 512278df9f4SMinchan Kim * 513278df9f4SMinchan Kim * If the page isn't page_mapped and dirty/writeback, the page 514278df9f4SMinchan Kim * could reclaim asap using PG_reclaim. 515278df9f4SMinchan Kim * 516278df9f4SMinchan Kim * 1. active, mapped page -> none 517278df9f4SMinchan Kim * 2. active, dirty/writeback page -> inactive, head, PG_reclaim 518278df9f4SMinchan Kim * 3. inactive, mapped page -> none 519278df9f4SMinchan Kim * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim 520278df9f4SMinchan Kim * 5. inactive, clean -> inactive, tail 521278df9f4SMinchan Kim * 6. Others -> none 522278df9f4SMinchan Kim * 523278df9f4SMinchan Kim * In 4, why it moves inactive's head, the VM expects the page would 524278df9f4SMinchan Kim * be write it out by flusher threads as this is much more effective 525278df9f4SMinchan Kim * than the single-page writeout from reclaim. 52631560180SMinchan Kim */ 527cc5993bdSMinchan Kim static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec, 528fa9add64SHugh Dickins void *arg) 52931560180SMinchan Kim { 53031560180SMinchan Kim int lru, file; 531278df9f4SMinchan Kim bool active; 53231560180SMinchan Kim 533278df9f4SMinchan Kim if (!PageLRU(page)) 53431560180SMinchan Kim return; 53531560180SMinchan Kim 536bad49d9cSMinchan Kim if (PageUnevictable(page)) 537bad49d9cSMinchan Kim return; 538bad49d9cSMinchan Kim 53931560180SMinchan Kim /* Some processes are using the page */ 54031560180SMinchan Kim if (page_mapped(page)) 54131560180SMinchan Kim return; 54231560180SMinchan Kim 543278df9f4SMinchan Kim active = PageActive(page); 54431560180SMinchan Kim file = page_is_file_cache(page); 54531560180SMinchan Kim lru = page_lru_base_type(page); 546fa9add64SHugh Dickins 547fa9add64SHugh Dickins del_page_from_lru_list(page, lruvec, lru + active); 54831560180SMinchan Kim ClearPageActive(page); 54931560180SMinchan Kim ClearPageReferenced(page); 550fa9add64SHugh Dickins add_page_to_lru_list(page, lruvec, lru); 55131560180SMinchan Kim 552278df9f4SMinchan Kim if (PageWriteback(page) || PageDirty(page)) { 553278df9f4SMinchan Kim /* 554278df9f4SMinchan Kim * PG_reclaim could be raced with end_page_writeback 555278df9f4SMinchan Kim * It can make readahead confusing. But race window 556278df9f4SMinchan Kim * is _really_ small and it's non-critical problem. 557278df9f4SMinchan Kim */ 558278df9f4SMinchan Kim SetPageReclaim(page); 559278df9f4SMinchan Kim } else { 560278df9f4SMinchan Kim /* 561278df9f4SMinchan Kim * The page's writeback ends up during pagevec 562278df9f4SMinchan Kim * We moves tha page into tail of inactive. 563278df9f4SMinchan Kim */ 564925b7673SJohannes Weiner list_move_tail(&page->lru, &lruvec->lists[lru]); 565278df9f4SMinchan Kim __count_vm_event(PGROTATED); 566278df9f4SMinchan Kim } 567278df9f4SMinchan Kim 568278df9f4SMinchan Kim if (active) 569278df9f4SMinchan Kim __count_vm_event(PGDEACTIVATE); 570fa9add64SHugh Dickins update_page_reclaim_stat(lruvec, file, 0); 57131560180SMinchan Kim } 57231560180SMinchan Kim 57310853a03SMinchan Kim 574f7ad2a6cSShaohua Li static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec, 57510853a03SMinchan Kim void *arg) 57610853a03SMinchan Kim { 577f7ad2a6cSShaohua Li if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) && 578f7ad2a6cSShaohua Li !PageUnevictable(page)) { 579f7ad2a6cSShaohua Li bool active = PageActive(page); 58010853a03SMinchan Kim 581f7ad2a6cSShaohua Li del_page_from_lru_list(page, lruvec, 582f7ad2a6cSShaohua Li LRU_INACTIVE_ANON + active); 58310853a03SMinchan Kim ClearPageActive(page); 58410853a03SMinchan Kim ClearPageReferenced(page); 585f7ad2a6cSShaohua Li /* 586f7ad2a6cSShaohua Li * lazyfree pages are clean anonymous pages. They have 587f7ad2a6cSShaohua Li * SwapBacked flag cleared to distinguish normal anonymous 588f7ad2a6cSShaohua Li * pages 589f7ad2a6cSShaohua Li */ 590f7ad2a6cSShaohua Li ClearPageSwapBacked(page); 591f7ad2a6cSShaohua Li add_page_to_lru_list(page, lruvec, LRU_INACTIVE_FILE); 59210853a03SMinchan Kim 593f7ad2a6cSShaohua Li __count_vm_events(PGLAZYFREE, hpage_nr_pages(page)); 5942262185cSRoman Gushchin count_memcg_page_event(page, PGLAZYFREE); 595f7ad2a6cSShaohua Li update_page_reclaim_stat(lruvec, 1, 0); 59610853a03SMinchan Kim } 59710853a03SMinchan Kim } 59810853a03SMinchan Kim 59931560180SMinchan Kim /* 600902aaed0SHisashi Hifumi * Drain pages out of the cpu's pagevecs. 601902aaed0SHisashi Hifumi * Either "cpu" is the current CPU, and preemption has already been 602902aaed0SHisashi Hifumi * disabled; or "cpu" is being hot-unplugged, and is already dead. 603902aaed0SHisashi Hifumi */ 604f0cb3c76SKonstantin Khlebnikov void lru_add_drain_cpu(int cpu) 6051da177e4SLinus Torvalds { 60613f7f789SMel Gorman struct pagevec *pvec = &per_cpu(lru_add_pvec, cpu); 6071da177e4SLinus Torvalds 6081da177e4SLinus Torvalds if (pagevec_count(pvec)) 609a0b8cab3SMel Gorman __pagevec_lru_add(pvec); 610902aaed0SHisashi Hifumi 611902aaed0SHisashi Hifumi pvec = &per_cpu(lru_rotate_pvecs, cpu); 612902aaed0SHisashi Hifumi if (pagevec_count(pvec)) { 613902aaed0SHisashi Hifumi unsigned long flags; 614902aaed0SHisashi Hifumi 615902aaed0SHisashi Hifumi /* No harm done if a racing interrupt already did this */ 616902aaed0SHisashi Hifumi local_irq_save(flags); 617902aaed0SHisashi Hifumi pagevec_move_tail(pvec); 618902aaed0SHisashi Hifumi local_irq_restore(flags); 619902aaed0SHisashi Hifumi } 62031560180SMinchan Kim 621cc5993bdSMinchan Kim pvec = &per_cpu(lru_deactivate_file_pvecs, cpu); 62231560180SMinchan Kim if (pagevec_count(pvec)) 623cc5993bdSMinchan Kim pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL); 624eb709b0dSShaohua Li 625f7ad2a6cSShaohua Li pvec = &per_cpu(lru_lazyfree_pvecs, cpu); 62610853a03SMinchan Kim if (pagevec_count(pvec)) 627f7ad2a6cSShaohua Li pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL); 62810853a03SMinchan Kim 629eb709b0dSShaohua Li activate_page_drain(cpu); 63031560180SMinchan Kim } 63131560180SMinchan Kim 63231560180SMinchan Kim /** 633cc5993bdSMinchan Kim * deactivate_file_page - forcefully deactivate a file page 63431560180SMinchan Kim * @page: page to deactivate 63531560180SMinchan Kim * 63631560180SMinchan Kim * This function hints the VM that @page is a good reclaim candidate, 63731560180SMinchan Kim * for example if its invalidation fails due to the page being dirty 63831560180SMinchan Kim * or under writeback. 63931560180SMinchan Kim */ 640cc5993bdSMinchan Kim void deactivate_file_page(struct page *page) 64131560180SMinchan Kim { 642821ed6bbSMinchan Kim /* 643cc5993bdSMinchan Kim * In a workload with many unevictable page such as mprotect, 644cc5993bdSMinchan Kim * unevictable page deactivation for accelerating reclaim is pointless. 645821ed6bbSMinchan Kim */ 646821ed6bbSMinchan Kim if (PageUnevictable(page)) 647821ed6bbSMinchan Kim return; 648821ed6bbSMinchan Kim 64931560180SMinchan Kim if (likely(get_page_unless_zero(page))) { 650cc5993bdSMinchan Kim struct pagevec *pvec = &get_cpu_var(lru_deactivate_file_pvecs); 65131560180SMinchan Kim 6528f182270SLukasz Odzioba if (!pagevec_add(pvec, page) || PageCompound(page)) 653cc5993bdSMinchan Kim pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL); 654cc5993bdSMinchan Kim put_cpu_var(lru_deactivate_file_pvecs); 65531560180SMinchan Kim } 65680bfed90SAndrew Morton } 65780bfed90SAndrew Morton 65810853a03SMinchan Kim /** 659f7ad2a6cSShaohua Li * mark_page_lazyfree - make an anon page lazyfree 66010853a03SMinchan Kim * @page: page to deactivate 66110853a03SMinchan Kim * 662f7ad2a6cSShaohua Li * mark_page_lazyfree() moves @page to the inactive file list. 663f7ad2a6cSShaohua Li * This is done to accelerate the reclaim of @page. 66410853a03SMinchan Kim */ 665f7ad2a6cSShaohua Li void mark_page_lazyfree(struct page *page) 66610853a03SMinchan Kim { 667f7ad2a6cSShaohua Li if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) && 668f7ad2a6cSShaohua Li !PageUnevictable(page)) { 669f7ad2a6cSShaohua Li struct pagevec *pvec = &get_cpu_var(lru_lazyfree_pvecs); 67010853a03SMinchan Kim 67109cbfeafSKirill A. Shutemov get_page(page); 6728f182270SLukasz Odzioba if (!pagevec_add(pvec, page) || PageCompound(page)) 673f7ad2a6cSShaohua Li pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL); 674f7ad2a6cSShaohua Li put_cpu_var(lru_lazyfree_pvecs); 67510853a03SMinchan Kim } 67610853a03SMinchan Kim } 67710853a03SMinchan Kim 67880bfed90SAndrew Morton void lru_add_drain(void) 67980bfed90SAndrew Morton { 680f0cb3c76SKonstantin Khlebnikov lru_add_drain_cpu(get_cpu()); 68180bfed90SAndrew Morton put_cpu(); 6821da177e4SLinus Torvalds } 6831da177e4SLinus Torvalds 684c4028958SDavid Howells static void lru_add_drain_per_cpu(struct work_struct *dummy) 685053837fcSNick Piggin { 686053837fcSNick Piggin lru_add_drain(); 687053837fcSNick Piggin } 688053837fcSNick Piggin 6895fbc4616SChris Metcalf static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work); 6905fbc4616SChris Metcalf 691a47fed5bSThomas Gleixner void lru_add_drain_all_cpuslocked(void) 692053837fcSNick Piggin { 6935fbc4616SChris Metcalf static DEFINE_MUTEX(lock); 6945fbc4616SChris Metcalf static struct cpumask has_work; 6955fbc4616SChris Metcalf int cpu; 6965fbc4616SChris Metcalf 697ce612879SMichal Hocko /* 698ce612879SMichal Hocko * Make sure nobody triggers this path before mm_percpu_wq is fully 699ce612879SMichal Hocko * initialized. 700ce612879SMichal Hocko */ 701ce612879SMichal Hocko if (WARN_ON(!mm_percpu_wq)) 702ce612879SMichal Hocko return; 703ce612879SMichal Hocko 7045fbc4616SChris Metcalf mutex_lock(&lock); 7055fbc4616SChris Metcalf cpumask_clear(&has_work); 7065fbc4616SChris Metcalf 7075fbc4616SChris Metcalf for_each_online_cpu(cpu) { 7085fbc4616SChris Metcalf struct work_struct *work = &per_cpu(lru_add_drain_work, cpu); 7095fbc4616SChris Metcalf 7105fbc4616SChris Metcalf if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) || 7115fbc4616SChris Metcalf pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) || 712cc5993bdSMinchan Kim pagevec_count(&per_cpu(lru_deactivate_file_pvecs, cpu)) || 713f7ad2a6cSShaohua Li pagevec_count(&per_cpu(lru_lazyfree_pvecs, cpu)) || 7145fbc4616SChris Metcalf need_activate_page_drain(cpu)) { 7155fbc4616SChris Metcalf INIT_WORK(work, lru_add_drain_per_cpu); 716ce612879SMichal Hocko queue_work_on(cpu, mm_percpu_wq, work); 7175fbc4616SChris Metcalf cpumask_set_cpu(cpu, &has_work); 7185fbc4616SChris Metcalf } 7195fbc4616SChris Metcalf } 7205fbc4616SChris Metcalf 7215fbc4616SChris Metcalf for_each_cpu(cpu, &has_work) 7225fbc4616SChris Metcalf flush_work(&per_cpu(lru_add_drain_work, cpu)); 7235fbc4616SChris Metcalf 7245fbc4616SChris Metcalf mutex_unlock(&lock); 725053837fcSNick Piggin } 726053837fcSNick Piggin 727a47fed5bSThomas Gleixner void lru_add_drain_all(void) 728a47fed5bSThomas Gleixner { 729a47fed5bSThomas Gleixner get_online_cpus(); 730a47fed5bSThomas Gleixner lru_add_drain_all_cpuslocked(); 731a47fed5bSThomas Gleixner put_online_cpus(); 732a47fed5bSThomas Gleixner } 733a47fed5bSThomas Gleixner 734aabfb572SMichal Hocko /** 735ea1754a0SKirill A. Shutemov * release_pages - batched put_page() 736aabfb572SMichal Hocko * @pages: array of pages to release 737aabfb572SMichal Hocko * @nr: number of pages 738aabfb572SMichal Hocko * @cold: whether the pages are cache cold 7391da177e4SLinus Torvalds * 740aabfb572SMichal Hocko * Decrement the reference count on all the pages in @pages. If it 741aabfb572SMichal Hocko * fell to zero, remove the page from the LRU and free it. 7421da177e4SLinus Torvalds */ 743b745bc85SMel Gorman void release_pages(struct page **pages, int nr, bool cold) 7441da177e4SLinus Torvalds { 7451da177e4SLinus Torvalds int i; 746cc59850eSKonstantin Khlebnikov LIST_HEAD(pages_to_free); 747599d0c95SMel Gorman struct pglist_data *locked_pgdat = NULL; 748fa9add64SHugh Dickins struct lruvec *lruvec; 749902aaed0SHisashi Hifumi unsigned long uninitialized_var(flags); 750aabfb572SMichal Hocko unsigned int uninitialized_var(lock_batch); 7511da177e4SLinus Torvalds 7521da177e4SLinus Torvalds for (i = 0; i < nr; i++) { 7531da177e4SLinus Torvalds struct page *page = pages[i]; 7541da177e4SLinus Torvalds 755aabfb572SMichal Hocko /* 756aabfb572SMichal Hocko * Make sure the IRQ-safe lock-holding time does not get 757aabfb572SMichal Hocko * excessive with a continuous string of pages from the 758599d0c95SMel Gorman * same pgdat. The lock is held only if pgdat != NULL. 759aabfb572SMichal Hocko */ 760599d0c95SMel Gorman if (locked_pgdat && ++lock_batch == SWAP_CLUSTER_MAX) { 761599d0c95SMel Gorman spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); 762599d0c95SMel Gorman locked_pgdat = NULL; 763aabfb572SMichal Hocko } 764aabfb572SMichal Hocko 7656fcb52a5SAaron Lu if (is_huge_zero_page(page)) 766aa88b68cSKirill A. Shutemov continue; 767aa88b68cSKirill A. Shutemov 768*df6ad698SJérôme Glisse /* Device public page can not be huge page */ 769*df6ad698SJérôme Glisse if (is_device_public_page(page)) { 770*df6ad698SJérôme Glisse if (locked_pgdat) { 771*df6ad698SJérôme Glisse spin_unlock_irqrestore(&locked_pgdat->lru_lock, 772*df6ad698SJérôme Glisse flags); 773*df6ad698SJérôme Glisse locked_pgdat = NULL; 774*df6ad698SJérôme Glisse } 775*df6ad698SJérôme Glisse put_zone_device_private_or_public_page(page); 776*df6ad698SJérôme Glisse continue; 777*df6ad698SJérôme Glisse } 778*df6ad698SJérôme Glisse 779ddc58f27SKirill A. Shutemov page = compound_head(page); 780b5810039SNick Piggin if (!put_page_testzero(page)) 7811da177e4SLinus Torvalds continue; 7821da177e4SLinus Torvalds 783ddc58f27SKirill A. Shutemov if (PageCompound(page)) { 784599d0c95SMel Gorman if (locked_pgdat) { 785599d0c95SMel Gorman spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); 786599d0c95SMel Gorman locked_pgdat = NULL; 787ddc58f27SKirill A. Shutemov } 788ddc58f27SKirill A. Shutemov __put_compound_page(page); 789ddc58f27SKirill A. Shutemov continue; 790ddc58f27SKirill A. Shutemov } 791ddc58f27SKirill A. Shutemov 79246453a6eSNick Piggin if (PageLRU(page)) { 793599d0c95SMel Gorman struct pglist_data *pgdat = page_pgdat(page); 794894bc310SLee Schermerhorn 795599d0c95SMel Gorman if (pgdat != locked_pgdat) { 796599d0c95SMel Gorman if (locked_pgdat) 797599d0c95SMel Gorman spin_unlock_irqrestore(&locked_pgdat->lru_lock, 798902aaed0SHisashi Hifumi flags); 799aabfb572SMichal Hocko lock_batch = 0; 800599d0c95SMel Gorman locked_pgdat = pgdat; 801599d0c95SMel Gorman spin_lock_irqsave(&locked_pgdat->lru_lock, flags); 8021da177e4SLinus Torvalds } 803fa9add64SHugh Dickins 804599d0c95SMel Gorman lruvec = mem_cgroup_page_lruvec(page, locked_pgdat); 805309381feSSasha Levin VM_BUG_ON_PAGE(!PageLRU(page), page); 80667453911SNick Piggin __ClearPageLRU(page); 807fa9add64SHugh Dickins del_page_from_lru_list(page, lruvec, page_off_lru(page)); 80846453a6eSNick Piggin } 80946453a6eSNick Piggin 810c53954a0SMel Gorman /* Clear Active bit in case of parallel mark_page_accessed */ 811e3741b50SMel Gorman __ClearPageActive(page); 81262906027SNicholas Piggin __ClearPageWaiters(page); 813c53954a0SMel Gorman 814cc59850eSKonstantin Khlebnikov list_add(&page->lru, &pages_to_free); 8151da177e4SLinus Torvalds } 816599d0c95SMel Gorman if (locked_pgdat) 817599d0c95SMel Gorman spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); 8181da177e4SLinus Torvalds 819747db954SJohannes Weiner mem_cgroup_uncharge_list(&pages_to_free); 820cc59850eSKonstantin Khlebnikov free_hot_cold_page_list(&pages_to_free, cold); 8211da177e4SLinus Torvalds } 8220be8557bSMiklos Szeredi EXPORT_SYMBOL(release_pages); 8231da177e4SLinus Torvalds 8241da177e4SLinus Torvalds /* 8251da177e4SLinus Torvalds * The pages which we're about to release may be in the deferred lru-addition 8261da177e4SLinus Torvalds * queues. That would prevent them from really being freed right now. That's 8271da177e4SLinus Torvalds * OK from a correctness point of view but is inefficient - those pages may be 8281da177e4SLinus Torvalds * cache-warm and we want to give them back to the page allocator ASAP. 8291da177e4SLinus Torvalds * 8301da177e4SLinus Torvalds * So __pagevec_release() will drain those queues here. __pagevec_lru_add() 8311da177e4SLinus Torvalds * and __pagevec_lru_add_active() call release_pages() directly to avoid 8321da177e4SLinus Torvalds * mutual recursion. 8331da177e4SLinus Torvalds */ 8341da177e4SLinus Torvalds void __pagevec_release(struct pagevec *pvec) 8351da177e4SLinus Torvalds { 8361da177e4SLinus Torvalds lru_add_drain(); 8371da177e4SLinus Torvalds release_pages(pvec->pages, pagevec_count(pvec), pvec->cold); 8381da177e4SLinus Torvalds pagevec_reinit(pvec); 8391da177e4SLinus Torvalds } 8407f285701SSteve French EXPORT_SYMBOL(__pagevec_release); 8417f285701SSteve French 84212d27107SHugh Dickins #ifdef CONFIG_TRANSPARENT_HUGEPAGE 84371e3aac0SAndrea Arcangeli /* used by __split_huge_page_refcount() */ 844fa9add64SHugh Dickins void lru_add_page_tail(struct page *page, struct page *page_tail, 8455bc7b8acSShaohua Li struct lruvec *lruvec, struct list_head *list) 84671e3aac0SAndrea Arcangeli { 84771e3aac0SAndrea Arcangeli const int file = 0; 84871e3aac0SAndrea Arcangeli 849309381feSSasha Levin VM_BUG_ON_PAGE(!PageHead(page), page); 850309381feSSasha Levin VM_BUG_ON_PAGE(PageCompound(page_tail), page); 851309381feSSasha Levin VM_BUG_ON_PAGE(PageLRU(page_tail), page); 852fa9add64SHugh Dickins VM_BUG_ON(NR_CPUS != 1 && 853599d0c95SMel Gorman !spin_is_locked(&lruvec_pgdat(lruvec)->lru_lock)); 85471e3aac0SAndrea Arcangeli 8555bc7b8acSShaohua Li if (!list) 85671e3aac0SAndrea Arcangeli SetPageLRU(page_tail); 85771e3aac0SAndrea Arcangeli 85812d27107SHugh Dickins if (likely(PageLRU(page))) 85912d27107SHugh Dickins list_add_tail(&page_tail->lru, &page->lru); 8605bc7b8acSShaohua Li else if (list) { 8615bc7b8acSShaohua Li /* page reclaim is reclaiming a huge page */ 8625bc7b8acSShaohua Li get_page(page_tail); 8635bc7b8acSShaohua Li list_add_tail(&page_tail->lru, list); 8645bc7b8acSShaohua Li } else { 86512d27107SHugh Dickins struct list_head *list_head; 86612d27107SHugh Dickins /* 86712d27107SHugh Dickins * Head page has not yet been counted, as an hpage, 86812d27107SHugh Dickins * so we must account for each subpage individually. 86912d27107SHugh Dickins * 87012d27107SHugh Dickins * Use the standard add function to put page_tail on the list, 87112d27107SHugh Dickins * but then correct its position so they all end up in order. 87212d27107SHugh Dickins */ 873e180cf80SKirill A. Shutemov add_page_to_lru_list(page_tail, lruvec, page_lru(page_tail)); 87412d27107SHugh Dickins list_head = page_tail->lru.prev; 87512d27107SHugh Dickins list_move_tail(&page_tail->lru, list_head); 87671e3aac0SAndrea Arcangeli } 8777512102cSHugh Dickins 8787512102cSHugh Dickins if (!PageUnevictable(page)) 879e180cf80SKirill A. Shutemov update_page_reclaim_stat(lruvec, file, PageActive(page_tail)); 88071e3aac0SAndrea Arcangeli } 88112d27107SHugh Dickins #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 88271e3aac0SAndrea Arcangeli 883fa9add64SHugh Dickins static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec, 884fa9add64SHugh Dickins void *arg) 8853dd7ae8eSShaohua Li { 88613f7f789SMel Gorman int file = page_is_file_cache(page); 88713f7f789SMel Gorman int active = PageActive(page); 88813f7f789SMel Gorman enum lru_list lru = page_lru(page); 8893dd7ae8eSShaohua Li 890309381feSSasha Levin VM_BUG_ON_PAGE(PageLRU(page), page); 8913dd7ae8eSShaohua Li 8923dd7ae8eSShaohua Li SetPageLRU(page); 893fa9add64SHugh Dickins add_page_to_lru_list(page, lruvec, lru); 894fa9add64SHugh Dickins update_page_reclaim_stat(lruvec, file, active); 89524b7e581SMel Gorman trace_mm_lru_insertion(page, lru); 8963dd7ae8eSShaohua Li } 8973dd7ae8eSShaohua Li 8981da177e4SLinus Torvalds /* 8991da177e4SLinus Torvalds * Add the passed pages to the LRU, then drop the caller's refcount 9001da177e4SLinus Torvalds * on them. Reinitialises the caller's pagevec. 9011da177e4SLinus Torvalds */ 902a0b8cab3SMel Gorman void __pagevec_lru_add(struct pagevec *pvec) 9031da177e4SLinus Torvalds { 904a0b8cab3SMel Gorman pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, NULL); 9051da177e4SLinus Torvalds } 9065095ae83SHugh Dickins EXPORT_SYMBOL(__pagevec_lru_add); 907f04e9ebbSKOSAKI Motohiro 9081da177e4SLinus Torvalds /** 9090cd6144aSJohannes Weiner * pagevec_lookup_entries - gang pagecache lookup 9100cd6144aSJohannes Weiner * @pvec: Where the resulting entries are placed 9110cd6144aSJohannes Weiner * @mapping: The address_space to search 9120cd6144aSJohannes Weiner * @start: The starting entry index 9130cd6144aSJohannes Weiner * @nr_entries: The maximum number of entries 9140cd6144aSJohannes Weiner * @indices: The cache indices corresponding to the entries in @pvec 9150cd6144aSJohannes Weiner * 9160cd6144aSJohannes Weiner * pagevec_lookup_entries() will search for and return a group of up 9170cd6144aSJohannes Weiner * to @nr_entries pages and shadow entries in the mapping. All 9180cd6144aSJohannes Weiner * entries are placed in @pvec. pagevec_lookup_entries() takes a 9190cd6144aSJohannes Weiner * reference against actual pages in @pvec. 9200cd6144aSJohannes Weiner * 9210cd6144aSJohannes Weiner * The search returns a group of mapping-contiguous entries with 9220cd6144aSJohannes Weiner * ascending indexes. There may be holes in the indices due to 9230cd6144aSJohannes Weiner * not-present entries. 9240cd6144aSJohannes Weiner * 9250cd6144aSJohannes Weiner * pagevec_lookup_entries() returns the number of entries which were 9260cd6144aSJohannes Weiner * found. 9270cd6144aSJohannes Weiner */ 9280cd6144aSJohannes Weiner unsigned pagevec_lookup_entries(struct pagevec *pvec, 9290cd6144aSJohannes Weiner struct address_space *mapping, 9300cd6144aSJohannes Weiner pgoff_t start, unsigned nr_pages, 9310cd6144aSJohannes Weiner pgoff_t *indices) 9320cd6144aSJohannes Weiner { 9330cd6144aSJohannes Weiner pvec->nr = find_get_entries(mapping, start, nr_pages, 9340cd6144aSJohannes Weiner pvec->pages, indices); 9350cd6144aSJohannes Weiner return pagevec_count(pvec); 9360cd6144aSJohannes Weiner } 9370cd6144aSJohannes Weiner 9380cd6144aSJohannes Weiner /** 9390cd6144aSJohannes Weiner * pagevec_remove_exceptionals - pagevec exceptionals pruning 9400cd6144aSJohannes Weiner * @pvec: The pagevec to prune 9410cd6144aSJohannes Weiner * 9420cd6144aSJohannes Weiner * pagevec_lookup_entries() fills both pages and exceptional radix 9430cd6144aSJohannes Weiner * tree entries into the pagevec. This function prunes all 9440cd6144aSJohannes Weiner * exceptionals from @pvec without leaving holes, so that it can be 9450cd6144aSJohannes Weiner * passed on to page-only pagevec operations. 9460cd6144aSJohannes Weiner */ 9470cd6144aSJohannes Weiner void pagevec_remove_exceptionals(struct pagevec *pvec) 9480cd6144aSJohannes Weiner { 9490cd6144aSJohannes Weiner int i, j; 9500cd6144aSJohannes Weiner 9510cd6144aSJohannes Weiner for (i = 0, j = 0; i < pagevec_count(pvec); i++) { 9520cd6144aSJohannes Weiner struct page *page = pvec->pages[i]; 9530cd6144aSJohannes Weiner if (!radix_tree_exceptional_entry(page)) 9540cd6144aSJohannes Weiner pvec->pages[j++] = page; 9550cd6144aSJohannes Weiner } 9560cd6144aSJohannes Weiner pvec->nr = j; 9570cd6144aSJohannes Weiner } 9580cd6144aSJohannes Weiner 9590cd6144aSJohannes Weiner /** 960b947cee4SJan Kara * pagevec_lookup_range - gang pagecache lookup 9611da177e4SLinus Torvalds * @pvec: Where the resulting pages are placed 9621da177e4SLinus Torvalds * @mapping: The address_space to search 9631da177e4SLinus Torvalds * @start: The starting page index 964b947cee4SJan Kara * @end: The final page index 9651da177e4SLinus Torvalds * @nr_pages: The maximum number of pages 9661da177e4SLinus Torvalds * 967b947cee4SJan Kara * pagevec_lookup_range() will search for and return a group of up to @nr_pages 968b947cee4SJan Kara * pages in the mapping starting from index @start and upto index @end 969b947cee4SJan Kara * (inclusive). The pages are placed in @pvec. pagevec_lookup() takes a 9701da177e4SLinus Torvalds * reference against the pages in @pvec. 9711da177e4SLinus Torvalds * 9721da177e4SLinus Torvalds * The search returns a group of mapping-contiguous pages with ascending 973d72dc8a2SJan Kara * indexes. There may be holes in the indices due to not-present pages. We 974d72dc8a2SJan Kara * also update @start to index the next page for the traversal. 9751da177e4SLinus Torvalds * 976b947cee4SJan Kara * pagevec_lookup_range() returns the number of pages which were found. If this 977b947cee4SJan Kara * number is smaller than @nr_pages, the end of specified range has been 978b947cee4SJan Kara * reached. 9791da177e4SLinus Torvalds */ 980b947cee4SJan Kara unsigned pagevec_lookup_range(struct pagevec *pvec, 981397162ffSJan Kara struct address_space *mapping, pgoff_t *start, pgoff_t end) 9821da177e4SLinus Torvalds { 983397162ffSJan Kara pvec->nr = find_get_pages_range(mapping, start, end, PAGEVEC_SIZE, 984b947cee4SJan Kara pvec->pages); 9851da177e4SLinus Torvalds return pagevec_count(pvec); 9861da177e4SLinus Torvalds } 987b947cee4SJan Kara EXPORT_SYMBOL(pagevec_lookup_range); 98878539fdfSChristoph Hellwig 9891da177e4SLinus Torvalds unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping, 9901da177e4SLinus Torvalds pgoff_t *index, int tag, unsigned nr_pages) 9911da177e4SLinus Torvalds { 9921da177e4SLinus Torvalds pvec->nr = find_get_pages_tag(mapping, index, tag, 9931da177e4SLinus Torvalds nr_pages, pvec->pages); 9941da177e4SLinus Torvalds return pagevec_count(pvec); 9951da177e4SLinus Torvalds } 9967f285701SSteve French EXPORT_SYMBOL(pagevec_lookup_tag); 9971da177e4SLinus Torvalds 9981da177e4SLinus Torvalds /* 9991da177e4SLinus Torvalds * Perform any setup for the swap system 10001da177e4SLinus Torvalds */ 10011da177e4SLinus Torvalds void __init swap_setup(void) 10021da177e4SLinus Torvalds { 10034481374cSJan Beulich unsigned long megs = totalram_pages >> (20 - PAGE_SHIFT); 1004e0bf68ddSPeter Zijlstra 10051da177e4SLinus Torvalds /* Use a smaller cluster for small-memory machines */ 10061da177e4SLinus Torvalds if (megs < 16) 10071da177e4SLinus Torvalds page_cluster = 2; 10081da177e4SLinus Torvalds else 10091da177e4SLinus Torvalds page_cluster = 3; 10101da177e4SLinus Torvalds /* 10111da177e4SLinus Torvalds * Right now other parts of the system means that we 10121da177e4SLinus Torvalds * _really_ don't want to cluster much more 10131da177e4SLinus Torvalds */ 10141da177e4SLinus Torvalds } 1015