11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * linux/mm/swap.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 51da177e4SLinus Torvalds */ 61da177e4SLinus Torvalds 71da177e4SLinus Torvalds /* 8183ff22bSSimon Arlott * This file contains the default values for the operation of the 91da177e4SLinus Torvalds * Linux VM subsystem. Fine-tuning documentation can be found in 101da177e4SLinus Torvalds * Documentation/sysctl/vm.txt. 111da177e4SLinus Torvalds * Started 18.12.91 121da177e4SLinus Torvalds * Swap aging added 23.2.95, Stephen Tweedie. 131da177e4SLinus Torvalds * Buffermem limits added 12.3.98, Rik van Riel. 141da177e4SLinus Torvalds */ 151da177e4SLinus Torvalds 161da177e4SLinus Torvalds #include <linux/mm.h> 171da177e4SLinus Torvalds #include <linux/sched.h> 181da177e4SLinus Torvalds #include <linux/kernel_stat.h> 191da177e4SLinus Torvalds #include <linux/swap.h> 201da177e4SLinus Torvalds #include <linux/mman.h> 211da177e4SLinus Torvalds #include <linux/pagemap.h> 221da177e4SLinus Torvalds #include <linux/pagevec.h> 231da177e4SLinus Torvalds #include <linux/init.h> 241da177e4SLinus Torvalds #include <linux/module.h> 251da177e4SLinus Torvalds #include <linux/mm_inline.h> 261da177e4SLinus Torvalds #include <linux/buffer_head.h> /* for try_to_release_page() */ 271da177e4SLinus Torvalds #include <linux/percpu_counter.h> 281da177e4SLinus Torvalds #include <linux/percpu.h> 291da177e4SLinus Torvalds #include <linux/cpu.h> 301da177e4SLinus Torvalds #include <linux/notifier.h> 31e0bf68ddSPeter Zijlstra #include <linux/backing-dev.h> 3266e1707bSBalbir Singh #include <linux/memcontrol.h> 335a0e3ad6STejun Heo #include <linux/gfp.h> 341da177e4SLinus Torvalds 3564d6519dSLee Schermerhorn #include "internal.h" 3664d6519dSLee Schermerhorn 371da177e4SLinus Torvalds /* How many pages do we try to swap or page in/out together? */ 381da177e4SLinus Torvalds int page_cluster; 391da177e4SLinus Torvalds 40f04e9ebbSKOSAKI Motohiro static DEFINE_PER_CPU(struct pagevec[NR_LRU_LISTS], lru_add_pvecs); 41f84f9504SVegard Nossum static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs); 4231560180SMinchan Kim static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs); 43902aaed0SHisashi Hifumi 44b221385bSAdrian Bunk /* 45b221385bSAdrian Bunk * This path almost never happens for VM activity - pages are normally 46b221385bSAdrian Bunk * freed via pagevecs. But it gets used by networking. 47b221385bSAdrian Bunk */ 48920c7a5dSHarvey Harrison static void __page_cache_release(struct page *page) 49b221385bSAdrian Bunk { 50b221385bSAdrian Bunk if (PageLRU(page)) { 51b221385bSAdrian Bunk unsigned long flags; 52b221385bSAdrian Bunk struct zone *zone = page_zone(page); 53b221385bSAdrian Bunk 54b221385bSAdrian Bunk spin_lock_irqsave(&zone->lru_lock, flags); 55b221385bSAdrian Bunk VM_BUG_ON(!PageLRU(page)); 56b221385bSAdrian Bunk __ClearPageLRU(page); 57b221385bSAdrian Bunk del_page_from_lru(zone, page); 58b221385bSAdrian Bunk spin_unlock_irqrestore(&zone->lru_lock, flags); 59b221385bSAdrian Bunk } 6091807063SAndrea Arcangeli } 6191807063SAndrea Arcangeli 6291807063SAndrea Arcangeli static void __put_single_page(struct page *page) 6391807063SAndrea Arcangeli { 6491807063SAndrea Arcangeli __page_cache_release(page); 65fc91668eSLi Hong free_hot_cold_page(page, 0); 66b221385bSAdrian Bunk } 67b221385bSAdrian Bunk 6891807063SAndrea Arcangeli static void __put_compound_page(struct page *page) 6991807063SAndrea Arcangeli { 7091807063SAndrea Arcangeli compound_page_dtor *dtor; 7191807063SAndrea Arcangeli 7291807063SAndrea Arcangeli __page_cache_release(page); 7391807063SAndrea Arcangeli dtor = get_compound_page_dtor(page); 7491807063SAndrea Arcangeli (*dtor)(page); 7591807063SAndrea Arcangeli } 7691807063SAndrea Arcangeli 778519fb30SNick Piggin static void put_compound_page(struct page *page) 781da177e4SLinus Torvalds { 7991807063SAndrea Arcangeli if (unlikely(PageTail(page))) { 8091807063SAndrea Arcangeli /* __split_huge_page_refcount can run under us */ 8191807063SAndrea Arcangeli struct page *page_head = page->first_page; 8291807063SAndrea Arcangeli smp_rmb(); 8391807063SAndrea Arcangeli /* 8491807063SAndrea Arcangeli * If PageTail is still set after smp_rmb() we can be sure 8591807063SAndrea Arcangeli * that the page->first_page we read wasn't a dangling pointer. 8691807063SAndrea Arcangeli * See __split_huge_page_refcount() smp_wmb(). 8791807063SAndrea Arcangeli */ 8891807063SAndrea Arcangeli if (likely(PageTail(page) && get_page_unless_zero(page_head))) { 8991807063SAndrea Arcangeli unsigned long flags; 9091807063SAndrea Arcangeli /* 9191807063SAndrea Arcangeli * Verify that our page_head wasn't converted 9291807063SAndrea Arcangeli * to a a regular page before we got a 9391807063SAndrea Arcangeli * reference on it. 9491807063SAndrea Arcangeli */ 9591807063SAndrea Arcangeli if (unlikely(!PageHead(page_head))) { 9691807063SAndrea Arcangeli /* PageHead is cleared after PageTail */ 9791807063SAndrea Arcangeli smp_rmb(); 9891807063SAndrea Arcangeli VM_BUG_ON(PageTail(page)); 9991807063SAndrea Arcangeli goto out_put_head; 10091807063SAndrea Arcangeli } 10191807063SAndrea Arcangeli /* 10291807063SAndrea Arcangeli * Only run compound_lock on a valid PageHead, 10391807063SAndrea Arcangeli * after having it pinned with 10491807063SAndrea Arcangeli * get_page_unless_zero() above. 10591807063SAndrea Arcangeli */ 10691807063SAndrea Arcangeli smp_mb(); 10791807063SAndrea Arcangeli /* page_head wasn't a dangling pointer */ 10891807063SAndrea Arcangeli flags = compound_lock_irqsave(page_head); 10991807063SAndrea Arcangeli if (unlikely(!PageTail(page))) { 11091807063SAndrea Arcangeli /* __split_huge_page_refcount run before us */ 11191807063SAndrea Arcangeli compound_unlock_irqrestore(page_head, flags); 11291807063SAndrea Arcangeli VM_BUG_ON(PageHead(page_head)); 11391807063SAndrea Arcangeli out_put_head: 11491807063SAndrea Arcangeli if (put_page_testzero(page_head)) 11591807063SAndrea Arcangeli __put_single_page(page_head); 11691807063SAndrea Arcangeli out_put_single: 11791807063SAndrea Arcangeli if (put_page_testzero(page)) 11891807063SAndrea Arcangeli __put_single_page(page); 11991807063SAndrea Arcangeli return; 12091807063SAndrea Arcangeli } 12191807063SAndrea Arcangeli VM_BUG_ON(page_head != page->first_page); 12291807063SAndrea Arcangeli /* 12391807063SAndrea Arcangeli * We can release the refcount taken by 12491807063SAndrea Arcangeli * get_page_unless_zero now that 12591807063SAndrea Arcangeli * split_huge_page_refcount is blocked on the 12691807063SAndrea Arcangeli * compound_lock. 12791807063SAndrea Arcangeli */ 12891807063SAndrea Arcangeli if (put_page_testzero(page_head)) 12991807063SAndrea Arcangeli VM_BUG_ON(1); 13091807063SAndrea Arcangeli /* __split_huge_page_refcount will wait now */ 13191807063SAndrea Arcangeli VM_BUG_ON(atomic_read(&page->_count) <= 0); 13291807063SAndrea Arcangeli atomic_dec(&page->_count); 13391807063SAndrea Arcangeli VM_BUG_ON(atomic_read(&page_head->_count) <= 0); 13491807063SAndrea Arcangeli compound_unlock_irqrestore(page_head, flags); 135a95a82e9SAndrea Arcangeli if (put_page_testzero(page_head)) { 136a95a82e9SAndrea Arcangeli if (PageHead(page_head)) 13791807063SAndrea Arcangeli __put_compound_page(page_head); 138a95a82e9SAndrea Arcangeli else 139a95a82e9SAndrea Arcangeli __put_single_page(page_head); 140a95a82e9SAndrea Arcangeli } 14191807063SAndrea Arcangeli } else { 14291807063SAndrea Arcangeli /* page_head is a dangling pointer */ 14391807063SAndrea Arcangeli VM_BUG_ON(PageTail(page)); 14491807063SAndrea Arcangeli goto out_put_single; 14591807063SAndrea Arcangeli } 14691807063SAndrea Arcangeli } else if (put_page_testzero(page)) { 14791807063SAndrea Arcangeli if (PageHead(page)) 14891807063SAndrea Arcangeli __put_compound_page(page); 14991807063SAndrea Arcangeli else 15091807063SAndrea Arcangeli __put_single_page(page); 1511da177e4SLinus Torvalds } 1521da177e4SLinus Torvalds } 1538519fb30SNick Piggin 1548519fb30SNick Piggin void put_page(struct page *page) 1558519fb30SNick Piggin { 1568519fb30SNick Piggin if (unlikely(PageCompound(page))) 1578519fb30SNick Piggin put_compound_page(page); 1588519fb30SNick Piggin else if (put_page_testzero(page)) 15991807063SAndrea Arcangeli __put_single_page(page); 1601da177e4SLinus Torvalds } 1611da177e4SLinus Torvalds EXPORT_SYMBOL(put_page); 1621da177e4SLinus Torvalds 1631d7ea732SAlexander Zarochentsev /** 1647682486bSRandy Dunlap * put_pages_list() - release a list of pages 1657682486bSRandy Dunlap * @pages: list of pages threaded on page->lru 1661d7ea732SAlexander Zarochentsev * 1671d7ea732SAlexander Zarochentsev * Release a list of pages which are strung together on page.lru. Currently 1681d7ea732SAlexander Zarochentsev * used by read_cache_pages() and related error recovery code. 1691d7ea732SAlexander Zarochentsev */ 1701d7ea732SAlexander Zarochentsev void put_pages_list(struct list_head *pages) 1711d7ea732SAlexander Zarochentsev { 1721d7ea732SAlexander Zarochentsev while (!list_empty(pages)) { 1731d7ea732SAlexander Zarochentsev struct page *victim; 1741d7ea732SAlexander Zarochentsev 1751d7ea732SAlexander Zarochentsev victim = list_entry(pages->prev, struct page, lru); 1761d7ea732SAlexander Zarochentsev list_del(&victim->lru); 1771d7ea732SAlexander Zarochentsev page_cache_release(victim); 1781d7ea732SAlexander Zarochentsev } 1791d7ea732SAlexander Zarochentsev } 1801d7ea732SAlexander Zarochentsev EXPORT_SYMBOL(put_pages_list); 1811d7ea732SAlexander Zarochentsev 1823dd7ae8eSShaohua Li static void pagevec_lru_move_fn(struct pagevec *pvec, 1833dd7ae8eSShaohua Li void (*move_fn)(struct page *page, void *arg), 1843dd7ae8eSShaohua Li void *arg) 185902aaed0SHisashi Hifumi { 186902aaed0SHisashi Hifumi int i; 187902aaed0SHisashi Hifumi struct zone *zone = NULL; 1883dd7ae8eSShaohua Li unsigned long flags = 0; 189902aaed0SHisashi Hifumi 190902aaed0SHisashi Hifumi for (i = 0; i < pagevec_count(pvec); i++) { 191902aaed0SHisashi Hifumi struct page *page = pvec->pages[i]; 192902aaed0SHisashi Hifumi struct zone *pagezone = page_zone(page); 193902aaed0SHisashi Hifumi 194902aaed0SHisashi Hifumi if (pagezone != zone) { 195902aaed0SHisashi Hifumi if (zone) 1963dd7ae8eSShaohua Li spin_unlock_irqrestore(&zone->lru_lock, flags); 197902aaed0SHisashi Hifumi zone = pagezone; 1983dd7ae8eSShaohua Li spin_lock_irqsave(&zone->lru_lock, flags); 199902aaed0SHisashi Hifumi } 2003dd7ae8eSShaohua Li 2013dd7ae8eSShaohua Li (*move_fn)(page, arg); 2023dd7ae8eSShaohua Li } 2033dd7ae8eSShaohua Li if (zone) 2043dd7ae8eSShaohua Li spin_unlock_irqrestore(&zone->lru_lock, flags); 2053dd7ae8eSShaohua Li release_pages(pvec->pages, pvec->nr, pvec->cold); 2063dd7ae8eSShaohua Li pagevec_reinit(pvec); 2073dd7ae8eSShaohua Li } 2083dd7ae8eSShaohua Li 2093dd7ae8eSShaohua Li static void pagevec_move_tail_fn(struct page *page, void *arg) 2103dd7ae8eSShaohua Li { 2113dd7ae8eSShaohua Li int *pgmoved = arg; 2123dd7ae8eSShaohua Li struct zone *zone = page_zone(page); 2133dd7ae8eSShaohua Li 214894bc310SLee Schermerhorn if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { 2153f58a829SMinchan Kim enum lru_list lru = page_lru_base_type(page); 2164f98a2feSRik van Riel list_move_tail(&page->lru, &zone->lru[lru].list); 2173f58a829SMinchan Kim mem_cgroup_rotate_reclaimable_page(page); 2183dd7ae8eSShaohua Li (*pgmoved)++; 219902aaed0SHisashi Hifumi } 220902aaed0SHisashi Hifumi } 2213dd7ae8eSShaohua Li 2223dd7ae8eSShaohua Li /* 2233dd7ae8eSShaohua Li * pagevec_move_tail() must be called with IRQ disabled. 2243dd7ae8eSShaohua Li * Otherwise this may cause nasty races. 2253dd7ae8eSShaohua Li */ 2263dd7ae8eSShaohua Li static void pagevec_move_tail(struct pagevec *pvec) 2273dd7ae8eSShaohua Li { 2283dd7ae8eSShaohua Li int pgmoved = 0; 2293dd7ae8eSShaohua Li 2303dd7ae8eSShaohua Li pagevec_lru_move_fn(pvec, pagevec_move_tail_fn, &pgmoved); 231902aaed0SHisashi Hifumi __count_vm_events(PGROTATED, pgmoved); 232902aaed0SHisashi Hifumi } 233902aaed0SHisashi Hifumi 234902aaed0SHisashi Hifumi /* 2351da177e4SLinus Torvalds * Writeback is about to end against a page which has been marked for immediate 2361da177e4SLinus Torvalds * reclaim. If it still appears to be reclaimable, move it to the tail of the 237902aaed0SHisashi Hifumi * inactive list. 2381da177e4SLinus Torvalds */ 239ac6aadb2SMiklos Szeredi void rotate_reclaimable_page(struct page *page) 2401da177e4SLinus Torvalds { 241ac6aadb2SMiklos Szeredi if (!PageLocked(page) && !PageDirty(page) && !PageActive(page) && 242894bc310SLee Schermerhorn !PageUnevictable(page) && PageLRU(page)) { 243902aaed0SHisashi Hifumi struct pagevec *pvec; 2441da177e4SLinus Torvalds unsigned long flags; 2451da177e4SLinus Torvalds 246902aaed0SHisashi Hifumi page_cache_get(page); 247902aaed0SHisashi Hifumi local_irq_save(flags); 248902aaed0SHisashi Hifumi pvec = &__get_cpu_var(lru_rotate_pvecs); 249902aaed0SHisashi Hifumi if (!pagevec_add(pvec, page)) 250902aaed0SHisashi Hifumi pagevec_move_tail(pvec); 251902aaed0SHisashi Hifumi local_irq_restore(flags); 252ac6aadb2SMiklos Szeredi } 2531da177e4SLinus Torvalds } 2541da177e4SLinus Torvalds 2553e2f41f1SKOSAKI Motohiro static void update_page_reclaim_stat(struct zone *zone, struct page *page, 2563e2f41f1SKOSAKI Motohiro int file, int rotated) 2573e2f41f1SKOSAKI Motohiro { 2583e2f41f1SKOSAKI Motohiro struct zone_reclaim_stat *reclaim_stat = &zone->reclaim_stat; 2593e2f41f1SKOSAKI Motohiro struct zone_reclaim_stat *memcg_reclaim_stat; 2603e2f41f1SKOSAKI Motohiro 2613e2f41f1SKOSAKI Motohiro memcg_reclaim_stat = mem_cgroup_get_reclaim_stat_from_page(page); 2623e2f41f1SKOSAKI Motohiro 2633e2f41f1SKOSAKI Motohiro reclaim_stat->recent_scanned[file]++; 2643e2f41f1SKOSAKI Motohiro if (rotated) 2653e2f41f1SKOSAKI Motohiro reclaim_stat->recent_rotated[file]++; 2663e2f41f1SKOSAKI Motohiro 2673e2f41f1SKOSAKI Motohiro if (!memcg_reclaim_stat) 2683e2f41f1SKOSAKI Motohiro return; 2693e2f41f1SKOSAKI Motohiro 2703e2f41f1SKOSAKI Motohiro memcg_reclaim_stat->recent_scanned[file]++; 2713e2f41f1SKOSAKI Motohiro if (rotated) 2723e2f41f1SKOSAKI Motohiro memcg_reclaim_stat->recent_rotated[file]++; 2733e2f41f1SKOSAKI Motohiro } 2743e2f41f1SKOSAKI Motohiro 275*eb709b0dSShaohua Li static void __activate_page(struct page *page, void *arg) 276744ed144SShaohua Li { 277744ed144SShaohua Li struct zone *zone = page_zone(page); 2787a608572SLinus Torvalds 2797a608572SLinus Torvalds if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { 280744ed144SShaohua Li int file = page_is_file_cache(page); 281744ed144SShaohua Li int lru = page_lru_base_type(page); 282744ed144SShaohua Li del_page_from_lru_list(zone, page, lru); 283744ed144SShaohua Li 284744ed144SShaohua Li SetPageActive(page); 285744ed144SShaohua Li lru += LRU_ACTIVE; 286744ed144SShaohua Li add_page_to_lru_list(zone, page, lru); 287744ed144SShaohua Li __count_vm_event(PGACTIVATE); 2887a608572SLinus Torvalds 289744ed144SShaohua Li update_page_reclaim_stat(zone, page, file, 1); 290744ed144SShaohua Li } 291*eb709b0dSShaohua Li } 292*eb709b0dSShaohua Li 293*eb709b0dSShaohua Li #ifdef CONFIG_SMP 294*eb709b0dSShaohua Li static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs); 295*eb709b0dSShaohua Li 296*eb709b0dSShaohua Li static void activate_page_drain(int cpu) 297*eb709b0dSShaohua Li { 298*eb709b0dSShaohua Li struct pagevec *pvec = &per_cpu(activate_page_pvecs, cpu); 299*eb709b0dSShaohua Li 300*eb709b0dSShaohua Li if (pagevec_count(pvec)) 301*eb709b0dSShaohua Li pagevec_lru_move_fn(pvec, __activate_page, NULL); 302*eb709b0dSShaohua Li } 303*eb709b0dSShaohua Li 304*eb709b0dSShaohua Li void activate_page(struct page *page) 305*eb709b0dSShaohua Li { 306*eb709b0dSShaohua Li if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { 307*eb709b0dSShaohua Li struct pagevec *pvec = &get_cpu_var(activate_page_pvecs); 308*eb709b0dSShaohua Li 309*eb709b0dSShaohua Li page_cache_get(page); 310*eb709b0dSShaohua Li if (!pagevec_add(pvec, page)) 311*eb709b0dSShaohua Li pagevec_lru_move_fn(pvec, __activate_page, NULL); 312*eb709b0dSShaohua Li put_cpu_var(activate_page_pvecs); 313*eb709b0dSShaohua Li } 314*eb709b0dSShaohua Li } 315*eb709b0dSShaohua Li 316*eb709b0dSShaohua Li #else 317*eb709b0dSShaohua Li static inline void activate_page_drain(int cpu) 318*eb709b0dSShaohua Li { 319*eb709b0dSShaohua Li } 320*eb709b0dSShaohua Li 321*eb709b0dSShaohua Li void activate_page(struct page *page) 322*eb709b0dSShaohua Li { 323*eb709b0dSShaohua Li struct zone *zone = page_zone(page); 324*eb709b0dSShaohua Li 325*eb709b0dSShaohua Li spin_lock_irq(&zone->lru_lock); 326*eb709b0dSShaohua Li __activate_page(page, NULL); 3271da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 3281da177e4SLinus Torvalds } 329*eb709b0dSShaohua Li #endif 3301da177e4SLinus Torvalds 3311da177e4SLinus Torvalds /* 3321da177e4SLinus Torvalds * Mark a page as having seen activity. 3331da177e4SLinus Torvalds * 3341da177e4SLinus Torvalds * inactive,unreferenced -> inactive,referenced 3351da177e4SLinus Torvalds * inactive,referenced -> active,unreferenced 3361da177e4SLinus Torvalds * active,unreferenced -> active,referenced 3371da177e4SLinus Torvalds */ 338920c7a5dSHarvey Harrison void mark_page_accessed(struct page *page) 3391da177e4SLinus Torvalds { 340894bc310SLee Schermerhorn if (!PageActive(page) && !PageUnevictable(page) && 341894bc310SLee Schermerhorn PageReferenced(page) && PageLRU(page)) { 3421da177e4SLinus Torvalds activate_page(page); 3431da177e4SLinus Torvalds ClearPageReferenced(page); 3441da177e4SLinus Torvalds } else if (!PageReferenced(page)) { 3451da177e4SLinus Torvalds SetPageReferenced(page); 3461da177e4SLinus Torvalds } 3471da177e4SLinus Torvalds } 3481da177e4SLinus Torvalds 3491da177e4SLinus Torvalds EXPORT_SYMBOL(mark_page_accessed); 3501da177e4SLinus Torvalds 351f04e9ebbSKOSAKI Motohiro void __lru_cache_add(struct page *page, enum lru_list lru) 3521da177e4SLinus Torvalds { 353f04e9ebbSKOSAKI Motohiro struct pagevec *pvec = &get_cpu_var(lru_add_pvecs)[lru]; 3541da177e4SLinus Torvalds 3551da177e4SLinus Torvalds page_cache_get(page); 3561da177e4SLinus Torvalds if (!pagevec_add(pvec, page)) 357f04e9ebbSKOSAKI Motohiro ____pagevec_lru_add(pvec, lru); 3581da177e4SLinus Torvalds put_cpu_var(lru_add_pvecs); 3591da177e4SLinus Torvalds } 36047846b06SMiklos Szeredi EXPORT_SYMBOL(__lru_cache_add); 3611da177e4SLinus Torvalds 362f04e9ebbSKOSAKI Motohiro /** 363f04e9ebbSKOSAKI Motohiro * lru_cache_add_lru - add a page to a page list 364f04e9ebbSKOSAKI Motohiro * @page: the page to be added to the LRU. 365f04e9ebbSKOSAKI Motohiro * @lru: the LRU list to which the page is added. 366f04e9ebbSKOSAKI Motohiro */ 367f04e9ebbSKOSAKI Motohiro void lru_cache_add_lru(struct page *page, enum lru_list lru) 3681da177e4SLinus Torvalds { 369f04e9ebbSKOSAKI Motohiro if (PageActive(page)) { 370894bc310SLee Schermerhorn VM_BUG_ON(PageUnevictable(page)); 371f04e9ebbSKOSAKI Motohiro ClearPageActive(page); 372894bc310SLee Schermerhorn } else if (PageUnevictable(page)) { 373894bc310SLee Schermerhorn VM_BUG_ON(PageActive(page)); 374894bc310SLee Schermerhorn ClearPageUnevictable(page); 375f04e9ebbSKOSAKI Motohiro } 3761da177e4SLinus Torvalds 377894bc310SLee Schermerhorn VM_BUG_ON(PageLRU(page) || PageActive(page) || PageUnevictable(page)); 378f04e9ebbSKOSAKI Motohiro __lru_cache_add(page, lru); 3791da177e4SLinus Torvalds } 3801da177e4SLinus Torvalds 381894bc310SLee Schermerhorn /** 382894bc310SLee Schermerhorn * add_page_to_unevictable_list - add a page to the unevictable list 383894bc310SLee Schermerhorn * @page: the page to be added to the unevictable list 384894bc310SLee Schermerhorn * 385894bc310SLee Schermerhorn * Add page directly to its zone's unevictable list. To avoid races with 386894bc310SLee Schermerhorn * tasks that might be making the page evictable, through eg. munlock, 387894bc310SLee Schermerhorn * munmap or exit, while it's not on the lru, we want to add the page 388894bc310SLee Schermerhorn * while it's locked or otherwise "invisible" to other tasks. This is 389894bc310SLee Schermerhorn * difficult to do when using the pagevec cache, so bypass that. 390894bc310SLee Schermerhorn */ 391894bc310SLee Schermerhorn void add_page_to_unevictable_list(struct page *page) 392894bc310SLee Schermerhorn { 393894bc310SLee Schermerhorn struct zone *zone = page_zone(page); 394894bc310SLee Schermerhorn 395894bc310SLee Schermerhorn spin_lock_irq(&zone->lru_lock); 396894bc310SLee Schermerhorn SetPageUnevictable(page); 397894bc310SLee Schermerhorn SetPageLRU(page); 398894bc310SLee Schermerhorn add_page_to_lru_list(zone, page, LRU_UNEVICTABLE); 399894bc310SLee Schermerhorn spin_unlock_irq(&zone->lru_lock); 400894bc310SLee Schermerhorn } 401894bc310SLee Schermerhorn 402902aaed0SHisashi Hifumi /* 40331560180SMinchan Kim * If the page can not be invalidated, it is moved to the 40431560180SMinchan Kim * inactive list to speed up its reclaim. It is moved to the 40531560180SMinchan Kim * head of the list, rather than the tail, to give the flusher 40631560180SMinchan Kim * threads some time to write it out, as this is much more 40731560180SMinchan Kim * effective than the single-page writeout from reclaim. 408278df9f4SMinchan Kim * 409278df9f4SMinchan Kim * If the page isn't page_mapped and dirty/writeback, the page 410278df9f4SMinchan Kim * could reclaim asap using PG_reclaim. 411278df9f4SMinchan Kim * 412278df9f4SMinchan Kim * 1. active, mapped page -> none 413278df9f4SMinchan Kim * 2. active, dirty/writeback page -> inactive, head, PG_reclaim 414278df9f4SMinchan Kim * 3. inactive, mapped page -> none 415278df9f4SMinchan Kim * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim 416278df9f4SMinchan Kim * 5. inactive, clean -> inactive, tail 417278df9f4SMinchan Kim * 6. Others -> none 418278df9f4SMinchan Kim * 419278df9f4SMinchan Kim * In 4, why it moves inactive's head, the VM expects the page would 420278df9f4SMinchan Kim * be write it out by flusher threads as this is much more effective 421278df9f4SMinchan Kim * than the single-page writeout from reclaim. 42231560180SMinchan Kim */ 4233dd7ae8eSShaohua Li static void lru_deactivate_fn(struct page *page, void *arg) 42431560180SMinchan Kim { 42531560180SMinchan Kim int lru, file; 426278df9f4SMinchan Kim bool active; 4273dd7ae8eSShaohua Li struct zone *zone = page_zone(page); 42831560180SMinchan Kim 429278df9f4SMinchan Kim if (!PageLRU(page)) 43031560180SMinchan Kim return; 43131560180SMinchan Kim 432bad49d9cSMinchan Kim if (PageUnevictable(page)) 433bad49d9cSMinchan Kim return; 434bad49d9cSMinchan Kim 43531560180SMinchan Kim /* Some processes are using the page */ 43631560180SMinchan Kim if (page_mapped(page)) 43731560180SMinchan Kim return; 43831560180SMinchan Kim 439278df9f4SMinchan Kim active = PageActive(page); 440278df9f4SMinchan Kim 44131560180SMinchan Kim file = page_is_file_cache(page); 44231560180SMinchan Kim lru = page_lru_base_type(page); 443278df9f4SMinchan Kim del_page_from_lru_list(zone, page, lru + active); 44431560180SMinchan Kim ClearPageActive(page); 44531560180SMinchan Kim ClearPageReferenced(page); 44631560180SMinchan Kim add_page_to_lru_list(zone, page, lru); 44731560180SMinchan Kim 448278df9f4SMinchan Kim if (PageWriteback(page) || PageDirty(page)) { 449278df9f4SMinchan Kim /* 450278df9f4SMinchan Kim * PG_reclaim could be raced with end_page_writeback 451278df9f4SMinchan Kim * It can make readahead confusing. But race window 452278df9f4SMinchan Kim * is _really_ small and it's non-critical problem. 453278df9f4SMinchan Kim */ 454278df9f4SMinchan Kim SetPageReclaim(page); 455278df9f4SMinchan Kim } else { 456278df9f4SMinchan Kim /* 457278df9f4SMinchan Kim * The page's writeback ends up during pagevec 458278df9f4SMinchan Kim * We moves tha page into tail of inactive. 459278df9f4SMinchan Kim */ 460278df9f4SMinchan Kim list_move_tail(&page->lru, &zone->lru[lru].list); 461278df9f4SMinchan Kim mem_cgroup_rotate_reclaimable_page(page); 462278df9f4SMinchan Kim __count_vm_event(PGROTATED); 463278df9f4SMinchan Kim } 464278df9f4SMinchan Kim 465278df9f4SMinchan Kim if (active) 466278df9f4SMinchan Kim __count_vm_event(PGDEACTIVATE); 46731560180SMinchan Kim update_page_reclaim_stat(zone, page, file, 0); 46831560180SMinchan Kim } 46931560180SMinchan Kim 47031560180SMinchan Kim /* 471902aaed0SHisashi Hifumi * Drain pages out of the cpu's pagevecs. 472902aaed0SHisashi Hifumi * Either "cpu" is the current CPU, and preemption has already been 473902aaed0SHisashi Hifumi * disabled; or "cpu" is being hot-unplugged, and is already dead. 474902aaed0SHisashi Hifumi */ 475902aaed0SHisashi Hifumi static void drain_cpu_pagevecs(int cpu) 4761da177e4SLinus Torvalds { 477f04e9ebbSKOSAKI Motohiro struct pagevec *pvecs = per_cpu(lru_add_pvecs, cpu); 478902aaed0SHisashi Hifumi struct pagevec *pvec; 479f04e9ebbSKOSAKI Motohiro int lru; 4801da177e4SLinus Torvalds 481f04e9ebbSKOSAKI Motohiro for_each_lru(lru) { 482f04e9ebbSKOSAKI Motohiro pvec = &pvecs[lru - LRU_BASE]; 4831da177e4SLinus Torvalds if (pagevec_count(pvec)) 484f04e9ebbSKOSAKI Motohiro ____pagevec_lru_add(pvec, lru); 485f04e9ebbSKOSAKI Motohiro } 486902aaed0SHisashi Hifumi 487902aaed0SHisashi Hifumi pvec = &per_cpu(lru_rotate_pvecs, cpu); 488902aaed0SHisashi Hifumi if (pagevec_count(pvec)) { 489902aaed0SHisashi Hifumi unsigned long flags; 490902aaed0SHisashi Hifumi 491902aaed0SHisashi Hifumi /* No harm done if a racing interrupt already did this */ 492902aaed0SHisashi Hifumi local_irq_save(flags); 493902aaed0SHisashi Hifumi pagevec_move_tail(pvec); 494902aaed0SHisashi Hifumi local_irq_restore(flags); 495902aaed0SHisashi Hifumi } 49631560180SMinchan Kim 49731560180SMinchan Kim pvec = &per_cpu(lru_deactivate_pvecs, cpu); 49831560180SMinchan Kim if (pagevec_count(pvec)) 4993dd7ae8eSShaohua Li pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL); 500*eb709b0dSShaohua Li 501*eb709b0dSShaohua Li activate_page_drain(cpu); 50231560180SMinchan Kim } 50331560180SMinchan Kim 50431560180SMinchan Kim /** 50531560180SMinchan Kim * deactivate_page - forcefully deactivate a page 50631560180SMinchan Kim * @page: page to deactivate 50731560180SMinchan Kim * 50831560180SMinchan Kim * This function hints the VM that @page is a good reclaim candidate, 50931560180SMinchan Kim * for example if its invalidation fails due to the page being dirty 51031560180SMinchan Kim * or under writeback. 51131560180SMinchan Kim */ 51231560180SMinchan Kim void deactivate_page(struct page *page) 51331560180SMinchan Kim { 514821ed6bbSMinchan Kim /* 515821ed6bbSMinchan Kim * In a workload with many unevictable page such as mprotect, unevictable 516821ed6bbSMinchan Kim * page deactivation for accelerating reclaim is pointless. 517821ed6bbSMinchan Kim */ 518821ed6bbSMinchan Kim if (PageUnevictable(page)) 519821ed6bbSMinchan Kim return; 520821ed6bbSMinchan Kim 52131560180SMinchan Kim if (likely(get_page_unless_zero(page))) { 52231560180SMinchan Kim struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs); 52331560180SMinchan Kim 52431560180SMinchan Kim if (!pagevec_add(pvec, page)) 5253dd7ae8eSShaohua Li pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL); 52631560180SMinchan Kim put_cpu_var(lru_deactivate_pvecs); 52731560180SMinchan Kim } 52880bfed90SAndrew Morton } 52980bfed90SAndrew Morton 53080bfed90SAndrew Morton void lru_add_drain(void) 53180bfed90SAndrew Morton { 532902aaed0SHisashi Hifumi drain_cpu_pagevecs(get_cpu()); 53380bfed90SAndrew Morton put_cpu(); 5341da177e4SLinus Torvalds } 5351da177e4SLinus Torvalds 536c4028958SDavid Howells static void lru_add_drain_per_cpu(struct work_struct *dummy) 537053837fcSNick Piggin { 538053837fcSNick Piggin lru_add_drain(); 539053837fcSNick Piggin } 540053837fcSNick Piggin 541053837fcSNick Piggin /* 542053837fcSNick Piggin * Returns 0 for success 543053837fcSNick Piggin */ 544053837fcSNick Piggin int lru_add_drain_all(void) 545053837fcSNick Piggin { 546c4028958SDavid Howells return schedule_on_each_cpu(lru_add_drain_per_cpu); 547053837fcSNick Piggin } 548053837fcSNick Piggin 5491da177e4SLinus Torvalds /* 5501da177e4SLinus Torvalds * Batched page_cache_release(). Decrement the reference count on all the 5511da177e4SLinus Torvalds * passed pages. If it fell to zero then remove the page from the LRU and 5521da177e4SLinus Torvalds * free it. 5531da177e4SLinus Torvalds * 5541da177e4SLinus Torvalds * Avoid taking zone->lru_lock if possible, but if it is taken, retain it 5551da177e4SLinus Torvalds * for the remainder of the operation. 5561da177e4SLinus Torvalds * 557ab33dc09SFernando Luis Vazquez Cao * The locking in this function is against shrink_inactive_list(): we recheck 558ab33dc09SFernando Luis Vazquez Cao * the page count inside the lock to see whether shrink_inactive_list() 559ab33dc09SFernando Luis Vazquez Cao * grabbed the page via the LRU. If it did, give up: shrink_inactive_list() 560ab33dc09SFernando Luis Vazquez Cao * will free it. 5611da177e4SLinus Torvalds */ 5621da177e4SLinus Torvalds void release_pages(struct page **pages, int nr, int cold) 5631da177e4SLinus Torvalds { 5641da177e4SLinus Torvalds int i; 5651da177e4SLinus Torvalds struct pagevec pages_to_free; 5661da177e4SLinus Torvalds struct zone *zone = NULL; 567902aaed0SHisashi Hifumi unsigned long uninitialized_var(flags); 5681da177e4SLinus Torvalds 5691da177e4SLinus Torvalds pagevec_init(&pages_to_free, cold); 5701da177e4SLinus Torvalds for (i = 0; i < nr; i++) { 5711da177e4SLinus Torvalds struct page *page = pages[i]; 5721da177e4SLinus Torvalds 5738519fb30SNick Piggin if (unlikely(PageCompound(page))) { 5748519fb30SNick Piggin if (zone) { 575902aaed0SHisashi Hifumi spin_unlock_irqrestore(&zone->lru_lock, flags); 5768519fb30SNick Piggin zone = NULL; 5778519fb30SNick Piggin } 5788519fb30SNick Piggin put_compound_page(page); 5798519fb30SNick Piggin continue; 5808519fb30SNick Piggin } 5818519fb30SNick Piggin 582b5810039SNick Piggin if (!put_page_testzero(page)) 5831da177e4SLinus Torvalds continue; 5841da177e4SLinus Torvalds 58546453a6eSNick Piggin if (PageLRU(page)) { 58646453a6eSNick Piggin struct zone *pagezone = page_zone(page); 587894bc310SLee Schermerhorn 5881da177e4SLinus Torvalds if (pagezone != zone) { 5891da177e4SLinus Torvalds if (zone) 590902aaed0SHisashi Hifumi spin_unlock_irqrestore(&zone->lru_lock, 591902aaed0SHisashi Hifumi flags); 5921da177e4SLinus Torvalds zone = pagezone; 593902aaed0SHisashi Hifumi spin_lock_irqsave(&zone->lru_lock, flags); 5941da177e4SLinus Torvalds } 595725d704eSNick Piggin VM_BUG_ON(!PageLRU(page)); 59667453911SNick Piggin __ClearPageLRU(page); 5971da177e4SLinus Torvalds del_page_from_lru(zone, page); 59846453a6eSNick Piggin } 59946453a6eSNick Piggin 6001da177e4SLinus Torvalds if (!pagevec_add(&pages_to_free, page)) { 60146453a6eSNick Piggin if (zone) { 602902aaed0SHisashi Hifumi spin_unlock_irqrestore(&zone->lru_lock, flags); 60346453a6eSNick Piggin zone = NULL; 60446453a6eSNick Piggin } 6051da177e4SLinus Torvalds __pagevec_free(&pages_to_free); 6061da177e4SLinus Torvalds pagevec_reinit(&pages_to_free); 6071da177e4SLinus Torvalds } 6081da177e4SLinus Torvalds } 6091da177e4SLinus Torvalds if (zone) 610902aaed0SHisashi Hifumi spin_unlock_irqrestore(&zone->lru_lock, flags); 6111da177e4SLinus Torvalds 6121da177e4SLinus Torvalds pagevec_free(&pages_to_free); 6131da177e4SLinus Torvalds } 6140be8557bSMiklos Szeredi EXPORT_SYMBOL(release_pages); 6151da177e4SLinus Torvalds 6161da177e4SLinus Torvalds /* 6171da177e4SLinus Torvalds * The pages which we're about to release may be in the deferred lru-addition 6181da177e4SLinus Torvalds * queues. That would prevent them from really being freed right now. That's 6191da177e4SLinus Torvalds * OK from a correctness point of view but is inefficient - those pages may be 6201da177e4SLinus Torvalds * cache-warm and we want to give them back to the page allocator ASAP. 6211da177e4SLinus Torvalds * 6221da177e4SLinus Torvalds * So __pagevec_release() will drain those queues here. __pagevec_lru_add() 6231da177e4SLinus Torvalds * and __pagevec_lru_add_active() call release_pages() directly to avoid 6241da177e4SLinus Torvalds * mutual recursion. 6251da177e4SLinus Torvalds */ 6261da177e4SLinus Torvalds void __pagevec_release(struct pagevec *pvec) 6271da177e4SLinus Torvalds { 6281da177e4SLinus Torvalds lru_add_drain(); 6291da177e4SLinus Torvalds release_pages(pvec->pages, pagevec_count(pvec), pvec->cold); 6301da177e4SLinus Torvalds pagevec_reinit(pvec); 6311da177e4SLinus Torvalds } 6321da177e4SLinus Torvalds 6337f285701SSteve French EXPORT_SYMBOL(__pagevec_release); 6347f285701SSteve French 63571e3aac0SAndrea Arcangeli /* used by __split_huge_page_refcount() */ 63671e3aac0SAndrea Arcangeli void lru_add_page_tail(struct zone* zone, 63771e3aac0SAndrea Arcangeli struct page *page, struct page *page_tail) 63871e3aac0SAndrea Arcangeli { 63971e3aac0SAndrea Arcangeli int active; 64071e3aac0SAndrea Arcangeli enum lru_list lru; 64171e3aac0SAndrea Arcangeli const int file = 0; 64271e3aac0SAndrea Arcangeli struct list_head *head; 64371e3aac0SAndrea Arcangeli 64471e3aac0SAndrea Arcangeli VM_BUG_ON(!PageHead(page)); 64571e3aac0SAndrea Arcangeli VM_BUG_ON(PageCompound(page_tail)); 64671e3aac0SAndrea Arcangeli VM_BUG_ON(PageLRU(page_tail)); 64771e3aac0SAndrea Arcangeli VM_BUG_ON(!spin_is_locked(&zone->lru_lock)); 64871e3aac0SAndrea Arcangeli 64971e3aac0SAndrea Arcangeli SetPageLRU(page_tail); 65071e3aac0SAndrea Arcangeli 65171e3aac0SAndrea Arcangeli if (page_evictable(page_tail, NULL)) { 65271e3aac0SAndrea Arcangeli if (PageActive(page)) { 65371e3aac0SAndrea Arcangeli SetPageActive(page_tail); 65471e3aac0SAndrea Arcangeli active = 1; 65571e3aac0SAndrea Arcangeli lru = LRU_ACTIVE_ANON; 65671e3aac0SAndrea Arcangeli } else { 65771e3aac0SAndrea Arcangeli active = 0; 65871e3aac0SAndrea Arcangeli lru = LRU_INACTIVE_ANON; 65971e3aac0SAndrea Arcangeli } 66071e3aac0SAndrea Arcangeli update_page_reclaim_stat(zone, page_tail, file, active); 66171e3aac0SAndrea Arcangeli if (likely(PageLRU(page))) 66271e3aac0SAndrea Arcangeli head = page->lru.prev; 66371e3aac0SAndrea Arcangeli else 66471e3aac0SAndrea Arcangeli head = &zone->lru[lru].list; 66571e3aac0SAndrea Arcangeli __add_page_to_lru_list(zone, page_tail, lru, head); 66671e3aac0SAndrea Arcangeli } else { 66771e3aac0SAndrea Arcangeli SetPageUnevictable(page_tail); 66871e3aac0SAndrea Arcangeli add_page_to_lru_list(zone, page_tail, LRU_UNEVICTABLE); 66971e3aac0SAndrea Arcangeli } 67071e3aac0SAndrea Arcangeli } 67171e3aac0SAndrea Arcangeli 6723dd7ae8eSShaohua Li static void ____pagevec_lru_add_fn(struct page *page, void *arg) 6733dd7ae8eSShaohua Li { 6743dd7ae8eSShaohua Li enum lru_list lru = (enum lru_list)arg; 6753dd7ae8eSShaohua Li struct zone *zone = page_zone(page); 6763dd7ae8eSShaohua Li int file = is_file_lru(lru); 6773dd7ae8eSShaohua Li int active = is_active_lru(lru); 6783dd7ae8eSShaohua Li 6793dd7ae8eSShaohua Li VM_BUG_ON(PageActive(page)); 6803dd7ae8eSShaohua Li VM_BUG_ON(PageUnevictable(page)); 6813dd7ae8eSShaohua Li VM_BUG_ON(PageLRU(page)); 6823dd7ae8eSShaohua Li 6833dd7ae8eSShaohua Li SetPageLRU(page); 6843dd7ae8eSShaohua Li if (active) 6853dd7ae8eSShaohua Li SetPageActive(page); 6863dd7ae8eSShaohua Li update_page_reclaim_stat(zone, page, file, active); 6873dd7ae8eSShaohua Li add_page_to_lru_list(zone, page, lru); 6883dd7ae8eSShaohua Li } 6893dd7ae8eSShaohua Li 6901da177e4SLinus Torvalds /* 6911da177e4SLinus Torvalds * Add the passed pages to the LRU, then drop the caller's refcount 6921da177e4SLinus Torvalds * on them. Reinitialises the caller's pagevec. 6931da177e4SLinus Torvalds */ 694f04e9ebbSKOSAKI Motohiro void ____pagevec_lru_add(struct pagevec *pvec, enum lru_list lru) 6951da177e4SLinus Torvalds { 696894bc310SLee Schermerhorn VM_BUG_ON(is_unevictable_lru(lru)); 6971da177e4SLinus Torvalds 6983dd7ae8eSShaohua Li pagevec_lru_move_fn(pvec, ____pagevec_lru_add_fn, (void *)lru); 6991da177e4SLinus Torvalds } 7001da177e4SLinus Torvalds 701f04e9ebbSKOSAKI Motohiro EXPORT_SYMBOL(____pagevec_lru_add); 702f04e9ebbSKOSAKI Motohiro 7031da177e4SLinus Torvalds /* 7041da177e4SLinus Torvalds * Try to drop buffers from the pages in a pagevec 7051da177e4SLinus Torvalds */ 7061da177e4SLinus Torvalds void pagevec_strip(struct pagevec *pvec) 7071da177e4SLinus Torvalds { 7081da177e4SLinus Torvalds int i; 7091da177e4SLinus Torvalds 7101da177e4SLinus Torvalds for (i = 0; i < pagevec_count(pvec); i++) { 7111da177e4SLinus Torvalds struct page *page = pvec->pages[i]; 7121da177e4SLinus Torvalds 713266cf658SDavid Howells if (page_has_private(page) && trylock_page(page)) { 714266cf658SDavid Howells if (page_has_private(page)) 7151da177e4SLinus Torvalds try_to_release_page(page, 0); 7161da177e4SLinus Torvalds unlock_page(page); 7171da177e4SLinus Torvalds } 7181da177e4SLinus Torvalds } 7191da177e4SLinus Torvalds } 7201da177e4SLinus Torvalds 7211da177e4SLinus Torvalds /** 7221da177e4SLinus Torvalds * pagevec_lookup - gang pagecache lookup 7231da177e4SLinus Torvalds * @pvec: Where the resulting pages are placed 7241da177e4SLinus Torvalds * @mapping: The address_space to search 7251da177e4SLinus Torvalds * @start: The starting page index 7261da177e4SLinus Torvalds * @nr_pages: The maximum number of pages 7271da177e4SLinus Torvalds * 7281da177e4SLinus Torvalds * pagevec_lookup() will search for and return a group of up to @nr_pages pages 7291da177e4SLinus Torvalds * in the mapping. The pages are placed in @pvec. pagevec_lookup() takes a 7301da177e4SLinus Torvalds * reference against the pages in @pvec. 7311da177e4SLinus Torvalds * 7321da177e4SLinus Torvalds * The search returns a group of mapping-contiguous pages with ascending 7331da177e4SLinus Torvalds * indexes. There may be holes in the indices due to not-present pages. 7341da177e4SLinus Torvalds * 7351da177e4SLinus Torvalds * pagevec_lookup() returns the number of pages which were found. 7361da177e4SLinus Torvalds */ 7371da177e4SLinus Torvalds unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping, 7381da177e4SLinus Torvalds pgoff_t start, unsigned nr_pages) 7391da177e4SLinus Torvalds { 7401da177e4SLinus Torvalds pvec->nr = find_get_pages(mapping, start, nr_pages, pvec->pages); 7411da177e4SLinus Torvalds return pagevec_count(pvec); 7421da177e4SLinus Torvalds } 7431da177e4SLinus Torvalds 74478539fdfSChristoph Hellwig EXPORT_SYMBOL(pagevec_lookup); 74578539fdfSChristoph Hellwig 7461da177e4SLinus Torvalds unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping, 7471da177e4SLinus Torvalds pgoff_t *index, int tag, unsigned nr_pages) 7481da177e4SLinus Torvalds { 7491da177e4SLinus Torvalds pvec->nr = find_get_pages_tag(mapping, index, tag, 7501da177e4SLinus Torvalds nr_pages, pvec->pages); 7511da177e4SLinus Torvalds return pagevec_count(pvec); 7521da177e4SLinus Torvalds } 7531da177e4SLinus Torvalds 7547f285701SSteve French EXPORT_SYMBOL(pagevec_lookup_tag); 7551da177e4SLinus Torvalds 7561da177e4SLinus Torvalds /* 7571da177e4SLinus Torvalds * Perform any setup for the swap system 7581da177e4SLinus Torvalds */ 7591da177e4SLinus Torvalds void __init swap_setup(void) 7601da177e4SLinus Torvalds { 7614481374cSJan Beulich unsigned long megs = totalram_pages >> (20 - PAGE_SHIFT); 7621da177e4SLinus Torvalds 763e0bf68ddSPeter Zijlstra #ifdef CONFIG_SWAP 764e0bf68ddSPeter Zijlstra bdi_init(swapper_space.backing_dev_info); 765e0bf68ddSPeter Zijlstra #endif 766e0bf68ddSPeter Zijlstra 7671da177e4SLinus Torvalds /* Use a smaller cluster for small-memory machines */ 7681da177e4SLinus Torvalds if (megs < 16) 7691da177e4SLinus Torvalds page_cluster = 2; 7701da177e4SLinus Torvalds else 7711da177e4SLinus Torvalds page_cluster = 3; 7721da177e4SLinus Torvalds /* 7731da177e4SLinus Torvalds * Right now other parts of the system means that we 7741da177e4SLinus Torvalds * _really_ don't want to cluster much more 7751da177e4SLinus Torvalds */ 7761da177e4SLinus Torvalds } 777