11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * linux/mm/swap.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 51da177e4SLinus Torvalds */ 61da177e4SLinus Torvalds 71da177e4SLinus Torvalds /* 8183ff22bSSimon Arlott * This file contains the default values for the operation of the 91da177e4SLinus Torvalds * Linux VM subsystem. Fine-tuning documentation can be found in 101da177e4SLinus Torvalds * Documentation/sysctl/vm.txt. 111da177e4SLinus Torvalds * Started 18.12.91 121da177e4SLinus Torvalds * Swap aging added 23.2.95, Stephen Tweedie. 131da177e4SLinus Torvalds * Buffermem limits added 12.3.98, Rik van Riel. 141da177e4SLinus Torvalds */ 151da177e4SLinus Torvalds 161da177e4SLinus Torvalds #include <linux/mm.h> 171da177e4SLinus Torvalds #include <linux/sched.h> 181da177e4SLinus Torvalds #include <linux/kernel_stat.h> 191da177e4SLinus Torvalds #include <linux/swap.h> 201da177e4SLinus Torvalds #include <linux/mman.h> 211da177e4SLinus Torvalds #include <linux/pagemap.h> 221da177e4SLinus Torvalds #include <linux/pagevec.h> 231da177e4SLinus Torvalds #include <linux/init.h> 241da177e4SLinus Torvalds #include <linux/module.h> 251da177e4SLinus Torvalds #include <linux/mm_inline.h> 261da177e4SLinus Torvalds #include <linux/buffer_head.h> /* for try_to_release_page() */ 271da177e4SLinus Torvalds #include <linux/percpu_counter.h> 281da177e4SLinus Torvalds #include <linux/percpu.h> 291da177e4SLinus Torvalds #include <linux/cpu.h> 301da177e4SLinus Torvalds #include <linux/notifier.h> 31e0bf68ddSPeter Zijlstra #include <linux/backing-dev.h> 32*66e1707bSBalbir Singh #include <linux/memcontrol.h> 331da177e4SLinus Torvalds 341da177e4SLinus Torvalds /* How many pages do we try to swap or page in/out together? */ 351da177e4SLinus Torvalds int page_cluster; 361da177e4SLinus Torvalds 37902aaed0SHisashi Hifumi static DEFINE_PER_CPU(struct pagevec, lru_add_pvecs) = { 0, }; 38902aaed0SHisashi Hifumi static DEFINE_PER_CPU(struct pagevec, lru_add_active_pvecs) = { 0, }; 39902aaed0SHisashi Hifumi static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs) = { 0, }; 40902aaed0SHisashi Hifumi 41b221385bSAdrian Bunk /* 42b221385bSAdrian Bunk * This path almost never happens for VM activity - pages are normally 43b221385bSAdrian Bunk * freed via pagevecs. But it gets used by networking. 44b221385bSAdrian Bunk */ 45920c7a5dSHarvey Harrison static void __page_cache_release(struct page *page) 46b221385bSAdrian Bunk { 47b221385bSAdrian Bunk if (PageLRU(page)) { 48b221385bSAdrian Bunk unsigned long flags; 49b221385bSAdrian Bunk struct zone *zone = page_zone(page); 50b221385bSAdrian Bunk 51b221385bSAdrian Bunk spin_lock_irqsave(&zone->lru_lock, flags); 52b221385bSAdrian Bunk VM_BUG_ON(!PageLRU(page)); 53b221385bSAdrian Bunk __ClearPageLRU(page); 54b221385bSAdrian Bunk del_page_from_lru(zone, page); 55b221385bSAdrian Bunk spin_unlock_irqrestore(&zone->lru_lock, flags); 56b221385bSAdrian Bunk } 57b221385bSAdrian Bunk free_hot_page(page); 58b221385bSAdrian Bunk } 59b221385bSAdrian Bunk 608519fb30SNick Piggin static void put_compound_page(struct page *page) 611da177e4SLinus Torvalds { 62d85f3385SChristoph Lameter page = compound_head(page); 631da177e4SLinus Torvalds if (put_page_testzero(page)) { 6433f2ef89SAndy Whitcroft compound_page_dtor *dtor; 651da177e4SLinus Torvalds 6633f2ef89SAndy Whitcroft dtor = get_compound_page_dtor(page); 671da177e4SLinus Torvalds (*dtor)(page); 681da177e4SLinus Torvalds } 691da177e4SLinus Torvalds } 708519fb30SNick Piggin 718519fb30SNick Piggin void put_page(struct page *page) 728519fb30SNick Piggin { 738519fb30SNick Piggin if (unlikely(PageCompound(page))) 748519fb30SNick Piggin put_compound_page(page); 758519fb30SNick Piggin else if (put_page_testzero(page)) 761da177e4SLinus Torvalds __page_cache_release(page); 771da177e4SLinus Torvalds } 781da177e4SLinus Torvalds EXPORT_SYMBOL(put_page); 791da177e4SLinus Torvalds 801d7ea732SAlexander Zarochentsev /** 811d7ea732SAlexander Zarochentsev * put_pages_list(): release a list of pages 821d7ea732SAlexander Zarochentsev * 831d7ea732SAlexander Zarochentsev * Release a list of pages which are strung together on page.lru. Currently 841d7ea732SAlexander Zarochentsev * used by read_cache_pages() and related error recovery code. 851d7ea732SAlexander Zarochentsev * 861d7ea732SAlexander Zarochentsev * @pages: list of pages threaded on page->lru 871d7ea732SAlexander Zarochentsev */ 881d7ea732SAlexander Zarochentsev void put_pages_list(struct list_head *pages) 891d7ea732SAlexander Zarochentsev { 901d7ea732SAlexander Zarochentsev while (!list_empty(pages)) { 911d7ea732SAlexander Zarochentsev struct page *victim; 921d7ea732SAlexander Zarochentsev 931d7ea732SAlexander Zarochentsev victim = list_entry(pages->prev, struct page, lru); 941d7ea732SAlexander Zarochentsev list_del(&victim->lru); 951d7ea732SAlexander Zarochentsev page_cache_release(victim); 961d7ea732SAlexander Zarochentsev } 971d7ea732SAlexander Zarochentsev } 981d7ea732SAlexander Zarochentsev EXPORT_SYMBOL(put_pages_list); 991d7ea732SAlexander Zarochentsev 1001da177e4SLinus Torvalds /* 101902aaed0SHisashi Hifumi * pagevec_move_tail() must be called with IRQ disabled. 102902aaed0SHisashi Hifumi * Otherwise this may cause nasty races. 103902aaed0SHisashi Hifumi */ 104902aaed0SHisashi Hifumi static void pagevec_move_tail(struct pagevec *pvec) 105902aaed0SHisashi Hifumi { 106902aaed0SHisashi Hifumi int i; 107902aaed0SHisashi Hifumi int pgmoved = 0; 108902aaed0SHisashi Hifumi struct zone *zone = NULL; 109902aaed0SHisashi Hifumi 110902aaed0SHisashi Hifumi for (i = 0; i < pagevec_count(pvec); i++) { 111902aaed0SHisashi Hifumi struct page *page = pvec->pages[i]; 112902aaed0SHisashi Hifumi struct zone *pagezone = page_zone(page); 113902aaed0SHisashi Hifumi 114902aaed0SHisashi Hifumi if (pagezone != zone) { 115902aaed0SHisashi Hifumi if (zone) 116902aaed0SHisashi Hifumi spin_unlock(&zone->lru_lock); 117902aaed0SHisashi Hifumi zone = pagezone; 118902aaed0SHisashi Hifumi spin_lock(&zone->lru_lock); 119902aaed0SHisashi Hifumi } 120902aaed0SHisashi Hifumi if (PageLRU(page) && !PageActive(page)) { 121902aaed0SHisashi Hifumi list_move_tail(&page->lru, &zone->inactive_list); 122902aaed0SHisashi Hifumi pgmoved++; 123902aaed0SHisashi Hifumi } 124902aaed0SHisashi Hifumi } 125902aaed0SHisashi Hifumi if (zone) 126902aaed0SHisashi Hifumi spin_unlock(&zone->lru_lock); 127902aaed0SHisashi Hifumi __count_vm_events(PGROTATED, pgmoved); 128902aaed0SHisashi Hifumi release_pages(pvec->pages, pvec->nr, pvec->cold); 129902aaed0SHisashi Hifumi pagevec_reinit(pvec); 130902aaed0SHisashi Hifumi } 131902aaed0SHisashi Hifumi 132902aaed0SHisashi Hifumi /* 1331da177e4SLinus Torvalds * Writeback is about to end against a page which has been marked for immediate 1341da177e4SLinus Torvalds * reclaim. If it still appears to be reclaimable, move it to the tail of the 135902aaed0SHisashi Hifumi * inactive list. 1361da177e4SLinus Torvalds * 1371da177e4SLinus Torvalds * Returns zero if it cleared PG_writeback. 1381da177e4SLinus Torvalds */ 1391da177e4SLinus Torvalds int rotate_reclaimable_page(struct page *page) 1401da177e4SLinus Torvalds { 141902aaed0SHisashi Hifumi struct pagevec *pvec; 1421da177e4SLinus Torvalds unsigned long flags; 1431da177e4SLinus Torvalds 1441da177e4SLinus Torvalds if (PageLocked(page)) 1451da177e4SLinus Torvalds return 1; 1461da177e4SLinus Torvalds if (PageDirty(page)) 1471da177e4SLinus Torvalds return 1; 1481da177e4SLinus Torvalds if (PageActive(page)) 1491da177e4SLinus Torvalds return 1; 1501da177e4SLinus Torvalds if (!PageLRU(page)) 1511da177e4SLinus Torvalds return 1; 1521da177e4SLinus Torvalds 153902aaed0SHisashi Hifumi page_cache_get(page); 154902aaed0SHisashi Hifumi local_irq_save(flags); 155902aaed0SHisashi Hifumi pvec = &__get_cpu_var(lru_rotate_pvecs); 156902aaed0SHisashi Hifumi if (!pagevec_add(pvec, page)) 157902aaed0SHisashi Hifumi pagevec_move_tail(pvec); 158902aaed0SHisashi Hifumi local_irq_restore(flags); 159902aaed0SHisashi Hifumi 1601da177e4SLinus Torvalds if (!test_clear_page_writeback(page)) 1611da177e4SLinus Torvalds BUG(); 162902aaed0SHisashi Hifumi 1631da177e4SLinus Torvalds return 0; 1641da177e4SLinus Torvalds } 1651da177e4SLinus Torvalds 1661da177e4SLinus Torvalds /* 1671da177e4SLinus Torvalds * FIXME: speed this up? 1681da177e4SLinus Torvalds */ 169920c7a5dSHarvey Harrison void activate_page(struct page *page) 1701da177e4SLinus Torvalds { 1711da177e4SLinus Torvalds struct zone *zone = page_zone(page); 1721da177e4SLinus Torvalds 1731da177e4SLinus Torvalds spin_lock_irq(&zone->lru_lock); 1741da177e4SLinus Torvalds if (PageLRU(page) && !PageActive(page)) { 1751da177e4SLinus Torvalds del_page_from_inactive_list(zone, page); 1761da177e4SLinus Torvalds SetPageActive(page); 1771da177e4SLinus Torvalds add_page_to_active_list(zone, page); 178f8891e5eSChristoph Lameter __count_vm_event(PGACTIVATE); 179*66e1707bSBalbir Singh mem_cgroup_move_lists(page_get_page_cgroup(page), true); 1801da177e4SLinus Torvalds } 1811da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 1821da177e4SLinus Torvalds } 1831da177e4SLinus Torvalds 1841da177e4SLinus Torvalds /* 1851da177e4SLinus Torvalds * Mark a page as having seen activity. 1861da177e4SLinus Torvalds * 1871da177e4SLinus Torvalds * inactive,unreferenced -> inactive,referenced 1881da177e4SLinus Torvalds * inactive,referenced -> active,unreferenced 1891da177e4SLinus Torvalds * active,unreferenced -> active,referenced 1901da177e4SLinus Torvalds */ 191920c7a5dSHarvey Harrison void mark_page_accessed(struct page *page) 1921da177e4SLinus Torvalds { 1931da177e4SLinus Torvalds if (!PageActive(page) && PageReferenced(page) && PageLRU(page)) { 1941da177e4SLinus Torvalds activate_page(page); 1951da177e4SLinus Torvalds ClearPageReferenced(page); 1961da177e4SLinus Torvalds } else if (!PageReferenced(page)) { 1971da177e4SLinus Torvalds SetPageReferenced(page); 1981da177e4SLinus Torvalds } 1991da177e4SLinus Torvalds } 2001da177e4SLinus Torvalds 2011da177e4SLinus Torvalds EXPORT_SYMBOL(mark_page_accessed); 2021da177e4SLinus Torvalds 2031da177e4SLinus Torvalds /** 2041da177e4SLinus Torvalds * lru_cache_add: add a page to the page lists 2051da177e4SLinus Torvalds * @page: the page to add 2061da177e4SLinus Torvalds */ 207920c7a5dSHarvey Harrison void lru_cache_add(struct page *page) 2081da177e4SLinus Torvalds { 2091da177e4SLinus Torvalds struct pagevec *pvec = &get_cpu_var(lru_add_pvecs); 2101da177e4SLinus Torvalds 2111da177e4SLinus Torvalds page_cache_get(page); 2121da177e4SLinus Torvalds if (!pagevec_add(pvec, page)) 2131da177e4SLinus Torvalds __pagevec_lru_add(pvec); 2141da177e4SLinus Torvalds put_cpu_var(lru_add_pvecs); 2151da177e4SLinus Torvalds } 2161da177e4SLinus Torvalds 217920c7a5dSHarvey Harrison void lru_cache_add_active(struct page *page) 2181da177e4SLinus Torvalds { 2191da177e4SLinus Torvalds struct pagevec *pvec = &get_cpu_var(lru_add_active_pvecs); 2201da177e4SLinus Torvalds 2211da177e4SLinus Torvalds page_cache_get(page); 2221da177e4SLinus Torvalds if (!pagevec_add(pvec, page)) 2231da177e4SLinus Torvalds __pagevec_lru_add_active(pvec); 2241da177e4SLinus Torvalds put_cpu_var(lru_add_active_pvecs); 2251da177e4SLinus Torvalds } 2261da177e4SLinus Torvalds 227902aaed0SHisashi Hifumi /* 228902aaed0SHisashi Hifumi * Drain pages out of the cpu's pagevecs. 229902aaed0SHisashi Hifumi * Either "cpu" is the current CPU, and preemption has already been 230902aaed0SHisashi Hifumi * disabled; or "cpu" is being hot-unplugged, and is already dead. 231902aaed0SHisashi Hifumi */ 232902aaed0SHisashi Hifumi static void drain_cpu_pagevecs(int cpu) 2331da177e4SLinus Torvalds { 234902aaed0SHisashi Hifumi struct pagevec *pvec; 2351da177e4SLinus Torvalds 236902aaed0SHisashi Hifumi pvec = &per_cpu(lru_add_pvecs, cpu); 2371da177e4SLinus Torvalds if (pagevec_count(pvec)) 2381da177e4SLinus Torvalds __pagevec_lru_add(pvec); 239902aaed0SHisashi Hifumi 24080bfed90SAndrew Morton pvec = &per_cpu(lru_add_active_pvecs, cpu); 2411da177e4SLinus Torvalds if (pagevec_count(pvec)) 2421da177e4SLinus Torvalds __pagevec_lru_add_active(pvec); 243902aaed0SHisashi Hifumi 244902aaed0SHisashi Hifumi pvec = &per_cpu(lru_rotate_pvecs, cpu); 245902aaed0SHisashi Hifumi if (pagevec_count(pvec)) { 246902aaed0SHisashi Hifumi unsigned long flags; 247902aaed0SHisashi Hifumi 248902aaed0SHisashi Hifumi /* No harm done if a racing interrupt already did this */ 249902aaed0SHisashi Hifumi local_irq_save(flags); 250902aaed0SHisashi Hifumi pagevec_move_tail(pvec); 251902aaed0SHisashi Hifumi local_irq_restore(flags); 252902aaed0SHisashi Hifumi } 25380bfed90SAndrew Morton } 25480bfed90SAndrew Morton 25580bfed90SAndrew Morton void lru_add_drain(void) 25680bfed90SAndrew Morton { 257902aaed0SHisashi Hifumi drain_cpu_pagevecs(get_cpu()); 25880bfed90SAndrew Morton put_cpu(); 2591da177e4SLinus Torvalds } 2601da177e4SLinus Torvalds 261053837fcSNick Piggin #ifdef CONFIG_NUMA 262c4028958SDavid Howells static void lru_add_drain_per_cpu(struct work_struct *dummy) 263053837fcSNick Piggin { 264053837fcSNick Piggin lru_add_drain(); 265053837fcSNick Piggin } 266053837fcSNick Piggin 267053837fcSNick Piggin /* 268053837fcSNick Piggin * Returns 0 for success 269053837fcSNick Piggin */ 270053837fcSNick Piggin int lru_add_drain_all(void) 271053837fcSNick Piggin { 272c4028958SDavid Howells return schedule_on_each_cpu(lru_add_drain_per_cpu); 273053837fcSNick Piggin } 274053837fcSNick Piggin 275053837fcSNick Piggin #else 276053837fcSNick Piggin 277053837fcSNick Piggin /* 278053837fcSNick Piggin * Returns 0 for success 279053837fcSNick Piggin */ 280053837fcSNick Piggin int lru_add_drain_all(void) 281053837fcSNick Piggin { 282053837fcSNick Piggin lru_add_drain(); 283053837fcSNick Piggin return 0; 284053837fcSNick Piggin } 285053837fcSNick Piggin #endif 286053837fcSNick Piggin 2871da177e4SLinus Torvalds /* 2881da177e4SLinus Torvalds * Batched page_cache_release(). Decrement the reference count on all the 2891da177e4SLinus Torvalds * passed pages. If it fell to zero then remove the page from the LRU and 2901da177e4SLinus Torvalds * free it. 2911da177e4SLinus Torvalds * 2921da177e4SLinus Torvalds * Avoid taking zone->lru_lock if possible, but if it is taken, retain it 2931da177e4SLinus Torvalds * for the remainder of the operation. 2941da177e4SLinus Torvalds * 2951da177e4SLinus Torvalds * The locking in this function is against shrink_cache(): we recheck the 2961da177e4SLinus Torvalds * page count inside the lock to see whether shrink_cache grabbed the page 2971da177e4SLinus Torvalds * via the LRU. If it did, give up: shrink_cache will free it. 2981da177e4SLinus Torvalds */ 2991da177e4SLinus Torvalds void release_pages(struct page **pages, int nr, int cold) 3001da177e4SLinus Torvalds { 3011da177e4SLinus Torvalds int i; 3021da177e4SLinus Torvalds struct pagevec pages_to_free; 3031da177e4SLinus Torvalds struct zone *zone = NULL; 304902aaed0SHisashi Hifumi unsigned long uninitialized_var(flags); 3051da177e4SLinus Torvalds 3061da177e4SLinus Torvalds pagevec_init(&pages_to_free, cold); 3071da177e4SLinus Torvalds for (i = 0; i < nr; i++) { 3081da177e4SLinus Torvalds struct page *page = pages[i]; 3091da177e4SLinus Torvalds 3108519fb30SNick Piggin if (unlikely(PageCompound(page))) { 3118519fb30SNick Piggin if (zone) { 312902aaed0SHisashi Hifumi spin_unlock_irqrestore(&zone->lru_lock, flags); 3138519fb30SNick Piggin zone = NULL; 3148519fb30SNick Piggin } 3158519fb30SNick Piggin put_compound_page(page); 3168519fb30SNick Piggin continue; 3178519fb30SNick Piggin } 3188519fb30SNick Piggin 319b5810039SNick Piggin if (!put_page_testzero(page)) 3201da177e4SLinus Torvalds continue; 3211da177e4SLinus Torvalds 32246453a6eSNick Piggin if (PageLRU(page)) { 32346453a6eSNick Piggin struct zone *pagezone = page_zone(page); 3241da177e4SLinus Torvalds if (pagezone != zone) { 3251da177e4SLinus Torvalds if (zone) 326902aaed0SHisashi Hifumi spin_unlock_irqrestore(&zone->lru_lock, 327902aaed0SHisashi Hifumi flags); 3281da177e4SLinus Torvalds zone = pagezone; 329902aaed0SHisashi Hifumi spin_lock_irqsave(&zone->lru_lock, flags); 3301da177e4SLinus Torvalds } 331725d704eSNick Piggin VM_BUG_ON(!PageLRU(page)); 33267453911SNick Piggin __ClearPageLRU(page); 3331da177e4SLinus Torvalds del_page_from_lru(zone, page); 33446453a6eSNick Piggin } 33546453a6eSNick Piggin 3361da177e4SLinus Torvalds if (!pagevec_add(&pages_to_free, page)) { 33746453a6eSNick Piggin if (zone) { 338902aaed0SHisashi Hifumi spin_unlock_irqrestore(&zone->lru_lock, flags); 33946453a6eSNick Piggin zone = NULL; 34046453a6eSNick Piggin } 3411da177e4SLinus Torvalds __pagevec_free(&pages_to_free); 3421da177e4SLinus Torvalds pagevec_reinit(&pages_to_free); 3431da177e4SLinus Torvalds } 3441da177e4SLinus Torvalds } 3451da177e4SLinus Torvalds if (zone) 346902aaed0SHisashi Hifumi spin_unlock_irqrestore(&zone->lru_lock, flags); 3471da177e4SLinus Torvalds 3481da177e4SLinus Torvalds pagevec_free(&pages_to_free); 3491da177e4SLinus Torvalds } 3501da177e4SLinus Torvalds 3511da177e4SLinus Torvalds /* 3521da177e4SLinus Torvalds * The pages which we're about to release may be in the deferred lru-addition 3531da177e4SLinus Torvalds * queues. That would prevent them from really being freed right now. That's 3541da177e4SLinus Torvalds * OK from a correctness point of view but is inefficient - those pages may be 3551da177e4SLinus Torvalds * cache-warm and we want to give them back to the page allocator ASAP. 3561da177e4SLinus Torvalds * 3571da177e4SLinus Torvalds * So __pagevec_release() will drain those queues here. __pagevec_lru_add() 3581da177e4SLinus Torvalds * and __pagevec_lru_add_active() call release_pages() directly to avoid 3591da177e4SLinus Torvalds * mutual recursion. 3601da177e4SLinus Torvalds */ 3611da177e4SLinus Torvalds void __pagevec_release(struct pagevec *pvec) 3621da177e4SLinus Torvalds { 3631da177e4SLinus Torvalds lru_add_drain(); 3641da177e4SLinus Torvalds release_pages(pvec->pages, pagevec_count(pvec), pvec->cold); 3651da177e4SLinus Torvalds pagevec_reinit(pvec); 3661da177e4SLinus Torvalds } 3671da177e4SLinus Torvalds 3687f285701SSteve French EXPORT_SYMBOL(__pagevec_release); 3697f285701SSteve French 3701da177e4SLinus Torvalds /* 3711da177e4SLinus Torvalds * pagevec_release() for pages which are known to not be on the LRU 3721da177e4SLinus Torvalds * 3731da177e4SLinus Torvalds * This function reinitialises the caller's pagevec. 3741da177e4SLinus Torvalds */ 3751da177e4SLinus Torvalds void __pagevec_release_nonlru(struct pagevec *pvec) 3761da177e4SLinus Torvalds { 3771da177e4SLinus Torvalds int i; 3781da177e4SLinus Torvalds struct pagevec pages_to_free; 3791da177e4SLinus Torvalds 3801da177e4SLinus Torvalds pagevec_init(&pages_to_free, pvec->cold); 3811da177e4SLinus Torvalds for (i = 0; i < pagevec_count(pvec); i++) { 3821da177e4SLinus Torvalds struct page *page = pvec->pages[i]; 3831da177e4SLinus Torvalds 384725d704eSNick Piggin VM_BUG_ON(PageLRU(page)); 3851da177e4SLinus Torvalds if (put_page_testzero(page)) 3861da177e4SLinus Torvalds pagevec_add(&pages_to_free, page); 3871da177e4SLinus Torvalds } 3881da177e4SLinus Torvalds pagevec_free(&pages_to_free); 3891da177e4SLinus Torvalds pagevec_reinit(pvec); 3901da177e4SLinus Torvalds } 3911da177e4SLinus Torvalds 3921da177e4SLinus Torvalds /* 3931da177e4SLinus Torvalds * Add the passed pages to the LRU, then drop the caller's refcount 3941da177e4SLinus Torvalds * on them. Reinitialises the caller's pagevec. 3951da177e4SLinus Torvalds */ 3961da177e4SLinus Torvalds void __pagevec_lru_add(struct pagevec *pvec) 3971da177e4SLinus Torvalds { 3981da177e4SLinus Torvalds int i; 3991da177e4SLinus Torvalds struct zone *zone = NULL; 4001da177e4SLinus Torvalds 4011da177e4SLinus Torvalds for (i = 0; i < pagevec_count(pvec); i++) { 4021da177e4SLinus Torvalds struct page *page = pvec->pages[i]; 4031da177e4SLinus Torvalds struct zone *pagezone = page_zone(page); 4041da177e4SLinus Torvalds 4051da177e4SLinus Torvalds if (pagezone != zone) { 4061da177e4SLinus Torvalds if (zone) 4071da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 4081da177e4SLinus Torvalds zone = pagezone; 4091da177e4SLinus Torvalds spin_lock_irq(&zone->lru_lock); 4101da177e4SLinus Torvalds } 411725d704eSNick Piggin VM_BUG_ON(PageLRU(page)); 4128d438f96SNick Piggin SetPageLRU(page); 4131da177e4SLinus Torvalds add_page_to_inactive_list(zone, page); 4141da177e4SLinus Torvalds } 4151da177e4SLinus Torvalds if (zone) 4161da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 4171da177e4SLinus Torvalds release_pages(pvec->pages, pvec->nr, pvec->cold); 4181da177e4SLinus Torvalds pagevec_reinit(pvec); 4191da177e4SLinus Torvalds } 4201da177e4SLinus Torvalds 4211da177e4SLinus Torvalds EXPORT_SYMBOL(__pagevec_lru_add); 4221da177e4SLinus Torvalds 4231da177e4SLinus Torvalds void __pagevec_lru_add_active(struct pagevec *pvec) 4241da177e4SLinus Torvalds { 4251da177e4SLinus Torvalds int i; 4261da177e4SLinus Torvalds struct zone *zone = NULL; 4271da177e4SLinus Torvalds 4281da177e4SLinus Torvalds for (i = 0; i < pagevec_count(pvec); i++) { 4291da177e4SLinus Torvalds struct page *page = pvec->pages[i]; 4301da177e4SLinus Torvalds struct zone *pagezone = page_zone(page); 4311da177e4SLinus Torvalds 4321da177e4SLinus Torvalds if (pagezone != zone) { 4331da177e4SLinus Torvalds if (zone) 4341da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 4351da177e4SLinus Torvalds zone = pagezone; 4361da177e4SLinus Torvalds spin_lock_irq(&zone->lru_lock); 4371da177e4SLinus Torvalds } 438725d704eSNick Piggin VM_BUG_ON(PageLRU(page)); 4398d438f96SNick Piggin SetPageLRU(page); 440725d704eSNick Piggin VM_BUG_ON(PageActive(page)); 4414c84cacfSNick Piggin SetPageActive(page); 4421da177e4SLinus Torvalds add_page_to_active_list(zone, page); 4431da177e4SLinus Torvalds } 4441da177e4SLinus Torvalds if (zone) 4451da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 4461da177e4SLinus Torvalds release_pages(pvec->pages, pvec->nr, pvec->cold); 4471da177e4SLinus Torvalds pagevec_reinit(pvec); 4481da177e4SLinus Torvalds } 4491da177e4SLinus Torvalds 4501da177e4SLinus Torvalds /* 4511da177e4SLinus Torvalds * Try to drop buffers from the pages in a pagevec 4521da177e4SLinus Torvalds */ 4531da177e4SLinus Torvalds void pagevec_strip(struct pagevec *pvec) 4541da177e4SLinus Torvalds { 4551da177e4SLinus Torvalds int i; 4561da177e4SLinus Torvalds 4571da177e4SLinus Torvalds for (i = 0; i < pagevec_count(pvec); i++) { 4581da177e4SLinus Torvalds struct page *page = pvec->pages[i]; 4591da177e4SLinus Torvalds 4601da177e4SLinus Torvalds if (PagePrivate(page) && !TestSetPageLocked(page)) { 4615b40dc78SChristoph Lameter if (PagePrivate(page)) 4621da177e4SLinus Torvalds try_to_release_page(page, 0); 4631da177e4SLinus Torvalds unlock_page(page); 4641da177e4SLinus Torvalds } 4651da177e4SLinus Torvalds } 4661da177e4SLinus Torvalds } 4671da177e4SLinus Torvalds 4681da177e4SLinus Torvalds /** 4691da177e4SLinus Torvalds * pagevec_lookup - gang pagecache lookup 4701da177e4SLinus Torvalds * @pvec: Where the resulting pages are placed 4711da177e4SLinus Torvalds * @mapping: The address_space to search 4721da177e4SLinus Torvalds * @start: The starting page index 4731da177e4SLinus Torvalds * @nr_pages: The maximum number of pages 4741da177e4SLinus Torvalds * 4751da177e4SLinus Torvalds * pagevec_lookup() will search for and return a group of up to @nr_pages pages 4761da177e4SLinus Torvalds * in the mapping. The pages are placed in @pvec. pagevec_lookup() takes a 4771da177e4SLinus Torvalds * reference against the pages in @pvec. 4781da177e4SLinus Torvalds * 4791da177e4SLinus Torvalds * The search returns a group of mapping-contiguous pages with ascending 4801da177e4SLinus Torvalds * indexes. There may be holes in the indices due to not-present pages. 4811da177e4SLinus Torvalds * 4821da177e4SLinus Torvalds * pagevec_lookup() returns the number of pages which were found. 4831da177e4SLinus Torvalds */ 4841da177e4SLinus Torvalds unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping, 4851da177e4SLinus Torvalds pgoff_t start, unsigned nr_pages) 4861da177e4SLinus Torvalds { 4871da177e4SLinus Torvalds pvec->nr = find_get_pages(mapping, start, nr_pages, pvec->pages); 4881da177e4SLinus Torvalds return pagevec_count(pvec); 4891da177e4SLinus Torvalds } 4901da177e4SLinus Torvalds 49178539fdfSChristoph Hellwig EXPORT_SYMBOL(pagevec_lookup); 49278539fdfSChristoph Hellwig 4931da177e4SLinus Torvalds unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping, 4941da177e4SLinus Torvalds pgoff_t *index, int tag, unsigned nr_pages) 4951da177e4SLinus Torvalds { 4961da177e4SLinus Torvalds pvec->nr = find_get_pages_tag(mapping, index, tag, 4971da177e4SLinus Torvalds nr_pages, pvec->pages); 4981da177e4SLinus Torvalds return pagevec_count(pvec); 4991da177e4SLinus Torvalds } 5001da177e4SLinus Torvalds 5017f285701SSteve French EXPORT_SYMBOL(pagevec_lookup_tag); 5021da177e4SLinus Torvalds 5031da177e4SLinus Torvalds #ifdef CONFIG_SMP 5041da177e4SLinus Torvalds /* 5051da177e4SLinus Torvalds * We tolerate a little inaccuracy to avoid ping-ponging the counter between 5061da177e4SLinus Torvalds * CPUs 5071da177e4SLinus Torvalds */ 5081da177e4SLinus Torvalds #define ACCT_THRESHOLD max(16, NR_CPUS * 2) 5091da177e4SLinus Torvalds 5101da177e4SLinus Torvalds static DEFINE_PER_CPU(long, committed_space) = 0; 5111da177e4SLinus Torvalds 5121da177e4SLinus Torvalds void vm_acct_memory(long pages) 5131da177e4SLinus Torvalds { 5141da177e4SLinus Torvalds long *local; 5151da177e4SLinus Torvalds 5161da177e4SLinus Torvalds preempt_disable(); 5171da177e4SLinus Torvalds local = &__get_cpu_var(committed_space); 5181da177e4SLinus Torvalds *local += pages; 5191da177e4SLinus Torvalds if (*local > ACCT_THRESHOLD || *local < -ACCT_THRESHOLD) { 5201da177e4SLinus Torvalds atomic_add(*local, &vm_committed_space); 5211da177e4SLinus Torvalds *local = 0; 5221da177e4SLinus Torvalds } 5231da177e4SLinus Torvalds preempt_enable(); 5241da177e4SLinus Torvalds } 5251da177e4SLinus Torvalds 5261da177e4SLinus Torvalds #ifdef CONFIG_HOTPLUG_CPU 5271da177e4SLinus Torvalds 5281da177e4SLinus Torvalds /* Drop the CPU's cached committed space back into the central pool. */ 5291da177e4SLinus Torvalds static int cpu_swap_callback(struct notifier_block *nfb, 5301da177e4SLinus Torvalds unsigned long action, 5311da177e4SLinus Torvalds void *hcpu) 5321da177e4SLinus Torvalds { 5331da177e4SLinus Torvalds long *committed; 5341da177e4SLinus Torvalds 5351da177e4SLinus Torvalds committed = &per_cpu(committed_space, (long)hcpu); 5368bb78442SRafael J. Wysocki if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { 5371da177e4SLinus Torvalds atomic_add(*committed, &vm_committed_space); 5381da177e4SLinus Torvalds *committed = 0; 539902aaed0SHisashi Hifumi drain_cpu_pagevecs((long)hcpu); 5401da177e4SLinus Torvalds } 5411da177e4SLinus Torvalds return NOTIFY_OK; 5421da177e4SLinus Torvalds } 5431da177e4SLinus Torvalds #endif /* CONFIG_HOTPLUG_CPU */ 5441da177e4SLinus Torvalds #endif /* CONFIG_SMP */ 5451da177e4SLinus Torvalds 5461da177e4SLinus Torvalds /* 5471da177e4SLinus Torvalds * Perform any setup for the swap system 5481da177e4SLinus Torvalds */ 5491da177e4SLinus Torvalds void __init swap_setup(void) 5501da177e4SLinus Torvalds { 5511da177e4SLinus Torvalds unsigned long megs = num_physpages >> (20 - PAGE_SHIFT); 5521da177e4SLinus Torvalds 553e0bf68ddSPeter Zijlstra #ifdef CONFIG_SWAP 554e0bf68ddSPeter Zijlstra bdi_init(swapper_space.backing_dev_info); 555e0bf68ddSPeter Zijlstra #endif 556e0bf68ddSPeter Zijlstra 5571da177e4SLinus Torvalds /* Use a smaller cluster for small-memory machines */ 5581da177e4SLinus Torvalds if (megs < 16) 5591da177e4SLinus Torvalds page_cluster = 2; 5601da177e4SLinus Torvalds else 5611da177e4SLinus Torvalds page_cluster = 3; 5621da177e4SLinus Torvalds /* 5631da177e4SLinus Torvalds * Right now other parts of the system means that we 5641da177e4SLinus Torvalds * _really_ don't want to cluster much more 5651da177e4SLinus Torvalds */ 56602316067SIngo Molnar #ifdef CONFIG_HOTPLUG_CPU 5671da177e4SLinus Torvalds hotcpu_notifier(cpu_swap_callback, 0); 56802316067SIngo Molnar #endif 5691da177e4SLinus Torvalds } 570