11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * linux/mm/swap.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 51da177e4SLinus Torvalds */ 61da177e4SLinus Torvalds 71da177e4SLinus Torvalds /* 8183ff22bSSimon Arlott * This file contains the default values for the operation of the 91da177e4SLinus Torvalds * Linux VM subsystem. Fine-tuning documentation can be found in 101da177e4SLinus Torvalds * Documentation/sysctl/vm.txt. 111da177e4SLinus Torvalds * Started 18.12.91 121da177e4SLinus Torvalds * Swap aging added 23.2.95, Stephen Tweedie. 131da177e4SLinus Torvalds * Buffermem limits added 12.3.98, Rik van Riel. 141da177e4SLinus Torvalds */ 151da177e4SLinus Torvalds 161da177e4SLinus Torvalds #include <linux/mm.h> 171da177e4SLinus Torvalds #include <linux/sched.h> 181da177e4SLinus Torvalds #include <linux/kernel_stat.h> 191da177e4SLinus Torvalds #include <linux/swap.h> 201da177e4SLinus Torvalds #include <linux/mman.h> 211da177e4SLinus Torvalds #include <linux/pagemap.h> 221da177e4SLinus Torvalds #include <linux/pagevec.h> 231da177e4SLinus Torvalds #include <linux/init.h> 241da177e4SLinus Torvalds #include <linux/module.h> 251da177e4SLinus Torvalds #include <linux/mm_inline.h> 261da177e4SLinus Torvalds #include <linux/buffer_head.h> /* for try_to_release_page() */ 271da177e4SLinus Torvalds #include <linux/percpu_counter.h> 281da177e4SLinus Torvalds #include <linux/percpu.h> 291da177e4SLinus Torvalds #include <linux/cpu.h> 301da177e4SLinus Torvalds #include <linux/notifier.h> 31e0bf68ddSPeter Zijlstra #include <linux/backing-dev.h> 3266e1707bSBalbir Singh #include <linux/memcontrol.h> 331da177e4SLinus Torvalds 341da177e4SLinus Torvalds /* How many pages do we try to swap or page in/out together? */ 351da177e4SLinus Torvalds int page_cluster; 361da177e4SLinus Torvalds 37902aaed0SHisashi Hifumi static DEFINE_PER_CPU(struct pagevec, lru_add_pvecs) = { 0, }; 38902aaed0SHisashi Hifumi static DEFINE_PER_CPU(struct pagevec, lru_add_active_pvecs) = { 0, }; 39902aaed0SHisashi Hifumi static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs) = { 0, }; 40902aaed0SHisashi Hifumi 41b221385bSAdrian Bunk /* 42b221385bSAdrian Bunk * This path almost never happens for VM activity - pages are normally 43b221385bSAdrian Bunk * freed via pagevecs. But it gets used by networking. 44b221385bSAdrian Bunk */ 45920c7a5dSHarvey Harrison static void __page_cache_release(struct page *page) 46b221385bSAdrian Bunk { 47b221385bSAdrian Bunk if (PageLRU(page)) { 48b221385bSAdrian Bunk unsigned long flags; 49b221385bSAdrian Bunk struct zone *zone = page_zone(page); 50b221385bSAdrian Bunk 51b221385bSAdrian Bunk spin_lock_irqsave(&zone->lru_lock, flags); 52b221385bSAdrian Bunk VM_BUG_ON(!PageLRU(page)); 53b221385bSAdrian Bunk __ClearPageLRU(page); 54b221385bSAdrian Bunk del_page_from_lru(zone, page); 55b221385bSAdrian Bunk spin_unlock_irqrestore(&zone->lru_lock, flags); 56b221385bSAdrian Bunk } 57b221385bSAdrian Bunk free_hot_page(page); 58b221385bSAdrian Bunk } 59b221385bSAdrian Bunk 608519fb30SNick Piggin static void put_compound_page(struct page *page) 611da177e4SLinus Torvalds { 62d85f3385SChristoph Lameter page = compound_head(page); 631da177e4SLinus Torvalds if (put_page_testzero(page)) { 6433f2ef89SAndy Whitcroft compound_page_dtor *dtor; 651da177e4SLinus Torvalds 6633f2ef89SAndy Whitcroft dtor = get_compound_page_dtor(page); 671da177e4SLinus Torvalds (*dtor)(page); 681da177e4SLinus Torvalds } 691da177e4SLinus Torvalds } 708519fb30SNick Piggin 718519fb30SNick Piggin void put_page(struct page *page) 728519fb30SNick Piggin { 738519fb30SNick Piggin if (unlikely(PageCompound(page))) 748519fb30SNick Piggin put_compound_page(page); 758519fb30SNick Piggin else if (put_page_testzero(page)) 761da177e4SLinus Torvalds __page_cache_release(page); 771da177e4SLinus Torvalds } 781da177e4SLinus Torvalds EXPORT_SYMBOL(put_page); 791da177e4SLinus Torvalds 801d7ea732SAlexander Zarochentsev /** 81*7682486bSRandy Dunlap * put_pages_list() - release a list of pages 82*7682486bSRandy Dunlap * @pages: list of pages threaded on page->lru 831d7ea732SAlexander Zarochentsev * 841d7ea732SAlexander Zarochentsev * Release a list of pages which are strung together on page.lru. Currently 851d7ea732SAlexander Zarochentsev * used by read_cache_pages() and related error recovery code. 861d7ea732SAlexander Zarochentsev */ 871d7ea732SAlexander Zarochentsev void put_pages_list(struct list_head *pages) 881d7ea732SAlexander Zarochentsev { 891d7ea732SAlexander Zarochentsev while (!list_empty(pages)) { 901d7ea732SAlexander Zarochentsev struct page *victim; 911d7ea732SAlexander Zarochentsev 921d7ea732SAlexander Zarochentsev victim = list_entry(pages->prev, struct page, lru); 931d7ea732SAlexander Zarochentsev list_del(&victim->lru); 941d7ea732SAlexander Zarochentsev page_cache_release(victim); 951d7ea732SAlexander Zarochentsev } 961d7ea732SAlexander Zarochentsev } 971d7ea732SAlexander Zarochentsev EXPORT_SYMBOL(put_pages_list); 981d7ea732SAlexander Zarochentsev 991da177e4SLinus Torvalds /* 100902aaed0SHisashi Hifumi * pagevec_move_tail() must be called with IRQ disabled. 101902aaed0SHisashi Hifumi * Otherwise this may cause nasty races. 102902aaed0SHisashi Hifumi */ 103902aaed0SHisashi Hifumi static void pagevec_move_tail(struct pagevec *pvec) 104902aaed0SHisashi Hifumi { 105902aaed0SHisashi Hifumi int i; 106902aaed0SHisashi Hifumi int pgmoved = 0; 107902aaed0SHisashi Hifumi struct zone *zone = NULL; 108902aaed0SHisashi Hifumi 109902aaed0SHisashi Hifumi for (i = 0; i < pagevec_count(pvec); i++) { 110902aaed0SHisashi Hifumi struct page *page = pvec->pages[i]; 111902aaed0SHisashi Hifumi struct zone *pagezone = page_zone(page); 112902aaed0SHisashi Hifumi 113902aaed0SHisashi Hifumi if (pagezone != zone) { 114902aaed0SHisashi Hifumi if (zone) 115902aaed0SHisashi Hifumi spin_unlock(&zone->lru_lock); 116902aaed0SHisashi Hifumi zone = pagezone; 117902aaed0SHisashi Hifumi spin_lock(&zone->lru_lock); 118902aaed0SHisashi Hifumi } 119902aaed0SHisashi Hifumi if (PageLRU(page) && !PageActive(page)) { 120902aaed0SHisashi Hifumi list_move_tail(&page->lru, &zone->inactive_list); 121902aaed0SHisashi Hifumi pgmoved++; 122902aaed0SHisashi Hifumi } 123902aaed0SHisashi Hifumi } 124902aaed0SHisashi Hifumi if (zone) 125902aaed0SHisashi Hifumi spin_unlock(&zone->lru_lock); 126902aaed0SHisashi Hifumi __count_vm_events(PGROTATED, pgmoved); 127902aaed0SHisashi Hifumi release_pages(pvec->pages, pvec->nr, pvec->cold); 128902aaed0SHisashi Hifumi pagevec_reinit(pvec); 129902aaed0SHisashi Hifumi } 130902aaed0SHisashi Hifumi 131902aaed0SHisashi Hifumi /* 1321da177e4SLinus Torvalds * Writeback is about to end against a page which has been marked for immediate 1331da177e4SLinus Torvalds * reclaim. If it still appears to be reclaimable, move it to the tail of the 134902aaed0SHisashi Hifumi * inactive list. 1351da177e4SLinus Torvalds * 1361da177e4SLinus Torvalds * Returns zero if it cleared PG_writeback. 1371da177e4SLinus Torvalds */ 1381da177e4SLinus Torvalds int rotate_reclaimable_page(struct page *page) 1391da177e4SLinus Torvalds { 140902aaed0SHisashi Hifumi struct pagevec *pvec; 1411da177e4SLinus Torvalds unsigned long flags; 1421da177e4SLinus Torvalds 1431da177e4SLinus Torvalds if (PageLocked(page)) 1441da177e4SLinus Torvalds return 1; 1451da177e4SLinus Torvalds if (PageDirty(page)) 1461da177e4SLinus Torvalds return 1; 1471da177e4SLinus Torvalds if (PageActive(page)) 1481da177e4SLinus Torvalds return 1; 1491da177e4SLinus Torvalds if (!PageLRU(page)) 1501da177e4SLinus Torvalds return 1; 1511da177e4SLinus Torvalds 152902aaed0SHisashi Hifumi page_cache_get(page); 153902aaed0SHisashi Hifumi local_irq_save(flags); 154902aaed0SHisashi Hifumi pvec = &__get_cpu_var(lru_rotate_pvecs); 155902aaed0SHisashi Hifumi if (!pagevec_add(pvec, page)) 156902aaed0SHisashi Hifumi pagevec_move_tail(pvec); 157902aaed0SHisashi Hifumi local_irq_restore(flags); 158902aaed0SHisashi Hifumi 1591da177e4SLinus Torvalds if (!test_clear_page_writeback(page)) 1601da177e4SLinus Torvalds BUG(); 161902aaed0SHisashi Hifumi 1621da177e4SLinus Torvalds return 0; 1631da177e4SLinus Torvalds } 1641da177e4SLinus Torvalds 1651da177e4SLinus Torvalds /* 1661da177e4SLinus Torvalds * FIXME: speed this up? 1671da177e4SLinus Torvalds */ 168920c7a5dSHarvey Harrison void activate_page(struct page *page) 1691da177e4SLinus Torvalds { 1701da177e4SLinus Torvalds struct zone *zone = page_zone(page); 1711da177e4SLinus Torvalds 1721da177e4SLinus Torvalds spin_lock_irq(&zone->lru_lock); 1731da177e4SLinus Torvalds if (PageLRU(page) && !PageActive(page)) { 1741da177e4SLinus Torvalds del_page_from_inactive_list(zone, page); 1751da177e4SLinus Torvalds SetPageActive(page); 1761da177e4SLinus Torvalds add_page_to_active_list(zone, page); 177f8891e5eSChristoph Lameter __count_vm_event(PGACTIVATE); 178427d5416SHugh Dickins mem_cgroup_move_lists(page, true); 1791da177e4SLinus Torvalds } 1801da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 1811da177e4SLinus Torvalds } 1821da177e4SLinus Torvalds 1831da177e4SLinus Torvalds /* 1841da177e4SLinus Torvalds * Mark a page as having seen activity. 1851da177e4SLinus Torvalds * 1861da177e4SLinus Torvalds * inactive,unreferenced -> inactive,referenced 1871da177e4SLinus Torvalds * inactive,referenced -> active,unreferenced 1881da177e4SLinus Torvalds * active,unreferenced -> active,referenced 1891da177e4SLinus Torvalds */ 190920c7a5dSHarvey Harrison void mark_page_accessed(struct page *page) 1911da177e4SLinus Torvalds { 1921da177e4SLinus Torvalds if (!PageActive(page) && PageReferenced(page) && PageLRU(page)) { 1931da177e4SLinus Torvalds activate_page(page); 1941da177e4SLinus Torvalds ClearPageReferenced(page); 1951da177e4SLinus Torvalds } else if (!PageReferenced(page)) { 1961da177e4SLinus Torvalds SetPageReferenced(page); 1971da177e4SLinus Torvalds } 1981da177e4SLinus Torvalds } 1991da177e4SLinus Torvalds 2001da177e4SLinus Torvalds EXPORT_SYMBOL(mark_page_accessed); 2011da177e4SLinus Torvalds 2021da177e4SLinus Torvalds /** 2031da177e4SLinus Torvalds * lru_cache_add: add a page to the page lists 2041da177e4SLinus Torvalds * @page: the page to add 2051da177e4SLinus Torvalds */ 206920c7a5dSHarvey Harrison void lru_cache_add(struct page *page) 2071da177e4SLinus Torvalds { 2081da177e4SLinus Torvalds struct pagevec *pvec = &get_cpu_var(lru_add_pvecs); 2091da177e4SLinus Torvalds 2101da177e4SLinus Torvalds page_cache_get(page); 2111da177e4SLinus Torvalds if (!pagevec_add(pvec, page)) 2121da177e4SLinus Torvalds __pagevec_lru_add(pvec); 2131da177e4SLinus Torvalds put_cpu_var(lru_add_pvecs); 2141da177e4SLinus Torvalds } 2151da177e4SLinus Torvalds 216920c7a5dSHarvey Harrison void lru_cache_add_active(struct page *page) 2171da177e4SLinus Torvalds { 2181da177e4SLinus Torvalds struct pagevec *pvec = &get_cpu_var(lru_add_active_pvecs); 2191da177e4SLinus Torvalds 2201da177e4SLinus Torvalds page_cache_get(page); 2211da177e4SLinus Torvalds if (!pagevec_add(pvec, page)) 2221da177e4SLinus Torvalds __pagevec_lru_add_active(pvec); 2231da177e4SLinus Torvalds put_cpu_var(lru_add_active_pvecs); 2241da177e4SLinus Torvalds } 2251da177e4SLinus Torvalds 226902aaed0SHisashi Hifumi /* 227902aaed0SHisashi Hifumi * Drain pages out of the cpu's pagevecs. 228902aaed0SHisashi Hifumi * Either "cpu" is the current CPU, and preemption has already been 229902aaed0SHisashi Hifumi * disabled; or "cpu" is being hot-unplugged, and is already dead. 230902aaed0SHisashi Hifumi */ 231902aaed0SHisashi Hifumi static void drain_cpu_pagevecs(int cpu) 2321da177e4SLinus Torvalds { 233902aaed0SHisashi Hifumi struct pagevec *pvec; 2341da177e4SLinus Torvalds 235902aaed0SHisashi Hifumi pvec = &per_cpu(lru_add_pvecs, cpu); 2361da177e4SLinus Torvalds if (pagevec_count(pvec)) 2371da177e4SLinus Torvalds __pagevec_lru_add(pvec); 238902aaed0SHisashi Hifumi 23980bfed90SAndrew Morton pvec = &per_cpu(lru_add_active_pvecs, cpu); 2401da177e4SLinus Torvalds if (pagevec_count(pvec)) 2411da177e4SLinus Torvalds __pagevec_lru_add_active(pvec); 242902aaed0SHisashi Hifumi 243902aaed0SHisashi Hifumi pvec = &per_cpu(lru_rotate_pvecs, cpu); 244902aaed0SHisashi Hifumi if (pagevec_count(pvec)) { 245902aaed0SHisashi Hifumi unsigned long flags; 246902aaed0SHisashi Hifumi 247902aaed0SHisashi Hifumi /* No harm done if a racing interrupt already did this */ 248902aaed0SHisashi Hifumi local_irq_save(flags); 249902aaed0SHisashi Hifumi pagevec_move_tail(pvec); 250902aaed0SHisashi Hifumi local_irq_restore(flags); 251902aaed0SHisashi Hifumi } 25280bfed90SAndrew Morton } 25380bfed90SAndrew Morton 25480bfed90SAndrew Morton void lru_add_drain(void) 25580bfed90SAndrew Morton { 256902aaed0SHisashi Hifumi drain_cpu_pagevecs(get_cpu()); 25780bfed90SAndrew Morton put_cpu(); 2581da177e4SLinus Torvalds } 2591da177e4SLinus Torvalds 260053837fcSNick Piggin #ifdef CONFIG_NUMA 261c4028958SDavid Howells static void lru_add_drain_per_cpu(struct work_struct *dummy) 262053837fcSNick Piggin { 263053837fcSNick Piggin lru_add_drain(); 264053837fcSNick Piggin } 265053837fcSNick Piggin 266053837fcSNick Piggin /* 267053837fcSNick Piggin * Returns 0 for success 268053837fcSNick Piggin */ 269053837fcSNick Piggin int lru_add_drain_all(void) 270053837fcSNick Piggin { 271c4028958SDavid Howells return schedule_on_each_cpu(lru_add_drain_per_cpu); 272053837fcSNick Piggin } 273053837fcSNick Piggin 274053837fcSNick Piggin #else 275053837fcSNick Piggin 276053837fcSNick Piggin /* 277053837fcSNick Piggin * Returns 0 for success 278053837fcSNick Piggin */ 279053837fcSNick Piggin int lru_add_drain_all(void) 280053837fcSNick Piggin { 281053837fcSNick Piggin lru_add_drain(); 282053837fcSNick Piggin return 0; 283053837fcSNick Piggin } 284053837fcSNick Piggin #endif 285053837fcSNick Piggin 2861da177e4SLinus Torvalds /* 2871da177e4SLinus Torvalds * Batched page_cache_release(). Decrement the reference count on all the 2881da177e4SLinus Torvalds * passed pages. If it fell to zero then remove the page from the LRU and 2891da177e4SLinus Torvalds * free it. 2901da177e4SLinus Torvalds * 2911da177e4SLinus Torvalds * Avoid taking zone->lru_lock if possible, but if it is taken, retain it 2921da177e4SLinus Torvalds * for the remainder of the operation. 2931da177e4SLinus Torvalds * 2941da177e4SLinus Torvalds * The locking in this function is against shrink_cache(): we recheck the 2951da177e4SLinus Torvalds * page count inside the lock to see whether shrink_cache grabbed the page 2961da177e4SLinus Torvalds * via the LRU. If it did, give up: shrink_cache will free it. 2971da177e4SLinus Torvalds */ 2981da177e4SLinus Torvalds void release_pages(struct page **pages, int nr, int cold) 2991da177e4SLinus Torvalds { 3001da177e4SLinus Torvalds int i; 3011da177e4SLinus Torvalds struct pagevec pages_to_free; 3021da177e4SLinus Torvalds struct zone *zone = NULL; 303902aaed0SHisashi Hifumi unsigned long uninitialized_var(flags); 3041da177e4SLinus Torvalds 3051da177e4SLinus Torvalds pagevec_init(&pages_to_free, cold); 3061da177e4SLinus Torvalds for (i = 0; i < nr; i++) { 3071da177e4SLinus Torvalds struct page *page = pages[i]; 3081da177e4SLinus Torvalds 3098519fb30SNick Piggin if (unlikely(PageCompound(page))) { 3108519fb30SNick Piggin if (zone) { 311902aaed0SHisashi Hifumi spin_unlock_irqrestore(&zone->lru_lock, flags); 3128519fb30SNick Piggin zone = NULL; 3138519fb30SNick Piggin } 3148519fb30SNick Piggin put_compound_page(page); 3158519fb30SNick Piggin continue; 3168519fb30SNick Piggin } 3178519fb30SNick Piggin 318b5810039SNick Piggin if (!put_page_testzero(page)) 3191da177e4SLinus Torvalds continue; 3201da177e4SLinus Torvalds 32146453a6eSNick Piggin if (PageLRU(page)) { 32246453a6eSNick Piggin struct zone *pagezone = page_zone(page); 3231da177e4SLinus Torvalds if (pagezone != zone) { 3241da177e4SLinus Torvalds if (zone) 325902aaed0SHisashi Hifumi spin_unlock_irqrestore(&zone->lru_lock, 326902aaed0SHisashi Hifumi flags); 3271da177e4SLinus Torvalds zone = pagezone; 328902aaed0SHisashi Hifumi spin_lock_irqsave(&zone->lru_lock, flags); 3291da177e4SLinus Torvalds } 330725d704eSNick Piggin VM_BUG_ON(!PageLRU(page)); 33167453911SNick Piggin __ClearPageLRU(page); 3321da177e4SLinus Torvalds del_page_from_lru(zone, page); 33346453a6eSNick Piggin } 33446453a6eSNick Piggin 3351da177e4SLinus Torvalds if (!pagevec_add(&pages_to_free, page)) { 33646453a6eSNick Piggin if (zone) { 337902aaed0SHisashi Hifumi spin_unlock_irqrestore(&zone->lru_lock, flags); 33846453a6eSNick Piggin zone = NULL; 33946453a6eSNick Piggin } 3401da177e4SLinus Torvalds __pagevec_free(&pages_to_free); 3411da177e4SLinus Torvalds pagevec_reinit(&pages_to_free); 3421da177e4SLinus Torvalds } 3431da177e4SLinus Torvalds } 3441da177e4SLinus Torvalds if (zone) 345902aaed0SHisashi Hifumi spin_unlock_irqrestore(&zone->lru_lock, flags); 3461da177e4SLinus Torvalds 3471da177e4SLinus Torvalds pagevec_free(&pages_to_free); 3481da177e4SLinus Torvalds } 3491da177e4SLinus Torvalds 3501da177e4SLinus Torvalds /* 3511da177e4SLinus Torvalds * The pages which we're about to release may be in the deferred lru-addition 3521da177e4SLinus Torvalds * queues. That would prevent them from really being freed right now. That's 3531da177e4SLinus Torvalds * OK from a correctness point of view but is inefficient - those pages may be 3541da177e4SLinus Torvalds * cache-warm and we want to give them back to the page allocator ASAP. 3551da177e4SLinus Torvalds * 3561da177e4SLinus Torvalds * So __pagevec_release() will drain those queues here. __pagevec_lru_add() 3571da177e4SLinus Torvalds * and __pagevec_lru_add_active() call release_pages() directly to avoid 3581da177e4SLinus Torvalds * mutual recursion. 3591da177e4SLinus Torvalds */ 3601da177e4SLinus Torvalds void __pagevec_release(struct pagevec *pvec) 3611da177e4SLinus Torvalds { 3621da177e4SLinus Torvalds lru_add_drain(); 3631da177e4SLinus Torvalds release_pages(pvec->pages, pagevec_count(pvec), pvec->cold); 3641da177e4SLinus Torvalds pagevec_reinit(pvec); 3651da177e4SLinus Torvalds } 3661da177e4SLinus Torvalds 3677f285701SSteve French EXPORT_SYMBOL(__pagevec_release); 3687f285701SSteve French 3691da177e4SLinus Torvalds /* 3701da177e4SLinus Torvalds * pagevec_release() for pages which are known to not be on the LRU 3711da177e4SLinus Torvalds * 3721da177e4SLinus Torvalds * This function reinitialises the caller's pagevec. 3731da177e4SLinus Torvalds */ 3741da177e4SLinus Torvalds void __pagevec_release_nonlru(struct pagevec *pvec) 3751da177e4SLinus Torvalds { 3761da177e4SLinus Torvalds int i; 3771da177e4SLinus Torvalds struct pagevec pages_to_free; 3781da177e4SLinus Torvalds 3791da177e4SLinus Torvalds pagevec_init(&pages_to_free, pvec->cold); 3801da177e4SLinus Torvalds for (i = 0; i < pagevec_count(pvec); i++) { 3811da177e4SLinus Torvalds struct page *page = pvec->pages[i]; 3821da177e4SLinus Torvalds 383725d704eSNick Piggin VM_BUG_ON(PageLRU(page)); 3841da177e4SLinus Torvalds if (put_page_testzero(page)) 3851da177e4SLinus Torvalds pagevec_add(&pages_to_free, page); 3861da177e4SLinus Torvalds } 3871da177e4SLinus Torvalds pagevec_free(&pages_to_free); 3881da177e4SLinus Torvalds pagevec_reinit(pvec); 3891da177e4SLinus Torvalds } 3901da177e4SLinus Torvalds 3911da177e4SLinus Torvalds /* 3921da177e4SLinus Torvalds * Add the passed pages to the LRU, then drop the caller's refcount 3931da177e4SLinus Torvalds * on them. Reinitialises the caller's pagevec. 3941da177e4SLinus Torvalds */ 3951da177e4SLinus Torvalds void __pagevec_lru_add(struct pagevec *pvec) 3961da177e4SLinus Torvalds { 3971da177e4SLinus Torvalds int i; 3981da177e4SLinus Torvalds struct zone *zone = NULL; 3991da177e4SLinus Torvalds 4001da177e4SLinus Torvalds for (i = 0; i < pagevec_count(pvec); i++) { 4011da177e4SLinus Torvalds struct page *page = pvec->pages[i]; 4021da177e4SLinus Torvalds struct zone *pagezone = page_zone(page); 4031da177e4SLinus Torvalds 4041da177e4SLinus Torvalds if (pagezone != zone) { 4051da177e4SLinus Torvalds if (zone) 4061da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 4071da177e4SLinus Torvalds zone = pagezone; 4081da177e4SLinus Torvalds spin_lock_irq(&zone->lru_lock); 4091da177e4SLinus Torvalds } 410725d704eSNick Piggin VM_BUG_ON(PageLRU(page)); 4118d438f96SNick Piggin SetPageLRU(page); 4121da177e4SLinus Torvalds add_page_to_inactive_list(zone, page); 4131da177e4SLinus Torvalds } 4141da177e4SLinus Torvalds if (zone) 4151da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 4161da177e4SLinus Torvalds release_pages(pvec->pages, pvec->nr, pvec->cold); 4171da177e4SLinus Torvalds pagevec_reinit(pvec); 4181da177e4SLinus Torvalds } 4191da177e4SLinus Torvalds 4201da177e4SLinus Torvalds EXPORT_SYMBOL(__pagevec_lru_add); 4211da177e4SLinus Torvalds 4221da177e4SLinus Torvalds void __pagevec_lru_add_active(struct pagevec *pvec) 4231da177e4SLinus Torvalds { 4241da177e4SLinus Torvalds int i; 4251da177e4SLinus Torvalds struct zone *zone = NULL; 4261da177e4SLinus Torvalds 4271da177e4SLinus Torvalds for (i = 0; i < pagevec_count(pvec); i++) { 4281da177e4SLinus Torvalds struct page *page = pvec->pages[i]; 4291da177e4SLinus Torvalds struct zone *pagezone = page_zone(page); 4301da177e4SLinus Torvalds 4311da177e4SLinus Torvalds if (pagezone != zone) { 4321da177e4SLinus Torvalds if (zone) 4331da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 4341da177e4SLinus Torvalds zone = pagezone; 4351da177e4SLinus Torvalds spin_lock_irq(&zone->lru_lock); 4361da177e4SLinus Torvalds } 437725d704eSNick Piggin VM_BUG_ON(PageLRU(page)); 4388d438f96SNick Piggin SetPageLRU(page); 439725d704eSNick Piggin VM_BUG_ON(PageActive(page)); 4404c84cacfSNick Piggin SetPageActive(page); 4411da177e4SLinus Torvalds add_page_to_active_list(zone, page); 4421da177e4SLinus Torvalds } 4431da177e4SLinus Torvalds if (zone) 4441da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 4451da177e4SLinus Torvalds release_pages(pvec->pages, pvec->nr, pvec->cold); 4461da177e4SLinus Torvalds pagevec_reinit(pvec); 4471da177e4SLinus Torvalds } 4481da177e4SLinus Torvalds 4491da177e4SLinus Torvalds /* 4501da177e4SLinus Torvalds * Try to drop buffers from the pages in a pagevec 4511da177e4SLinus Torvalds */ 4521da177e4SLinus Torvalds void pagevec_strip(struct pagevec *pvec) 4531da177e4SLinus Torvalds { 4541da177e4SLinus Torvalds int i; 4551da177e4SLinus Torvalds 4561da177e4SLinus Torvalds for (i = 0; i < pagevec_count(pvec); i++) { 4571da177e4SLinus Torvalds struct page *page = pvec->pages[i]; 4581da177e4SLinus Torvalds 4591da177e4SLinus Torvalds if (PagePrivate(page) && !TestSetPageLocked(page)) { 4605b40dc78SChristoph Lameter if (PagePrivate(page)) 4611da177e4SLinus Torvalds try_to_release_page(page, 0); 4621da177e4SLinus Torvalds unlock_page(page); 4631da177e4SLinus Torvalds } 4641da177e4SLinus Torvalds } 4651da177e4SLinus Torvalds } 4661da177e4SLinus Torvalds 4671da177e4SLinus Torvalds /** 4681da177e4SLinus Torvalds * pagevec_lookup - gang pagecache lookup 4691da177e4SLinus Torvalds * @pvec: Where the resulting pages are placed 4701da177e4SLinus Torvalds * @mapping: The address_space to search 4711da177e4SLinus Torvalds * @start: The starting page index 4721da177e4SLinus Torvalds * @nr_pages: The maximum number of pages 4731da177e4SLinus Torvalds * 4741da177e4SLinus Torvalds * pagevec_lookup() will search for and return a group of up to @nr_pages pages 4751da177e4SLinus Torvalds * in the mapping. The pages are placed in @pvec. pagevec_lookup() takes a 4761da177e4SLinus Torvalds * reference against the pages in @pvec. 4771da177e4SLinus Torvalds * 4781da177e4SLinus Torvalds * The search returns a group of mapping-contiguous pages with ascending 4791da177e4SLinus Torvalds * indexes. There may be holes in the indices due to not-present pages. 4801da177e4SLinus Torvalds * 4811da177e4SLinus Torvalds * pagevec_lookup() returns the number of pages which were found. 4821da177e4SLinus Torvalds */ 4831da177e4SLinus Torvalds unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping, 4841da177e4SLinus Torvalds pgoff_t start, unsigned nr_pages) 4851da177e4SLinus Torvalds { 4861da177e4SLinus Torvalds pvec->nr = find_get_pages(mapping, start, nr_pages, pvec->pages); 4871da177e4SLinus Torvalds return pagevec_count(pvec); 4881da177e4SLinus Torvalds } 4891da177e4SLinus Torvalds 49078539fdfSChristoph Hellwig EXPORT_SYMBOL(pagevec_lookup); 49178539fdfSChristoph Hellwig 4921da177e4SLinus Torvalds unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping, 4931da177e4SLinus Torvalds pgoff_t *index, int tag, unsigned nr_pages) 4941da177e4SLinus Torvalds { 4951da177e4SLinus Torvalds pvec->nr = find_get_pages_tag(mapping, index, tag, 4961da177e4SLinus Torvalds nr_pages, pvec->pages); 4971da177e4SLinus Torvalds return pagevec_count(pvec); 4981da177e4SLinus Torvalds } 4991da177e4SLinus Torvalds 5007f285701SSteve French EXPORT_SYMBOL(pagevec_lookup_tag); 5011da177e4SLinus Torvalds 5021da177e4SLinus Torvalds #ifdef CONFIG_SMP 5031da177e4SLinus Torvalds /* 5041da177e4SLinus Torvalds * We tolerate a little inaccuracy to avoid ping-ponging the counter between 5051da177e4SLinus Torvalds * CPUs 5061da177e4SLinus Torvalds */ 5071da177e4SLinus Torvalds #define ACCT_THRESHOLD max(16, NR_CPUS * 2) 5081da177e4SLinus Torvalds 5091da177e4SLinus Torvalds static DEFINE_PER_CPU(long, committed_space) = 0; 5101da177e4SLinus Torvalds 5111da177e4SLinus Torvalds void vm_acct_memory(long pages) 5121da177e4SLinus Torvalds { 5131da177e4SLinus Torvalds long *local; 5141da177e4SLinus Torvalds 5151da177e4SLinus Torvalds preempt_disable(); 5161da177e4SLinus Torvalds local = &__get_cpu_var(committed_space); 5171da177e4SLinus Torvalds *local += pages; 5181da177e4SLinus Torvalds if (*local > ACCT_THRESHOLD || *local < -ACCT_THRESHOLD) { 5191da177e4SLinus Torvalds atomic_add(*local, &vm_committed_space); 5201da177e4SLinus Torvalds *local = 0; 5211da177e4SLinus Torvalds } 5221da177e4SLinus Torvalds preempt_enable(); 5231da177e4SLinus Torvalds } 5241da177e4SLinus Torvalds 5251da177e4SLinus Torvalds #ifdef CONFIG_HOTPLUG_CPU 5261da177e4SLinus Torvalds 5271da177e4SLinus Torvalds /* Drop the CPU's cached committed space back into the central pool. */ 5281da177e4SLinus Torvalds static int cpu_swap_callback(struct notifier_block *nfb, 5291da177e4SLinus Torvalds unsigned long action, 5301da177e4SLinus Torvalds void *hcpu) 5311da177e4SLinus Torvalds { 5321da177e4SLinus Torvalds long *committed; 5331da177e4SLinus Torvalds 5341da177e4SLinus Torvalds committed = &per_cpu(committed_space, (long)hcpu); 5358bb78442SRafael J. Wysocki if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { 5361da177e4SLinus Torvalds atomic_add(*committed, &vm_committed_space); 5371da177e4SLinus Torvalds *committed = 0; 538902aaed0SHisashi Hifumi drain_cpu_pagevecs((long)hcpu); 5391da177e4SLinus Torvalds } 5401da177e4SLinus Torvalds return NOTIFY_OK; 5411da177e4SLinus Torvalds } 5421da177e4SLinus Torvalds #endif /* CONFIG_HOTPLUG_CPU */ 5431da177e4SLinus Torvalds #endif /* CONFIG_SMP */ 5441da177e4SLinus Torvalds 5451da177e4SLinus Torvalds /* 5461da177e4SLinus Torvalds * Perform any setup for the swap system 5471da177e4SLinus Torvalds */ 5481da177e4SLinus Torvalds void __init swap_setup(void) 5491da177e4SLinus Torvalds { 5501da177e4SLinus Torvalds unsigned long megs = num_physpages >> (20 - PAGE_SHIFT); 5511da177e4SLinus Torvalds 552e0bf68ddSPeter Zijlstra #ifdef CONFIG_SWAP 553e0bf68ddSPeter Zijlstra bdi_init(swapper_space.backing_dev_info); 554e0bf68ddSPeter Zijlstra #endif 555e0bf68ddSPeter Zijlstra 5561da177e4SLinus Torvalds /* Use a smaller cluster for small-memory machines */ 5571da177e4SLinus Torvalds if (megs < 16) 5581da177e4SLinus Torvalds page_cluster = 2; 5591da177e4SLinus Torvalds else 5601da177e4SLinus Torvalds page_cluster = 3; 5611da177e4SLinus Torvalds /* 5621da177e4SLinus Torvalds * Right now other parts of the system means that we 5631da177e4SLinus Torvalds * _really_ don't want to cluster much more 5641da177e4SLinus Torvalds */ 56502316067SIngo Molnar #ifdef CONFIG_HOTPLUG_CPU 5661da177e4SLinus Torvalds hotcpu_notifier(cpu_swap_callback, 0); 56702316067SIngo Molnar #endif 5681da177e4SLinus Torvalds } 569