11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * linux/mm/swap.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 51da177e4SLinus Torvalds */ 61da177e4SLinus Torvalds 71da177e4SLinus Torvalds /* 81da177e4SLinus Torvalds * This file contains the default values for the opereation of the 91da177e4SLinus Torvalds * Linux VM subsystem. Fine-tuning documentation can be found in 101da177e4SLinus Torvalds * Documentation/sysctl/vm.txt. 111da177e4SLinus Torvalds * Started 18.12.91 121da177e4SLinus Torvalds * Swap aging added 23.2.95, Stephen Tweedie. 131da177e4SLinus Torvalds * Buffermem limits added 12.3.98, Rik van Riel. 141da177e4SLinus Torvalds */ 151da177e4SLinus Torvalds 161da177e4SLinus Torvalds #include <linux/mm.h> 171da177e4SLinus Torvalds #include <linux/sched.h> 181da177e4SLinus Torvalds #include <linux/kernel_stat.h> 191da177e4SLinus Torvalds #include <linux/swap.h> 201da177e4SLinus Torvalds #include <linux/mman.h> 211da177e4SLinus Torvalds #include <linux/pagemap.h> 221da177e4SLinus Torvalds #include <linux/pagevec.h> 231da177e4SLinus Torvalds #include <linux/init.h> 241da177e4SLinus Torvalds #include <linux/module.h> 251da177e4SLinus Torvalds #include <linux/mm_inline.h> 261da177e4SLinus Torvalds #include <linux/buffer_head.h> /* for try_to_release_page() */ 271da177e4SLinus Torvalds #include <linux/module.h> 281da177e4SLinus Torvalds #include <linux/percpu_counter.h> 291da177e4SLinus Torvalds #include <linux/percpu.h> 301da177e4SLinus Torvalds #include <linux/cpu.h> 311da177e4SLinus Torvalds #include <linux/notifier.h> 321da177e4SLinus Torvalds #include <linux/init.h> 331da177e4SLinus Torvalds 341da177e4SLinus Torvalds /* How many pages do we try to swap or page in/out together? */ 351da177e4SLinus Torvalds int page_cluster; 361da177e4SLinus Torvalds 37*b221385bSAdrian Bunk /* 38*b221385bSAdrian Bunk * This path almost never happens for VM activity - pages are normally 39*b221385bSAdrian Bunk * freed via pagevecs. But it gets used by networking. 40*b221385bSAdrian Bunk */ 41*b221385bSAdrian Bunk static void fastcall __page_cache_release(struct page *page) 42*b221385bSAdrian Bunk { 43*b221385bSAdrian Bunk if (PageLRU(page)) { 44*b221385bSAdrian Bunk unsigned long flags; 45*b221385bSAdrian Bunk struct zone *zone = page_zone(page); 46*b221385bSAdrian Bunk 47*b221385bSAdrian Bunk spin_lock_irqsave(&zone->lru_lock, flags); 48*b221385bSAdrian Bunk VM_BUG_ON(!PageLRU(page)); 49*b221385bSAdrian Bunk __ClearPageLRU(page); 50*b221385bSAdrian Bunk del_page_from_lru(zone, page); 51*b221385bSAdrian Bunk spin_unlock_irqrestore(&zone->lru_lock, flags); 52*b221385bSAdrian Bunk } 53*b221385bSAdrian Bunk free_hot_page(page); 54*b221385bSAdrian Bunk } 55*b221385bSAdrian Bunk 568519fb30SNick Piggin static void put_compound_page(struct page *page) 571da177e4SLinus Torvalds { 584c21e2f2SHugh Dickins page = (struct page *)page_private(page); 591da177e4SLinus Torvalds if (put_page_testzero(page)) { 601da177e4SLinus Torvalds void (*dtor)(struct page *page); 611da177e4SLinus Torvalds 6241d78ba5SHugh Dickins dtor = (void (*)(struct page *))page[1].lru.next; 631da177e4SLinus Torvalds (*dtor)(page); 641da177e4SLinus Torvalds } 651da177e4SLinus Torvalds } 668519fb30SNick Piggin 678519fb30SNick Piggin void put_page(struct page *page) 688519fb30SNick Piggin { 698519fb30SNick Piggin if (unlikely(PageCompound(page))) 708519fb30SNick Piggin put_compound_page(page); 718519fb30SNick Piggin else if (put_page_testzero(page)) 721da177e4SLinus Torvalds __page_cache_release(page); 731da177e4SLinus Torvalds } 741da177e4SLinus Torvalds EXPORT_SYMBOL(put_page); 751da177e4SLinus Torvalds 761d7ea732SAlexander Zarochentsev /** 771d7ea732SAlexander Zarochentsev * put_pages_list(): release a list of pages 781d7ea732SAlexander Zarochentsev * 791d7ea732SAlexander Zarochentsev * Release a list of pages which are strung together on page.lru. Currently 801d7ea732SAlexander Zarochentsev * used by read_cache_pages() and related error recovery code. 811d7ea732SAlexander Zarochentsev * 821d7ea732SAlexander Zarochentsev * @pages: list of pages threaded on page->lru 831d7ea732SAlexander Zarochentsev */ 841d7ea732SAlexander Zarochentsev void put_pages_list(struct list_head *pages) 851d7ea732SAlexander Zarochentsev { 861d7ea732SAlexander Zarochentsev while (!list_empty(pages)) { 871d7ea732SAlexander Zarochentsev struct page *victim; 881d7ea732SAlexander Zarochentsev 891d7ea732SAlexander Zarochentsev victim = list_entry(pages->prev, struct page, lru); 901d7ea732SAlexander Zarochentsev list_del(&victim->lru); 911d7ea732SAlexander Zarochentsev page_cache_release(victim); 921d7ea732SAlexander Zarochentsev } 931d7ea732SAlexander Zarochentsev } 941d7ea732SAlexander Zarochentsev EXPORT_SYMBOL(put_pages_list); 951d7ea732SAlexander Zarochentsev 961da177e4SLinus Torvalds /* 971da177e4SLinus Torvalds * Writeback is about to end against a page which has been marked for immediate 981da177e4SLinus Torvalds * reclaim. If it still appears to be reclaimable, move it to the tail of the 991da177e4SLinus Torvalds * inactive list. The page still has PageWriteback set, which will pin it. 1001da177e4SLinus Torvalds * 1011da177e4SLinus Torvalds * We don't expect many pages to come through here, so don't bother batching 1021da177e4SLinus Torvalds * things up. 1031da177e4SLinus Torvalds * 1041da177e4SLinus Torvalds * To avoid placing the page at the tail of the LRU while PG_writeback is still 1051da177e4SLinus Torvalds * set, this function will clear PG_writeback before performing the page 1061da177e4SLinus Torvalds * motion. Do that inside the lru lock because once PG_writeback is cleared 1071da177e4SLinus Torvalds * we may not touch the page. 1081da177e4SLinus Torvalds * 1091da177e4SLinus Torvalds * Returns zero if it cleared PG_writeback. 1101da177e4SLinus Torvalds */ 1111da177e4SLinus Torvalds int rotate_reclaimable_page(struct page *page) 1121da177e4SLinus Torvalds { 1131da177e4SLinus Torvalds struct zone *zone; 1141da177e4SLinus Torvalds unsigned long flags; 1151da177e4SLinus Torvalds 1161da177e4SLinus Torvalds if (PageLocked(page)) 1171da177e4SLinus Torvalds return 1; 1181da177e4SLinus Torvalds if (PageDirty(page)) 1191da177e4SLinus Torvalds return 1; 1201da177e4SLinus Torvalds if (PageActive(page)) 1211da177e4SLinus Torvalds return 1; 1221da177e4SLinus Torvalds if (!PageLRU(page)) 1231da177e4SLinus Torvalds return 1; 1241da177e4SLinus Torvalds 1251da177e4SLinus Torvalds zone = page_zone(page); 1261da177e4SLinus Torvalds spin_lock_irqsave(&zone->lru_lock, flags); 1271da177e4SLinus Torvalds if (PageLRU(page) && !PageActive(page)) { 1281bfba4e8SAkinobu Mita list_move_tail(&page->lru, &zone->inactive_list); 129f8891e5eSChristoph Lameter __count_vm_event(PGROTATED); 1301da177e4SLinus Torvalds } 1311da177e4SLinus Torvalds if (!test_clear_page_writeback(page)) 1321da177e4SLinus Torvalds BUG(); 1331da177e4SLinus Torvalds spin_unlock_irqrestore(&zone->lru_lock, flags); 1341da177e4SLinus Torvalds return 0; 1351da177e4SLinus Torvalds } 1361da177e4SLinus Torvalds 1371da177e4SLinus Torvalds /* 1381da177e4SLinus Torvalds * FIXME: speed this up? 1391da177e4SLinus Torvalds */ 1401da177e4SLinus Torvalds void fastcall activate_page(struct page *page) 1411da177e4SLinus Torvalds { 1421da177e4SLinus Torvalds struct zone *zone = page_zone(page); 1431da177e4SLinus Torvalds 1441da177e4SLinus Torvalds spin_lock_irq(&zone->lru_lock); 1451da177e4SLinus Torvalds if (PageLRU(page) && !PageActive(page)) { 1461da177e4SLinus Torvalds del_page_from_inactive_list(zone, page); 1471da177e4SLinus Torvalds SetPageActive(page); 1481da177e4SLinus Torvalds add_page_to_active_list(zone, page); 149f8891e5eSChristoph Lameter __count_vm_event(PGACTIVATE); 1501da177e4SLinus Torvalds } 1511da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 1521da177e4SLinus Torvalds } 1531da177e4SLinus Torvalds 1541da177e4SLinus Torvalds /* 1551da177e4SLinus Torvalds * Mark a page as having seen activity. 1561da177e4SLinus Torvalds * 1571da177e4SLinus Torvalds * inactive,unreferenced -> inactive,referenced 1581da177e4SLinus Torvalds * inactive,referenced -> active,unreferenced 1591da177e4SLinus Torvalds * active,unreferenced -> active,referenced 1601da177e4SLinus Torvalds */ 1611da177e4SLinus Torvalds void fastcall mark_page_accessed(struct page *page) 1621da177e4SLinus Torvalds { 1631da177e4SLinus Torvalds if (!PageActive(page) && PageReferenced(page) && PageLRU(page)) { 1641da177e4SLinus Torvalds activate_page(page); 1651da177e4SLinus Torvalds ClearPageReferenced(page); 1661da177e4SLinus Torvalds } else if (!PageReferenced(page)) { 1671da177e4SLinus Torvalds SetPageReferenced(page); 1681da177e4SLinus Torvalds } 1691da177e4SLinus Torvalds } 1701da177e4SLinus Torvalds 1711da177e4SLinus Torvalds EXPORT_SYMBOL(mark_page_accessed); 1721da177e4SLinus Torvalds 1731da177e4SLinus Torvalds /** 1741da177e4SLinus Torvalds * lru_cache_add: add a page to the page lists 1751da177e4SLinus Torvalds * @page: the page to add 1761da177e4SLinus Torvalds */ 1771da177e4SLinus Torvalds static DEFINE_PER_CPU(struct pagevec, lru_add_pvecs) = { 0, }; 1781da177e4SLinus Torvalds static DEFINE_PER_CPU(struct pagevec, lru_add_active_pvecs) = { 0, }; 1791da177e4SLinus Torvalds 1801da177e4SLinus Torvalds void fastcall lru_cache_add(struct page *page) 1811da177e4SLinus Torvalds { 1821da177e4SLinus Torvalds struct pagevec *pvec = &get_cpu_var(lru_add_pvecs); 1831da177e4SLinus Torvalds 1841da177e4SLinus Torvalds page_cache_get(page); 1851da177e4SLinus Torvalds if (!pagevec_add(pvec, page)) 1861da177e4SLinus Torvalds __pagevec_lru_add(pvec); 1871da177e4SLinus Torvalds put_cpu_var(lru_add_pvecs); 1881da177e4SLinus Torvalds } 1891da177e4SLinus Torvalds 1901da177e4SLinus Torvalds void fastcall lru_cache_add_active(struct page *page) 1911da177e4SLinus Torvalds { 1921da177e4SLinus Torvalds struct pagevec *pvec = &get_cpu_var(lru_add_active_pvecs); 1931da177e4SLinus Torvalds 1941da177e4SLinus Torvalds page_cache_get(page); 1951da177e4SLinus Torvalds if (!pagevec_add(pvec, page)) 1961da177e4SLinus Torvalds __pagevec_lru_add_active(pvec); 1971da177e4SLinus Torvalds put_cpu_var(lru_add_active_pvecs); 1981da177e4SLinus Torvalds } 1991da177e4SLinus Torvalds 20080bfed90SAndrew Morton static void __lru_add_drain(int cpu) 2011da177e4SLinus Torvalds { 20280bfed90SAndrew Morton struct pagevec *pvec = &per_cpu(lru_add_pvecs, cpu); 2031da177e4SLinus Torvalds 20480bfed90SAndrew Morton /* CPU is dead, so no locking needed. */ 2051da177e4SLinus Torvalds if (pagevec_count(pvec)) 2061da177e4SLinus Torvalds __pagevec_lru_add(pvec); 20780bfed90SAndrew Morton pvec = &per_cpu(lru_add_active_pvecs, cpu); 2081da177e4SLinus Torvalds if (pagevec_count(pvec)) 2091da177e4SLinus Torvalds __pagevec_lru_add_active(pvec); 21080bfed90SAndrew Morton } 21180bfed90SAndrew Morton 21280bfed90SAndrew Morton void lru_add_drain(void) 21380bfed90SAndrew Morton { 21480bfed90SAndrew Morton __lru_add_drain(get_cpu()); 21580bfed90SAndrew Morton put_cpu(); 2161da177e4SLinus Torvalds } 2171da177e4SLinus Torvalds 218053837fcSNick Piggin #ifdef CONFIG_NUMA 219053837fcSNick Piggin static void lru_add_drain_per_cpu(void *dummy) 220053837fcSNick Piggin { 221053837fcSNick Piggin lru_add_drain(); 222053837fcSNick Piggin } 223053837fcSNick Piggin 224053837fcSNick Piggin /* 225053837fcSNick Piggin * Returns 0 for success 226053837fcSNick Piggin */ 227053837fcSNick Piggin int lru_add_drain_all(void) 228053837fcSNick Piggin { 229053837fcSNick Piggin return schedule_on_each_cpu(lru_add_drain_per_cpu, NULL); 230053837fcSNick Piggin } 231053837fcSNick Piggin 232053837fcSNick Piggin #else 233053837fcSNick Piggin 234053837fcSNick Piggin /* 235053837fcSNick Piggin * Returns 0 for success 236053837fcSNick Piggin */ 237053837fcSNick Piggin int lru_add_drain_all(void) 238053837fcSNick Piggin { 239053837fcSNick Piggin lru_add_drain(); 240053837fcSNick Piggin return 0; 241053837fcSNick Piggin } 242053837fcSNick Piggin #endif 243053837fcSNick Piggin 2441da177e4SLinus Torvalds /* 2451da177e4SLinus Torvalds * Batched page_cache_release(). Decrement the reference count on all the 2461da177e4SLinus Torvalds * passed pages. If it fell to zero then remove the page from the LRU and 2471da177e4SLinus Torvalds * free it. 2481da177e4SLinus Torvalds * 2491da177e4SLinus Torvalds * Avoid taking zone->lru_lock if possible, but if it is taken, retain it 2501da177e4SLinus Torvalds * for the remainder of the operation. 2511da177e4SLinus Torvalds * 2521da177e4SLinus Torvalds * The locking in this function is against shrink_cache(): we recheck the 2531da177e4SLinus Torvalds * page count inside the lock to see whether shrink_cache grabbed the page 2541da177e4SLinus Torvalds * via the LRU. If it did, give up: shrink_cache will free it. 2551da177e4SLinus Torvalds */ 2561da177e4SLinus Torvalds void release_pages(struct page **pages, int nr, int cold) 2571da177e4SLinus Torvalds { 2581da177e4SLinus Torvalds int i; 2591da177e4SLinus Torvalds struct pagevec pages_to_free; 2601da177e4SLinus Torvalds struct zone *zone = NULL; 2611da177e4SLinus Torvalds 2621da177e4SLinus Torvalds pagevec_init(&pages_to_free, cold); 2631da177e4SLinus Torvalds for (i = 0; i < nr; i++) { 2641da177e4SLinus Torvalds struct page *page = pages[i]; 2651da177e4SLinus Torvalds 2668519fb30SNick Piggin if (unlikely(PageCompound(page))) { 2678519fb30SNick Piggin if (zone) { 2688519fb30SNick Piggin spin_unlock_irq(&zone->lru_lock); 2698519fb30SNick Piggin zone = NULL; 2708519fb30SNick Piggin } 2718519fb30SNick Piggin put_compound_page(page); 2728519fb30SNick Piggin continue; 2738519fb30SNick Piggin } 2748519fb30SNick Piggin 275b5810039SNick Piggin if (!put_page_testzero(page)) 2761da177e4SLinus Torvalds continue; 2771da177e4SLinus Torvalds 27846453a6eSNick Piggin if (PageLRU(page)) { 27946453a6eSNick Piggin struct zone *pagezone = page_zone(page); 2801da177e4SLinus Torvalds if (pagezone != zone) { 2811da177e4SLinus Torvalds if (zone) 2821da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 2831da177e4SLinus Torvalds zone = pagezone; 2841da177e4SLinus Torvalds spin_lock_irq(&zone->lru_lock); 2851da177e4SLinus Torvalds } 286725d704eSNick Piggin VM_BUG_ON(!PageLRU(page)); 28767453911SNick Piggin __ClearPageLRU(page); 2881da177e4SLinus Torvalds del_page_from_lru(zone, page); 28946453a6eSNick Piggin } 29046453a6eSNick Piggin 2911da177e4SLinus Torvalds if (!pagevec_add(&pages_to_free, page)) { 29246453a6eSNick Piggin if (zone) { 2931da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 29446453a6eSNick Piggin zone = NULL; 29546453a6eSNick Piggin } 2961da177e4SLinus Torvalds __pagevec_free(&pages_to_free); 2971da177e4SLinus Torvalds pagevec_reinit(&pages_to_free); 2981da177e4SLinus Torvalds } 2991da177e4SLinus Torvalds } 3001da177e4SLinus Torvalds if (zone) 3011da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 3021da177e4SLinus Torvalds 3031da177e4SLinus Torvalds pagevec_free(&pages_to_free); 3041da177e4SLinus Torvalds } 3051da177e4SLinus Torvalds 3061da177e4SLinus Torvalds /* 3071da177e4SLinus Torvalds * The pages which we're about to release may be in the deferred lru-addition 3081da177e4SLinus Torvalds * queues. That would prevent them from really being freed right now. That's 3091da177e4SLinus Torvalds * OK from a correctness point of view but is inefficient - those pages may be 3101da177e4SLinus Torvalds * cache-warm and we want to give them back to the page allocator ASAP. 3111da177e4SLinus Torvalds * 3121da177e4SLinus Torvalds * So __pagevec_release() will drain those queues here. __pagevec_lru_add() 3131da177e4SLinus Torvalds * and __pagevec_lru_add_active() call release_pages() directly to avoid 3141da177e4SLinus Torvalds * mutual recursion. 3151da177e4SLinus Torvalds */ 3161da177e4SLinus Torvalds void __pagevec_release(struct pagevec *pvec) 3171da177e4SLinus Torvalds { 3181da177e4SLinus Torvalds lru_add_drain(); 3191da177e4SLinus Torvalds release_pages(pvec->pages, pagevec_count(pvec), pvec->cold); 3201da177e4SLinus Torvalds pagevec_reinit(pvec); 3211da177e4SLinus Torvalds } 3221da177e4SLinus Torvalds 3237f285701SSteve French EXPORT_SYMBOL(__pagevec_release); 3247f285701SSteve French 3251da177e4SLinus Torvalds /* 3261da177e4SLinus Torvalds * pagevec_release() for pages which are known to not be on the LRU 3271da177e4SLinus Torvalds * 3281da177e4SLinus Torvalds * This function reinitialises the caller's pagevec. 3291da177e4SLinus Torvalds */ 3301da177e4SLinus Torvalds void __pagevec_release_nonlru(struct pagevec *pvec) 3311da177e4SLinus Torvalds { 3321da177e4SLinus Torvalds int i; 3331da177e4SLinus Torvalds struct pagevec pages_to_free; 3341da177e4SLinus Torvalds 3351da177e4SLinus Torvalds pagevec_init(&pages_to_free, pvec->cold); 3361da177e4SLinus Torvalds for (i = 0; i < pagevec_count(pvec); i++) { 3371da177e4SLinus Torvalds struct page *page = pvec->pages[i]; 3381da177e4SLinus Torvalds 339725d704eSNick Piggin VM_BUG_ON(PageLRU(page)); 3401da177e4SLinus Torvalds if (put_page_testzero(page)) 3411da177e4SLinus Torvalds pagevec_add(&pages_to_free, page); 3421da177e4SLinus Torvalds } 3431da177e4SLinus Torvalds pagevec_free(&pages_to_free); 3441da177e4SLinus Torvalds pagevec_reinit(pvec); 3451da177e4SLinus Torvalds } 3461da177e4SLinus Torvalds 3471da177e4SLinus Torvalds /* 3481da177e4SLinus Torvalds * Add the passed pages to the LRU, then drop the caller's refcount 3491da177e4SLinus Torvalds * on them. Reinitialises the caller's pagevec. 3501da177e4SLinus Torvalds */ 3511da177e4SLinus Torvalds void __pagevec_lru_add(struct pagevec *pvec) 3521da177e4SLinus Torvalds { 3531da177e4SLinus Torvalds int i; 3541da177e4SLinus Torvalds struct zone *zone = NULL; 3551da177e4SLinus Torvalds 3561da177e4SLinus Torvalds for (i = 0; i < pagevec_count(pvec); i++) { 3571da177e4SLinus Torvalds struct page *page = pvec->pages[i]; 3581da177e4SLinus Torvalds struct zone *pagezone = page_zone(page); 3591da177e4SLinus Torvalds 3601da177e4SLinus Torvalds if (pagezone != zone) { 3611da177e4SLinus Torvalds if (zone) 3621da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 3631da177e4SLinus Torvalds zone = pagezone; 3641da177e4SLinus Torvalds spin_lock_irq(&zone->lru_lock); 3651da177e4SLinus Torvalds } 366725d704eSNick Piggin VM_BUG_ON(PageLRU(page)); 3678d438f96SNick Piggin SetPageLRU(page); 3681da177e4SLinus Torvalds add_page_to_inactive_list(zone, page); 3691da177e4SLinus Torvalds } 3701da177e4SLinus Torvalds if (zone) 3711da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 3721da177e4SLinus Torvalds release_pages(pvec->pages, pvec->nr, pvec->cold); 3731da177e4SLinus Torvalds pagevec_reinit(pvec); 3741da177e4SLinus Torvalds } 3751da177e4SLinus Torvalds 3761da177e4SLinus Torvalds EXPORT_SYMBOL(__pagevec_lru_add); 3771da177e4SLinus Torvalds 3781da177e4SLinus Torvalds void __pagevec_lru_add_active(struct pagevec *pvec) 3791da177e4SLinus Torvalds { 3801da177e4SLinus Torvalds int i; 3811da177e4SLinus Torvalds struct zone *zone = NULL; 3821da177e4SLinus Torvalds 3831da177e4SLinus Torvalds for (i = 0; i < pagevec_count(pvec); i++) { 3841da177e4SLinus Torvalds struct page *page = pvec->pages[i]; 3851da177e4SLinus Torvalds struct zone *pagezone = page_zone(page); 3861da177e4SLinus Torvalds 3871da177e4SLinus Torvalds if (pagezone != zone) { 3881da177e4SLinus Torvalds if (zone) 3891da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 3901da177e4SLinus Torvalds zone = pagezone; 3911da177e4SLinus Torvalds spin_lock_irq(&zone->lru_lock); 3921da177e4SLinus Torvalds } 393725d704eSNick Piggin VM_BUG_ON(PageLRU(page)); 3948d438f96SNick Piggin SetPageLRU(page); 395725d704eSNick Piggin VM_BUG_ON(PageActive(page)); 3964c84cacfSNick Piggin SetPageActive(page); 3971da177e4SLinus Torvalds add_page_to_active_list(zone, page); 3981da177e4SLinus Torvalds } 3991da177e4SLinus Torvalds if (zone) 4001da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 4011da177e4SLinus Torvalds release_pages(pvec->pages, pvec->nr, pvec->cold); 4021da177e4SLinus Torvalds pagevec_reinit(pvec); 4031da177e4SLinus Torvalds } 4041da177e4SLinus Torvalds 4051da177e4SLinus Torvalds /* 4061da177e4SLinus Torvalds * Try to drop buffers from the pages in a pagevec 4071da177e4SLinus Torvalds */ 4081da177e4SLinus Torvalds void pagevec_strip(struct pagevec *pvec) 4091da177e4SLinus Torvalds { 4101da177e4SLinus Torvalds int i; 4111da177e4SLinus Torvalds 4121da177e4SLinus Torvalds for (i = 0; i < pagevec_count(pvec); i++) { 4131da177e4SLinus Torvalds struct page *page = pvec->pages[i]; 4141da177e4SLinus Torvalds 4151da177e4SLinus Torvalds if (PagePrivate(page) && !TestSetPageLocked(page)) { 4165b40dc78SChristoph Lameter if (PagePrivate(page)) 4171da177e4SLinus Torvalds try_to_release_page(page, 0); 4181da177e4SLinus Torvalds unlock_page(page); 4191da177e4SLinus Torvalds } 4201da177e4SLinus Torvalds } 4211da177e4SLinus Torvalds } 4221da177e4SLinus Torvalds 4231da177e4SLinus Torvalds /** 4241da177e4SLinus Torvalds * pagevec_lookup - gang pagecache lookup 4251da177e4SLinus Torvalds * @pvec: Where the resulting pages are placed 4261da177e4SLinus Torvalds * @mapping: The address_space to search 4271da177e4SLinus Torvalds * @start: The starting page index 4281da177e4SLinus Torvalds * @nr_pages: The maximum number of pages 4291da177e4SLinus Torvalds * 4301da177e4SLinus Torvalds * pagevec_lookup() will search for and return a group of up to @nr_pages pages 4311da177e4SLinus Torvalds * in the mapping. The pages are placed in @pvec. pagevec_lookup() takes a 4321da177e4SLinus Torvalds * reference against the pages in @pvec. 4331da177e4SLinus Torvalds * 4341da177e4SLinus Torvalds * The search returns a group of mapping-contiguous pages with ascending 4351da177e4SLinus Torvalds * indexes. There may be holes in the indices due to not-present pages. 4361da177e4SLinus Torvalds * 4371da177e4SLinus Torvalds * pagevec_lookup() returns the number of pages which were found. 4381da177e4SLinus Torvalds */ 4391da177e4SLinus Torvalds unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping, 4401da177e4SLinus Torvalds pgoff_t start, unsigned nr_pages) 4411da177e4SLinus Torvalds { 4421da177e4SLinus Torvalds pvec->nr = find_get_pages(mapping, start, nr_pages, pvec->pages); 4431da177e4SLinus Torvalds return pagevec_count(pvec); 4441da177e4SLinus Torvalds } 4451da177e4SLinus Torvalds 44678539fdfSChristoph Hellwig EXPORT_SYMBOL(pagevec_lookup); 44778539fdfSChristoph Hellwig 4481da177e4SLinus Torvalds unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping, 4491da177e4SLinus Torvalds pgoff_t *index, int tag, unsigned nr_pages) 4501da177e4SLinus Torvalds { 4511da177e4SLinus Torvalds pvec->nr = find_get_pages_tag(mapping, index, tag, 4521da177e4SLinus Torvalds nr_pages, pvec->pages); 4531da177e4SLinus Torvalds return pagevec_count(pvec); 4541da177e4SLinus Torvalds } 4551da177e4SLinus Torvalds 4567f285701SSteve French EXPORT_SYMBOL(pagevec_lookup_tag); 4571da177e4SLinus Torvalds 4581da177e4SLinus Torvalds #ifdef CONFIG_SMP 4591da177e4SLinus Torvalds /* 4601da177e4SLinus Torvalds * We tolerate a little inaccuracy to avoid ping-ponging the counter between 4611da177e4SLinus Torvalds * CPUs 4621da177e4SLinus Torvalds */ 4631da177e4SLinus Torvalds #define ACCT_THRESHOLD max(16, NR_CPUS * 2) 4641da177e4SLinus Torvalds 4651da177e4SLinus Torvalds static DEFINE_PER_CPU(long, committed_space) = 0; 4661da177e4SLinus Torvalds 4671da177e4SLinus Torvalds void vm_acct_memory(long pages) 4681da177e4SLinus Torvalds { 4691da177e4SLinus Torvalds long *local; 4701da177e4SLinus Torvalds 4711da177e4SLinus Torvalds preempt_disable(); 4721da177e4SLinus Torvalds local = &__get_cpu_var(committed_space); 4731da177e4SLinus Torvalds *local += pages; 4741da177e4SLinus Torvalds if (*local > ACCT_THRESHOLD || *local < -ACCT_THRESHOLD) { 4751da177e4SLinus Torvalds atomic_add(*local, &vm_committed_space); 4761da177e4SLinus Torvalds *local = 0; 4771da177e4SLinus Torvalds } 4781da177e4SLinus Torvalds preempt_enable(); 4791da177e4SLinus Torvalds } 4801da177e4SLinus Torvalds 4811da177e4SLinus Torvalds #ifdef CONFIG_HOTPLUG_CPU 4821da177e4SLinus Torvalds 4831da177e4SLinus Torvalds /* Drop the CPU's cached committed space back into the central pool. */ 4841da177e4SLinus Torvalds static int cpu_swap_callback(struct notifier_block *nfb, 4851da177e4SLinus Torvalds unsigned long action, 4861da177e4SLinus Torvalds void *hcpu) 4871da177e4SLinus Torvalds { 4881da177e4SLinus Torvalds long *committed; 4891da177e4SLinus Torvalds 4901da177e4SLinus Torvalds committed = &per_cpu(committed_space, (long)hcpu); 4911da177e4SLinus Torvalds if (action == CPU_DEAD) { 4921da177e4SLinus Torvalds atomic_add(*committed, &vm_committed_space); 4931da177e4SLinus Torvalds *committed = 0; 49480bfed90SAndrew Morton __lru_add_drain((long)hcpu); 4951da177e4SLinus Torvalds } 4961da177e4SLinus Torvalds return NOTIFY_OK; 4971da177e4SLinus Torvalds } 4981da177e4SLinus Torvalds #endif /* CONFIG_HOTPLUG_CPU */ 4991da177e4SLinus Torvalds #endif /* CONFIG_SMP */ 5001da177e4SLinus Torvalds 5011da177e4SLinus Torvalds /* 5021da177e4SLinus Torvalds * Perform any setup for the swap system 5031da177e4SLinus Torvalds */ 5041da177e4SLinus Torvalds void __init swap_setup(void) 5051da177e4SLinus Torvalds { 5061da177e4SLinus Torvalds unsigned long megs = num_physpages >> (20 - PAGE_SHIFT); 5071da177e4SLinus Torvalds 5081da177e4SLinus Torvalds /* Use a smaller cluster for small-memory machines */ 5091da177e4SLinus Torvalds if (megs < 16) 5101da177e4SLinus Torvalds page_cluster = 2; 5111da177e4SLinus Torvalds else 5121da177e4SLinus Torvalds page_cluster = 3; 5131da177e4SLinus Torvalds /* 5141da177e4SLinus Torvalds * Right now other parts of the system means that we 5151da177e4SLinus Torvalds * _really_ don't want to cluster much more 5161da177e4SLinus Torvalds */ 5171da177e4SLinus Torvalds hotcpu_notifier(cpu_swap_callback, 0); 5181da177e4SLinus Torvalds } 519