11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * linux/mm/swap.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 51da177e4SLinus Torvalds */ 61da177e4SLinus Torvalds 71da177e4SLinus Torvalds /* 81da177e4SLinus Torvalds * This file contains the default values for the opereation of the 91da177e4SLinus Torvalds * Linux VM subsystem. Fine-tuning documentation can be found in 101da177e4SLinus Torvalds * Documentation/sysctl/vm.txt. 111da177e4SLinus Torvalds * Started 18.12.91 121da177e4SLinus Torvalds * Swap aging added 23.2.95, Stephen Tweedie. 131da177e4SLinus Torvalds * Buffermem limits added 12.3.98, Rik van Riel. 141da177e4SLinus Torvalds */ 151da177e4SLinus Torvalds 161da177e4SLinus Torvalds #include <linux/mm.h> 171da177e4SLinus Torvalds #include <linux/sched.h> 181da177e4SLinus Torvalds #include <linux/kernel_stat.h> 191da177e4SLinus Torvalds #include <linux/swap.h> 201da177e4SLinus Torvalds #include <linux/mman.h> 211da177e4SLinus Torvalds #include <linux/pagemap.h> 221da177e4SLinus Torvalds #include <linux/pagevec.h> 231da177e4SLinus Torvalds #include <linux/init.h> 241da177e4SLinus Torvalds #include <linux/module.h> 251da177e4SLinus Torvalds #include <linux/mm_inline.h> 261da177e4SLinus Torvalds #include <linux/buffer_head.h> /* for try_to_release_page() */ 271da177e4SLinus Torvalds #include <linux/module.h> 281da177e4SLinus Torvalds #include <linux/percpu_counter.h> 291da177e4SLinus Torvalds #include <linux/percpu.h> 301da177e4SLinus Torvalds #include <linux/cpu.h> 311da177e4SLinus Torvalds #include <linux/notifier.h> 321da177e4SLinus Torvalds #include <linux/init.h> 331da177e4SLinus Torvalds 341da177e4SLinus Torvalds /* How many pages do we try to swap or page in/out together? */ 351da177e4SLinus Torvalds int page_cluster; 361da177e4SLinus Torvalds 371da177e4SLinus Torvalds #ifdef CONFIG_HUGETLB_PAGE 381da177e4SLinus Torvalds 391da177e4SLinus Torvalds void put_page(struct page *page) 401da177e4SLinus Torvalds { 411da177e4SLinus Torvalds if (unlikely(PageCompound(page))) { 424c21e2f2SHugh Dickins page = (struct page *)page_private(page); 431da177e4SLinus Torvalds if (put_page_testzero(page)) { 441da177e4SLinus Torvalds void (*dtor)(struct page *page); 451da177e4SLinus Torvalds 461da177e4SLinus Torvalds dtor = (void (*)(struct page *))page[1].mapping; 471da177e4SLinus Torvalds (*dtor)(page); 481da177e4SLinus Torvalds } 491da177e4SLinus Torvalds return; 501da177e4SLinus Torvalds } 51b5810039SNick Piggin if (put_page_testzero(page)) 521da177e4SLinus Torvalds __page_cache_release(page); 531da177e4SLinus Torvalds } 541da177e4SLinus Torvalds EXPORT_SYMBOL(put_page); 551da177e4SLinus Torvalds #endif 561da177e4SLinus Torvalds 571da177e4SLinus Torvalds /* 581da177e4SLinus Torvalds * Writeback is about to end against a page which has been marked for immediate 591da177e4SLinus Torvalds * reclaim. If it still appears to be reclaimable, move it to the tail of the 601da177e4SLinus Torvalds * inactive list. The page still has PageWriteback set, which will pin it. 611da177e4SLinus Torvalds * 621da177e4SLinus Torvalds * We don't expect many pages to come through here, so don't bother batching 631da177e4SLinus Torvalds * things up. 641da177e4SLinus Torvalds * 651da177e4SLinus Torvalds * To avoid placing the page at the tail of the LRU while PG_writeback is still 661da177e4SLinus Torvalds * set, this function will clear PG_writeback before performing the page 671da177e4SLinus Torvalds * motion. Do that inside the lru lock because once PG_writeback is cleared 681da177e4SLinus Torvalds * we may not touch the page. 691da177e4SLinus Torvalds * 701da177e4SLinus Torvalds * Returns zero if it cleared PG_writeback. 711da177e4SLinus Torvalds */ 721da177e4SLinus Torvalds int rotate_reclaimable_page(struct page *page) 731da177e4SLinus Torvalds { 741da177e4SLinus Torvalds struct zone *zone; 751da177e4SLinus Torvalds unsigned long flags; 761da177e4SLinus Torvalds 771da177e4SLinus Torvalds if (PageLocked(page)) 781da177e4SLinus Torvalds return 1; 791da177e4SLinus Torvalds if (PageDirty(page)) 801da177e4SLinus Torvalds return 1; 811da177e4SLinus Torvalds if (PageActive(page)) 821da177e4SLinus Torvalds return 1; 831da177e4SLinus Torvalds if (!PageLRU(page)) 841da177e4SLinus Torvalds return 1; 851da177e4SLinus Torvalds 861da177e4SLinus Torvalds zone = page_zone(page); 871da177e4SLinus Torvalds spin_lock_irqsave(&zone->lru_lock, flags); 881da177e4SLinus Torvalds if (PageLRU(page) && !PageActive(page)) { 891da177e4SLinus Torvalds list_del(&page->lru); 901da177e4SLinus Torvalds list_add_tail(&page->lru, &zone->inactive_list); 911da177e4SLinus Torvalds inc_page_state(pgrotated); 921da177e4SLinus Torvalds } 931da177e4SLinus Torvalds if (!test_clear_page_writeback(page)) 941da177e4SLinus Torvalds BUG(); 951da177e4SLinus Torvalds spin_unlock_irqrestore(&zone->lru_lock, flags); 961da177e4SLinus Torvalds return 0; 971da177e4SLinus Torvalds } 981da177e4SLinus Torvalds 991da177e4SLinus Torvalds /* 1001da177e4SLinus Torvalds * FIXME: speed this up? 1011da177e4SLinus Torvalds */ 1021da177e4SLinus Torvalds void fastcall activate_page(struct page *page) 1031da177e4SLinus Torvalds { 1041da177e4SLinus Torvalds struct zone *zone = page_zone(page); 1051da177e4SLinus Torvalds 1061da177e4SLinus Torvalds spin_lock_irq(&zone->lru_lock); 1071da177e4SLinus Torvalds if (PageLRU(page) && !PageActive(page)) { 1081da177e4SLinus Torvalds del_page_from_inactive_list(zone, page); 1091da177e4SLinus Torvalds SetPageActive(page); 1101da177e4SLinus Torvalds add_page_to_active_list(zone, page); 1111da177e4SLinus Torvalds inc_page_state(pgactivate); 1121da177e4SLinus Torvalds } 1131da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 1141da177e4SLinus Torvalds } 1151da177e4SLinus Torvalds 1161da177e4SLinus Torvalds /* 1171da177e4SLinus Torvalds * Mark a page as having seen activity. 1181da177e4SLinus Torvalds * 1191da177e4SLinus Torvalds * inactive,unreferenced -> inactive,referenced 1201da177e4SLinus Torvalds * inactive,referenced -> active,unreferenced 1211da177e4SLinus Torvalds * active,unreferenced -> active,referenced 1221da177e4SLinus Torvalds */ 1231da177e4SLinus Torvalds void fastcall mark_page_accessed(struct page *page) 1241da177e4SLinus Torvalds { 1251da177e4SLinus Torvalds if (!PageActive(page) && PageReferenced(page) && PageLRU(page)) { 1261da177e4SLinus Torvalds activate_page(page); 1271da177e4SLinus Torvalds ClearPageReferenced(page); 1281da177e4SLinus Torvalds } else if (!PageReferenced(page)) { 1291da177e4SLinus Torvalds SetPageReferenced(page); 1301da177e4SLinus Torvalds } 1311da177e4SLinus Torvalds } 1321da177e4SLinus Torvalds 1331da177e4SLinus Torvalds EXPORT_SYMBOL(mark_page_accessed); 1341da177e4SLinus Torvalds 1351da177e4SLinus Torvalds /** 1361da177e4SLinus Torvalds * lru_cache_add: add a page to the page lists 1371da177e4SLinus Torvalds * @page: the page to add 1381da177e4SLinus Torvalds */ 1391da177e4SLinus Torvalds static DEFINE_PER_CPU(struct pagevec, lru_add_pvecs) = { 0, }; 1401da177e4SLinus Torvalds static DEFINE_PER_CPU(struct pagevec, lru_add_active_pvecs) = { 0, }; 1411da177e4SLinus Torvalds 1421da177e4SLinus Torvalds void fastcall lru_cache_add(struct page *page) 1431da177e4SLinus Torvalds { 1441da177e4SLinus Torvalds struct pagevec *pvec = &get_cpu_var(lru_add_pvecs); 1451da177e4SLinus Torvalds 1461da177e4SLinus Torvalds page_cache_get(page); 1471da177e4SLinus Torvalds if (!pagevec_add(pvec, page)) 1481da177e4SLinus Torvalds __pagevec_lru_add(pvec); 1491da177e4SLinus Torvalds put_cpu_var(lru_add_pvecs); 1501da177e4SLinus Torvalds } 1511da177e4SLinus Torvalds 1521da177e4SLinus Torvalds void fastcall lru_cache_add_active(struct page *page) 1531da177e4SLinus Torvalds { 1541da177e4SLinus Torvalds struct pagevec *pvec = &get_cpu_var(lru_add_active_pvecs); 1551da177e4SLinus Torvalds 1561da177e4SLinus Torvalds page_cache_get(page); 1571da177e4SLinus Torvalds if (!pagevec_add(pvec, page)) 1581da177e4SLinus Torvalds __pagevec_lru_add_active(pvec); 1591da177e4SLinus Torvalds put_cpu_var(lru_add_active_pvecs); 1601da177e4SLinus Torvalds } 1611da177e4SLinus Torvalds 1621da177e4SLinus Torvalds void lru_add_drain(void) 1631da177e4SLinus Torvalds { 1641da177e4SLinus Torvalds struct pagevec *pvec = &get_cpu_var(lru_add_pvecs); 1651da177e4SLinus Torvalds 1661da177e4SLinus Torvalds if (pagevec_count(pvec)) 1671da177e4SLinus Torvalds __pagevec_lru_add(pvec); 1681da177e4SLinus Torvalds pvec = &__get_cpu_var(lru_add_active_pvecs); 1691da177e4SLinus Torvalds if (pagevec_count(pvec)) 1701da177e4SLinus Torvalds __pagevec_lru_add_active(pvec); 1711da177e4SLinus Torvalds put_cpu_var(lru_add_pvecs); 1721da177e4SLinus Torvalds } 1731da177e4SLinus Torvalds 1741da177e4SLinus Torvalds /* 1751da177e4SLinus Torvalds * This path almost never happens for VM activity - pages are normally 1761da177e4SLinus Torvalds * freed via pagevecs. But it gets used by networking. 1771da177e4SLinus Torvalds */ 1781da177e4SLinus Torvalds void fastcall __page_cache_release(struct page *page) 1791da177e4SLinus Torvalds { 1801da177e4SLinus Torvalds unsigned long flags; 1811da177e4SLinus Torvalds struct zone *zone = page_zone(page); 1821da177e4SLinus Torvalds 1831da177e4SLinus Torvalds spin_lock_irqsave(&zone->lru_lock, flags); 1841da177e4SLinus Torvalds if (TestClearPageLRU(page)) 1851da177e4SLinus Torvalds del_page_from_lru(zone, page); 1861da177e4SLinus Torvalds if (page_count(page) != 0) 1871da177e4SLinus Torvalds page = NULL; 1881da177e4SLinus Torvalds spin_unlock_irqrestore(&zone->lru_lock, flags); 1891da177e4SLinus Torvalds if (page) 1901da177e4SLinus Torvalds free_hot_page(page); 1911da177e4SLinus Torvalds } 1921da177e4SLinus Torvalds 1931da177e4SLinus Torvalds EXPORT_SYMBOL(__page_cache_release); 1941da177e4SLinus Torvalds 1951da177e4SLinus Torvalds /* 1961da177e4SLinus Torvalds * Batched page_cache_release(). Decrement the reference count on all the 1971da177e4SLinus Torvalds * passed pages. If it fell to zero then remove the page from the LRU and 1981da177e4SLinus Torvalds * free it. 1991da177e4SLinus Torvalds * 2001da177e4SLinus Torvalds * Avoid taking zone->lru_lock if possible, but if it is taken, retain it 2011da177e4SLinus Torvalds * for the remainder of the operation. 2021da177e4SLinus Torvalds * 2031da177e4SLinus Torvalds * The locking in this function is against shrink_cache(): we recheck the 2041da177e4SLinus Torvalds * page count inside the lock to see whether shrink_cache grabbed the page 2051da177e4SLinus Torvalds * via the LRU. If it did, give up: shrink_cache will free it. 2061da177e4SLinus Torvalds */ 2071da177e4SLinus Torvalds void release_pages(struct page **pages, int nr, int cold) 2081da177e4SLinus Torvalds { 2091da177e4SLinus Torvalds int i; 2101da177e4SLinus Torvalds struct pagevec pages_to_free; 2111da177e4SLinus Torvalds struct zone *zone = NULL; 2121da177e4SLinus Torvalds 2131da177e4SLinus Torvalds pagevec_init(&pages_to_free, cold); 2141da177e4SLinus Torvalds for (i = 0; i < nr; i++) { 2151da177e4SLinus Torvalds struct page *page = pages[i]; 2161da177e4SLinus Torvalds struct zone *pagezone; 2171da177e4SLinus Torvalds 218b5810039SNick Piggin if (!put_page_testzero(page)) 2191da177e4SLinus Torvalds continue; 2201da177e4SLinus Torvalds 2211da177e4SLinus Torvalds pagezone = page_zone(page); 2221da177e4SLinus Torvalds if (pagezone != zone) { 2231da177e4SLinus Torvalds if (zone) 2241da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 2251da177e4SLinus Torvalds zone = pagezone; 2261da177e4SLinus Torvalds spin_lock_irq(&zone->lru_lock); 2271da177e4SLinus Torvalds } 2281da177e4SLinus Torvalds if (TestClearPageLRU(page)) 2291da177e4SLinus Torvalds del_page_from_lru(zone, page); 2301da177e4SLinus Torvalds if (page_count(page) == 0) { 2311da177e4SLinus Torvalds if (!pagevec_add(&pages_to_free, page)) { 2321da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 2331da177e4SLinus Torvalds __pagevec_free(&pages_to_free); 2341da177e4SLinus Torvalds pagevec_reinit(&pages_to_free); 2351da177e4SLinus Torvalds zone = NULL; /* No lock is held */ 2361da177e4SLinus Torvalds } 2371da177e4SLinus Torvalds } 2381da177e4SLinus Torvalds } 2391da177e4SLinus Torvalds if (zone) 2401da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 2411da177e4SLinus Torvalds 2421da177e4SLinus Torvalds pagevec_free(&pages_to_free); 2431da177e4SLinus Torvalds } 2441da177e4SLinus Torvalds 2451da177e4SLinus Torvalds /* 2461da177e4SLinus Torvalds * The pages which we're about to release may be in the deferred lru-addition 2471da177e4SLinus Torvalds * queues. That would prevent them from really being freed right now. That's 2481da177e4SLinus Torvalds * OK from a correctness point of view but is inefficient - those pages may be 2491da177e4SLinus Torvalds * cache-warm and we want to give them back to the page allocator ASAP. 2501da177e4SLinus Torvalds * 2511da177e4SLinus Torvalds * So __pagevec_release() will drain those queues here. __pagevec_lru_add() 2521da177e4SLinus Torvalds * and __pagevec_lru_add_active() call release_pages() directly to avoid 2531da177e4SLinus Torvalds * mutual recursion. 2541da177e4SLinus Torvalds */ 2551da177e4SLinus Torvalds void __pagevec_release(struct pagevec *pvec) 2561da177e4SLinus Torvalds { 2571da177e4SLinus Torvalds lru_add_drain(); 2581da177e4SLinus Torvalds release_pages(pvec->pages, pagevec_count(pvec), pvec->cold); 2591da177e4SLinus Torvalds pagevec_reinit(pvec); 2601da177e4SLinus Torvalds } 2611da177e4SLinus Torvalds 262*7f285701SSteve French EXPORT_SYMBOL(__pagevec_release); 263*7f285701SSteve French 2641da177e4SLinus Torvalds /* 2651da177e4SLinus Torvalds * pagevec_release() for pages which are known to not be on the LRU 2661da177e4SLinus Torvalds * 2671da177e4SLinus Torvalds * This function reinitialises the caller's pagevec. 2681da177e4SLinus Torvalds */ 2691da177e4SLinus Torvalds void __pagevec_release_nonlru(struct pagevec *pvec) 2701da177e4SLinus Torvalds { 2711da177e4SLinus Torvalds int i; 2721da177e4SLinus Torvalds struct pagevec pages_to_free; 2731da177e4SLinus Torvalds 2741da177e4SLinus Torvalds pagevec_init(&pages_to_free, pvec->cold); 2751da177e4SLinus Torvalds for (i = 0; i < pagevec_count(pvec); i++) { 2761da177e4SLinus Torvalds struct page *page = pvec->pages[i]; 2771da177e4SLinus Torvalds 2781da177e4SLinus Torvalds BUG_ON(PageLRU(page)); 2791da177e4SLinus Torvalds if (put_page_testzero(page)) 2801da177e4SLinus Torvalds pagevec_add(&pages_to_free, page); 2811da177e4SLinus Torvalds } 2821da177e4SLinus Torvalds pagevec_free(&pages_to_free); 2831da177e4SLinus Torvalds pagevec_reinit(pvec); 2841da177e4SLinus Torvalds } 2851da177e4SLinus Torvalds 2861da177e4SLinus Torvalds /* 2871da177e4SLinus Torvalds * Add the passed pages to the LRU, then drop the caller's refcount 2881da177e4SLinus Torvalds * on them. Reinitialises the caller's pagevec. 2891da177e4SLinus Torvalds */ 2901da177e4SLinus Torvalds void __pagevec_lru_add(struct pagevec *pvec) 2911da177e4SLinus Torvalds { 2921da177e4SLinus Torvalds int i; 2931da177e4SLinus Torvalds struct zone *zone = NULL; 2941da177e4SLinus Torvalds 2951da177e4SLinus Torvalds for (i = 0; i < pagevec_count(pvec); i++) { 2961da177e4SLinus Torvalds struct page *page = pvec->pages[i]; 2971da177e4SLinus Torvalds struct zone *pagezone = page_zone(page); 2981da177e4SLinus Torvalds 2991da177e4SLinus Torvalds if (pagezone != zone) { 3001da177e4SLinus Torvalds if (zone) 3011da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 3021da177e4SLinus Torvalds zone = pagezone; 3031da177e4SLinus Torvalds spin_lock_irq(&zone->lru_lock); 3041da177e4SLinus Torvalds } 3051da177e4SLinus Torvalds if (TestSetPageLRU(page)) 3061da177e4SLinus Torvalds BUG(); 3071da177e4SLinus Torvalds add_page_to_inactive_list(zone, page); 3081da177e4SLinus Torvalds } 3091da177e4SLinus Torvalds if (zone) 3101da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 3111da177e4SLinus Torvalds release_pages(pvec->pages, pvec->nr, pvec->cold); 3121da177e4SLinus Torvalds pagevec_reinit(pvec); 3131da177e4SLinus Torvalds } 3141da177e4SLinus Torvalds 3151da177e4SLinus Torvalds EXPORT_SYMBOL(__pagevec_lru_add); 3161da177e4SLinus Torvalds 3171da177e4SLinus Torvalds void __pagevec_lru_add_active(struct pagevec *pvec) 3181da177e4SLinus Torvalds { 3191da177e4SLinus Torvalds int i; 3201da177e4SLinus Torvalds struct zone *zone = NULL; 3211da177e4SLinus Torvalds 3221da177e4SLinus Torvalds for (i = 0; i < pagevec_count(pvec); i++) { 3231da177e4SLinus Torvalds struct page *page = pvec->pages[i]; 3241da177e4SLinus Torvalds struct zone *pagezone = page_zone(page); 3251da177e4SLinus Torvalds 3261da177e4SLinus Torvalds if (pagezone != zone) { 3271da177e4SLinus Torvalds if (zone) 3281da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 3291da177e4SLinus Torvalds zone = pagezone; 3301da177e4SLinus Torvalds spin_lock_irq(&zone->lru_lock); 3311da177e4SLinus Torvalds } 3321da177e4SLinus Torvalds if (TestSetPageLRU(page)) 3331da177e4SLinus Torvalds BUG(); 3341da177e4SLinus Torvalds if (TestSetPageActive(page)) 3351da177e4SLinus Torvalds BUG(); 3361da177e4SLinus Torvalds add_page_to_active_list(zone, page); 3371da177e4SLinus Torvalds } 3381da177e4SLinus Torvalds if (zone) 3391da177e4SLinus Torvalds spin_unlock_irq(&zone->lru_lock); 3401da177e4SLinus Torvalds release_pages(pvec->pages, pvec->nr, pvec->cold); 3411da177e4SLinus Torvalds pagevec_reinit(pvec); 3421da177e4SLinus Torvalds } 3431da177e4SLinus Torvalds 3441da177e4SLinus Torvalds /* 3451da177e4SLinus Torvalds * Try to drop buffers from the pages in a pagevec 3461da177e4SLinus Torvalds */ 3471da177e4SLinus Torvalds void pagevec_strip(struct pagevec *pvec) 3481da177e4SLinus Torvalds { 3491da177e4SLinus Torvalds int i; 3501da177e4SLinus Torvalds 3511da177e4SLinus Torvalds for (i = 0; i < pagevec_count(pvec); i++) { 3521da177e4SLinus Torvalds struct page *page = pvec->pages[i]; 3531da177e4SLinus Torvalds 3541da177e4SLinus Torvalds if (PagePrivate(page) && !TestSetPageLocked(page)) { 3551da177e4SLinus Torvalds try_to_release_page(page, 0); 3561da177e4SLinus Torvalds unlock_page(page); 3571da177e4SLinus Torvalds } 3581da177e4SLinus Torvalds } 3591da177e4SLinus Torvalds } 3601da177e4SLinus Torvalds 3611da177e4SLinus Torvalds /** 3621da177e4SLinus Torvalds * pagevec_lookup - gang pagecache lookup 3631da177e4SLinus Torvalds * @pvec: Where the resulting pages are placed 3641da177e4SLinus Torvalds * @mapping: The address_space to search 3651da177e4SLinus Torvalds * @start: The starting page index 3661da177e4SLinus Torvalds * @nr_pages: The maximum number of pages 3671da177e4SLinus Torvalds * 3681da177e4SLinus Torvalds * pagevec_lookup() will search for and return a group of up to @nr_pages pages 3691da177e4SLinus Torvalds * in the mapping. The pages are placed in @pvec. pagevec_lookup() takes a 3701da177e4SLinus Torvalds * reference against the pages in @pvec. 3711da177e4SLinus Torvalds * 3721da177e4SLinus Torvalds * The search returns a group of mapping-contiguous pages with ascending 3731da177e4SLinus Torvalds * indexes. There may be holes in the indices due to not-present pages. 3741da177e4SLinus Torvalds * 3751da177e4SLinus Torvalds * pagevec_lookup() returns the number of pages which were found. 3761da177e4SLinus Torvalds */ 3771da177e4SLinus Torvalds unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping, 3781da177e4SLinus Torvalds pgoff_t start, unsigned nr_pages) 3791da177e4SLinus Torvalds { 3801da177e4SLinus Torvalds pvec->nr = find_get_pages(mapping, start, nr_pages, pvec->pages); 3811da177e4SLinus Torvalds return pagevec_count(pvec); 3821da177e4SLinus Torvalds } 3831da177e4SLinus Torvalds 3841da177e4SLinus Torvalds unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping, 3851da177e4SLinus Torvalds pgoff_t *index, int tag, unsigned nr_pages) 3861da177e4SLinus Torvalds { 3871da177e4SLinus Torvalds pvec->nr = find_get_pages_tag(mapping, index, tag, 3881da177e4SLinus Torvalds nr_pages, pvec->pages); 3891da177e4SLinus Torvalds return pagevec_count(pvec); 3901da177e4SLinus Torvalds } 3911da177e4SLinus Torvalds 392*7f285701SSteve French EXPORT_SYMBOL(pagevec_lookup_tag); 3931da177e4SLinus Torvalds 3941da177e4SLinus Torvalds #ifdef CONFIG_SMP 3951da177e4SLinus Torvalds /* 3961da177e4SLinus Torvalds * We tolerate a little inaccuracy to avoid ping-ponging the counter between 3971da177e4SLinus Torvalds * CPUs 3981da177e4SLinus Torvalds */ 3991da177e4SLinus Torvalds #define ACCT_THRESHOLD max(16, NR_CPUS * 2) 4001da177e4SLinus Torvalds 4011da177e4SLinus Torvalds static DEFINE_PER_CPU(long, committed_space) = 0; 4021da177e4SLinus Torvalds 4031da177e4SLinus Torvalds void vm_acct_memory(long pages) 4041da177e4SLinus Torvalds { 4051da177e4SLinus Torvalds long *local; 4061da177e4SLinus Torvalds 4071da177e4SLinus Torvalds preempt_disable(); 4081da177e4SLinus Torvalds local = &__get_cpu_var(committed_space); 4091da177e4SLinus Torvalds *local += pages; 4101da177e4SLinus Torvalds if (*local > ACCT_THRESHOLD || *local < -ACCT_THRESHOLD) { 4111da177e4SLinus Torvalds atomic_add(*local, &vm_committed_space); 4121da177e4SLinus Torvalds *local = 0; 4131da177e4SLinus Torvalds } 4141da177e4SLinus Torvalds preempt_enable(); 4151da177e4SLinus Torvalds } 4161da177e4SLinus Torvalds EXPORT_SYMBOL(vm_acct_memory); 4171da177e4SLinus Torvalds 4181da177e4SLinus Torvalds #ifdef CONFIG_HOTPLUG_CPU 4191da177e4SLinus Torvalds static void lru_drain_cache(unsigned int cpu) 4201da177e4SLinus Torvalds { 4211da177e4SLinus Torvalds struct pagevec *pvec = &per_cpu(lru_add_pvecs, cpu); 4221da177e4SLinus Torvalds 4231da177e4SLinus Torvalds /* CPU is dead, so no locking needed. */ 4241da177e4SLinus Torvalds if (pagevec_count(pvec)) 4251da177e4SLinus Torvalds __pagevec_lru_add(pvec); 4261da177e4SLinus Torvalds pvec = &per_cpu(lru_add_active_pvecs, cpu); 4271da177e4SLinus Torvalds if (pagevec_count(pvec)) 4281da177e4SLinus Torvalds __pagevec_lru_add_active(pvec); 4291da177e4SLinus Torvalds } 4301da177e4SLinus Torvalds 4311da177e4SLinus Torvalds /* Drop the CPU's cached committed space back into the central pool. */ 4321da177e4SLinus Torvalds static int cpu_swap_callback(struct notifier_block *nfb, 4331da177e4SLinus Torvalds unsigned long action, 4341da177e4SLinus Torvalds void *hcpu) 4351da177e4SLinus Torvalds { 4361da177e4SLinus Torvalds long *committed; 4371da177e4SLinus Torvalds 4381da177e4SLinus Torvalds committed = &per_cpu(committed_space, (long)hcpu); 4391da177e4SLinus Torvalds if (action == CPU_DEAD) { 4401da177e4SLinus Torvalds atomic_add(*committed, &vm_committed_space); 4411da177e4SLinus Torvalds *committed = 0; 4421da177e4SLinus Torvalds lru_drain_cache((long)hcpu); 4431da177e4SLinus Torvalds } 4441da177e4SLinus Torvalds return NOTIFY_OK; 4451da177e4SLinus Torvalds } 4461da177e4SLinus Torvalds #endif /* CONFIG_HOTPLUG_CPU */ 4471da177e4SLinus Torvalds #endif /* CONFIG_SMP */ 4481da177e4SLinus Torvalds 4491da177e4SLinus Torvalds #ifdef CONFIG_SMP 4501da177e4SLinus Torvalds void percpu_counter_mod(struct percpu_counter *fbc, long amount) 4511da177e4SLinus Torvalds { 4521da177e4SLinus Torvalds long count; 4531da177e4SLinus Torvalds long *pcount; 4541da177e4SLinus Torvalds int cpu = get_cpu(); 4551da177e4SLinus Torvalds 4561da177e4SLinus Torvalds pcount = per_cpu_ptr(fbc->counters, cpu); 4571da177e4SLinus Torvalds count = *pcount + amount; 4581da177e4SLinus Torvalds if (count >= FBC_BATCH || count <= -FBC_BATCH) { 4591da177e4SLinus Torvalds spin_lock(&fbc->lock); 4601da177e4SLinus Torvalds fbc->count += count; 4611da177e4SLinus Torvalds spin_unlock(&fbc->lock); 4621da177e4SLinus Torvalds count = 0; 4631da177e4SLinus Torvalds } 4641da177e4SLinus Torvalds *pcount = count; 4651da177e4SLinus Torvalds put_cpu(); 4661da177e4SLinus Torvalds } 4671da177e4SLinus Torvalds EXPORT_SYMBOL(percpu_counter_mod); 4681da177e4SLinus Torvalds #endif 4691da177e4SLinus Torvalds 4701da177e4SLinus Torvalds /* 4711da177e4SLinus Torvalds * Perform any setup for the swap system 4721da177e4SLinus Torvalds */ 4731da177e4SLinus Torvalds void __init swap_setup(void) 4741da177e4SLinus Torvalds { 4751da177e4SLinus Torvalds unsigned long megs = num_physpages >> (20 - PAGE_SHIFT); 4761da177e4SLinus Torvalds 4771da177e4SLinus Torvalds /* Use a smaller cluster for small-memory machines */ 4781da177e4SLinus Torvalds if (megs < 16) 4791da177e4SLinus Torvalds page_cluster = 2; 4801da177e4SLinus Torvalds else 4811da177e4SLinus Torvalds page_cluster = 3; 4821da177e4SLinus Torvalds /* 4831da177e4SLinus Torvalds * Right now other parts of the system means that we 4841da177e4SLinus Torvalds * _really_ don't want to cluster much more 4851da177e4SLinus Torvalds */ 4861da177e4SLinus Torvalds hotcpu_notifier(cpu_swap_callback, 0); 4871da177e4SLinus Torvalds } 488