1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * linux/mm/swap.c 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 61da177e4SLinus Torvalds */ 71da177e4SLinus Torvalds 81da177e4SLinus Torvalds /* 9183ff22bSSimon Arlott * This file contains the default values for the operation of the 101da177e4SLinus Torvalds * Linux VM subsystem. Fine-tuning documentation can be found in 1157043247SMauro Carvalho Chehab * Documentation/admin-guide/sysctl/vm.rst. 121da177e4SLinus Torvalds * Started 18.12.91 131da177e4SLinus Torvalds * Swap aging added 23.2.95, Stephen Tweedie. 141da177e4SLinus Torvalds * Buffermem limits added 12.3.98, Rik van Riel. 151da177e4SLinus Torvalds */ 161da177e4SLinus Torvalds 171da177e4SLinus Torvalds #include <linux/mm.h> 181da177e4SLinus Torvalds #include <linux/sched.h> 191da177e4SLinus Torvalds #include <linux/kernel_stat.h> 201da177e4SLinus Torvalds #include <linux/swap.h> 211da177e4SLinus Torvalds #include <linux/mman.h> 221da177e4SLinus Torvalds #include <linux/pagemap.h> 231da177e4SLinus Torvalds #include <linux/pagevec.h> 241da177e4SLinus Torvalds #include <linux/init.h> 25b95f1b31SPaul Gortmaker #include <linux/export.h> 261da177e4SLinus Torvalds #include <linux/mm_inline.h> 271da177e4SLinus Torvalds #include <linux/percpu_counter.h> 283565fce3SDan Williams #include <linux/memremap.h> 291da177e4SLinus Torvalds #include <linux/percpu.h> 301da177e4SLinus Torvalds #include <linux/cpu.h> 311da177e4SLinus Torvalds #include <linux/notifier.h> 32e0bf68ddSPeter Zijlstra #include <linux/backing-dev.h> 3366e1707bSBalbir Singh #include <linux/memcontrol.h> 345a0e3ad6STejun Heo #include <linux/gfp.h> 35a27bb332SKent Overstreet #include <linux/uio.h> 36822fc613SNaoya Horiguchi #include <linux/hugetlb.h> 3733c3fc71SVladimir Davydov #include <linux/page_idle.h> 38b01b2141SIngo Molnar #include <linux/local_lock.h> 398cc621d2SMinchan Kim #include <linux/buffer_head.h> 401da177e4SLinus Torvalds 4164d6519dSLee Schermerhorn #include "internal.h" 4264d6519dSLee Schermerhorn 43c6286c98SMel Gorman #define CREATE_TRACE_POINTS 44c6286c98SMel Gorman #include <trace/events/pagemap.h> 45c6286c98SMel Gorman 46ea0ffd0cSKairui Song /* How many pages do we try to swap or page in/out together? As a power of 2 */ 471da177e4SLinus Torvalds int page_cluster; 48ea0ffd0cSKairui Song const int page_cluster_max = 31; 491da177e4SLinus Torvalds 50c2bc1681SMatthew Wilcox (Oracle) /* Protecting only lru_rotate.fbatch which requires disabling interrupts */ 51b01b2141SIngo Molnar struct lru_rotate { 52b01b2141SIngo Molnar local_lock_t lock; 53c2bc1681SMatthew Wilcox (Oracle) struct folio_batch fbatch; 54b01b2141SIngo Molnar }; 55b01b2141SIngo Molnar static DEFINE_PER_CPU(struct lru_rotate, lru_rotate) = { 56b01b2141SIngo Molnar .lock = INIT_LOCAL_LOCK(lock), 57b01b2141SIngo Molnar }; 58b01b2141SIngo Molnar 59b01b2141SIngo Molnar /* 6082ac64d8SMatthew Wilcox (Oracle) * The following folio batches are grouped together because they are protected 61b01b2141SIngo Molnar * by disabling preemption (and interrupts remain enabled). 62b01b2141SIngo Molnar */ 6382ac64d8SMatthew Wilcox (Oracle) struct cpu_fbatches { 64b01b2141SIngo Molnar local_lock_t lock; 6570dea534SMatthew Wilcox (Oracle) struct folio_batch lru_add; 667a3dbfe8SMatthew Wilcox (Oracle) struct folio_batch lru_deactivate_file; 6785cd7791SMatthew Wilcox (Oracle) struct folio_batch lru_deactivate; 68cec394baSMatthew Wilcox (Oracle) struct folio_batch lru_lazyfree; 69a4a921aaSMing Li #ifdef CONFIG_SMP 703a44610bSMatthew Wilcox (Oracle) struct folio_batch activate; 71a4a921aaSMing Li #endif 72b01b2141SIngo Molnar }; 7382ac64d8SMatthew Wilcox (Oracle) static DEFINE_PER_CPU(struct cpu_fbatches, cpu_fbatches) = { 74b01b2141SIngo Molnar .lock = INIT_LOCAL_LOCK(lock), 75b01b2141SIngo Molnar }; 76902aaed0SHisashi Hifumi 77b221385bSAdrian Bunk /* 78b109b870SHugh Dickins * This path almost never happens for VM activity - pages are normally freed 791fec6890SMatthew Wilcox (Oracle) * in batches. But it gets used by networking - and for compound pages. 80b221385bSAdrian Bunk */ 81188e8caeSMatthew Wilcox (Oracle) static void __page_cache_release(struct folio *folio) 82b221385bSAdrian Bunk { 83188e8caeSMatthew Wilcox (Oracle) if (folio_test_lru(folio)) { 84fa9add64SHugh Dickins struct lruvec *lruvec; 85fa9add64SHugh Dickins unsigned long flags; 86b221385bSAdrian Bunk 87e809c3feSMatthew Wilcox (Oracle) lruvec = folio_lruvec_lock_irqsave(folio, &flags); 88188e8caeSMatthew Wilcox (Oracle) lruvec_del_folio(lruvec, folio); 89188e8caeSMatthew Wilcox (Oracle) __folio_clear_lru_flags(folio); 906168d0daSAlex Shi unlock_page_lruvec_irqrestore(lruvec, flags); 91b221385bSAdrian Bunk } 92*99fbb6bfSMatthew Wilcox (Oracle) /* See comment on folio_test_mlocked in folios_put() */ 93188e8caeSMatthew Wilcox (Oracle) if (unlikely(folio_test_mlocked(folio))) { 94188e8caeSMatthew Wilcox (Oracle) long nr_pages = folio_nr_pages(folio); 95b109b870SHugh Dickins 96188e8caeSMatthew Wilcox (Oracle) __folio_clear_mlocked(folio); 97188e8caeSMatthew Wilcox (Oracle) zone_stat_mod_folio(folio, NR_MLOCK, -nr_pages); 98b109b870SHugh Dickins count_vm_events(UNEVICTABLE_PGCLEARED, nr_pages); 99b109b870SHugh Dickins } 10091807063SAndrea Arcangeli } 10191807063SAndrea Arcangeli 10283d99659SMatthew Wilcox (Oracle) static void __folio_put_small(struct folio *folio) 10391807063SAndrea Arcangeli { 104188e8caeSMatthew Wilcox (Oracle) __page_cache_release(folio); 10583d99659SMatthew Wilcox (Oracle) mem_cgroup_uncharge(folio); 10683d99659SMatthew Wilcox (Oracle) free_unref_page(&folio->page, 0); 107b221385bSAdrian Bunk } 108b221385bSAdrian Bunk 1095ef82fe7SMatthew Wilcox (Oracle) static void __folio_put_large(struct folio *folio) 11091807063SAndrea Arcangeli { 111822fc613SNaoya Horiguchi /* 112822fc613SNaoya Horiguchi * __page_cache_release() is supposed to be called for thp, not for 113822fc613SNaoya Horiguchi * hugetlb. This is because hugetlb page does never have PageLRU set 114822fc613SNaoya Horiguchi * (it's never listed to any LRU lists) and no memcg routines should 115822fc613SNaoya Horiguchi * be called for hugetlb (it has a separate hugetlb_cgroup.) 116822fc613SNaoya Horiguchi */ 1175ef82fe7SMatthew Wilcox (Oracle) if (!folio_test_hugetlb(folio)) 118188e8caeSMatthew Wilcox (Oracle) __page_cache_release(folio); 1195375336cSMatthew Wilcox (Oracle) destroy_large_folio(folio); 12091807063SAndrea Arcangeli } 12191807063SAndrea Arcangeli 1228d29c703SMatthew Wilcox (Oracle) void __folio_put(struct folio *folio) 123c747ce79SJianyu Zhan { 1248d29c703SMatthew Wilcox (Oracle) if (unlikely(folio_is_zone_device(folio))) 1258d29c703SMatthew Wilcox (Oracle) free_zone_device_page(&folio->page); 1268d29c703SMatthew Wilcox (Oracle) else if (unlikely(folio_test_large(folio))) 1275ef82fe7SMatthew Wilcox (Oracle) __folio_put_large(folio); 12826296ad2SAndrew Morton else 12983d99659SMatthew Wilcox (Oracle) __folio_put_small(folio); 13026296ad2SAndrew Morton } 1318d29c703SMatthew Wilcox (Oracle) EXPORT_SYMBOL(__folio_put); 13270b50f94SAndrea Arcangeli 1331d7ea732SAlexander Zarochentsev /** 1347682486bSRandy Dunlap * put_pages_list() - release a list of pages 1357682486bSRandy Dunlap * @pages: list of pages threaded on page->lru 1361d7ea732SAlexander Zarochentsev * 137988c69f1SMatthew Wilcox (Oracle) * Release a list of pages which are strung together on page.lru. 1381d7ea732SAlexander Zarochentsev */ 1391d7ea732SAlexander Zarochentsev void put_pages_list(struct list_head *pages) 1401d7ea732SAlexander Zarochentsev { 1412f58e5deSMatthew Wilcox (Oracle) struct folio *folio, *next; 1421d7ea732SAlexander Zarochentsev 1432f58e5deSMatthew Wilcox (Oracle) list_for_each_entry_safe(folio, next, pages, lru) { 1442f58e5deSMatthew Wilcox (Oracle) if (!folio_put_testzero(folio)) { 1452f58e5deSMatthew Wilcox (Oracle) list_del(&folio->lru); 146988c69f1SMatthew Wilcox (Oracle) continue; 1471d7ea732SAlexander Zarochentsev } 1482f58e5deSMatthew Wilcox (Oracle) if (folio_test_large(folio)) { 1492f58e5deSMatthew Wilcox (Oracle) list_del(&folio->lru); 1505ef82fe7SMatthew Wilcox (Oracle) __folio_put_large(folio); 151988c69f1SMatthew Wilcox (Oracle) continue; 152988c69f1SMatthew Wilcox (Oracle) } 1532f58e5deSMatthew Wilcox (Oracle) /* LRU flag must be clear because it's passed using the lru */ 154988c69f1SMatthew Wilcox (Oracle) } 155988c69f1SMatthew Wilcox (Oracle) 156988c69f1SMatthew Wilcox (Oracle) free_unref_page_list(pages); 1573cd018b4SMatthew Wilcox INIT_LIST_HEAD(pages); 1581d7ea732SAlexander Zarochentsev } 1591d7ea732SAlexander Zarochentsev EXPORT_SYMBOL(put_pages_list); 1601d7ea732SAlexander Zarochentsev 161c2bc1681SMatthew Wilcox (Oracle) typedef void (*move_fn_t)(struct lruvec *lruvec, struct folio *folio); 162c2bc1681SMatthew Wilcox (Oracle) 16370dea534SMatthew Wilcox (Oracle) static void lru_add_fn(struct lruvec *lruvec, struct folio *folio) 1647d80dd09SMatthew Wilcox (Oracle) { 1657d80dd09SMatthew Wilcox (Oracle) int was_unevictable = folio_test_clear_unevictable(folio); 1667d80dd09SMatthew Wilcox (Oracle) long nr_pages = folio_nr_pages(folio); 1677d80dd09SMatthew Wilcox (Oracle) 1687d80dd09SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); 1697d80dd09SMatthew Wilcox (Oracle) 1707d80dd09SMatthew Wilcox (Oracle) /* 1717d80dd09SMatthew Wilcox (Oracle) * Is an smp_mb__after_atomic() still required here, before 172188e8caeSMatthew Wilcox (Oracle) * folio_evictable() tests the mlocked flag, to rule out the possibility 1737d80dd09SMatthew Wilcox (Oracle) * of stranding an evictable folio on an unevictable LRU? I think 174e0650a41SMatthew Wilcox (Oracle) * not, because __munlock_folio() only clears the mlocked flag 175188e8caeSMatthew Wilcox (Oracle) * while the LRU lock is held. 1767d80dd09SMatthew Wilcox (Oracle) * 1777d80dd09SMatthew Wilcox (Oracle) * (That is not true of __page_cache_release(), and not necessarily 178*99fbb6bfSMatthew Wilcox (Oracle) * true of folios_put(): but those only clear the mlocked flag after 179188e8caeSMatthew Wilcox (Oracle) * folio_put_testzero() has excluded any other users of the folio.) 1807d80dd09SMatthew Wilcox (Oracle) */ 1817d80dd09SMatthew Wilcox (Oracle) if (folio_evictable(folio)) { 1827d80dd09SMatthew Wilcox (Oracle) if (was_unevictable) 1837d80dd09SMatthew Wilcox (Oracle) __count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages); 1847d80dd09SMatthew Wilcox (Oracle) } else { 1857d80dd09SMatthew Wilcox (Oracle) folio_clear_active(folio); 1867d80dd09SMatthew Wilcox (Oracle) folio_set_unevictable(folio); 1877d80dd09SMatthew Wilcox (Oracle) /* 1887d80dd09SMatthew Wilcox (Oracle) * folio->mlock_count = !!folio_test_mlocked(folio)? 189e0650a41SMatthew Wilcox (Oracle) * But that leaves __mlock_folio() in doubt whether another 1907d80dd09SMatthew Wilcox (Oracle) * actor has already counted the mlock or not. Err on the 1917d80dd09SMatthew Wilcox (Oracle) * safe side, underestimate, let page reclaim fix it, rather 1927d80dd09SMatthew Wilcox (Oracle) * than leaving a page on the unevictable LRU indefinitely. 1937d80dd09SMatthew Wilcox (Oracle) */ 1947d80dd09SMatthew Wilcox (Oracle) folio->mlock_count = 0; 1957d80dd09SMatthew Wilcox (Oracle) if (!was_unevictable) 1967d80dd09SMatthew Wilcox (Oracle) __count_vm_events(UNEVICTABLE_PGCULLED, nr_pages); 1977d80dd09SMatthew Wilcox (Oracle) } 1987d80dd09SMatthew Wilcox (Oracle) 1997d80dd09SMatthew Wilcox (Oracle) lruvec_add_folio(lruvec, folio); 2007d80dd09SMatthew Wilcox (Oracle) trace_mm_lru_insertion(folio); 2017d80dd09SMatthew Wilcox (Oracle) } 2027d80dd09SMatthew Wilcox (Oracle) 203c2bc1681SMatthew Wilcox (Oracle) static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn) 204902aaed0SHisashi Hifumi { 205902aaed0SHisashi Hifumi int i; 2066168d0daSAlex Shi struct lruvec *lruvec = NULL; 2073dd7ae8eSShaohua Li unsigned long flags = 0; 208902aaed0SHisashi Hifumi 209c2bc1681SMatthew Wilcox (Oracle) for (i = 0; i < folio_batch_count(fbatch); i++) { 210c2bc1681SMatthew Wilcox (Oracle) struct folio *folio = fbatch->folios[i]; 2113dd7ae8eSShaohua Li 212c2bc1681SMatthew Wilcox (Oracle) /* block memcg migration while the folio moves between lru */ 21370dea534SMatthew Wilcox (Oracle) if (move_fn != lru_add_fn && !folio_test_clear_lru(folio)) 214fc574c23SAlex Shi continue; 215fc574c23SAlex Shi 2160de340cbSMatthew Wilcox (Oracle) lruvec = folio_lruvec_relock_irqsave(folio, lruvec, &flags); 217c2bc1681SMatthew Wilcox (Oracle) move_fn(lruvec, folio); 218fc574c23SAlex Shi 219c2bc1681SMatthew Wilcox (Oracle) folio_set_lru(folio); 2203dd7ae8eSShaohua Li } 221c2bc1681SMatthew Wilcox (Oracle) 2226168d0daSAlex Shi if (lruvec) 2236168d0daSAlex Shi unlock_page_lruvec_irqrestore(lruvec, flags); 224*99fbb6bfSMatthew Wilcox (Oracle) folios_put(fbatch); 2253dd7ae8eSShaohua Li } 2263dd7ae8eSShaohua Li 227c2bc1681SMatthew Wilcox (Oracle) static void folio_batch_add_and_move(struct folio_batch *fbatch, 228c2bc1681SMatthew Wilcox (Oracle) struct folio *folio, move_fn_t move_fn) 2293dd7ae8eSShaohua Li { 230c2bc1681SMatthew Wilcox (Oracle) if (folio_batch_add(fbatch, folio) && !folio_test_large(folio) && 231c2bc1681SMatthew Wilcox (Oracle) !lru_cache_disabled()) 232c2bc1681SMatthew Wilcox (Oracle) return; 233c2bc1681SMatthew Wilcox (Oracle) folio_batch_move_lru(fbatch, move_fn); 234c2bc1681SMatthew Wilcox (Oracle) } 235575ced1cSMatthew Wilcox (Oracle) 236c2bc1681SMatthew Wilcox (Oracle) static void lru_move_tail_fn(struct lruvec *lruvec, struct folio *folio) 237c2bc1681SMatthew Wilcox (Oracle) { 238575ced1cSMatthew Wilcox (Oracle) if (!folio_test_unevictable(folio)) { 239575ced1cSMatthew Wilcox (Oracle) lruvec_del_folio(lruvec, folio); 240575ced1cSMatthew Wilcox (Oracle) folio_clear_active(folio); 241575ced1cSMatthew Wilcox (Oracle) lruvec_add_folio_tail(lruvec, folio); 242575ced1cSMatthew Wilcox (Oracle) __count_vm_events(PGROTATED, folio_nr_pages(folio)); 243902aaed0SHisashi Hifumi } 244902aaed0SHisashi Hifumi } 2453dd7ae8eSShaohua Li 2463dd7ae8eSShaohua Li /* 247575ced1cSMatthew Wilcox (Oracle) * Writeback is about to end against a folio which has been marked for 248575ced1cSMatthew Wilcox (Oracle) * immediate reclaim. If it still appears to be reclaimable, move it 249575ced1cSMatthew Wilcox (Oracle) * to the tail of the inactive list. 250c7c7b80cSAlex Shi * 251575ced1cSMatthew Wilcox (Oracle) * folio_rotate_reclaimable() must disable IRQs, to prevent nasty races. 2521da177e4SLinus Torvalds */ 253575ced1cSMatthew Wilcox (Oracle) void folio_rotate_reclaimable(struct folio *folio) 2541da177e4SLinus Torvalds { 255575ced1cSMatthew Wilcox (Oracle) if (!folio_test_locked(folio) && !folio_test_dirty(folio) && 256575ced1cSMatthew Wilcox (Oracle) !folio_test_unevictable(folio) && folio_test_lru(folio)) { 257c2bc1681SMatthew Wilcox (Oracle) struct folio_batch *fbatch; 2581da177e4SLinus Torvalds unsigned long flags; 2591da177e4SLinus Torvalds 260575ced1cSMatthew Wilcox (Oracle) folio_get(folio); 261b01b2141SIngo Molnar local_lock_irqsave(&lru_rotate.lock, flags); 262c2bc1681SMatthew Wilcox (Oracle) fbatch = this_cpu_ptr(&lru_rotate.fbatch); 263c2bc1681SMatthew Wilcox (Oracle) folio_batch_add_and_move(fbatch, folio, lru_move_tail_fn); 264b01b2141SIngo Molnar local_unlock_irqrestore(&lru_rotate.lock, flags); 265ac6aadb2SMiklos Szeredi } 2661da177e4SLinus Torvalds } 2671da177e4SLinus Torvalds 2680538a82cSJohannes Weiner void lru_note_cost(struct lruvec *lruvec, bool file, 2690538a82cSJohannes Weiner unsigned int nr_io, unsigned int nr_rotated) 2703e2f41f1SKOSAKI Motohiro { 2710538a82cSJohannes Weiner unsigned long cost; 2720538a82cSJohannes Weiner 2730538a82cSJohannes Weiner /* 2740538a82cSJohannes Weiner * Reflect the relative cost of incurring IO and spending CPU 2750538a82cSJohannes Weiner * time on rotations. This doesn't attempt to make a precise 2760538a82cSJohannes Weiner * comparison, it just says: if reloads are about comparable 2770538a82cSJohannes Weiner * between the LRU lists, or rotations are overwhelmingly 2780538a82cSJohannes Weiner * different between them, adjust scan balance for CPU work. 2790538a82cSJohannes Weiner */ 2800538a82cSJohannes Weiner cost = nr_io * SWAP_CLUSTER_MAX + nr_rotated; 2810538a82cSJohannes Weiner 2827cf111bcSJohannes Weiner do { 2837cf111bcSJohannes Weiner unsigned long lrusize; 2847cf111bcSJohannes Weiner 2856168d0daSAlex Shi /* 2866168d0daSAlex Shi * Hold lruvec->lru_lock is safe here, since 2876168d0daSAlex Shi * 1) The pinned lruvec in reclaim, or 2886168d0daSAlex Shi * 2) From a pre-LRU page during refault (which also holds the 2896168d0daSAlex Shi * rcu lock, so would be safe even if the page was on the LRU 2906168d0daSAlex Shi * and could move simultaneously to a new lruvec). 2916168d0daSAlex Shi */ 2926168d0daSAlex Shi spin_lock_irq(&lruvec->lru_lock); 2937cf111bcSJohannes Weiner /* Record cost event */ 29496f8bf4fSJohannes Weiner if (file) 2950538a82cSJohannes Weiner lruvec->file_cost += cost; 2961431d4d1SJohannes Weiner else 2970538a82cSJohannes Weiner lruvec->anon_cost += cost; 2987cf111bcSJohannes Weiner 2997cf111bcSJohannes Weiner /* 3007cf111bcSJohannes Weiner * Decay previous events 3017cf111bcSJohannes Weiner * 3027cf111bcSJohannes Weiner * Because workloads change over time (and to avoid 3037cf111bcSJohannes Weiner * overflow) we keep these statistics as a floating 3047cf111bcSJohannes Weiner * average, which ends up weighing recent refaults 3057cf111bcSJohannes Weiner * more than old ones. 3067cf111bcSJohannes Weiner */ 3077cf111bcSJohannes Weiner lrusize = lruvec_page_state(lruvec, NR_INACTIVE_ANON) + 3087cf111bcSJohannes Weiner lruvec_page_state(lruvec, NR_ACTIVE_ANON) + 3097cf111bcSJohannes Weiner lruvec_page_state(lruvec, NR_INACTIVE_FILE) + 3107cf111bcSJohannes Weiner lruvec_page_state(lruvec, NR_ACTIVE_FILE); 3117cf111bcSJohannes Weiner 3127cf111bcSJohannes Weiner if (lruvec->file_cost + lruvec->anon_cost > lrusize / 4) { 3137cf111bcSJohannes Weiner lruvec->file_cost /= 2; 3147cf111bcSJohannes Weiner lruvec->anon_cost /= 2; 3157cf111bcSJohannes Weiner } 3166168d0daSAlex Shi spin_unlock_irq(&lruvec->lru_lock); 3177cf111bcSJohannes Weiner } while ((lruvec = parent_lruvec(lruvec))); 3183e2f41f1SKOSAKI Motohiro } 3193e2f41f1SKOSAKI Motohiro 3200538a82cSJohannes Weiner void lru_note_cost_refault(struct folio *folio) 32196f8bf4fSJohannes Weiner { 3220995d7e5SMatthew Wilcox (Oracle) lru_note_cost(folio_lruvec(folio), folio_is_file_lru(folio), 3230538a82cSJohannes Weiner folio_nr_pages(folio), 0); 32496f8bf4fSJohannes Weiner } 32596f8bf4fSJohannes Weiner 3263a44610bSMatthew Wilcox (Oracle) static void folio_activate_fn(struct lruvec *lruvec, struct folio *folio) 327744ed144SShaohua Li { 328f2d27392SMatthew Wilcox (Oracle) if (!folio_test_active(folio) && !folio_test_unevictable(folio)) { 329f2d27392SMatthew Wilcox (Oracle) long nr_pages = folio_nr_pages(folio); 330744ed144SShaohua Li 331f2d27392SMatthew Wilcox (Oracle) lruvec_del_folio(lruvec, folio); 332f2d27392SMatthew Wilcox (Oracle) folio_set_active(folio); 333f2d27392SMatthew Wilcox (Oracle) lruvec_add_folio(lruvec, folio); 334f2d27392SMatthew Wilcox (Oracle) trace_mm_lru_activate(folio); 3357a608572SLinus Torvalds 33621e330fcSShakeel Butt __count_vm_events(PGACTIVATE, nr_pages); 33721e330fcSShakeel Butt __count_memcg_events(lruvec_memcg(lruvec), PGACTIVATE, 33821e330fcSShakeel Butt nr_pages); 339744ed144SShaohua Li } 340eb709b0dSShaohua Li } 341eb709b0dSShaohua Li 342eb709b0dSShaohua Li #ifdef CONFIG_SMP 3433a44610bSMatthew Wilcox (Oracle) static void folio_activate_drain(int cpu) 344f2d27392SMatthew Wilcox (Oracle) { 34582ac64d8SMatthew Wilcox (Oracle) struct folio_batch *fbatch = &per_cpu(cpu_fbatches.activate, cpu); 346f2d27392SMatthew Wilcox (Oracle) 3473a44610bSMatthew Wilcox (Oracle) if (folio_batch_count(fbatch)) 3483a44610bSMatthew Wilcox (Oracle) folio_batch_move_lru(fbatch, folio_activate_fn); 3495fbc4616SChris Metcalf } 3505fbc4616SChris Metcalf 351018ee47fSYu Zhao void folio_activate(struct folio *folio) 352eb709b0dSShaohua Li { 353f2d27392SMatthew Wilcox (Oracle) if (folio_test_lru(folio) && !folio_test_active(folio) && 354f2d27392SMatthew Wilcox (Oracle) !folio_test_unevictable(folio)) { 3553a44610bSMatthew Wilcox (Oracle) struct folio_batch *fbatch; 356eb709b0dSShaohua Li 357f2d27392SMatthew Wilcox (Oracle) folio_get(folio); 35882ac64d8SMatthew Wilcox (Oracle) local_lock(&cpu_fbatches.lock); 35982ac64d8SMatthew Wilcox (Oracle) fbatch = this_cpu_ptr(&cpu_fbatches.activate); 3603a44610bSMatthew Wilcox (Oracle) folio_batch_add_and_move(fbatch, folio, folio_activate_fn); 36182ac64d8SMatthew Wilcox (Oracle) local_unlock(&cpu_fbatches.lock); 362eb709b0dSShaohua Li } 363eb709b0dSShaohua Li } 364eb709b0dSShaohua Li 365eb709b0dSShaohua Li #else 3663a44610bSMatthew Wilcox (Oracle) static inline void folio_activate_drain(int cpu) 367eb709b0dSShaohua Li { 368eb709b0dSShaohua Li } 369eb709b0dSShaohua Li 370018ee47fSYu Zhao void folio_activate(struct folio *folio) 371eb709b0dSShaohua Li { 3726168d0daSAlex Shi struct lruvec *lruvec; 373eb709b0dSShaohua Li 374f2d27392SMatthew Wilcox (Oracle) if (folio_test_clear_lru(folio)) { 375e809c3feSMatthew Wilcox (Oracle) lruvec = folio_lruvec_lock_irq(folio); 3763a44610bSMatthew Wilcox (Oracle) folio_activate_fn(lruvec, folio); 3776168d0daSAlex Shi unlock_page_lruvec_irq(lruvec); 378f2d27392SMatthew Wilcox (Oracle) folio_set_lru(folio); 3796168d0daSAlex Shi } 3801da177e4SLinus Torvalds } 381eb709b0dSShaohua Li #endif 3821da177e4SLinus Torvalds 38376580b65SMatthew Wilcox (Oracle) static void __lru_cache_activate_folio(struct folio *folio) 384059285a2SMel Gorman { 38570dea534SMatthew Wilcox (Oracle) struct folio_batch *fbatch; 386059285a2SMel Gorman int i; 387059285a2SMel Gorman 38882ac64d8SMatthew Wilcox (Oracle) local_lock(&cpu_fbatches.lock); 38982ac64d8SMatthew Wilcox (Oracle) fbatch = this_cpu_ptr(&cpu_fbatches.lru_add); 390b01b2141SIngo Molnar 391059285a2SMel Gorman /* 39270dea534SMatthew Wilcox (Oracle) * Search backwards on the optimistic assumption that the folio being 39370dea534SMatthew Wilcox (Oracle) * activated has just been added to this batch. Note that only 39470dea534SMatthew Wilcox (Oracle) * the local batch is examined as a !LRU folio could be in the 395059285a2SMel Gorman * process of being released, reclaimed, migrated or on a remote 39670dea534SMatthew Wilcox (Oracle) * batch that is currently being drained. Furthermore, marking 39770dea534SMatthew Wilcox (Oracle) * a remote batch's folio active potentially hits a race where 39870dea534SMatthew Wilcox (Oracle) * a folio is marked active just after it is added to the inactive 399059285a2SMel Gorman * list causing accounting errors and BUG_ON checks to trigger. 400059285a2SMel Gorman */ 40170dea534SMatthew Wilcox (Oracle) for (i = folio_batch_count(fbatch) - 1; i >= 0; i--) { 40270dea534SMatthew Wilcox (Oracle) struct folio *batch_folio = fbatch->folios[i]; 403059285a2SMel Gorman 40470dea534SMatthew Wilcox (Oracle) if (batch_folio == folio) { 40576580b65SMatthew Wilcox (Oracle) folio_set_active(folio); 406059285a2SMel Gorman break; 407059285a2SMel Gorman } 408059285a2SMel Gorman } 409059285a2SMel Gorman 41082ac64d8SMatthew Wilcox (Oracle) local_unlock(&cpu_fbatches.lock); 411059285a2SMel Gorman } 412059285a2SMel Gorman 413ac35a490SYu Zhao #ifdef CONFIG_LRU_GEN 414ac35a490SYu Zhao static void folio_inc_refs(struct folio *folio) 415ac35a490SYu Zhao { 416ac35a490SYu Zhao unsigned long new_flags, old_flags = READ_ONCE(folio->flags); 417ac35a490SYu Zhao 418ac35a490SYu Zhao if (folio_test_unevictable(folio)) 419ac35a490SYu Zhao return; 420ac35a490SYu Zhao 421ac35a490SYu Zhao if (!folio_test_referenced(folio)) { 422ac35a490SYu Zhao folio_set_referenced(folio); 423ac35a490SYu Zhao return; 424ac35a490SYu Zhao } 425ac35a490SYu Zhao 426ac35a490SYu Zhao if (!folio_test_workingset(folio)) { 427ac35a490SYu Zhao folio_set_workingset(folio); 428ac35a490SYu Zhao return; 429ac35a490SYu Zhao } 430ac35a490SYu Zhao 431ac35a490SYu Zhao /* see the comment on MAX_NR_TIERS */ 432ac35a490SYu Zhao do { 433ac35a490SYu Zhao new_flags = old_flags & LRU_REFS_MASK; 434ac35a490SYu Zhao if (new_flags == LRU_REFS_MASK) 435ac35a490SYu Zhao break; 436ac35a490SYu Zhao 437ac35a490SYu Zhao new_flags += BIT(LRU_REFS_PGOFF); 438ac35a490SYu Zhao new_flags |= old_flags & ~LRU_REFS_MASK; 439ac35a490SYu Zhao } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); 440ac35a490SYu Zhao } 441ac35a490SYu Zhao #else 442ac35a490SYu Zhao static void folio_inc_refs(struct folio *folio) 443ac35a490SYu Zhao { 444ac35a490SYu Zhao } 445ac35a490SYu Zhao #endif /* CONFIG_LRU_GEN */ 446ac35a490SYu Zhao 4471da177e4SLinus Torvalds /* 4481da177e4SLinus Torvalds * Mark a page as having seen activity. 4491da177e4SLinus Torvalds * 4501da177e4SLinus Torvalds * inactive,unreferenced -> inactive,referenced 4511da177e4SLinus Torvalds * inactive,referenced -> active,unreferenced 4521da177e4SLinus Torvalds * active,unreferenced -> active,referenced 453eb39d618SHugh Dickins * 454eb39d618SHugh Dickins * When a newly allocated page is not yet visible, so safe for non-atomic ops, 455eb39d618SHugh Dickins * __SetPageReferenced(page) may be substituted for mark_page_accessed(page). 4561da177e4SLinus Torvalds */ 45776580b65SMatthew Wilcox (Oracle) void folio_mark_accessed(struct folio *folio) 4581da177e4SLinus Torvalds { 459ac35a490SYu Zhao if (lru_gen_enabled()) { 460ac35a490SYu Zhao folio_inc_refs(folio); 461ac35a490SYu Zhao return; 462ac35a490SYu Zhao } 463ac35a490SYu Zhao 46476580b65SMatthew Wilcox (Oracle) if (!folio_test_referenced(folio)) { 46576580b65SMatthew Wilcox (Oracle) folio_set_referenced(folio); 46676580b65SMatthew Wilcox (Oracle) } else if (folio_test_unevictable(folio)) { 467a1100a74SFengguang Wu /* 468a1100a74SFengguang Wu * Unevictable pages are on the "LRU_UNEVICTABLE" list. But, 469a1100a74SFengguang Wu * this list is never rotated or maintained, so marking an 470914c32e4SBang Li * unevictable page accessed has no effect. 471a1100a74SFengguang Wu */ 47276580b65SMatthew Wilcox (Oracle) } else if (!folio_test_active(folio)) { 473059285a2SMel Gorman /* 4743a44610bSMatthew Wilcox (Oracle) * If the folio is on the LRU, queue it for activation via 47582ac64d8SMatthew Wilcox (Oracle) * cpu_fbatches.activate. Otherwise, assume the folio is in a 4763a44610bSMatthew Wilcox (Oracle) * folio_batch, mark it active and it'll be moved to the active 477059285a2SMel Gorman * LRU on the next drain. 478059285a2SMel Gorman */ 47976580b65SMatthew Wilcox (Oracle) if (folio_test_lru(folio)) 48076580b65SMatthew Wilcox (Oracle) folio_activate(folio); 481059285a2SMel Gorman else 48276580b65SMatthew Wilcox (Oracle) __lru_cache_activate_folio(folio); 48376580b65SMatthew Wilcox (Oracle) folio_clear_referenced(folio); 48476580b65SMatthew Wilcox (Oracle) workingset_activation(folio); 4851da177e4SLinus Torvalds } 48676580b65SMatthew Wilcox (Oracle) if (folio_test_idle(folio)) 48776580b65SMatthew Wilcox (Oracle) folio_clear_idle(folio); 4881da177e4SLinus Torvalds } 48976580b65SMatthew Wilcox (Oracle) EXPORT_SYMBOL(folio_mark_accessed); 4901da177e4SLinus Torvalds 491f04e9ebbSKOSAKI Motohiro /** 4920d31125dSMatthew Wilcox (Oracle) * folio_add_lru - Add a folio to an LRU list. 4930d31125dSMatthew Wilcox (Oracle) * @folio: The folio to be added to the LRU. 4942329d375SJianyu Zhan * 4950d31125dSMatthew Wilcox (Oracle) * Queue the folio for addition to the LRU. The decision on whether 4962329d375SJianyu Zhan * to add the page to the [in]active [file|anon] list is deferred until the 49782ac64d8SMatthew Wilcox (Oracle) * folio_batch is drained. This gives a chance for the caller of folio_add_lru() 4980d31125dSMatthew Wilcox (Oracle) * have the folio added to the active list using folio_mark_accessed(). 499f04e9ebbSKOSAKI Motohiro */ 5000d31125dSMatthew Wilcox (Oracle) void folio_add_lru(struct folio *folio) 5011da177e4SLinus Torvalds { 50270dea534SMatthew Wilcox (Oracle) struct folio_batch *fbatch; 5036058eaecSJohannes Weiner 50470dea534SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(folio_test_active(folio) && 50570dea534SMatthew Wilcox (Oracle) folio_test_unevictable(folio), folio); 5060d31125dSMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); 5076058eaecSJohannes Weiner 508ec1c86b2SYu Zhao /* see the comment in lru_gen_add_folio() */ 509ec1c86b2SYu Zhao if (lru_gen_enabled() && !folio_test_unevictable(folio) && 510ec1c86b2SYu Zhao lru_gen_in_fault() && !(current->flags & PF_MEMALLOC)) 511ec1c86b2SYu Zhao folio_set_active(folio); 512ec1c86b2SYu Zhao 5130d31125dSMatthew Wilcox (Oracle) folio_get(folio); 51482ac64d8SMatthew Wilcox (Oracle) local_lock(&cpu_fbatches.lock); 51582ac64d8SMatthew Wilcox (Oracle) fbatch = this_cpu_ptr(&cpu_fbatches.lru_add); 51670dea534SMatthew Wilcox (Oracle) folio_batch_add_and_move(fbatch, folio, lru_add_fn); 51782ac64d8SMatthew Wilcox (Oracle) local_unlock(&cpu_fbatches.lock); 5181da177e4SLinus Torvalds } 5190d31125dSMatthew Wilcox (Oracle) EXPORT_SYMBOL(folio_add_lru); 5201da177e4SLinus Torvalds 521894bc310SLee Schermerhorn /** 522681ecf63SMatthew Wilcox (Oracle) * folio_add_lru_vma() - Add a folio to the appropate LRU list for this VMA. 523681ecf63SMatthew Wilcox (Oracle) * @folio: The folio to be added to the LRU. 524681ecf63SMatthew Wilcox (Oracle) * @vma: VMA in which the folio is mapped. 52500501b53SJohannes Weiner * 526681ecf63SMatthew Wilcox (Oracle) * If the VMA is mlocked, @folio is added to the unevictable list. 527681ecf63SMatthew Wilcox (Oracle) * Otherwise, it is treated the same way as folio_add_lru(). 52800501b53SJohannes Weiner */ 529681ecf63SMatthew Wilcox (Oracle) void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma) 53000501b53SJohannes Weiner { 531681ecf63SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); 53200501b53SJohannes Weiner 5332fbb0c10SHugh Dickins if (unlikely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED)) 53496f97c43SLorenzo Stoakes mlock_new_folio(folio); 5352fbb0c10SHugh Dickins else 536681ecf63SMatthew Wilcox (Oracle) folio_add_lru(folio); 53700501b53SJohannes Weiner } 53800501b53SJohannes Weiner 539902aaed0SHisashi Hifumi /* 5407a3dbfe8SMatthew Wilcox (Oracle) * If the folio cannot be invalidated, it is moved to the 54131560180SMinchan Kim * inactive list to speed up its reclaim. It is moved to the 54231560180SMinchan Kim * head of the list, rather than the tail, to give the flusher 54331560180SMinchan Kim * threads some time to write it out, as this is much more 54431560180SMinchan Kim * effective than the single-page writeout from reclaim. 545278df9f4SMinchan Kim * 5467a3dbfe8SMatthew Wilcox (Oracle) * If the folio isn't mapped and dirty/writeback, the folio 5477a3dbfe8SMatthew Wilcox (Oracle) * could be reclaimed asap using the reclaim flag. 548278df9f4SMinchan Kim * 5497a3dbfe8SMatthew Wilcox (Oracle) * 1. active, mapped folio -> none 5507a3dbfe8SMatthew Wilcox (Oracle) * 2. active, dirty/writeback folio -> inactive, head, reclaim 5517a3dbfe8SMatthew Wilcox (Oracle) * 3. inactive, mapped folio -> none 5527a3dbfe8SMatthew Wilcox (Oracle) * 4. inactive, dirty/writeback folio -> inactive, head, reclaim 553278df9f4SMinchan Kim * 5. inactive, clean -> inactive, tail 554278df9f4SMinchan Kim * 6. Others -> none 555278df9f4SMinchan Kim * 5567a3dbfe8SMatthew Wilcox (Oracle) * In 4, it moves to the head of the inactive list so the folio is 5577a3dbfe8SMatthew Wilcox (Oracle) * written out by flusher threads as this is much more efficient 558278df9f4SMinchan Kim * than the single-page writeout from reclaim. 55931560180SMinchan Kim */ 5607a3dbfe8SMatthew Wilcox (Oracle) static void lru_deactivate_file_fn(struct lruvec *lruvec, struct folio *folio) 56131560180SMinchan Kim { 5627a3dbfe8SMatthew Wilcox (Oracle) bool active = folio_test_active(folio); 5637a3dbfe8SMatthew Wilcox (Oracle) long nr_pages = folio_nr_pages(folio); 56431560180SMinchan Kim 5657a3dbfe8SMatthew Wilcox (Oracle) if (folio_test_unevictable(folio)) 566bad49d9cSMinchan Kim return; 567bad49d9cSMinchan Kim 5687a3dbfe8SMatthew Wilcox (Oracle) /* Some processes are using the folio */ 5697a3dbfe8SMatthew Wilcox (Oracle) if (folio_mapped(folio)) 57031560180SMinchan Kim return; 57131560180SMinchan Kim 5727a3dbfe8SMatthew Wilcox (Oracle) lruvec_del_folio(lruvec, folio); 5737a3dbfe8SMatthew Wilcox (Oracle) folio_clear_active(folio); 5747a3dbfe8SMatthew Wilcox (Oracle) folio_clear_referenced(folio); 57531560180SMinchan Kim 5767a3dbfe8SMatthew Wilcox (Oracle) if (folio_test_writeback(folio) || folio_test_dirty(folio)) { 577278df9f4SMinchan Kim /* 5787a3dbfe8SMatthew Wilcox (Oracle) * Setting the reclaim flag could race with 5797a3dbfe8SMatthew Wilcox (Oracle) * folio_end_writeback() and confuse readahead. But the 5807a3dbfe8SMatthew Wilcox (Oracle) * race window is _really_ small and it's not a critical 5817a3dbfe8SMatthew Wilcox (Oracle) * problem. 582278df9f4SMinchan Kim */ 5837a3dbfe8SMatthew Wilcox (Oracle) lruvec_add_folio(lruvec, folio); 5847a3dbfe8SMatthew Wilcox (Oracle) folio_set_reclaim(folio); 585278df9f4SMinchan Kim } else { 586278df9f4SMinchan Kim /* 5877a3dbfe8SMatthew Wilcox (Oracle) * The folio's writeback ended while it was in the batch. 5887a3dbfe8SMatthew Wilcox (Oracle) * We move that folio to the tail of the inactive list. 589278df9f4SMinchan Kim */ 5907a3dbfe8SMatthew Wilcox (Oracle) lruvec_add_folio_tail(lruvec, folio); 5915d91f31fSShakeel Butt __count_vm_events(PGROTATED, nr_pages); 592278df9f4SMinchan Kim } 593278df9f4SMinchan Kim 59421e330fcSShakeel Butt if (active) { 5955d91f31fSShakeel Butt __count_vm_events(PGDEACTIVATE, nr_pages); 59621e330fcSShakeel Butt __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, 59721e330fcSShakeel Butt nr_pages); 59821e330fcSShakeel Butt } 59931560180SMinchan Kim } 60031560180SMinchan Kim 60185cd7791SMatthew Wilcox (Oracle) static void lru_deactivate_fn(struct lruvec *lruvec, struct folio *folio) 6029c276cc6SMinchan Kim { 603ec1c86b2SYu Zhao if (!folio_test_unevictable(folio) && (folio_test_active(folio) || lru_gen_enabled())) { 60485cd7791SMatthew Wilcox (Oracle) long nr_pages = folio_nr_pages(folio); 6059c276cc6SMinchan Kim 60685cd7791SMatthew Wilcox (Oracle) lruvec_del_folio(lruvec, folio); 60785cd7791SMatthew Wilcox (Oracle) folio_clear_active(folio); 60885cd7791SMatthew Wilcox (Oracle) folio_clear_referenced(folio); 60985cd7791SMatthew Wilcox (Oracle) lruvec_add_folio(lruvec, folio); 6109c276cc6SMinchan Kim 61121e330fcSShakeel Butt __count_vm_events(PGDEACTIVATE, nr_pages); 61221e330fcSShakeel Butt __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, 61321e330fcSShakeel Butt nr_pages); 6149c276cc6SMinchan Kim } 6159c276cc6SMinchan Kim } 61610853a03SMinchan Kim 617cec394baSMatthew Wilcox (Oracle) static void lru_lazyfree_fn(struct lruvec *lruvec, struct folio *folio) 61810853a03SMinchan Kim { 619cec394baSMatthew Wilcox (Oracle) if (folio_test_anon(folio) && folio_test_swapbacked(folio) && 620cec394baSMatthew Wilcox (Oracle) !folio_test_swapcache(folio) && !folio_test_unevictable(folio)) { 621cec394baSMatthew Wilcox (Oracle) long nr_pages = folio_nr_pages(folio); 62210853a03SMinchan Kim 623cec394baSMatthew Wilcox (Oracle) lruvec_del_folio(lruvec, folio); 624cec394baSMatthew Wilcox (Oracle) folio_clear_active(folio); 625cec394baSMatthew Wilcox (Oracle) folio_clear_referenced(folio); 626f7ad2a6cSShaohua Li /* 627cec394baSMatthew Wilcox (Oracle) * Lazyfree folios are clean anonymous folios. They have 628cec394baSMatthew Wilcox (Oracle) * the swapbacked flag cleared, to distinguish them from normal 629cec394baSMatthew Wilcox (Oracle) * anonymous folios 630f7ad2a6cSShaohua Li */ 631cec394baSMatthew Wilcox (Oracle) folio_clear_swapbacked(folio); 632cec394baSMatthew Wilcox (Oracle) lruvec_add_folio(lruvec, folio); 63310853a03SMinchan Kim 63421e330fcSShakeel Butt __count_vm_events(PGLAZYFREE, nr_pages); 63521e330fcSShakeel Butt __count_memcg_events(lruvec_memcg(lruvec), PGLAZYFREE, 63621e330fcSShakeel Butt nr_pages); 63710853a03SMinchan Kim } 63810853a03SMinchan Kim } 63910853a03SMinchan Kim 64031560180SMinchan Kim /* 64182ac64d8SMatthew Wilcox (Oracle) * Drain pages out of the cpu's folio_batch. 642902aaed0SHisashi Hifumi * Either "cpu" is the current CPU, and preemption has already been 643902aaed0SHisashi Hifumi * disabled; or "cpu" is being hot-unplugged, and is already dead. 644902aaed0SHisashi Hifumi */ 645f0cb3c76SKonstantin Khlebnikov void lru_add_drain_cpu(int cpu) 6461da177e4SLinus Torvalds { 647a2d33b5dSMatthew Wilcox (Oracle) struct cpu_fbatches *fbatches = &per_cpu(cpu_fbatches, cpu); 648a2d33b5dSMatthew Wilcox (Oracle) struct folio_batch *fbatch = &fbatches->lru_add; 6491da177e4SLinus Torvalds 65070dea534SMatthew Wilcox (Oracle) if (folio_batch_count(fbatch)) 65170dea534SMatthew Wilcox (Oracle) folio_batch_move_lru(fbatch, lru_add_fn); 652902aaed0SHisashi Hifumi 653c2bc1681SMatthew Wilcox (Oracle) fbatch = &per_cpu(lru_rotate.fbatch, cpu); 6547e0cc01eSQian Cai /* Disabling interrupts below acts as a compiler barrier. */ 655c2bc1681SMatthew Wilcox (Oracle) if (data_race(folio_batch_count(fbatch))) { 656902aaed0SHisashi Hifumi unsigned long flags; 657902aaed0SHisashi Hifumi 658902aaed0SHisashi Hifumi /* No harm done if a racing interrupt already did this */ 659b01b2141SIngo Molnar local_lock_irqsave(&lru_rotate.lock, flags); 660c2bc1681SMatthew Wilcox (Oracle) folio_batch_move_lru(fbatch, lru_move_tail_fn); 661b01b2141SIngo Molnar local_unlock_irqrestore(&lru_rotate.lock, flags); 662902aaed0SHisashi Hifumi } 66331560180SMinchan Kim 664a2d33b5dSMatthew Wilcox (Oracle) fbatch = &fbatches->lru_deactivate_file; 6657a3dbfe8SMatthew Wilcox (Oracle) if (folio_batch_count(fbatch)) 6667a3dbfe8SMatthew Wilcox (Oracle) folio_batch_move_lru(fbatch, lru_deactivate_file_fn); 667eb709b0dSShaohua Li 668a2d33b5dSMatthew Wilcox (Oracle) fbatch = &fbatches->lru_deactivate; 66985cd7791SMatthew Wilcox (Oracle) if (folio_batch_count(fbatch)) 67085cd7791SMatthew Wilcox (Oracle) folio_batch_move_lru(fbatch, lru_deactivate_fn); 6719c276cc6SMinchan Kim 672a2d33b5dSMatthew Wilcox (Oracle) fbatch = &fbatches->lru_lazyfree; 673cec394baSMatthew Wilcox (Oracle) if (folio_batch_count(fbatch)) 674cec394baSMatthew Wilcox (Oracle) folio_batch_move_lru(fbatch, lru_lazyfree_fn); 67510853a03SMinchan Kim 6763a44610bSMatthew Wilcox (Oracle) folio_activate_drain(cpu); 67731560180SMinchan Kim } 67831560180SMinchan Kim 67931560180SMinchan Kim /** 6807a3dbfe8SMatthew Wilcox (Oracle) * deactivate_file_folio() - Deactivate a file folio. 681261b6840SMatthew Wilcox (Oracle) * @folio: Folio to deactivate. 68231560180SMinchan Kim * 683261b6840SMatthew Wilcox (Oracle) * This function hints to the VM that @folio is a good reclaim candidate, 684261b6840SMatthew Wilcox (Oracle) * for example if its invalidation fails due to the folio being dirty 68531560180SMinchan Kim * or under writeback. 686261b6840SMatthew Wilcox (Oracle) * 6877a3dbfe8SMatthew Wilcox (Oracle) * Context: Caller holds a reference on the folio. 68831560180SMinchan Kim */ 689261b6840SMatthew Wilcox (Oracle) void deactivate_file_folio(struct folio *folio) 69031560180SMinchan Kim { 6917a3dbfe8SMatthew Wilcox (Oracle) struct folio_batch *fbatch; 692b01b2141SIngo Molnar 6937a3dbfe8SMatthew Wilcox (Oracle) /* Deactivating an unevictable folio will not accelerate reclaim */ 694261b6840SMatthew Wilcox (Oracle) if (folio_test_unevictable(folio)) 695261b6840SMatthew Wilcox (Oracle) return; 696261b6840SMatthew Wilcox (Oracle) 697261b6840SMatthew Wilcox (Oracle) folio_get(folio); 69882ac64d8SMatthew Wilcox (Oracle) local_lock(&cpu_fbatches.lock); 69982ac64d8SMatthew Wilcox (Oracle) fbatch = this_cpu_ptr(&cpu_fbatches.lru_deactivate_file); 7007a3dbfe8SMatthew Wilcox (Oracle) folio_batch_add_and_move(fbatch, folio, lru_deactivate_file_fn); 70182ac64d8SMatthew Wilcox (Oracle) local_unlock(&cpu_fbatches.lock); 70231560180SMinchan Kim } 70380bfed90SAndrew Morton 7049c276cc6SMinchan Kim /* 7055a9e3474SVishal Moola (Oracle) * folio_deactivate - deactivate a folio 7065a9e3474SVishal Moola (Oracle) * @folio: folio to deactivate 7079c276cc6SMinchan Kim * 7085a9e3474SVishal Moola (Oracle) * folio_deactivate() moves @folio to the inactive list if @folio was on the 7095a9e3474SVishal Moola (Oracle) * active list and was not unevictable. This is done to accelerate the 7105a9e3474SVishal Moola (Oracle) * reclaim of @folio. 7119c276cc6SMinchan Kim */ 7125a9e3474SVishal Moola (Oracle) void folio_deactivate(struct folio *folio) 7139c276cc6SMinchan Kim { 714ec1c86b2SYu Zhao if (folio_test_lru(folio) && !folio_test_unevictable(folio) && 715ec1c86b2SYu Zhao (folio_test_active(folio) || lru_gen_enabled())) { 71685cd7791SMatthew Wilcox (Oracle) struct folio_batch *fbatch; 71785cd7791SMatthew Wilcox (Oracle) 71885cd7791SMatthew Wilcox (Oracle) folio_get(folio); 71982ac64d8SMatthew Wilcox (Oracle) local_lock(&cpu_fbatches.lock); 72082ac64d8SMatthew Wilcox (Oracle) fbatch = this_cpu_ptr(&cpu_fbatches.lru_deactivate); 72185cd7791SMatthew Wilcox (Oracle) folio_batch_add_and_move(fbatch, folio, lru_deactivate_fn); 72282ac64d8SMatthew Wilcox (Oracle) local_unlock(&cpu_fbatches.lock); 7239c276cc6SMinchan Kim } 7249c276cc6SMinchan Kim } 7259c276cc6SMinchan Kim 72610853a03SMinchan Kim /** 7276a6fe9ebSKefeng Wang * folio_mark_lazyfree - make an anon folio lazyfree 7286a6fe9ebSKefeng Wang * @folio: folio to deactivate 72910853a03SMinchan Kim * 7306a6fe9ebSKefeng Wang * folio_mark_lazyfree() moves @folio to the inactive file list. 7316a6fe9ebSKefeng Wang * This is done to accelerate the reclaim of @folio. 73210853a03SMinchan Kim */ 7336a6fe9ebSKefeng Wang void folio_mark_lazyfree(struct folio *folio) 73410853a03SMinchan Kim { 735cec394baSMatthew Wilcox (Oracle) if (folio_test_lru(folio) && folio_test_anon(folio) && 736cec394baSMatthew Wilcox (Oracle) folio_test_swapbacked(folio) && !folio_test_swapcache(folio) && 737cec394baSMatthew Wilcox (Oracle) !folio_test_unevictable(folio)) { 738cec394baSMatthew Wilcox (Oracle) struct folio_batch *fbatch; 739cec394baSMatthew Wilcox (Oracle) 740cec394baSMatthew Wilcox (Oracle) folio_get(folio); 74182ac64d8SMatthew Wilcox (Oracle) local_lock(&cpu_fbatches.lock); 74282ac64d8SMatthew Wilcox (Oracle) fbatch = this_cpu_ptr(&cpu_fbatches.lru_lazyfree); 743cec394baSMatthew Wilcox (Oracle) folio_batch_add_and_move(fbatch, folio, lru_lazyfree_fn); 74482ac64d8SMatthew Wilcox (Oracle) local_unlock(&cpu_fbatches.lock); 74510853a03SMinchan Kim } 74610853a03SMinchan Kim } 74710853a03SMinchan Kim 74880bfed90SAndrew Morton void lru_add_drain(void) 74980bfed90SAndrew Morton { 75082ac64d8SMatthew Wilcox (Oracle) local_lock(&cpu_fbatches.lock); 751b01b2141SIngo Molnar lru_add_drain_cpu(smp_processor_id()); 75282ac64d8SMatthew Wilcox (Oracle) local_unlock(&cpu_fbatches.lock); 75396f97c43SLorenzo Stoakes mlock_drain_local(); 754b01b2141SIngo Molnar } 755b01b2141SIngo Molnar 756243418e3SMinchan Kim /* 757243418e3SMinchan Kim * It's called from per-cpu workqueue context in SMP case so 758243418e3SMinchan Kim * lru_add_drain_cpu and invalidate_bh_lrus_cpu should run on 759243418e3SMinchan Kim * the same cpu. It shouldn't be a problem in !SMP case since 760243418e3SMinchan Kim * the core is only one and the locks will disable preemption. 761243418e3SMinchan Kim */ 762243418e3SMinchan Kim static void lru_add_and_bh_lrus_drain(void) 763243418e3SMinchan Kim { 76482ac64d8SMatthew Wilcox (Oracle) local_lock(&cpu_fbatches.lock); 765243418e3SMinchan Kim lru_add_drain_cpu(smp_processor_id()); 76682ac64d8SMatthew Wilcox (Oracle) local_unlock(&cpu_fbatches.lock); 767243418e3SMinchan Kim invalidate_bh_lrus_cpu(); 76896f97c43SLorenzo Stoakes mlock_drain_local(); 769243418e3SMinchan Kim } 770243418e3SMinchan Kim 771b01b2141SIngo Molnar void lru_add_drain_cpu_zone(struct zone *zone) 772b01b2141SIngo Molnar { 77382ac64d8SMatthew Wilcox (Oracle) local_lock(&cpu_fbatches.lock); 774b01b2141SIngo Molnar lru_add_drain_cpu(smp_processor_id()); 775b01b2141SIngo Molnar drain_local_pages(zone); 77682ac64d8SMatthew Wilcox (Oracle) local_unlock(&cpu_fbatches.lock); 77796f97c43SLorenzo Stoakes mlock_drain_local(); 7781da177e4SLinus Torvalds } 7791da177e4SLinus Torvalds 7806ea183d6SMichal Hocko #ifdef CONFIG_SMP 7816ea183d6SMichal Hocko 7826ea183d6SMichal Hocko static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work); 7836ea183d6SMichal Hocko 784c4028958SDavid Howells static void lru_add_drain_per_cpu(struct work_struct *dummy) 785053837fcSNick Piggin { 786243418e3SMinchan Kim lru_add_and_bh_lrus_drain(); 787053837fcSNick Piggin } 788053837fcSNick Piggin 7894864545aSMatthew Wilcox (Oracle) static bool cpu_needs_drain(unsigned int cpu) 7904864545aSMatthew Wilcox (Oracle) { 7914864545aSMatthew Wilcox (Oracle) struct cpu_fbatches *fbatches = &per_cpu(cpu_fbatches, cpu); 7924864545aSMatthew Wilcox (Oracle) 7934864545aSMatthew Wilcox (Oracle) /* Check these in order of likelihood that they're not zero */ 7944864545aSMatthew Wilcox (Oracle) return folio_batch_count(&fbatches->lru_add) || 7954864545aSMatthew Wilcox (Oracle) data_race(folio_batch_count(&per_cpu(lru_rotate.fbatch, cpu))) || 7964864545aSMatthew Wilcox (Oracle) folio_batch_count(&fbatches->lru_deactivate_file) || 7974864545aSMatthew Wilcox (Oracle) folio_batch_count(&fbatches->lru_deactivate) || 7984864545aSMatthew Wilcox (Oracle) folio_batch_count(&fbatches->lru_lazyfree) || 7994864545aSMatthew Wilcox (Oracle) folio_batch_count(&fbatches->activate) || 80096f97c43SLorenzo Stoakes need_mlock_drain(cpu) || 8014864545aSMatthew Wilcox (Oracle) has_bh_in_lru(cpu, NULL); 8024864545aSMatthew Wilcox (Oracle) } 8034864545aSMatthew Wilcox (Oracle) 8049852a721SMichal Hocko /* 8059852a721SMichal Hocko * Doesn't need any cpu hotplug locking because we do rely on per-cpu 8069852a721SMichal Hocko * kworkers being shut down before our page_alloc_cpu_dead callback is 8079852a721SMichal Hocko * executed on the offlined cpu. 8089852a721SMichal Hocko * Calling this function with cpu hotplug locks held can actually lead 8099852a721SMichal Hocko * to obscure indirect dependencies via WQ context. 8109852a721SMichal Hocko */ 8113db3264dSMiaohe Lin static inline void __lru_add_drain_all(bool force_all_cpus) 812053837fcSNick Piggin { 8136446a513SAhmed S. Darwish /* 8146446a513SAhmed S. Darwish * lru_drain_gen - Global pages generation number 8156446a513SAhmed S. Darwish * 8166446a513SAhmed S. Darwish * (A) Definition: global lru_drain_gen = x implies that all generations 8176446a513SAhmed S. Darwish * 0 < n <= x are already *scheduled* for draining. 8186446a513SAhmed S. Darwish * 8196446a513SAhmed S. Darwish * This is an optimization for the highly-contended use case where a 8206446a513SAhmed S. Darwish * user space workload keeps constantly generating a flow of pages for 8216446a513SAhmed S. Darwish * each CPU. 8226446a513SAhmed S. Darwish */ 8236446a513SAhmed S. Darwish static unsigned int lru_drain_gen; 8245fbc4616SChris Metcalf static struct cpumask has_work; 8256446a513SAhmed S. Darwish static DEFINE_MUTEX(lock); 8266446a513SAhmed S. Darwish unsigned cpu, this_gen; 8275fbc4616SChris Metcalf 828ce612879SMichal Hocko /* 829ce612879SMichal Hocko * Make sure nobody triggers this path before mm_percpu_wq is fully 830ce612879SMichal Hocko * initialized. 831ce612879SMichal Hocko */ 832ce612879SMichal Hocko if (WARN_ON(!mm_percpu_wq)) 833ce612879SMichal Hocko return; 834ce612879SMichal Hocko 8356446a513SAhmed S. Darwish /* 83682ac64d8SMatthew Wilcox (Oracle) * Guarantee folio_batch counter stores visible by this CPU 83782ac64d8SMatthew Wilcox (Oracle) * are visible to other CPUs before loading the current drain 83882ac64d8SMatthew Wilcox (Oracle) * generation. 8396446a513SAhmed S. Darwish */ 8406446a513SAhmed S. Darwish smp_mb(); 8416446a513SAhmed S. Darwish 8426446a513SAhmed S. Darwish /* 8436446a513SAhmed S. Darwish * (B) Locally cache global LRU draining generation number 8446446a513SAhmed S. Darwish * 8456446a513SAhmed S. Darwish * The read barrier ensures that the counter is loaded before the mutex 8466446a513SAhmed S. Darwish * is taken. It pairs with smp_mb() inside the mutex critical section 8476446a513SAhmed S. Darwish * at (D). 8486446a513SAhmed S. Darwish */ 8496446a513SAhmed S. Darwish this_gen = smp_load_acquire(&lru_drain_gen); 850eef1a429SKonstantin Khlebnikov 8515fbc4616SChris Metcalf mutex_lock(&lock); 852eef1a429SKonstantin Khlebnikov 853eef1a429SKonstantin Khlebnikov /* 8546446a513SAhmed S. Darwish * (C) Exit the draining operation if a newer generation, from another 8556446a513SAhmed S. Darwish * lru_add_drain_all(), was already scheduled for draining. Check (A). 856eef1a429SKonstantin Khlebnikov */ 857d479960eSMinchan Kim if (unlikely(this_gen != lru_drain_gen && !force_all_cpus)) 858eef1a429SKonstantin Khlebnikov goto done; 859eef1a429SKonstantin Khlebnikov 8606446a513SAhmed S. Darwish /* 8616446a513SAhmed S. Darwish * (D) Increment global generation number 8626446a513SAhmed S. Darwish * 8636446a513SAhmed S. Darwish * Pairs with smp_load_acquire() at (B), outside of the critical 86482ac64d8SMatthew Wilcox (Oracle) * section. Use a full memory barrier to guarantee that the 86582ac64d8SMatthew Wilcox (Oracle) * new global drain generation number is stored before loading 86682ac64d8SMatthew Wilcox (Oracle) * folio_batch counters. 8676446a513SAhmed S. Darwish * 8686446a513SAhmed S. Darwish * This pairing must be done here, before the for_each_online_cpu loop 8696446a513SAhmed S. Darwish * below which drains the page vectors. 8706446a513SAhmed S. Darwish * 8716446a513SAhmed S. Darwish * Let x, y, and z represent some system CPU numbers, where x < y < z. 872cb152a1aSShijie Luo * Assume CPU #z is in the middle of the for_each_online_cpu loop 8736446a513SAhmed S. Darwish * below and has already reached CPU #y's per-cpu data. CPU #x comes 8746446a513SAhmed S. Darwish * along, adds some pages to its per-cpu vectors, then calls 8756446a513SAhmed S. Darwish * lru_add_drain_all(). 8766446a513SAhmed S. Darwish * 8776446a513SAhmed S. Darwish * If the paired barrier is done at any later step, e.g. after the 8786446a513SAhmed S. Darwish * loop, CPU #x will just exit at (C) and miss flushing out all of its 8796446a513SAhmed S. Darwish * added pages. 8806446a513SAhmed S. Darwish */ 8816446a513SAhmed S. Darwish WRITE_ONCE(lru_drain_gen, lru_drain_gen + 1); 8826446a513SAhmed S. Darwish smp_mb(); 883eef1a429SKonstantin Khlebnikov 8845fbc4616SChris Metcalf cpumask_clear(&has_work); 8855fbc4616SChris Metcalf for_each_online_cpu(cpu) { 8865fbc4616SChris Metcalf struct work_struct *work = &per_cpu(lru_add_drain_work, cpu); 8875fbc4616SChris Metcalf 8884864545aSMatthew Wilcox (Oracle) if (cpu_needs_drain(cpu)) { 8895fbc4616SChris Metcalf INIT_WORK(work, lru_add_drain_per_cpu); 890ce612879SMichal Hocko queue_work_on(cpu, mm_percpu_wq, work); 8916446a513SAhmed S. Darwish __cpumask_set_cpu(cpu, &has_work); 8925fbc4616SChris Metcalf } 8935fbc4616SChris Metcalf } 8945fbc4616SChris Metcalf 8955fbc4616SChris Metcalf for_each_cpu(cpu, &has_work) 8965fbc4616SChris Metcalf flush_work(&per_cpu(lru_add_drain_work, cpu)); 8975fbc4616SChris Metcalf 898eef1a429SKonstantin Khlebnikov done: 8995fbc4616SChris Metcalf mutex_unlock(&lock); 900053837fcSNick Piggin } 901d479960eSMinchan Kim 902d479960eSMinchan Kim void lru_add_drain_all(void) 903d479960eSMinchan Kim { 904d479960eSMinchan Kim __lru_add_drain_all(false); 905d479960eSMinchan Kim } 9066ea183d6SMichal Hocko #else 9076ea183d6SMichal Hocko void lru_add_drain_all(void) 9086ea183d6SMichal Hocko { 9096ea183d6SMichal Hocko lru_add_drain(); 9106ea183d6SMichal Hocko } 9116446a513SAhmed S. Darwish #endif /* CONFIG_SMP */ 912053837fcSNick Piggin 913d479960eSMinchan Kim atomic_t lru_disable_count = ATOMIC_INIT(0); 914d479960eSMinchan Kim 915d479960eSMinchan Kim /* 916d479960eSMinchan Kim * lru_cache_disable() needs to be called before we start compiling 917d479960eSMinchan Kim * a list of pages to be migrated using isolate_lru_page(). 918d479960eSMinchan Kim * It drains pages on LRU cache and then disable on all cpus until 919d479960eSMinchan Kim * lru_cache_enable is called. 920d479960eSMinchan Kim * 921d479960eSMinchan Kim * Must be paired with a call to lru_cache_enable(). 922d479960eSMinchan Kim */ 923d479960eSMinchan Kim void lru_cache_disable(void) 924d479960eSMinchan Kim { 925d479960eSMinchan Kim atomic_inc(&lru_disable_count); 926d479960eSMinchan Kim /* 927ff042f4aSMarcelo Tosatti * Readers of lru_disable_count are protected by either disabling 928ff042f4aSMarcelo Tosatti * preemption or rcu_read_lock: 929ff042f4aSMarcelo Tosatti * 930ff042f4aSMarcelo Tosatti * preempt_disable, local_irq_disable [bh_lru_lock()] 931ff042f4aSMarcelo Tosatti * rcu_read_lock [rt_spin_lock CONFIG_PREEMPT_RT] 932ff042f4aSMarcelo Tosatti * preempt_disable [local_lock !CONFIG_PREEMPT_RT] 933ff042f4aSMarcelo Tosatti * 934ff042f4aSMarcelo Tosatti * Since v5.1 kernel, synchronize_rcu() is guaranteed to wait on 935ff042f4aSMarcelo Tosatti * preempt_disable() regions of code. So any CPU which sees 936ff042f4aSMarcelo Tosatti * lru_disable_count = 0 will have exited the critical 937ff042f4aSMarcelo Tosatti * section when synchronize_rcu() returns. 938d479960eSMinchan Kim */ 93931733463SMarcelo Tosatti synchronize_rcu_expedited(); 940ff042f4aSMarcelo Tosatti #ifdef CONFIG_SMP 941d479960eSMinchan Kim __lru_add_drain_all(true); 942d479960eSMinchan Kim #else 943243418e3SMinchan Kim lru_add_and_bh_lrus_drain(); 944d479960eSMinchan Kim #endif 945d479960eSMinchan Kim } 946d479960eSMinchan Kim 947aabfb572SMichal Hocko /** 948*99fbb6bfSMatthew Wilcox (Oracle) * folios_put_refs - Reduce the reference count on a batch of folios. 949*99fbb6bfSMatthew Wilcox (Oracle) * @folios: The folios. 950*99fbb6bfSMatthew Wilcox (Oracle) * @refs: The number of refs to subtract from each folio. 9511da177e4SLinus Torvalds * 952*99fbb6bfSMatthew Wilcox (Oracle) * Like folio_put(), but for a batch of folios. This is more efficient 953*99fbb6bfSMatthew Wilcox (Oracle) * than writing the loop yourself as it will optimise the locks which need 954*99fbb6bfSMatthew Wilcox (Oracle) * to be taken if the folios are freed. The folios batch is returned 955*99fbb6bfSMatthew Wilcox (Oracle) * empty and ready to be reused for another batch; there is no need 956*99fbb6bfSMatthew Wilcox (Oracle) * to reinitialise it. If @refs is NULL, we subtract one from each 957*99fbb6bfSMatthew Wilcox (Oracle) * folio refcount. 958449c7967SLinus Torvalds * 959*99fbb6bfSMatthew Wilcox (Oracle) * Context: May be called in process or interrupt context, but not in NMI 960*99fbb6bfSMatthew Wilcox (Oracle) * context. May be called while holding a spinlock. 9611da177e4SLinus Torvalds */ 962*99fbb6bfSMatthew Wilcox (Oracle) void folios_put_refs(struct folio_batch *folios, unsigned int *refs) 9631da177e4SLinus Torvalds { 9641da177e4SLinus Torvalds int i; 965cc59850eSKonstantin Khlebnikov LIST_HEAD(pages_to_free); 9666168d0daSAlex Shi struct lruvec *lruvec = NULL; 9670de340cbSMatthew Wilcox (Oracle) unsigned long flags = 0; 9681da177e4SLinus Torvalds 969*99fbb6bfSMatthew Wilcox (Oracle) for (i = 0; i < folios->nr; i++) { 970*99fbb6bfSMatthew Wilcox (Oracle) struct folio *folio = folios->folios[i]; 971*99fbb6bfSMatthew Wilcox (Oracle) unsigned int nr_refs = refs ? refs[i] : 1; 972aabfb572SMichal Hocko 973ab5e653eSMatthew Wilcox (Oracle) if (is_huge_zero_page(&folio->page)) 974aa88b68cSKirill A. Shutemov continue; 975aa88b68cSKirill A. Shutemov 976ab5e653eSMatthew Wilcox (Oracle) if (folio_is_zone_device(folio)) { 9776168d0daSAlex Shi if (lruvec) { 9786168d0daSAlex Shi unlock_page_lruvec_irqrestore(lruvec, flags); 9796168d0daSAlex Shi lruvec = NULL; 980df6ad698SJérôme Glisse } 981d7f861b9SDavid Hildenbrand if (put_devmap_managed_page_refs(&folio->page, nr_refs)) 982df6ad698SJérôme Glisse continue; 983d7f861b9SDavid Hildenbrand if (folio_ref_sub_and_test(folio, nr_refs)) 984ab5e653eSMatthew Wilcox (Oracle) free_zone_device_page(&folio->page); 98543fbdeb3SRalph Campbell continue; 98607d80269SJohn Hubbard } 987df6ad698SJérôme Glisse 988d7f861b9SDavid Hildenbrand if (!folio_ref_sub_and_test(folio, nr_refs)) 9891da177e4SLinus Torvalds continue; 9901da177e4SLinus Torvalds 991ab5e653eSMatthew Wilcox (Oracle) if (folio_test_large(folio)) { 9926168d0daSAlex Shi if (lruvec) { 9936168d0daSAlex Shi unlock_page_lruvec_irqrestore(lruvec, flags); 9946168d0daSAlex Shi lruvec = NULL; 995ddc58f27SKirill A. Shutemov } 9965ef82fe7SMatthew Wilcox (Oracle) __folio_put_large(folio); 997ddc58f27SKirill A. Shutemov continue; 998ddc58f27SKirill A. Shutemov } 999ddc58f27SKirill A. Shutemov 1000ab5e653eSMatthew Wilcox (Oracle) if (folio_test_lru(folio)) { 10010de340cbSMatthew Wilcox (Oracle) lruvec = folio_lruvec_relock_irqsave(folio, lruvec, 10022a5e4e34SAlexander Duyck &flags); 1003ab5e653eSMatthew Wilcox (Oracle) lruvec_del_folio(lruvec, folio); 1004ab5e653eSMatthew Wilcox (Oracle) __folio_clear_lru_flags(folio); 100546453a6eSNick Piggin } 100646453a6eSNick Piggin 1007b109b870SHugh Dickins /* 1008b109b870SHugh Dickins * In rare cases, when truncation or holepunching raced with 1009b109b870SHugh Dickins * munlock after VM_LOCKED was cleared, Mlocked may still be 1010b109b870SHugh Dickins * found set here. This does not indicate a problem, unless 1011b109b870SHugh Dickins * "unevictable_pgs_cleared" appears worryingly large. 1012b109b870SHugh Dickins */ 1013ab5e653eSMatthew Wilcox (Oracle) if (unlikely(folio_test_mlocked(folio))) { 1014ab5e653eSMatthew Wilcox (Oracle) __folio_clear_mlocked(folio); 1015ab5e653eSMatthew Wilcox (Oracle) zone_stat_sub_folio(folio, NR_MLOCK); 1016b109b870SHugh Dickins count_vm_event(UNEVICTABLE_PGCLEARED); 1017b109b870SHugh Dickins } 1018b109b870SHugh Dickins 1019ab5e653eSMatthew Wilcox (Oracle) list_add(&folio->lru, &pages_to_free); 10201da177e4SLinus Torvalds } 10216168d0daSAlex Shi if (lruvec) 10226168d0daSAlex Shi unlock_page_lruvec_irqrestore(lruvec, flags); 10231da177e4SLinus Torvalds 1024747db954SJohannes Weiner mem_cgroup_uncharge_list(&pages_to_free); 10252d4894b5SMel Gorman free_unref_page_list(&pages_to_free); 1026*99fbb6bfSMatthew Wilcox (Oracle) folio_batch_reinit(folios); 1027*99fbb6bfSMatthew Wilcox (Oracle) } 1028*99fbb6bfSMatthew Wilcox (Oracle) EXPORT_SYMBOL(folios_put_refs); 1029*99fbb6bfSMatthew Wilcox (Oracle) 1030*99fbb6bfSMatthew Wilcox (Oracle) /** 1031*99fbb6bfSMatthew Wilcox (Oracle) * release_pages - batched put_page() 1032*99fbb6bfSMatthew Wilcox (Oracle) * @arg: array of pages to release 1033*99fbb6bfSMatthew Wilcox (Oracle) * @nr: number of pages 1034*99fbb6bfSMatthew Wilcox (Oracle) * 1035*99fbb6bfSMatthew Wilcox (Oracle) * Decrement the reference count on all the pages in @arg. If it 1036*99fbb6bfSMatthew Wilcox (Oracle) * fell to zero, remove the page from the LRU and free it. 1037*99fbb6bfSMatthew Wilcox (Oracle) * 1038*99fbb6bfSMatthew Wilcox (Oracle) * Note that the argument can be an array of pages, encoded pages, 1039*99fbb6bfSMatthew Wilcox (Oracle) * or folio pointers. We ignore any encoded bits, and turn any of 1040*99fbb6bfSMatthew Wilcox (Oracle) * them into just a folio that gets free'd. 1041*99fbb6bfSMatthew Wilcox (Oracle) */ 1042*99fbb6bfSMatthew Wilcox (Oracle) void release_pages(release_pages_arg arg, int nr) 1043*99fbb6bfSMatthew Wilcox (Oracle) { 1044*99fbb6bfSMatthew Wilcox (Oracle) struct folio_batch fbatch; 1045*99fbb6bfSMatthew Wilcox (Oracle) int refs[PAGEVEC_SIZE]; 1046*99fbb6bfSMatthew Wilcox (Oracle) struct encoded_page **encoded = arg.encoded_pages; 1047*99fbb6bfSMatthew Wilcox (Oracle) int i; 1048*99fbb6bfSMatthew Wilcox (Oracle) 1049*99fbb6bfSMatthew Wilcox (Oracle) folio_batch_init(&fbatch); 1050*99fbb6bfSMatthew Wilcox (Oracle) for (i = 0; i < nr; i++) { 1051*99fbb6bfSMatthew Wilcox (Oracle) /* Turn any of the argument types into a folio */ 1052*99fbb6bfSMatthew Wilcox (Oracle) struct folio *folio = page_folio(encoded_page_ptr(encoded[i])); 1053*99fbb6bfSMatthew Wilcox (Oracle) 1054*99fbb6bfSMatthew Wilcox (Oracle) /* Is our next entry actually "nr_pages" -> "nr_refs" ? */ 1055*99fbb6bfSMatthew Wilcox (Oracle) refs[fbatch.nr] = 1; 1056*99fbb6bfSMatthew Wilcox (Oracle) if (unlikely(encoded_page_flags(encoded[i]) & 1057*99fbb6bfSMatthew Wilcox (Oracle) ENCODED_PAGE_BIT_NR_PAGES_NEXT)) 1058*99fbb6bfSMatthew Wilcox (Oracle) refs[fbatch.nr] = encoded_nr_pages(encoded[++i]); 1059*99fbb6bfSMatthew Wilcox (Oracle) 1060*99fbb6bfSMatthew Wilcox (Oracle) if (folio_batch_add(&fbatch, folio) > 0) 1061*99fbb6bfSMatthew Wilcox (Oracle) continue; 1062*99fbb6bfSMatthew Wilcox (Oracle) folios_put_refs(&fbatch, refs); 1063*99fbb6bfSMatthew Wilcox (Oracle) } 1064*99fbb6bfSMatthew Wilcox (Oracle) 1065*99fbb6bfSMatthew Wilcox (Oracle) if (fbatch.nr) 1066*99fbb6bfSMatthew Wilcox (Oracle) folios_put_refs(&fbatch, refs); 10671da177e4SLinus Torvalds } 10680be8557bSMiklos Szeredi EXPORT_SYMBOL(release_pages); 10691da177e4SLinus Torvalds 10701da177e4SLinus Torvalds /* 10711e0877d5SMatthew Wilcox (Oracle) * The folios which we're about to release may be in the deferred lru-addition 10721da177e4SLinus Torvalds * queues. That would prevent them from really being freed right now. That's 10731e0877d5SMatthew Wilcox (Oracle) * OK from a correctness point of view but is inefficient - those folios may be 10741da177e4SLinus Torvalds * cache-warm and we want to give them back to the page allocator ASAP. 10751da177e4SLinus Torvalds * 10761e0877d5SMatthew Wilcox (Oracle) * So __folio_batch_release() will drain those queues here. 107770dea534SMatthew Wilcox (Oracle) * folio_batch_move_lru() calls folios_put() directly to avoid 10781da177e4SLinus Torvalds * mutual recursion. 10791da177e4SLinus Torvalds */ 10801e0877d5SMatthew Wilcox (Oracle) void __folio_batch_release(struct folio_batch *fbatch) 10811da177e4SLinus Torvalds { 10821e0877d5SMatthew Wilcox (Oracle) if (!fbatch->percpu_pvec_drained) { 10831da177e4SLinus Torvalds lru_add_drain(); 10841e0877d5SMatthew Wilcox (Oracle) fbatch->percpu_pvec_drained = true; 1085d9ed0d08SMel Gorman } 10861e0877d5SMatthew Wilcox (Oracle) release_pages(fbatch->folios, folio_batch_count(fbatch)); 10871e0877d5SMatthew Wilcox (Oracle) folio_batch_reinit(fbatch); 10881da177e4SLinus Torvalds } 10891e0877d5SMatthew Wilcox (Oracle) EXPORT_SYMBOL(__folio_batch_release); 10907f285701SSteve French 10911da177e4SLinus Torvalds /** 10921613fac9SMatthew Wilcox (Oracle) * folio_batch_remove_exceptionals() - Prune non-folios from a batch. 10931613fac9SMatthew Wilcox (Oracle) * @fbatch: The batch to prune 10940cd6144aSJohannes Weiner * 10951613fac9SMatthew Wilcox (Oracle) * find_get_entries() fills a batch with both folios and shadow/swap/DAX 10961613fac9SMatthew Wilcox (Oracle) * entries. This function prunes all the non-folio entries from @fbatch 10971613fac9SMatthew Wilcox (Oracle) * without leaving holes, so that it can be passed on to folio-only batch 10981613fac9SMatthew Wilcox (Oracle) * operations. 10990cd6144aSJohannes Weiner */ 11001613fac9SMatthew Wilcox (Oracle) void folio_batch_remove_exceptionals(struct folio_batch *fbatch) 11010cd6144aSJohannes Weiner { 11021613fac9SMatthew Wilcox (Oracle) unsigned int i, j; 11030cd6144aSJohannes Weiner 11041613fac9SMatthew Wilcox (Oracle) for (i = 0, j = 0; i < folio_batch_count(fbatch); i++) { 11051613fac9SMatthew Wilcox (Oracle) struct folio *folio = fbatch->folios[i]; 11061613fac9SMatthew Wilcox (Oracle) if (!xa_is_value(folio)) 11071613fac9SMatthew Wilcox (Oracle) fbatch->folios[j++] = folio; 11080cd6144aSJohannes Weiner } 11091613fac9SMatthew Wilcox (Oracle) fbatch->nr = j; 11100cd6144aSJohannes Weiner } 11110cd6144aSJohannes Weiner 11121da177e4SLinus Torvalds /* 11131da177e4SLinus Torvalds * Perform any setup for the swap system 11141da177e4SLinus Torvalds */ 11151da177e4SLinus Torvalds void __init swap_setup(void) 11161da177e4SLinus Torvalds { 1117ca79b0c2SArun KS unsigned long megs = totalram_pages() >> (20 - PAGE_SHIFT); 1118e0bf68ddSPeter Zijlstra 11191da177e4SLinus Torvalds /* Use a smaller cluster for small-memory machines */ 11201da177e4SLinus Torvalds if (megs < 16) 11211da177e4SLinus Torvalds page_cluster = 2; 11221da177e4SLinus Torvalds else 11231da177e4SLinus Torvalds page_cluster = 3; 11241da177e4SLinus Torvalds /* 11251da177e4SLinus Torvalds * Right now other parts of the system means that we 11261da177e4SLinus Torvalds * _really_ don't want to cluster much more 11271da177e4SLinus Torvalds */ 11281da177e4SLinus Torvalds } 1129