1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 4 * 5 * Swap reorganised 29.12.95, Stephen Tweedie. 6 * kswapd added: 7.1.96 sct 7 * Removed kswapd_ctl limits, and swap out as many pages as needed 8 * to bring the system back to freepages.high: 2.4.97, Rik van Riel. 9 * Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com). 10 * Multiqueue VM started 5.8.00, Rik van Riel. 11 */ 12 13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 14 15 #include <linux/mm.h> 16 #include <linux/sched/mm.h> 17 #include <linux/module.h> 18 #include <linux/gfp.h> 19 #include <linux/kernel_stat.h> 20 #include <linux/swap.h> 21 #include <linux/pagemap.h> 22 #include <linux/init.h> 23 #include <linux/highmem.h> 24 #include <linux/vmpressure.h> 25 #include <linux/vmstat.h> 26 #include <linux/file.h> 27 #include <linux/writeback.h> 28 #include <linux/blkdev.h> 29 #include <linux/buffer_head.h> /* for buffer_heads_over_limit */ 30 #include <linux/mm_inline.h> 31 #include <linux/backing-dev.h> 32 #include <linux/rmap.h> 33 #include <linux/topology.h> 34 #include <linux/cpu.h> 35 #include <linux/cpuset.h> 36 #include <linux/compaction.h> 37 #include <linux/notifier.h> 38 #include <linux/delay.h> 39 #include <linux/kthread.h> 40 #include <linux/freezer.h> 41 #include <linux/memcontrol.h> 42 #include <linux/migrate.h> 43 #include <linux/delayacct.h> 44 #include <linux/sysctl.h> 45 #include <linux/memory-tiers.h> 46 #include <linux/oom.h> 47 #include <linux/pagevec.h> 48 #include <linux/prefetch.h> 49 #include <linux/printk.h> 50 #include <linux/dax.h> 51 #include <linux/psi.h> 52 #include <linux/pagewalk.h> 53 #include <linux/shmem_fs.h> 54 #include <linux/ctype.h> 55 #include <linux/debugfs.h> 56 #include <linux/khugepaged.h> 57 #include <linux/rculist_nulls.h> 58 #include <linux/random.h> 59 #include <linux/mmu_notifier.h> 60 61 #include <asm/tlbflush.h> 62 #include <asm/div64.h> 63 64 #include <linux/swapops.h> 65 #include <linux/balloon_compaction.h> 66 #include <linux/sched/sysctl.h> 67 68 #include "internal.h" 69 #include "swap.h" 70 71 #define CREATE_TRACE_POINTS 72 #include <trace/events/vmscan.h> 73 74 struct scan_control { 75 /* How many pages shrink_list() should reclaim */ 76 unsigned long nr_to_reclaim; 77 78 /* 79 * Nodemask of nodes allowed by the caller. If NULL, all nodes 80 * are scanned. 81 */ 82 nodemask_t *nodemask; 83 84 /* 85 * The memory cgroup that hit its limit and as a result is the 86 * primary target of this reclaim invocation. 87 */ 88 struct mem_cgroup *target_mem_cgroup; 89 90 /* 91 * Scan pressure balancing between anon and file LRUs 92 */ 93 unsigned long anon_cost; 94 unsigned long file_cost; 95 96 #ifdef CONFIG_MEMCG 97 /* Swappiness value for proactive reclaim. Always use sc_swappiness()! */ 98 int *proactive_swappiness; 99 #endif 100 101 /* Can active folios be deactivated as part of reclaim? */ 102 #define DEACTIVATE_ANON 1 103 #define DEACTIVATE_FILE 2 104 unsigned int may_deactivate:2; 105 unsigned int force_deactivate:1; 106 unsigned int skipped_deactivate:1; 107 108 /* Writepage batching in laptop mode; RECLAIM_WRITE */ 109 unsigned int may_writepage:1; 110 111 /* Can mapped folios be reclaimed? */ 112 unsigned int may_unmap:1; 113 114 /* Can folios be swapped as part of reclaim? */ 115 unsigned int may_swap:1; 116 117 /* Not allow cache_trim_mode to be turned on as part of reclaim? */ 118 unsigned int no_cache_trim_mode:1; 119 120 /* Has cache_trim_mode failed at least once? */ 121 unsigned int cache_trim_mode_failed:1; 122 123 /* Proactive reclaim invoked by userspace through memory.reclaim */ 124 unsigned int proactive:1; 125 126 /* 127 * Cgroup memory below memory.low is protected as long as we 128 * don't threaten to OOM. If any cgroup is reclaimed at 129 * reduced force or passed over entirely due to its memory.low 130 * setting (memcg_low_skipped), and nothing is reclaimed as a 131 * result, then go back for one more cycle that reclaims the protected 132 * memory (memcg_low_reclaim) to avert OOM. 133 */ 134 unsigned int memcg_low_reclaim:1; 135 unsigned int memcg_low_skipped:1; 136 137 /* Shared cgroup tree walk failed, rescan the whole tree */ 138 unsigned int memcg_full_walk:1; 139 140 unsigned int hibernation_mode:1; 141 142 /* One of the zones is ready for compaction */ 143 unsigned int compaction_ready:1; 144 145 /* There is easily reclaimable cold cache in the current node */ 146 unsigned int cache_trim_mode:1; 147 148 /* The file folios on the current node are dangerously low */ 149 unsigned int file_is_tiny:1; 150 151 /* Always discard instead of demoting to lower tier memory */ 152 unsigned int no_demotion:1; 153 154 /* Allocation order */ 155 s8 order; 156 157 /* Scan (total_size >> priority) pages at once */ 158 s8 priority; 159 160 /* The highest zone to isolate folios for reclaim from */ 161 s8 reclaim_idx; 162 163 /* This context's GFP mask */ 164 gfp_t gfp_mask; 165 166 /* Incremented by the number of inactive pages that were scanned */ 167 unsigned long nr_scanned; 168 169 /* Number of pages freed so far during a call to shrink_zones() */ 170 unsigned long nr_reclaimed; 171 172 struct { 173 unsigned int dirty; 174 unsigned int unqueued_dirty; 175 unsigned int congested; 176 unsigned int writeback; 177 unsigned int immediate; 178 unsigned int file_taken; 179 unsigned int taken; 180 } nr; 181 182 /* for recording the reclaimed slab by now */ 183 struct reclaim_state reclaim_state; 184 }; 185 186 #ifdef ARCH_HAS_PREFETCHW 187 #define prefetchw_prev_lru_folio(_folio, _base, _field) \ 188 do { \ 189 if ((_folio)->lru.prev != _base) { \ 190 struct folio *prev; \ 191 \ 192 prev = lru_to_folio(&(_folio->lru)); \ 193 prefetchw(&prev->_field); \ 194 } \ 195 } while (0) 196 #else 197 #define prefetchw_prev_lru_folio(_folio, _base, _field) do { } while (0) 198 #endif 199 200 /* 201 * From 0 .. MAX_SWAPPINESS. Higher means more swappy. 202 */ 203 int vm_swappiness = 60; 204 205 #ifdef CONFIG_MEMCG 206 207 /* Returns true for reclaim through cgroup limits or cgroup interfaces. */ 208 static bool cgroup_reclaim(struct scan_control *sc) 209 { 210 return sc->target_mem_cgroup; 211 } 212 213 /* 214 * Returns true for reclaim on the root cgroup. This is true for direct 215 * allocator reclaim and reclaim through cgroup interfaces on the root cgroup. 216 */ 217 static bool root_reclaim(struct scan_control *sc) 218 { 219 return !sc->target_mem_cgroup || mem_cgroup_is_root(sc->target_mem_cgroup); 220 } 221 222 /** 223 * writeback_throttling_sane - is the usual dirty throttling mechanism available? 224 * @sc: scan_control in question 225 * 226 * The normal page dirty throttling mechanism in balance_dirty_pages() is 227 * completely broken with the legacy memcg and direct stalling in 228 * shrink_folio_list() is used for throttling instead, which lacks all the 229 * niceties such as fairness, adaptive pausing, bandwidth proportional 230 * allocation and configurability. 231 * 232 * This function tests whether the vmscan currently in progress can assume 233 * that the normal dirty throttling mechanism is operational. 234 */ 235 static bool writeback_throttling_sane(struct scan_control *sc) 236 { 237 if (!cgroup_reclaim(sc)) 238 return true; 239 #ifdef CONFIG_CGROUP_WRITEBACK 240 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 241 return true; 242 #endif 243 return false; 244 } 245 246 static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg) 247 { 248 if (sc->proactive && sc->proactive_swappiness) 249 return *sc->proactive_swappiness; 250 return mem_cgroup_swappiness(memcg); 251 } 252 #else 253 static bool cgroup_reclaim(struct scan_control *sc) 254 { 255 return false; 256 } 257 258 static bool root_reclaim(struct scan_control *sc) 259 { 260 return true; 261 } 262 263 static bool writeback_throttling_sane(struct scan_control *sc) 264 { 265 return true; 266 } 267 268 static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg) 269 { 270 return READ_ONCE(vm_swappiness); 271 } 272 #endif 273 274 /* for_each_managed_zone_pgdat - helper macro to iterate over all managed zones in a pgdat up to 275 * and including the specified highidx 276 * @zone: The current zone in the iterator 277 * @pgdat: The pgdat which node_zones are being iterated 278 * @idx: The index variable 279 * @highidx: The index of the highest zone to return 280 * 281 * This macro iterates through all managed zones up to and including the specified highidx. 282 * The zone iterator enters an invalid state after macro call and must be reinitialized 283 * before it can be used again. 284 */ 285 #define for_each_managed_zone_pgdat(zone, pgdat, idx, highidx) \ 286 for ((idx) = 0, (zone) = (pgdat)->node_zones; \ 287 (idx) <= (highidx); \ 288 (idx)++, (zone)++) \ 289 if (!managed_zone(zone)) \ 290 continue; \ 291 else 292 293 static void set_task_reclaim_state(struct task_struct *task, 294 struct reclaim_state *rs) 295 { 296 /* Check for an overwrite */ 297 WARN_ON_ONCE(rs && task->reclaim_state); 298 299 /* Check for the nulling of an already-nulled member */ 300 WARN_ON_ONCE(!rs && !task->reclaim_state); 301 302 task->reclaim_state = rs; 303 } 304 305 /* 306 * flush_reclaim_state(): add pages reclaimed outside of LRU-based reclaim to 307 * scan_control->nr_reclaimed. 308 */ 309 static void flush_reclaim_state(struct scan_control *sc) 310 { 311 /* 312 * Currently, reclaim_state->reclaimed includes three types of pages 313 * freed outside of vmscan: 314 * (1) Slab pages. 315 * (2) Clean file pages from pruned inodes (on highmem systems). 316 * (3) XFS freed buffer pages. 317 * 318 * For all of these cases, we cannot universally link the pages to a 319 * single memcg. For example, a memcg-aware shrinker can free one object 320 * charged to the target memcg, causing an entire page to be freed. 321 * If we count the entire page as reclaimed from the memcg, we end up 322 * overestimating the reclaimed amount (potentially under-reclaiming). 323 * 324 * Only count such pages for global reclaim to prevent under-reclaiming 325 * from the target memcg; preventing unnecessary retries during memcg 326 * charging and false positives from proactive reclaim. 327 * 328 * For uncommon cases where the freed pages were actually mostly 329 * charged to the target memcg, we end up underestimating the reclaimed 330 * amount. This should be fine. The freed pages will be uncharged 331 * anyway, even if they are not counted here properly, and we will be 332 * able to make forward progress in charging (which is usually in a 333 * retry loop). 334 * 335 * We can go one step further, and report the uncharged objcg pages in 336 * memcg reclaim, to make reporting more accurate and reduce 337 * underestimation, but it's probably not worth the complexity for now. 338 */ 339 if (current->reclaim_state && root_reclaim(sc)) { 340 sc->nr_reclaimed += current->reclaim_state->reclaimed; 341 current->reclaim_state->reclaimed = 0; 342 } 343 } 344 345 static bool can_demote(int nid, struct scan_control *sc, 346 struct mem_cgroup *memcg) 347 { 348 int demotion_nid; 349 350 if (!numa_demotion_enabled) 351 return false; 352 if (sc && sc->no_demotion) 353 return false; 354 355 demotion_nid = next_demotion_node(nid); 356 if (demotion_nid == NUMA_NO_NODE) 357 return false; 358 359 /* If demotion node isn't in the cgroup's mems_allowed, fall back */ 360 return mem_cgroup_node_allowed(memcg, demotion_nid); 361 } 362 363 static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg, 364 int nid, 365 struct scan_control *sc) 366 { 367 if (memcg == NULL) { 368 /* 369 * For non-memcg reclaim, is there 370 * space in any swap device? 371 */ 372 if (get_nr_swap_pages() > 0) 373 return true; 374 } else { 375 /* Is the memcg below its swap limit? */ 376 if (mem_cgroup_get_nr_swap_pages(memcg) > 0) 377 return true; 378 } 379 380 /* 381 * The page can not be swapped. 382 * 383 * Can it be reclaimed from this node via demotion? 384 */ 385 return can_demote(nid, sc, memcg); 386 } 387 388 /* 389 * This misses isolated folios which are not accounted for to save counters. 390 * As the data only determines if reclaim or compaction continues, it is 391 * not expected that isolated folios will be a dominating factor. 392 */ 393 unsigned long zone_reclaimable_pages(struct zone *zone) 394 { 395 unsigned long nr; 396 397 nr = zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_FILE) + 398 zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_FILE); 399 if (can_reclaim_anon_pages(NULL, zone_to_nid(zone), NULL)) 400 nr += zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_ANON) + 401 zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_ANON); 402 /* 403 * If there are no reclaimable file-backed or anonymous pages, 404 * ensure zones with sufficient free pages are not skipped. 405 * This prevents zones like DMA32 from being ignored in reclaim 406 * scenarios where they can still help alleviate memory pressure. 407 */ 408 if (nr == 0) 409 nr = zone_page_state_snapshot(zone, NR_FREE_PAGES); 410 return nr; 411 } 412 413 /** 414 * lruvec_lru_size - Returns the number of pages on the given LRU list. 415 * @lruvec: lru vector 416 * @lru: lru to use 417 * @zone_idx: zones to consider (use MAX_NR_ZONES - 1 for the whole LRU list) 418 */ 419 static unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, 420 int zone_idx) 421 { 422 unsigned long size = 0; 423 int zid; 424 struct zone *zone; 425 426 for_each_managed_zone_pgdat(zone, lruvec_pgdat(lruvec), zid, zone_idx) { 427 if (!mem_cgroup_disabled()) 428 size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid); 429 else 430 size += zone_page_state(zone, NR_ZONE_LRU_BASE + lru); 431 } 432 return size; 433 } 434 435 static unsigned long drop_slab_node(int nid) 436 { 437 unsigned long freed = 0; 438 struct mem_cgroup *memcg = NULL; 439 440 memcg = mem_cgroup_iter(NULL, NULL, NULL); 441 do { 442 freed += shrink_slab(GFP_KERNEL, nid, memcg, 0); 443 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL); 444 445 return freed; 446 } 447 448 void drop_slab(void) 449 { 450 int nid; 451 int shift = 0; 452 unsigned long freed; 453 454 do { 455 freed = 0; 456 for_each_online_node(nid) { 457 if (fatal_signal_pending(current)) 458 return; 459 460 freed += drop_slab_node(nid); 461 } 462 } while ((freed >> shift++) > 1); 463 } 464 465 #define CHECK_RECLAIMER_OFFSET(type) \ 466 do { \ 467 BUILD_BUG_ON(PGSTEAL_##type - PGSTEAL_KSWAPD != \ 468 PGDEMOTE_##type - PGDEMOTE_KSWAPD); \ 469 BUILD_BUG_ON(PGSTEAL_##type - PGSTEAL_KSWAPD != \ 470 PGSCAN_##type - PGSCAN_KSWAPD); \ 471 } while (0) 472 473 static int reclaimer_offset(struct scan_control *sc) 474 { 475 CHECK_RECLAIMER_OFFSET(DIRECT); 476 CHECK_RECLAIMER_OFFSET(KHUGEPAGED); 477 CHECK_RECLAIMER_OFFSET(PROACTIVE); 478 479 if (current_is_kswapd()) 480 return 0; 481 if (current_is_khugepaged()) 482 return PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD; 483 if (sc->proactive) 484 return PGSTEAL_PROACTIVE - PGSTEAL_KSWAPD; 485 return PGSTEAL_DIRECT - PGSTEAL_KSWAPD; 486 } 487 488 static inline int is_page_cache_freeable(struct folio *folio) 489 { 490 /* 491 * A freeable page cache folio is referenced only by the caller 492 * that isolated the folio, the page cache and optional filesystem 493 * private data at folio->private. 494 */ 495 return folio_ref_count(folio) - folio_test_private(folio) == 496 1 + folio_nr_pages(folio); 497 } 498 499 /* 500 * We detected a synchronous write error writing a folio out. Probably 501 * -ENOSPC. We need to propagate that into the address_space for a subsequent 502 * fsync(), msync() or close(). 503 * 504 * The tricky part is that after writepage we cannot touch the mapping: nothing 505 * prevents it from being freed up. But we have a ref on the folio and once 506 * that folio is locked, the mapping is pinned. 507 * 508 * We're allowed to run sleeping folio_lock() here because we know the caller has 509 * __GFP_FS. 510 */ 511 static void handle_write_error(struct address_space *mapping, 512 struct folio *folio, int error) 513 { 514 folio_lock(folio); 515 if (folio_mapping(folio) == mapping) 516 mapping_set_error(mapping, error); 517 folio_unlock(folio); 518 } 519 520 static bool skip_throttle_noprogress(pg_data_t *pgdat) 521 { 522 int reclaimable = 0, write_pending = 0; 523 int i; 524 struct zone *zone; 525 /* 526 * If kswapd is disabled, reschedule if necessary but do not 527 * throttle as the system is likely near OOM. 528 */ 529 if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES) 530 return true; 531 532 /* 533 * If there are a lot of dirty/writeback folios then do not 534 * throttle as throttling will occur when the folios cycle 535 * towards the end of the LRU if still under writeback. 536 */ 537 for_each_managed_zone_pgdat(zone, pgdat, i, MAX_NR_ZONES - 1) { 538 reclaimable += zone_reclaimable_pages(zone); 539 write_pending += zone_page_state_snapshot(zone, 540 NR_ZONE_WRITE_PENDING); 541 } 542 if (2 * write_pending <= reclaimable) 543 return true; 544 545 return false; 546 } 547 548 void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason) 549 { 550 wait_queue_head_t *wqh = &pgdat->reclaim_wait[reason]; 551 long timeout, ret; 552 DEFINE_WAIT(wait); 553 554 /* 555 * Do not throttle user workers, kthreads other than kswapd or 556 * workqueues. They may be required for reclaim to make 557 * forward progress (e.g. journalling workqueues or kthreads). 558 */ 559 if (!current_is_kswapd() && 560 current->flags & (PF_USER_WORKER|PF_KTHREAD)) { 561 cond_resched(); 562 return; 563 } 564 565 /* 566 * These figures are pulled out of thin air. 567 * VMSCAN_THROTTLE_ISOLATED is a transient condition based on too many 568 * parallel reclaimers which is a short-lived event so the timeout is 569 * short. Failing to make progress or waiting on writeback are 570 * potentially long-lived events so use a longer timeout. This is shaky 571 * logic as a failure to make progress could be due to anything from 572 * writeback to a slow device to excessive referenced folios at the tail 573 * of the inactive LRU. 574 */ 575 switch(reason) { 576 case VMSCAN_THROTTLE_WRITEBACK: 577 timeout = HZ/10; 578 579 if (atomic_inc_return(&pgdat->nr_writeback_throttled) == 1) { 580 WRITE_ONCE(pgdat->nr_reclaim_start, 581 node_page_state(pgdat, NR_THROTTLED_WRITTEN)); 582 } 583 584 break; 585 case VMSCAN_THROTTLE_CONGESTED: 586 fallthrough; 587 case VMSCAN_THROTTLE_NOPROGRESS: 588 if (skip_throttle_noprogress(pgdat)) { 589 cond_resched(); 590 return; 591 } 592 593 timeout = 1; 594 595 break; 596 case VMSCAN_THROTTLE_ISOLATED: 597 timeout = HZ/50; 598 break; 599 default: 600 WARN_ON_ONCE(1); 601 timeout = HZ; 602 break; 603 } 604 605 prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE); 606 ret = schedule_timeout(timeout); 607 finish_wait(wqh, &wait); 608 609 if (reason == VMSCAN_THROTTLE_WRITEBACK) 610 atomic_dec(&pgdat->nr_writeback_throttled); 611 612 trace_mm_vmscan_throttled(pgdat->node_id, jiffies_to_usecs(timeout), 613 jiffies_to_usecs(timeout - ret), 614 reason); 615 } 616 617 /* 618 * Account for folios written if tasks are throttled waiting on dirty 619 * folios to clean. If enough folios have been cleaned since throttling 620 * started then wakeup the throttled tasks. 621 */ 622 void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio, 623 int nr_throttled) 624 { 625 unsigned long nr_written; 626 627 node_stat_add_folio(folio, NR_THROTTLED_WRITTEN); 628 629 /* 630 * This is an inaccurate read as the per-cpu deltas may not 631 * be synchronised. However, given that the system is 632 * writeback throttled, it is not worth taking the penalty 633 * of getting an accurate count. At worst, the throttle 634 * timeout guarantees forward progress. 635 */ 636 nr_written = node_page_state(pgdat, NR_THROTTLED_WRITTEN) - 637 READ_ONCE(pgdat->nr_reclaim_start); 638 639 if (nr_written > SWAP_CLUSTER_MAX * nr_throttled) 640 wake_up(&pgdat->reclaim_wait[VMSCAN_THROTTLE_WRITEBACK]); 641 } 642 643 /* possible outcome of pageout() */ 644 typedef enum { 645 /* failed to write folio out, folio is locked */ 646 PAGE_KEEP, 647 /* move folio to the active list, folio is locked */ 648 PAGE_ACTIVATE, 649 /* folio has been sent to the disk successfully, folio is unlocked */ 650 PAGE_SUCCESS, 651 /* folio is clean and locked */ 652 PAGE_CLEAN, 653 } pageout_t; 654 655 /* 656 * pageout is called by shrink_folio_list() for each dirty folio. 657 */ 658 static pageout_t pageout(struct folio *folio, struct address_space *mapping, 659 struct swap_iocb **plug, struct list_head *folio_list) 660 { 661 int (*writeout)(struct folio *, struct writeback_control *); 662 663 /* 664 * We no longer attempt to writeback filesystem folios here, other 665 * than tmpfs/shmem. That's taken care of in page-writeback. 666 * If we find a dirty filesystem folio at the end of the LRU list, 667 * typically that means the filesystem is saturating the storage 668 * with contiguous writes and telling it to write a folio here 669 * would only make the situation worse by injecting an element 670 * of random access. 671 * 672 * If the folio is swapcache, write it back even if that would 673 * block, for some throttling. This happens by accident, because 674 * swap_backing_dev_info is bust: it doesn't reflect the 675 * congestion state of the swapdevs. Easy to fix, if needed. 676 */ 677 if (!is_page_cache_freeable(folio)) 678 return PAGE_KEEP; 679 if (!mapping) { 680 /* 681 * Some data journaling orphaned folios can have 682 * folio->mapping == NULL while being dirty with clean buffers. 683 */ 684 if (folio_test_private(folio)) { 685 if (try_to_free_buffers(folio)) { 686 folio_clear_dirty(folio); 687 pr_info("%s: orphaned folio\n", __func__); 688 return PAGE_CLEAN; 689 } 690 } 691 return PAGE_KEEP; 692 } 693 if (shmem_mapping(mapping)) 694 writeout = shmem_writeout; 695 else if (folio_test_anon(folio)) 696 writeout = swap_writeout; 697 else 698 return PAGE_ACTIVATE; 699 700 if (folio_clear_dirty_for_io(folio)) { 701 int res; 702 struct writeback_control wbc = { 703 .sync_mode = WB_SYNC_NONE, 704 .nr_to_write = SWAP_CLUSTER_MAX, 705 .range_start = 0, 706 .range_end = LLONG_MAX, 707 .for_reclaim = 1, 708 .swap_plug = plug, 709 }; 710 711 /* 712 * The large shmem folio can be split if CONFIG_THP_SWAP is 713 * not enabled or contiguous swap entries are failed to 714 * allocate. 715 */ 716 if (shmem_mapping(mapping) && folio_test_large(folio)) 717 wbc.list = folio_list; 718 719 folio_set_reclaim(folio); 720 res = writeout(folio, &wbc); 721 if (res < 0) 722 handle_write_error(mapping, folio, res); 723 if (res == AOP_WRITEPAGE_ACTIVATE) { 724 folio_clear_reclaim(folio); 725 return PAGE_ACTIVATE; 726 } 727 728 if (!folio_test_writeback(folio)) { 729 /* synchronous write? */ 730 folio_clear_reclaim(folio); 731 } 732 trace_mm_vmscan_write_folio(folio); 733 node_stat_add_folio(folio, NR_VMSCAN_WRITE); 734 return PAGE_SUCCESS; 735 } 736 737 return PAGE_CLEAN; 738 } 739 740 /* 741 * Same as remove_mapping, but if the folio is removed from the mapping, it 742 * gets returned with a refcount of 0. 743 */ 744 static int __remove_mapping(struct address_space *mapping, struct folio *folio, 745 bool reclaimed, struct mem_cgroup *target_memcg) 746 { 747 int refcount; 748 void *shadow = NULL; 749 750 BUG_ON(!folio_test_locked(folio)); 751 BUG_ON(mapping != folio_mapping(folio)); 752 753 if (!folio_test_swapcache(folio)) 754 spin_lock(&mapping->host->i_lock); 755 xa_lock_irq(&mapping->i_pages); 756 /* 757 * The non racy check for a busy folio. 758 * 759 * Must be careful with the order of the tests. When someone has 760 * a ref to the folio, it may be possible that they dirty it then 761 * drop the reference. So if the dirty flag is tested before the 762 * refcount here, then the following race may occur: 763 * 764 * get_user_pages(&page); 765 * [user mapping goes away] 766 * write_to(page); 767 * !folio_test_dirty(folio) [good] 768 * folio_set_dirty(folio); 769 * folio_put(folio); 770 * !refcount(folio) [good, discard it] 771 * 772 * [oops, our write_to data is lost] 773 * 774 * Reversing the order of the tests ensures such a situation cannot 775 * escape unnoticed. The smp_rmb is needed to ensure the folio->flags 776 * load is not satisfied before that of folio->_refcount. 777 * 778 * Note that if the dirty flag is always set via folio_mark_dirty, 779 * and thus under the i_pages lock, then this ordering is not required. 780 */ 781 refcount = 1 + folio_nr_pages(folio); 782 if (!folio_ref_freeze(folio, refcount)) 783 goto cannot_free; 784 /* note: atomic_cmpxchg in folio_ref_freeze provides the smp_rmb */ 785 if (unlikely(folio_test_dirty(folio))) { 786 folio_ref_unfreeze(folio, refcount); 787 goto cannot_free; 788 } 789 790 if (folio_test_swapcache(folio)) { 791 swp_entry_t swap = folio->swap; 792 793 if (reclaimed && !mapping_exiting(mapping)) 794 shadow = workingset_eviction(folio, target_memcg); 795 __delete_from_swap_cache(folio, swap, shadow); 796 memcg1_swapout(folio, swap); 797 xa_unlock_irq(&mapping->i_pages); 798 put_swap_folio(folio, swap); 799 } else { 800 void (*free_folio)(struct folio *); 801 802 free_folio = mapping->a_ops->free_folio; 803 /* 804 * Remember a shadow entry for reclaimed file cache in 805 * order to detect refaults, thus thrashing, later on. 806 * 807 * But don't store shadows in an address space that is 808 * already exiting. This is not just an optimization, 809 * inode reclaim needs to empty out the radix tree or 810 * the nodes are lost. Don't plant shadows behind its 811 * back. 812 * 813 * We also don't store shadows for DAX mappings because the 814 * only page cache folios found in these are zero pages 815 * covering holes, and because we don't want to mix DAX 816 * exceptional entries and shadow exceptional entries in the 817 * same address_space. 818 */ 819 if (reclaimed && folio_is_file_lru(folio) && 820 !mapping_exiting(mapping) && !dax_mapping(mapping)) 821 shadow = workingset_eviction(folio, target_memcg); 822 __filemap_remove_folio(folio, shadow); 823 xa_unlock_irq(&mapping->i_pages); 824 if (mapping_shrinkable(mapping)) 825 inode_add_lru(mapping->host); 826 spin_unlock(&mapping->host->i_lock); 827 828 if (free_folio) 829 free_folio(folio); 830 } 831 832 return 1; 833 834 cannot_free: 835 xa_unlock_irq(&mapping->i_pages); 836 if (!folio_test_swapcache(folio)) 837 spin_unlock(&mapping->host->i_lock); 838 return 0; 839 } 840 841 /** 842 * remove_mapping() - Attempt to remove a folio from its mapping. 843 * @mapping: The address space. 844 * @folio: The folio to remove. 845 * 846 * If the folio is dirty, under writeback or if someone else has a ref 847 * on it, removal will fail. 848 * Return: The number of pages removed from the mapping. 0 if the folio 849 * could not be removed. 850 * Context: The caller should have a single refcount on the folio and 851 * hold its lock. 852 */ 853 long remove_mapping(struct address_space *mapping, struct folio *folio) 854 { 855 if (__remove_mapping(mapping, folio, false, NULL)) { 856 /* 857 * Unfreezing the refcount with 1 effectively 858 * drops the pagecache ref for us without requiring another 859 * atomic operation. 860 */ 861 folio_ref_unfreeze(folio, 1); 862 return folio_nr_pages(folio); 863 } 864 return 0; 865 } 866 867 /** 868 * folio_putback_lru - Put previously isolated folio onto appropriate LRU list. 869 * @folio: Folio to be returned to an LRU list. 870 * 871 * Add previously isolated @folio to appropriate LRU list. 872 * The folio may still be unevictable for other reasons. 873 * 874 * Context: lru_lock must not be held, interrupts must be enabled. 875 */ 876 void folio_putback_lru(struct folio *folio) 877 { 878 folio_add_lru(folio); 879 folio_put(folio); /* drop ref from isolate */ 880 } 881 882 enum folio_references { 883 FOLIOREF_RECLAIM, 884 FOLIOREF_RECLAIM_CLEAN, 885 FOLIOREF_KEEP, 886 FOLIOREF_ACTIVATE, 887 }; 888 889 #ifdef CONFIG_LRU_GEN 890 /* 891 * Only used on a mapped folio in the eviction (rmap walk) path, where promotion 892 * needs to be done by taking the folio off the LRU list and then adding it back 893 * with PG_active set. In contrast, the aging (page table walk) path uses 894 * folio_update_gen(). 895 */ 896 static bool lru_gen_set_refs(struct folio *folio) 897 { 898 /* see the comment on LRU_REFS_FLAGS */ 899 if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { 900 set_mask_bits(&folio->flags, LRU_REFS_MASK, BIT(PG_referenced)); 901 return false; 902 } 903 904 set_mask_bits(&folio->flags, LRU_REFS_FLAGS, BIT(PG_workingset)); 905 return true; 906 } 907 #else 908 static bool lru_gen_set_refs(struct folio *folio) 909 { 910 return false; 911 } 912 #endif /* CONFIG_LRU_GEN */ 913 914 static enum folio_references folio_check_references(struct folio *folio, 915 struct scan_control *sc) 916 { 917 int referenced_ptes, referenced_folio; 918 unsigned long vm_flags; 919 920 referenced_ptes = folio_referenced(folio, 1, sc->target_mem_cgroup, 921 &vm_flags); 922 923 /* 924 * The supposedly reclaimable folio was found to be in a VM_LOCKED vma. 925 * Let the folio, now marked Mlocked, be moved to the unevictable list. 926 */ 927 if (vm_flags & VM_LOCKED) 928 return FOLIOREF_ACTIVATE; 929 930 /* 931 * There are two cases to consider. 932 * 1) Rmap lock contention: rotate. 933 * 2) Skip the non-shared swapbacked folio mapped solely by 934 * the exiting or OOM-reaped process. 935 */ 936 if (referenced_ptes == -1) 937 return FOLIOREF_KEEP; 938 939 if (lru_gen_enabled()) { 940 if (!referenced_ptes) 941 return FOLIOREF_RECLAIM; 942 943 return lru_gen_set_refs(folio) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP; 944 } 945 946 referenced_folio = folio_test_clear_referenced(folio); 947 948 if (referenced_ptes) { 949 /* 950 * All mapped folios start out with page table 951 * references from the instantiating fault, so we need 952 * to look twice if a mapped file/anon folio is used more 953 * than once. 954 * 955 * Mark it and spare it for another trip around the 956 * inactive list. Another page table reference will 957 * lead to its activation. 958 * 959 * Note: the mark is set for activated folios as well 960 * so that recently deactivated but used folios are 961 * quickly recovered. 962 */ 963 folio_set_referenced(folio); 964 965 if (referenced_folio || referenced_ptes > 1) 966 return FOLIOREF_ACTIVATE; 967 968 /* 969 * Activate file-backed executable folios after first usage. 970 */ 971 if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) 972 return FOLIOREF_ACTIVATE; 973 974 return FOLIOREF_KEEP; 975 } 976 977 /* Reclaim if clean, defer dirty folios to writeback */ 978 if (referenced_folio && folio_is_file_lru(folio)) 979 return FOLIOREF_RECLAIM_CLEAN; 980 981 return FOLIOREF_RECLAIM; 982 } 983 984 /* Check if a folio is dirty or under writeback */ 985 static void folio_check_dirty_writeback(struct folio *folio, 986 bool *dirty, bool *writeback) 987 { 988 struct address_space *mapping; 989 990 /* 991 * Anonymous folios are not handled by flushers and must be written 992 * from reclaim context. Do not stall reclaim based on them. 993 * MADV_FREE anonymous folios are put into inactive file list too. 994 * They could be mistakenly treated as file lru. So further anon 995 * test is needed. 996 */ 997 if (!folio_is_file_lru(folio) || 998 (folio_test_anon(folio) && !folio_test_swapbacked(folio))) { 999 *dirty = false; 1000 *writeback = false; 1001 return; 1002 } 1003 1004 /* By default assume that the folio flags are accurate */ 1005 *dirty = folio_test_dirty(folio); 1006 *writeback = folio_test_writeback(folio); 1007 1008 /* Verify dirty/writeback state if the filesystem supports it */ 1009 if (!folio_test_private(folio)) 1010 return; 1011 1012 mapping = folio_mapping(folio); 1013 if (mapping && mapping->a_ops->is_dirty_writeback) 1014 mapping->a_ops->is_dirty_writeback(folio, dirty, writeback); 1015 } 1016 1017 struct folio *alloc_migrate_folio(struct folio *src, unsigned long private) 1018 { 1019 struct folio *dst; 1020 nodemask_t *allowed_mask; 1021 struct migration_target_control *mtc; 1022 1023 mtc = (struct migration_target_control *)private; 1024 1025 allowed_mask = mtc->nmask; 1026 /* 1027 * make sure we allocate from the target node first also trying to 1028 * demote or reclaim pages from the target node via kswapd if we are 1029 * low on free memory on target node. If we don't do this and if 1030 * we have free memory on the slower(lower) memtier, we would start 1031 * allocating pages from slower(lower) memory tiers without even forcing 1032 * a demotion of cold pages from the target memtier. This can result 1033 * in the kernel placing hot pages in slower(lower) memory tiers. 1034 */ 1035 mtc->nmask = NULL; 1036 mtc->gfp_mask |= __GFP_THISNODE; 1037 dst = alloc_migration_target(src, (unsigned long)mtc); 1038 if (dst) 1039 return dst; 1040 1041 mtc->gfp_mask &= ~__GFP_THISNODE; 1042 mtc->nmask = allowed_mask; 1043 1044 return alloc_migration_target(src, (unsigned long)mtc); 1045 } 1046 1047 /* 1048 * Take folios on @demote_folios and attempt to demote them to another node. 1049 * Folios which are not demoted are left on @demote_folios. 1050 */ 1051 static unsigned int demote_folio_list(struct list_head *demote_folios, 1052 struct pglist_data *pgdat) 1053 { 1054 int target_nid = next_demotion_node(pgdat->node_id); 1055 unsigned int nr_succeeded; 1056 nodemask_t allowed_mask; 1057 1058 struct migration_target_control mtc = { 1059 /* 1060 * Allocate from 'node', or fail quickly and quietly. 1061 * When this happens, 'page' will likely just be discarded 1062 * instead of migrated. 1063 */ 1064 .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | __GFP_NOWARN | 1065 __GFP_NOMEMALLOC | GFP_NOWAIT, 1066 .nid = target_nid, 1067 .nmask = &allowed_mask, 1068 .reason = MR_DEMOTION, 1069 }; 1070 1071 if (list_empty(demote_folios)) 1072 return 0; 1073 1074 if (target_nid == NUMA_NO_NODE) 1075 return 0; 1076 1077 node_get_allowed_targets(pgdat, &allowed_mask); 1078 1079 /* Demotion ignores all cpuset and mempolicy settings */ 1080 migrate_pages(demote_folios, alloc_migrate_folio, NULL, 1081 (unsigned long)&mtc, MIGRATE_ASYNC, MR_DEMOTION, 1082 &nr_succeeded); 1083 1084 return nr_succeeded; 1085 } 1086 1087 static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask) 1088 { 1089 if (gfp_mask & __GFP_FS) 1090 return true; 1091 if (!folio_test_swapcache(folio) || !(gfp_mask & __GFP_IO)) 1092 return false; 1093 /* 1094 * We can "enter_fs" for swap-cache with only __GFP_IO 1095 * providing this isn't SWP_FS_OPS. 1096 * ->flags can be updated non-atomicially (scan_swap_map_slots), 1097 * but that will never affect SWP_FS_OPS, so the data_race 1098 * is safe. 1099 */ 1100 return !data_race(folio_swap_flags(folio) & SWP_FS_OPS); 1101 } 1102 1103 /* 1104 * shrink_folio_list() returns the number of reclaimed pages 1105 */ 1106 static unsigned int shrink_folio_list(struct list_head *folio_list, 1107 struct pglist_data *pgdat, struct scan_control *sc, 1108 struct reclaim_stat *stat, bool ignore_references, 1109 struct mem_cgroup *memcg) 1110 { 1111 struct folio_batch free_folios; 1112 LIST_HEAD(ret_folios); 1113 LIST_HEAD(demote_folios); 1114 unsigned int nr_reclaimed = 0, nr_demoted = 0; 1115 unsigned int pgactivate = 0; 1116 bool do_demote_pass; 1117 struct swap_iocb *plug = NULL; 1118 1119 folio_batch_init(&free_folios); 1120 memset(stat, 0, sizeof(*stat)); 1121 cond_resched(); 1122 do_demote_pass = can_demote(pgdat->node_id, sc, memcg); 1123 1124 retry: 1125 while (!list_empty(folio_list)) { 1126 struct address_space *mapping; 1127 struct folio *folio; 1128 enum folio_references references = FOLIOREF_RECLAIM; 1129 bool dirty, writeback; 1130 unsigned int nr_pages; 1131 1132 cond_resched(); 1133 1134 folio = lru_to_folio(folio_list); 1135 list_del(&folio->lru); 1136 1137 if (!folio_trylock(folio)) 1138 goto keep; 1139 1140 if (folio_contain_hwpoisoned_page(folio)) { 1141 unmap_poisoned_folio(folio, folio_pfn(folio), false); 1142 folio_unlock(folio); 1143 folio_put(folio); 1144 continue; 1145 } 1146 1147 VM_BUG_ON_FOLIO(folio_test_active(folio), folio); 1148 1149 nr_pages = folio_nr_pages(folio); 1150 1151 /* Account the number of base pages */ 1152 sc->nr_scanned += nr_pages; 1153 1154 if (unlikely(!folio_evictable(folio))) 1155 goto activate_locked; 1156 1157 if (!sc->may_unmap && folio_mapped(folio)) 1158 goto keep_locked; 1159 1160 /* 1161 * The number of dirty pages determines if a node is marked 1162 * reclaim_congested. kswapd will stall and start writing 1163 * folios if the tail of the LRU is all dirty unqueued folios. 1164 */ 1165 folio_check_dirty_writeback(folio, &dirty, &writeback); 1166 if (dirty || writeback) 1167 stat->nr_dirty += nr_pages; 1168 1169 if (dirty && !writeback) 1170 stat->nr_unqueued_dirty += nr_pages; 1171 1172 /* 1173 * Treat this folio as congested if folios are cycling 1174 * through the LRU so quickly that the folios marked 1175 * for immediate reclaim are making it to the end of 1176 * the LRU a second time. 1177 */ 1178 if (writeback && folio_test_reclaim(folio)) 1179 stat->nr_congested += nr_pages; 1180 1181 /* 1182 * If a folio at the tail of the LRU is under writeback, there 1183 * are three cases to consider. 1184 * 1185 * 1) If reclaim is encountering an excessive number 1186 * of folios under writeback and this folio has both 1187 * the writeback and reclaim flags set, then it 1188 * indicates that folios are being queued for I/O but 1189 * are being recycled through the LRU before the I/O 1190 * can complete. Waiting on the folio itself risks an 1191 * indefinite stall if it is impossible to writeback 1192 * the folio due to I/O error or disconnected storage 1193 * so instead note that the LRU is being scanned too 1194 * quickly and the caller can stall after the folio 1195 * list has been processed. 1196 * 1197 * 2) Global or new memcg reclaim encounters a folio that is 1198 * not marked for immediate reclaim, or the caller does not 1199 * have __GFP_FS (or __GFP_IO if it's simply going to swap, 1200 * not to fs), or the folio belongs to a mapping where 1201 * waiting on writeback during reclaim may lead to a deadlock. 1202 * In this case mark the folio for immediate reclaim and 1203 * continue scanning. 1204 * 1205 * Require may_enter_fs() because we would wait on fs, which 1206 * may not have submitted I/O yet. And the loop driver might 1207 * enter reclaim, and deadlock if it waits on a folio for 1208 * which it is needed to do the write (loop masks off 1209 * __GFP_IO|__GFP_FS for this reason); but more thought 1210 * would probably show more reasons. 1211 * 1212 * 3) Legacy memcg encounters a folio that already has the 1213 * reclaim flag set. memcg does not have any dirty folio 1214 * throttling so we could easily OOM just because too many 1215 * folios are in writeback and there is nothing else to 1216 * reclaim. Wait for the writeback to complete. 1217 * 1218 * In cases 1) and 2) we activate the folios to get them out of 1219 * the way while we continue scanning for clean folios on the 1220 * inactive list and refilling from the active list. The 1221 * observation here is that waiting for disk writes is more 1222 * expensive than potentially causing reloads down the line. 1223 * Since they're marked for immediate reclaim, they won't put 1224 * memory pressure on the cache working set any longer than it 1225 * takes to write them to disk. 1226 */ 1227 if (folio_test_writeback(folio)) { 1228 mapping = folio_mapping(folio); 1229 1230 /* Case 1 above */ 1231 if (current_is_kswapd() && 1232 folio_test_reclaim(folio) && 1233 test_bit(PGDAT_WRITEBACK, &pgdat->flags)) { 1234 stat->nr_immediate += nr_pages; 1235 goto activate_locked; 1236 1237 /* Case 2 above */ 1238 } else if (writeback_throttling_sane(sc) || 1239 !folio_test_reclaim(folio) || 1240 !may_enter_fs(folio, sc->gfp_mask) || 1241 (mapping && 1242 mapping_writeback_may_deadlock_on_reclaim(mapping))) { 1243 /* 1244 * This is slightly racy - 1245 * folio_end_writeback() might have 1246 * just cleared the reclaim flag, then 1247 * setting the reclaim flag here ends up 1248 * interpreted as the readahead flag - but 1249 * that does not matter enough to care. 1250 * What we do want is for this folio to 1251 * have the reclaim flag set next time 1252 * memcg reclaim reaches the tests above, 1253 * so it will then wait for writeback to 1254 * avoid OOM; and it's also appropriate 1255 * in global reclaim. 1256 */ 1257 folio_set_reclaim(folio); 1258 stat->nr_writeback += nr_pages; 1259 goto activate_locked; 1260 1261 /* Case 3 above */ 1262 } else { 1263 folio_unlock(folio); 1264 folio_wait_writeback(folio); 1265 /* then go back and try same folio again */ 1266 list_add_tail(&folio->lru, folio_list); 1267 continue; 1268 } 1269 } 1270 1271 if (!ignore_references) 1272 references = folio_check_references(folio, sc); 1273 1274 switch (references) { 1275 case FOLIOREF_ACTIVATE: 1276 goto activate_locked; 1277 case FOLIOREF_KEEP: 1278 stat->nr_ref_keep += nr_pages; 1279 goto keep_locked; 1280 case FOLIOREF_RECLAIM: 1281 case FOLIOREF_RECLAIM_CLEAN: 1282 ; /* try to reclaim the folio below */ 1283 } 1284 1285 /* 1286 * Before reclaiming the folio, try to relocate 1287 * its contents to another node. 1288 */ 1289 if (do_demote_pass && 1290 (thp_migration_supported() || !folio_test_large(folio))) { 1291 list_add(&folio->lru, &demote_folios); 1292 folio_unlock(folio); 1293 continue; 1294 } 1295 1296 /* 1297 * Anonymous process memory has backing store? 1298 * Try to allocate it some swap space here. 1299 * Lazyfree folio could be freed directly 1300 */ 1301 if (folio_test_anon(folio) && folio_test_swapbacked(folio)) { 1302 if (!folio_test_swapcache(folio)) { 1303 if (!(sc->gfp_mask & __GFP_IO)) 1304 goto keep_locked; 1305 if (folio_maybe_dma_pinned(folio)) 1306 goto keep_locked; 1307 if (folio_test_large(folio)) { 1308 /* cannot split folio, skip it */ 1309 if (!can_split_folio(folio, 1, NULL)) 1310 goto activate_locked; 1311 /* 1312 * Split partially mapped folios right away. 1313 * We can free the unmapped pages without IO. 1314 */ 1315 if (data_race(!list_empty(&folio->_deferred_list) && 1316 folio_test_partially_mapped(folio)) && 1317 split_folio_to_list(folio, folio_list)) 1318 goto activate_locked; 1319 } 1320 if (folio_alloc_swap(folio, __GFP_HIGH | __GFP_NOWARN)) { 1321 int __maybe_unused order = folio_order(folio); 1322 1323 if (!folio_test_large(folio)) 1324 goto activate_locked_split; 1325 /* Fallback to swap normal pages */ 1326 if (split_folio_to_list(folio, folio_list)) 1327 goto activate_locked; 1328 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1329 if (nr_pages >= HPAGE_PMD_NR) { 1330 count_memcg_folio_events(folio, 1331 THP_SWPOUT_FALLBACK, 1); 1332 count_vm_event(THP_SWPOUT_FALLBACK); 1333 } 1334 #endif 1335 count_mthp_stat(order, MTHP_STAT_SWPOUT_FALLBACK); 1336 if (folio_alloc_swap(folio, __GFP_HIGH | __GFP_NOWARN)) 1337 goto activate_locked_split; 1338 } 1339 /* 1340 * Normally the folio will be dirtied in unmap because its 1341 * pte should be dirty. A special case is MADV_FREE page. The 1342 * page's pte could have dirty bit cleared but the folio's 1343 * SwapBacked flag is still set because clearing the dirty bit 1344 * and SwapBacked flag has no lock protected. For such folio, 1345 * unmap will not set dirty bit for it, so folio reclaim will 1346 * not write the folio out. This can cause data corruption when 1347 * the folio is swapped in later. Always setting the dirty flag 1348 * for the folio solves the problem. 1349 */ 1350 folio_mark_dirty(folio); 1351 } 1352 } 1353 1354 /* 1355 * If the folio was split above, the tail pages will make 1356 * their own pass through this function and be accounted 1357 * then. 1358 */ 1359 if ((nr_pages > 1) && !folio_test_large(folio)) { 1360 sc->nr_scanned -= (nr_pages - 1); 1361 nr_pages = 1; 1362 } 1363 1364 /* 1365 * The folio is mapped into the page tables of one or more 1366 * processes. Try to unmap it here. 1367 */ 1368 if (folio_mapped(folio)) { 1369 enum ttu_flags flags = TTU_BATCH_FLUSH; 1370 bool was_swapbacked = folio_test_swapbacked(folio); 1371 1372 if (folio_test_pmd_mappable(folio)) 1373 flags |= TTU_SPLIT_HUGE_PMD; 1374 /* 1375 * Without TTU_SYNC, try_to_unmap will only begin to 1376 * hold PTL from the first present PTE within a large 1377 * folio. Some initial PTEs might be skipped due to 1378 * races with parallel PTE writes in which PTEs can be 1379 * cleared temporarily before being written new present 1380 * values. This will lead to a large folio is still 1381 * mapped while some subpages have been partially 1382 * unmapped after try_to_unmap; TTU_SYNC helps 1383 * try_to_unmap acquire PTL from the first PTE, 1384 * eliminating the influence of temporary PTE values. 1385 */ 1386 if (folio_test_large(folio)) 1387 flags |= TTU_SYNC; 1388 1389 try_to_unmap(folio, flags); 1390 if (folio_mapped(folio)) { 1391 stat->nr_unmap_fail += nr_pages; 1392 if (!was_swapbacked && 1393 folio_test_swapbacked(folio)) 1394 stat->nr_lazyfree_fail += nr_pages; 1395 goto activate_locked; 1396 } 1397 } 1398 1399 /* 1400 * Folio is unmapped now so it cannot be newly pinned anymore. 1401 * No point in trying to reclaim folio if it is pinned. 1402 * Furthermore we don't want to reclaim underlying fs metadata 1403 * if the folio is pinned and thus potentially modified by the 1404 * pinning process as that may upset the filesystem. 1405 */ 1406 if (folio_maybe_dma_pinned(folio)) 1407 goto activate_locked; 1408 1409 mapping = folio_mapping(folio); 1410 if (folio_test_dirty(folio)) { 1411 /* 1412 * Only kswapd can writeback filesystem folios 1413 * to avoid risk of stack overflow. But avoid 1414 * injecting inefficient single-folio I/O into 1415 * flusher writeback as much as possible: only 1416 * write folios when we've encountered many 1417 * dirty folios, and when we've already scanned 1418 * the rest of the LRU for clean folios and see 1419 * the same dirty folios again (with the reclaim 1420 * flag set). 1421 */ 1422 if (folio_is_file_lru(folio) && 1423 (!current_is_kswapd() || 1424 !folio_test_reclaim(folio) || 1425 !test_bit(PGDAT_DIRTY, &pgdat->flags))) { 1426 /* 1427 * Immediately reclaim when written back. 1428 * Similar in principle to folio_deactivate() 1429 * except we already have the folio isolated 1430 * and know it's dirty 1431 */ 1432 node_stat_mod_folio(folio, NR_VMSCAN_IMMEDIATE, 1433 nr_pages); 1434 folio_set_reclaim(folio); 1435 1436 goto activate_locked; 1437 } 1438 1439 if (references == FOLIOREF_RECLAIM_CLEAN) 1440 goto keep_locked; 1441 if (!may_enter_fs(folio, sc->gfp_mask)) 1442 goto keep_locked; 1443 if (!sc->may_writepage) 1444 goto keep_locked; 1445 1446 /* 1447 * Folio is dirty. Flush the TLB if a writable entry 1448 * potentially exists to avoid CPU writes after I/O 1449 * starts and then write it out here. 1450 */ 1451 try_to_unmap_flush_dirty(); 1452 switch (pageout(folio, mapping, &plug, folio_list)) { 1453 case PAGE_KEEP: 1454 goto keep_locked; 1455 case PAGE_ACTIVATE: 1456 /* 1457 * If shmem folio is split when writeback to swap, 1458 * the tail pages will make their own pass through 1459 * this function and be accounted then. 1460 */ 1461 if (nr_pages > 1 && !folio_test_large(folio)) { 1462 sc->nr_scanned -= (nr_pages - 1); 1463 nr_pages = 1; 1464 } 1465 goto activate_locked; 1466 case PAGE_SUCCESS: 1467 if (nr_pages > 1 && !folio_test_large(folio)) { 1468 sc->nr_scanned -= (nr_pages - 1); 1469 nr_pages = 1; 1470 } 1471 stat->nr_pageout += nr_pages; 1472 1473 if (folio_test_writeback(folio)) 1474 goto keep; 1475 if (folio_test_dirty(folio)) 1476 goto keep; 1477 1478 /* 1479 * A synchronous write - probably a ramdisk. Go 1480 * ahead and try to reclaim the folio. 1481 */ 1482 if (!folio_trylock(folio)) 1483 goto keep; 1484 if (folio_test_dirty(folio) || 1485 folio_test_writeback(folio)) 1486 goto keep_locked; 1487 mapping = folio_mapping(folio); 1488 fallthrough; 1489 case PAGE_CLEAN: 1490 ; /* try to free the folio below */ 1491 } 1492 } 1493 1494 /* 1495 * If the folio has buffers, try to free the buffer 1496 * mappings associated with this folio. If we succeed 1497 * we try to free the folio as well. 1498 * 1499 * We do this even if the folio is dirty. 1500 * filemap_release_folio() does not perform I/O, but it 1501 * is possible for a folio to have the dirty flag set, 1502 * but it is actually clean (all its buffers are clean). 1503 * This happens if the buffers were written out directly, 1504 * with submit_bh(). ext3 will do this, as well as 1505 * the blockdev mapping. filemap_release_folio() will 1506 * discover that cleanness and will drop the buffers 1507 * and mark the folio clean - it can be freed. 1508 * 1509 * Rarely, folios can have buffers and no ->mapping. 1510 * These are the folios which were not successfully 1511 * invalidated in truncate_cleanup_folio(). We try to 1512 * drop those buffers here and if that worked, and the 1513 * folio is no longer mapped into process address space 1514 * (refcount == 1) it can be freed. Otherwise, leave 1515 * the folio on the LRU so it is swappable. 1516 */ 1517 if (folio_needs_release(folio)) { 1518 if (!filemap_release_folio(folio, sc->gfp_mask)) 1519 goto activate_locked; 1520 if (!mapping && folio_ref_count(folio) == 1) { 1521 folio_unlock(folio); 1522 if (folio_put_testzero(folio)) 1523 goto free_it; 1524 else { 1525 /* 1526 * rare race with speculative reference. 1527 * the speculative reference will free 1528 * this folio shortly, so we may 1529 * increment nr_reclaimed here (and 1530 * leave it off the LRU). 1531 */ 1532 nr_reclaimed += nr_pages; 1533 continue; 1534 } 1535 } 1536 } 1537 1538 if (folio_test_anon(folio) && !folio_test_swapbacked(folio)) { 1539 /* follow __remove_mapping for reference */ 1540 if (!folio_ref_freeze(folio, 1)) 1541 goto keep_locked; 1542 /* 1543 * The folio has only one reference left, which is 1544 * from the isolation. After the caller puts the 1545 * folio back on the lru and drops the reference, the 1546 * folio will be freed anyway. It doesn't matter 1547 * which lru it goes on. So we don't bother checking 1548 * the dirty flag here. 1549 */ 1550 count_vm_events(PGLAZYFREED, nr_pages); 1551 count_memcg_folio_events(folio, PGLAZYFREED, nr_pages); 1552 } else if (!mapping || !__remove_mapping(mapping, folio, true, 1553 sc->target_mem_cgroup)) 1554 goto keep_locked; 1555 1556 folio_unlock(folio); 1557 free_it: 1558 /* 1559 * Folio may get swapped out as a whole, need to account 1560 * all pages in it. 1561 */ 1562 nr_reclaimed += nr_pages; 1563 1564 folio_unqueue_deferred_split(folio); 1565 if (folio_batch_add(&free_folios, folio) == 0) { 1566 mem_cgroup_uncharge_folios(&free_folios); 1567 try_to_unmap_flush(); 1568 free_unref_folios(&free_folios); 1569 } 1570 continue; 1571 1572 activate_locked_split: 1573 /* 1574 * The tail pages that are failed to add into swap cache 1575 * reach here. Fixup nr_scanned and nr_pages. 1576 */ 1577 if (nr_pages > 1) { 1578 sc->nr_scanned -= (nr_pages - 1); 1579 nr_pages = 1; 1580 } 1581 activate_locked: 1582 /* Not a candidate for swapping, so reclaim swap space. */ 1583 if (folio_test_swapcache(folio) && 1584 (mem_cgroup_swap_full(folio) || folio_test_mlocked(folio))) 1585 folio_free_swap(folio); 1586 VM_BUG_ON_FOLIO(folio_test_active(folio), folio); 1587 if (!folio_test_mlocked(folio)) { 1588 int type = folio_is_file_lru(folio); 1589 folio_set_active(folio); 1590 stat->nr_activate[type] += nr_pages; 1591 count_memcg_folio_events(folio, PGACTIVATE, nr_pages); 1592 } 1593 keep_locked: 1594 folio_unlock(folio); 1595 keep: 1596 list_add(&folio->lru, &ret_folios); 1597 VM_BUG_ON_FOLIO(folio_test_lru(folio) || 1598 folio_test_unevictable(folio), folio); 1599 } 1600 /* 'folio_list' is always empty here */ 1601 1602 /* Migrate folios selected for demotion */ 1603 nr_demoted = demote_folio_list(&demote_folios, pgdat); 1604 nr_reclaimed += nr_demoted; 1605 stat->nr_demoted += nr_demoted; 1606 /* Folios that could not be demoted are still in @demote_folios */ 1607 if (!list_empty(&demote_folios)) { 1608 /* Folios which weren't demoted go back on @folio_list */ 1609 list_splice_init(&demote_folios, folio_list); 1610 1611 /* 1612 * goto retry to reclaim the undemoted folios in folio_list if 1613 * desired. 1614 * 1615 * Reclaiming directly from top tier nodes is not often desired 1616 * due to it breaking the LRU ordering: in general memory 1617 * should be reclaimed from lower tier nodes and demoted from 1618 * top tier nodes. 1619 * 1620 * However, disabling reclaim from top tier nodes entirely 1621 * would cause ooms in edge scenarios where lower tier memory 1622 * is unreclaimable for whatever reason, eg memory being 1623 * mlocked or too hot to reclaim. We can disable reclaim 1624 * from top tier nodes in proactive reclaim though as that is 1625 * not real memory pressure. 1626 */ 1627 if (!sc->proactive) { 1628 do_demote_pass = false; 1629 goto retry; 1630 } 1631 } 1632 1633 pgactivate = stat->nr_activate[0] + stat->nr_activate[1]; 1634 1635 mem_cgroup_uncharge_folios(&free_folios); 1636 try_to_unmap_flush(); 1637 free_unref_folios(&free_folios); 1638 1639 list_splice(&ret_folios, folio_list); 1640 count_vm_events(PGACTIVATE, pgactivate); 1641 1642 if (plug) 1643 swap_write_unplug(plug); 1644 return nr_reclaimed; 1645 } 1646 1647 unsigned int reclaim_clean_pages_from_list(struct zone *zone, 1648 struct list_head *folio_list) 1649 { 1650 struct scan_control sc = { 1651 .gfp_mask = GFP_KERNEL, 1652 .may_unmap = 1, 1653 }; 1654 struct reclaim_stat stat; 1655 unsigned int nr_reclaimed; 1656 struct folio *folio, *next; 1657 LIST_HEAD(clean_folios); 1658 unsigned int noreclaim_flag; 1659 1660 list_for_each_entry_safe(folio, next, folio_list, lru) { 1661 if (!folio_test_hugetlb(folio) && folio_is_file_lru(folio) && 1662 !folio_test_dirty(folio) && !__folio_test_movable(folio) && 1663 !folio_test_unevictable(folio)) { 1664 folio_clear_active(folio); 1665 list_move(&folio->lru, &clean_folios); 1666 } 1667 } 1668 1669 /* 1670 * We should be safe here since we are only dealing with file pages and 1671 * we are not kswapd and therefore cannot write dirty file pages. But 1672 * call memalloc_noreclaim_save() anyway, just in case these conditions 1673 * change in the future. 1674 */ 1675 noreclaim_flag = memalloc_noreclaim_save(); 1676 nr_reclaimed = shrink_folio_list(&clean_folios, zone->zone_pgdat, &sc, 1677 &stat, true, NULL); 1678 memalloc_noreclaim_restore(noreclaim_flag); 1679 1680 list_splice(&clean_folios, folio_list); 1681 mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, 1682 -(long)nr_reclaimed); 1683 /* 1684 * Since lazyfree pages are isolated from file LRU from the beginning, 1685 * they will rotate back to anonymous LRU in the end if it failed to 1686 * discard so isolated count will be mismatched. 1687 * Compensate the isolated count for both LRU lists. 1688 */ 1689 mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON, 1690 stat.nr_lazyfree_fail); 1691 mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, 1692 -(long)stat.nr_lazyfree_fail); 1693 return nr_reclaimed; 1694 } 1695 1696 /* 1697 * Update LRU sizes after isolating pages. The LRU size updates must 1698 * be complete before mem_cgroup_update_lru_size due to a sanity check. 1699 */ 1700 static __always_inline void update_lru_sizes(struct lruvec *lruvec, 1701 enum lru_list lru, unsigned long *nr_zone_taken) 1702 { 1703 int zid; 1704 1705 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1706 if (!nr_zone_taken[zid]) 1707 continue; 1708 1709 update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]); 1710 } 1711 1712 } 1713 1714 /* 1715 * Isolating page from the lruvec to fill in @dst list by nr_to_scan times. 1716 * 1717 * lruvec->lru_lock is heavily contended. Some of the functions that 1718 * shrink the lists perform better by taking out a batch of pages 1719 * and working on them outside the LRU lock. 1720 * 1721 * For pagecache intensive workloads, this function is the hottest 1722 * spot in the kernel (apart from copy_*_user functions). 1723 * 1724 * Lru_lock must be held before calling this function. 1725 * 1726 * @nr_to_scan: The number of eligible pages to look through on the list. 1727 * @lruvec: The LRU vector to pull pages from. 1728 * @dst: The temp list to put pages on to. 1729 * @nr_scanned: The number of pages that were scanned. 1730 * @sc: The scan_control struct for this reclaim session 1731 * @lru: LRU list id for isolating 1732 * 1733 * returns how many pages were moved onto *@dst. 1734 */ 1735 static unsigned long isolate_lru_folios(unsigned long nr_to_scan, 1736 struct lruvec *lruvec, struct list_head *dst, 1737 unsigned long *nr_scanned, struct scan_control *sc, 1738 enum lru_list lru) 1739 { 1740 struct list_head *src = &lruvec->lists[lru]; 1741 unsigned long nr_taken = 0; 1742 unsigned long nr_zone_taken[MAX_NR_ZONES] = { 0 }; 1743 unsigned long nr_skipped[MAX_NR_ZONES] = { 0, }; 1744 unsigned long skipped = 0, total_scan = 0, scan = 0; 1745 unsigned long nr_pages; 1746 unsigned long max_nr_skipped = 0; 1747 LIST_HEAD(folios_skipped); 1748 1749 while (scan < nr_to_scan && !list_empty(src)) { 1750 struct list_head *move_to = src; 1751 struct folio *folio; 1752 1753 folio = lru_to_folio(src); 1754 prefetchw_prev_lru_folio(folio, src, flags); 1755 1756 nr_pages = folio_nr_pages(folio); 1757 total_scan += nr_pages; 1758 1759 /* Using max_nr_skipped to prevent hard LOCKUP*/ 1760 if (max_nr_skipped < SWAP_CLUSTER_MAX_SKIPPED && 1761 (folio_zonenum(folio) > sc->reclaim_idx)) { 1762 nr_skipped[folio_zonenum(folio)] += nr_pages; 1763 move_to = &folios_skipped; 1764 max_nr_skipped++; 1765 goto move; 1766 } 1767 1768 /* 1769 * Do not count skipped folios because that makes the function 1770 * return with no isolated folios if the LRU mostly contains 1771 * ineligible folios. This causes the VM to not reclaim any 1772 * folios, triggering a premature OOM. 1773 * Account all pages in a folio. 1774 */ 1775 scan += nr_pages; 1776 1777 if (!folio_test_lru(folio)) 1778 goto move; 1779 if (!sc->may_unmap && folio_mapped(folio)) 1780 goto move; 1781 1782 /* 1783 * Be careful not to clear the lru flag until after we're 1784 * sure the folio is not being freed elsewhere -- the 1785 * folio release code relies on it. 1786 */ 1787 if (unlikely(!folio_try_get(folio))) 1788 goto move; 1789 1790 if (!folio_test_clear_lru(folio)) { 1791 /* Another thread is already isolating this folio */ 1792 folio_put(folio); 1793 goto move; 1794 } 1795 1796 nr_taken += nr_pages; 1797 nr_zone_taken[folio_zonenum(folio)] += nr_pages; 1798 move_to = dst; 1799 move: 1800 list_move(&folio->lru, move_to); 1801 } 1802 1803 /* 1804 * Splice any skipped folios to the start of the LRU list. Note that 1805 * this disrupts the LRU order when reclaiming for lower zones but 1806 * we cannot splice to the tail. If we did then the SWAP_CLUSTER_MAX 1807 * scanning would soon rescan the same folios to skip and waste lots 1808 * of cpu cycles. 1809 */ 1810 if (!list_empty(&folios_skipped)) { 1811 int zid; 1812 1813 list_splice(&folios_skipped, src); 1814 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1815 if (!nr_skipped[zid]) 1816 continue; 1817 1818 __count_zid_vm_events(PGSCAN_SKIP, zid, nr_skipped[zid]); 1819 skipped += nr_skipped[zid]; 1820 } 1821 } 1822 *nr_scanned = total_scan; 1823 trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan, 1824 total_scan, skipped, nr_taken, lru); 1825 update_lru_sizes(lruvec, lru, nr_zone_taken); 1826 return nr_taken; 1827 } 1828 1829 /** 1830 * folio_isolate_lru() - Try to isolate a folio from its LRU list. 1831 * @folio: Folio to isolate from its LRU list. 1832 * 1833 * Isolate a @folio from an LRU list and adjust the vmstat statistic 1834 * corresponding to whatever LRU list the folio was on. 1835 * 1836 * The folio will have its LRU flag cleared. If it was found on the 1837 * active list, it will have the Active flag set. If it was found on the 1838 * unevictable list, it will have the Unevictable flag set. These flags 1839 * may need to be cleared by the caller before letting the page go. 1840 * 1841 * Context: 1842 * 1843 * (1) Must be called with an elevated refcount on the folio. This is a 1844 * fundamental difference from isolate_lru_folios() (which is called 1845 * without a stable reference). 1846 * (2) The lru_lock must not be held. 1847 * (3) Interrupts must be enabled. 1848 * 1849 * Return: true if the folio was removed from an LRU list. 1850 * false if the folio was not on an LRU list. 1851 */ 1852 bool folio_isolate_lru(struct folio *folio) 1853 { 1854 bool ret = false; 1855 1856 VM_BUG_ON_FOLIO(!folio_ref_count(folio), folio); 1857 1858 if (folio_test_clear_lru(folio)) { 1859 struct lruvec *lruvec; 1860 1861 folio_get(folio); 1862 lruvec = folio_lruvec_lock_irq(folio); 1863 lruvec_del_folio(lruvec, folio); 1864 unlock_page_lruvec_irq(lruvec); 1865 ret = true; 1866 } 1867 1868 return ret; 1869 } 1870 1871 /* 1872 * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and 1873 * then get rescheduled. When there are massive number of tasks doing page 1874 * allocation, such sleeping direct reclaimers may keep piling up on each CPU, 1875 * the LRU list will go small and be scanned faster than necessary, leading to 1876 * unnecessary swapping, thrashing and OOM. 1877 */ 1878 static bool too_many_isolated(struct pglist_data *pgdat, int file, 1879 struct scan_control *sc) 1880 { 1881 unsigned long inactive, isolated; 1882 bool too_many; 1883 1884 if (current_is_kswapd()) 1885 return false; 1886 1887 if (!writeback_throttling_sane(sc)) 1888 return false; 1889 1890 if (file) { 1891 inactive = node_page_state(pgdat, NR_INACTIVE_FILE); 1892 isolated = node_page_state(pgdat, NR_ISOLATED_FILE); 1893 } else { 1894 inactive = node_page_state(pgdat, NR_INACTIVE_ANON); 1895 isolated = node_page_state(pgdat, NR_ISOLATED_ANON); 1896 } 1897 1898 /* 1899 * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they 1900 * won't get blocked by normal direct-reclaimers, forming a circular 1901 * deadlock. 1902 */ 1903 if (gfp_has_io_fs(sc->gfp_mask)) 1904 inactive >>= 3; 1905 1906 too_many = isolated > inactive; 1907 1908 /* Wake up tasks throttled due to too_many_isolated. */ 1909 if (!too_many) 1910 wake_throttle_isolated(pgdat); 1911 1912 return too_many; 1913 } 1914 1915 /* 1916 * move_folios_to_lru() moves folios from private @list to appropriate LRU list. 1917 * 1918 * Returns the number of pages moved to the given lruvec. 1919 */ 1920 static unsigned int move_folios_to_lru(struct lruvec *lruvec, 1921 struct list_head *list) 1922 { 1923 int nr_pages, nr_moved = 0; 1924 struct folio_batch free_folios; 1925 1926 folio_batch_init(&free_folios); 1927 while (!list_empty(list)) { 1928 struct folio *folio = lru_to_folio(list); 1929 1930 VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); 1931 list_del(&folio->lru); 1932 if (unlikely(!folio_evictable(folio))) { 1933 spin_unlock_irq(&lruvec->lru_lock); 1934 folio_putback_lru(folio); 1935 spin_lock_irq(&lruvec->lru_lock); 1936 continue; 1937 } 1938 1939 /* 1940 * The folio_set_lru needs to be kept here for list integrity. 1941 * Otherwise: 1942 * #0 move_folios_to_lru #1 release_pages 1943 * if (!folio_put_testzero()) 1944 * if (folio_put_testzero()) 1945 * !lru //skip lru_lock 1946 * folio_set_lru() 1947 * list_add(&folio->lru,) 1948 * list_add(&folio->lru,) 1949 */ 1950 folio_set_lru(folio); 1951 1952 if (unlikely(folio_put_testzero(folio))) { 1953 __folio_clear_lru_flags(folio); 1954 1955 folio_unqueue_deferred_split(folio); 1956 if (folio_batch_add(&free_folios, folio) == 0) { 1957 spin_unlock_irq(&lruvec->lru_lock); 1958 mem_cgroup_uncharge_folios(&free_folios); 1959 free_unref_folios(&free_folios); 1960 spin_lock_irq(&lruvec->lru_lock); 1961 } 1962 1963 continue; 1964 } 1965 1966 /* 1967 * All pages were isolated from the same lruvec (and isolation 1968 * inhibits memcg migration). 1969 */ 1970 VM_BUG_ON_FOLIO(!folio_matches_lruvec(folio, lruvec), folio); 1971 lruvec_add_folio(lruvec, folio); 1972 nr_pages = folio_nr_pages(folio); 1973 nr_moved += nr_pages; 1974 if (folio_test_active(folio)) 1975 workingset_age_nonresident(lruvec, nr_pages); 1976 } 1977 1978 if (free_folios.nr) { 1979 spin_unlock_irq(&lruvec->lru_lock); 1980 mem_cgroup_uncharge_folios(&free_folios); 1981 free_unref_folios(&free_folios); 1982 spin_lock_irq(&lruvec->lru_lock); 1983 } 1984 1985 return nr_moved; 1986 } 1987 1988 /* 1989 * If a kernel thread (such as nfsd for loop-back mounts) services a backing 1990 * device by writing to the page cache it sets PF_LOCAL_THROTTLE. In this case 1991 * we should not throttle. Otherwise it is safe to do so. 1992 */ 1993 static int current_may_throttle(void) 1994 { 1995 return !(current->flags & PF_LOCAL_THROTTLE); 1996 } 1997 1998 /* 1999 * shrink_inactive_list() is a helper for shrink_node(). It returns the number 2000 * of reclaimed pages 2001 */ 2002 static unsigned long shrink_inactive_list(unsigned long nr_to_scan, 2003 struct lruvec *lruvec, struct scan_control *sc, 2004 enum lru_list lru) 2005 { 2006 LIST_HEAD(folio_list); 2007 unsigned long nr_scanned; 2008 unsigned int nr_reclaimed = 0; 2009 unsigned long nr_taken; 2010 struct reclaim_stat stat; 2011 bool file = is_file_lru(lru); 2012 enum vm_event_item item; 2013 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 2014 bool stalled = false; 2015 2016 while (unlikely(too_many_isolated(pgdat, file, sc))) { 2017 if (stalled) 2018 return 0; 2019 2020 /* wait a bit for the reclaimer. */ 2021 stalled = true; 2022 reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED); 2023 2024 /* We are about to die and free our memory. Return now. */ 2025 if (fatal_signal_pending(current)) 2026 return SWAP_CLUSTER_MAX; 2027 } 2028 2029 lru_add_drain(); 2030 2031 spin_lock_irq(&lruvec->lru_lock); 2032 2033 nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &folio_list, 2034 &nr_scanned, sc, lru); 2035 2036 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken); 2037 item = PGSCAN_KSWAPD + reclaimer_offset(sc); 2038 if (!cgroup_reclaim(sc)) 2039 __count_vm_events(item, nr_scanned); 2040 count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned); 2041 __count_vm_events(PGSCAN_ANON + file, nr_scanned); 2042 2043 spin_unlock_irq(&lruvec->lru_lock); 2044 2045 if (nr_taken == 0) 2046 return 0; 2047 2048 nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false, 2049 lruvec_memcg(lruvec)); 2050 2051 spin_lock_irq(&lruvec->lru_lock); 2052 move_folios_to_lru(lruvec, &folio_list); 2053 2054 __mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc), 2055 stat.nr_demoted); 2056 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); 2057 item = PGSTEAL_KSWAPD + reclaimer_offset(sc); 2058 if (!cgroup_reclaim(sc)) 2059 __count_vm_events(item, nr_reclaimed); 2060 count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed); 2061 __count_vm_events(PGSTEAL_ANON + file, nr_reclaimed); 2062 spin_unlock_irq(&lruvec->lru_lock); 2063 2064 lru_note_cost(lruvec, file, stat.nr_pageout, nr_scanned - nr_reclaimed); 2065 2066 /* 2067 * If dirty folios are scanned that are not queued for IO, it 2068 * implies that flushers are not doing their job. This can 2069 * happen when memory pressure pushes dirty folios to the end of 2070 * the LRU before the dirty limits are breached and the dirty 2071 * data has expired. It can also happen when the proportion of 2072 * dirty folios grows not through writes but through memory 2073 * pressure reclaiming all the clean cache. And in some cases, 2074 * the flushers simply cannot keep up with the allocation 2075 * rate. Nudge the flusher threads in case they are asleep. 2076 */ 2077 if (stat.nr_unqueued_dirty == nr_taken) { 2078 wakeup_flusher_threads(WB_REASON_VMSCAN); 2079 /* 2080 * For cgroupv1 dirty throttling is achieved by waking up 2081 * the kernel flusher here and later waiting on folios 2082 * which are in writeback to finish (see shrink_folio_list()). 2083 * 2084 * Flusher may not be able to issue writeback quickly 2085 * enough for cgroupv1 writeback throttling to work 2086 * on a large system. 2087 */ 2088 if (!writeback_throttling_sane(sc)) 2089 reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK); 2090 } 2091 2092 sc->nr.dirty += stat.nr_dirty; 2093 sc->nr.congested += stat.nr_congested; 2094 sc->nr.unqueued_dirty += stat.nr_unqueued_dirty; 2095 sc->nr.writeback += stat.nr_writeback; 2096 sc->nr.immediate += stat.nr_immediate; 2097 sc->nr.taken += nr_taken; 2098 if (file) 2099 sc->nr.file_taken += nr_taken; 2100 2101 trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id, 2102 nr_scanned, nr_reclaimed, &stat, sc->priority, file); 2103 return nr_reclaimed; 2104 } 2105 2106 /* 2107 * shrink_active_list() moves folios from the active LRU to the inactive LRU. 2108 * 2109 * We move them the other way if the folio is referenced by one or more 2110 * processes. 2111 * 2112 * If the folios are mostly unmapped, the processing is fast and it is 2113 * appropriate to hold lru_lock across the whole operation. But if 2114 * the folios are mapped, the processing is slow (folio_referenced()), so 2115 * we should drop lru_lock around each folio. It's impossible to balance 2116 * this, so instead we remove the folios from the LRU while processing them. 2117 * It is safe to rely on the active flag against the non-LRU folios in here 2118 * because nobody will play with that bit on a non-LRU folio. 2119 * 2120 * The downside is that we have to touch folio->_refcount against each folio. 2121 * But we had to alter folio->flags anyway. 2122 */ 2123 static void shrink_active_list(unsigned long nr_to_scan, 2124 struct lruvec *lruvec, 2125 struct scan_control *sc, 2126 enum lru_list lru) 2127 { 2128 unsigned long nr_taken; 2129 unsigned long nr_scanned; 2130 unsigned long vm_flags; 2131 LIST_HEAD(l_hold); /* The folios which were snipped off */ 2132 LIST_HEAD(l_active); 2133 LIST_HEAD(l_inactive); 2134 unsigned nr_deactivate, nr_activate; 2135 unsigned nr_rotated = 0; 2136 bool file = is_file_lru(lru); 2137 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 2138 2139 lru_add_drain(); 2140 2141 spin_lock_irq(&lruvec->lru_lock); 2142 2143 nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &l_hold, 2144 &nr_scanned, sc, lru); 2145 2146 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken); 2147 2148 if (!cgroup_reclaim(sc)) 2149 __count_vm_events(PGREFILL, nr_scanned); 2150 count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned); 2151 2152 spin_unlock_irq(&lruvec->lru_lock); 2153 2154 while (!list_empty(&l_hold)) { 2155 struct folio *folio; 2156 2157 cond_resched(); 2158 folio = lru_to_folio(&l_hold); 2159 list_del(&folio->lru); 2160 2161 if (unlikely(!folio_evictable(folio))) { 2162 folio_putback_lru(folio); 2163 continue; 2164 } 2165 2166 if (unlikely(buffer_heads_over_limit)) { 2167 if (folio_needs_release(folio) && 2168 folio_trylock(folio)) { 2169 filemap_release_folio(folio, 0); 2170 folio_unlock(folio); 2171 } 2172 } 2173 2174 /* Referenced or rmap lock contention: rotate */ 2175 if (folio_referenced(folio, 0, sc->target_mem_cgroup, 2176 &vm_flags) != 0) { 2177 /* 2178 * Identify referenced, file-backed active folios and 2179 * give them one more trip around the active list. So 2180 * that executable code get better chances to stay in 2181 * memory under moderate memory pressure. Anon folios 2182 * are not likely to be evicted by use-once streaming 2183 * IO, plus JVM can create lots of anon VM_EXEC folios, 2184 * so we ignore them here. 2185 */ 2186 if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) { 2187 nr_rotated += folio_nr_pages(folio); 2188 list_add(&folio->lru, &l_active); 2189 continue; 2190 } 2191 } 2192 2193 folio_clear_active(folio); /* we are de-activating */ 2194 folio_set_workingset(folio); 2195 list_add(&folio->lru, &l_inactive); 2196 } 2197 2198 /* 2199 * Move folios back to the lru list. 2200 */ 2201 spin_lock_irq(&lruvec->lru_lock); 2202 2203 nr_activate = move_folios_to_lru(lruvec, &l_active); 2204 nr_deactivate = move_folios_to_lru(lruvec, &l_inactive); 2205 2206 __count_vm_events(PGDEACTIVATE, nr_deactivate); 2207 count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate); 2208 2209 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); 2210 spin_unlock_irq(&lruvec->lru_lock); 2211 2212 if (nr_rotated) 2213 lru_note_cost(lruvec, file, 0, nr_rotated); 2214 trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate, 2215 nr_deactivate, nr_rotated, sc->priority, file); 2216 } 2217 2218 static unsigned int reclaim_folio_list(struct list_head *folio_list, 2219 struct pglist_data *pgdat) 2220 { 2221 struct reclaim_stat stat; 2222 unsigned int nr_reclaimed; 2223 struct folio *folio; 2224 struct scan_control sc = { 2225 .gfp_mask = GFP_KERNEL, 2226 .may_writepage = 1, 2227 .may_unmap = 1, 2228 .may_swap = 1, 2229 .no_demotion = 1, 2230 }; 2231 2232 nr_reclaimed = shrink_folio_list(folio_list, pgdat, &sc, &stat, true, NULL); 2233 while (!list_empty(folio_list)) { 2234 folio = lru_to_folio(folio_list); 2235 list_del(&folio->lru); 2236 folio_putback_lru(folio); 2237 } 2238 trace_mm_vmscan_reclaim_pages(pgdat->node_id, sc.nr_scanned, nr_reclaimed, &stat); 2239 2240 return nr_reclaimed; 2241 } 2242 2243 unsigned long reclaim_pages(struct list_head *folio_list) 2244 { 2245 int nid; 2246 unsigned int nr_reclaimed = 0; 2247 LIST_HEAD(node_folio_list); 2248 unsigned int noreclaim_flag; 2249 2250 if (list_empty(folio_list)) 2251 return nr_reclaimed; 2252 2253 noreclaim_flag = memalloc_noreclaim_save(); 2254 2255 nid = folio_nid(lru_to_folio(folio_list)); 2256 do { 2257 struct folio *folio = lru_to_folio(folio_list); 2258 2259 if (nid == folio_nid(folio)) { 2260 folio_clear_active(folio); 2261 list_move(&folio->lru, &node_folio_list); 2262 continue; 2263 } 2264 2265 nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid)); 2266 nid = folio_nid(lru_to_folio(folio_list)); 2267 } while (!list_empty(folio_list)); 2268 2269 nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid)); 2270 2271 memalloc_noreclaim_restore(noreclaim_flag); 2272 2273 return nr_reclaimed; 2274 } 2275 2276 static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan, 2277 struct lruvec *lruvec, struct scan_control *sc) 2278 { 2279 if (is_active_lru(lru)) { 2280 if (sc->may_deactivate & (1 << is_file_lru(lru))) 2281 shrink_active_list(nr_to_scan, lruvec, sc, lru); 2282 else 2283 sc->skipped_deactivate = 1; 2284 return 0; 2285 } 2286 2287 return shrink_inactive_list(nr_to_scan, lruvec, sc, lru); 2288 } 2289 2290 /* 2291 * The inactive anon list should be small enough that the VM never has 2292 * to do too much work. 2293 * 2294 * The inactive file list should be small enough to leave most memory 2295 * to the established workingset on the scan-resistant active list, 2296 * but large enough to avoid thrashing the aggregate readahead window. 2297 * 2298 * Both inactive lists should also be large enough that each inactive 2299 * folio has a chance to be referenced again before it is reclaimed. 2300 * 2301 * If that fails and refaulting is observed, the inactive list grows. 2302 * 2303 * The inactive_ratio is the target ratio of ACTIVE to INACTIVE folios 2304 * on this LRU, maintained by the pageout code. An inactive_ratio 2305 * of 3 means 3:1 or 25% of the folios are kept on the inactive list. 2306 * 2307 * total target max 2308 * memory ratio inactive 2309 * ------------------------------------- 2310 * 10MB 1 5MB 2311 * 100MB 1 50MB 2312 * 1GB 3 250MB 2313 * 10GB 10 0.9GB 2314 * 100GB 31 3GB 2315 * 1TB 101 10GB 2316 * 10TB 320 32GB 2317 */ 2318 static bool inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru) 2319 { 2320 enum lru_list active_lru = inactive_lru + LRU_ACTIVE; 2321 unsigned long inactive, active; 2322 unsigned long inactive_ratio; 2323 unsigned long gb; 2324 2325 inactive = lruvec_page_state(lruvec, NR_LRU_BASE + inactive_lru); 2326 active = lruvec_page_state(lruvec, NR_LRU_BASE + active_lru); 2327 2328 gb = (inactive + active) >> (30 - PAGE_SHIFT); 2329 if (gb) 2330 inactive_ratio = int_sqrt(10 * gb); 2331 else 2332 inactive_ratio = 1; 2333 2334 return inactive * inactive_ratio < active; 2335 } 2336 2337 enum scan_balance { 2338 SCAN_EQUAL, 2339 SCAN_FRACT, 2340 SCAN_ANON, 2341 SCAN_FILE, 2342 }; 2343 2344 static void prepare_scan_control(pg_data_t *pgdat, struct scan_control *sc) 2345 { 2346 unsigned long file; 2347 struct lruvec *target_lruvec; 2348 2349 if (lru_gen_enabled()) 2350 return; 2351 2352 target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); 2353 2354 /* 2355 * Flush the memory cgroup stats in rate-limited way as we don't need 2356 * most accurate stats here. We may switch to regular stats flushing 2357 * in the future once it is cheap enough. 2358 */ 2359 mem_cgroup_flush_stats_ratelimited(sc->target_mem_cgroup); 2360 2361 /* 2362 * Determine the scan balance between anon and file LRUs. 2363 */ 2364 spin_lock_irq(&target_lruvec->lru_lock); 2365 sc->anon_cost = target_lruvec->anon_cost; 2366 sc->file_cost = target_lruvec->file_cost; 2367 spin_unlock_irq(&target_lruvec->lru_lock); 2368 2369 /* 2370 * Target desirable inactive:active list ratios for the anon 2371 * and file LRU lists. 2372 */ 2373 if (!sc->force_deactivate) { 2374 unsigned long refaults; 2375 2376 /* 2377 * When refaults are being observed, it means a new 2378 * workingset is being established. Deactivate to get 2379 * rid of any stale active pages quickly. 2380 */ 2381 refaults = lruvec_page_state(target_lruvec, 2382 WORKINGSET_ACTIVATE_ANON); 2383 if (refaults != target_lruvec->refaults[WORKINGSET_ANON] || 2384 inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) 2385 sc->may_deactivate |= DEACTIVATE_ANON; 2386 else 2387 sc->may_deactivate &= ~DEACTIVATE_ANON; 2388 2389 refaults = lruvec_page_state(target_lruvec, 2390 WORKINGSET_ACTIVATE_FILE); 2391 if (refaults != target_lruvec->refaults[WORKINGSET_FILE] || 2392 inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) 2393 sc->may_deactivate |= DEACTIVATE_FILE; 2394 else 2395 sc->may_deactivate &= ~DEACTIVATE_FILE; 2396 } else 2397 sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; 2398 2399 /* 2400 * If we have plenty of inactive file pages that aren't 2401 * thrashing, try to reclaim those first before touching 2402 * anonymous pages. 2403 */ 2404 file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); 2405 if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE) && 2406 !sc->no_cache_trim_mode) 2407 sc->cache_trim_mode = 1; 2408 else 2409 sc->cache_trim_mode = 0; 2410 2411 /* 2412 * Prevent the reclaimer from falling into the cache trap: as 2413 * cache pages start out inactive, every cache fault will tip 2414 * the scan balance towards the file LRU. And as the file LRU 2415 * shrinks, so does the window for rotation from references. 2416 * This means we have a runaway feedback loop where a tiny 2417 * thrashing file LRU becomes infinitely more attractive than 2418 * anon pages. Try to detect this based on file LRU size. 2419 */ 2420 if (!cgroup_reclaim(sc)) { 2421 unsigned long total_high_wmark = 0; 2422 unsigned long free, anon; 2423 int z; 2424 struct zone *zone; 2425 2426 free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); 2427 file = node_page_state(pgdat, NR_ACTIVE_FILE) + 2428 node_page_state(pgdat, NR_INACTIVE_FILE); 2429 2430 for_each_managed_zone_pgdat(zone, pgdat, z, MAX_NR_ZONES - 1) { 2431 total_high_wmark += high_wmark_pages(zone); 2432 } 2433 2434 /* 2435 * Consider anon: if that's low too, this isn't a 2436 * runaway file reclaim problem, but rather just 2437 * extreme pressure. Reclaim as per usual then. 2438 */ 2439 anon = node_page_state(pgdat, NR_INACTIVE_ANON); 2440 2441 sc->file_is_tiny = 2442 file + free <= total_high_wmark && 2443 !(sc->may_deactivate & DEACTIVATE_ANON) && 2444 anon >> sc->priority; 2445 } 2446 } 2447 2448 static inline void calculate_pressure_balance(struct scan_control *sc, 2449 int swappiness, u64 *fraction, u64 *denominator) 2450 { 2451 unsigned long anon_cost, file_cost, total_cost; 2452 unsigned long ap, fp; 2453 2454 /* 2455 * Calculate the pressure balance between anon and file pages. 2456 * 2457 * The amount of pressure we put on each LRU is inversely 2458 * proportional to the cost of reclaiming each list, as 2459 * determined by the share of pages that are refaulting, times 2460 * the relative IO cost of bringing back a swapped out 2461 * anonymous page vs reloading a filesystem page (swappiness). 2462 * 2463 * Although we limit that influence to ensure no list gets 2464 * left behind completely: at least a third of the pressure is 2465 * applied, before swappiness. 2466 * 2467 * With swappiness at 100, anon and file have equal IO cost. 2468 */ 2469 total_cost = sc->anon_cost + sc->file_cost; 2470 anon_cost = total_cost + sc->anon_cost; 2471 file_cost = total_cost + sc->file_cost; 2472 total_cost = anon_cost + file_cost; 2473 2474 ap = swappiness * (total_cost + 1); 2475 ap /= anon_cost + 1; 2476 2477 fp = (MAX_SWAPPINESS - swappiness) * (total_cost + 1); 2478 fp /= file_cost + 1; 2479 2480 fraction[WORKINGSET_ANON] = ap; 2481 fraction[WORKINGSET_FILE] = fp; 2482 *denominator = ap + fp; 2483 } 2484 2485 /* 2486 * Determine how aggressively the anon and file LRU lists should be 2487 * scanned. 2488 * 2489 * nr[0] = anon inactive folios to scan; nr[1] = anon active folios to scan 2490 * nr[2] = file inactive folios to scan; nr[3] = file active folios to scan 2491 */ 2492 static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, 2493 unsigned long *nr) 2494 { 2495 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 2496 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 2497 int swappiness = sc_swappiness(sc, memcg); 2498 u64 fraction[ANON_AND_FILE]; 2499 u64 denominator = 0; /* gcc */ 2500 enum scan_balance scan_balance; 2501 enum lru_list lru; 2502 2503 /* If we have no swap space, do not bother scanning anon folios. */ 2504 if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) { 2505 scan_balance = SCAN_FILE; 2506 goto out; 2507 } 2508 2509 /* 2510 * Global reclaim will swap to prevent OOM even with no 2511 * swappiness, but memcg users want to use this knob to 2512 * disable swapping for individual groups completely when 2513 * using the memory controller's swap limit feature would be 2514 * too expensive. 2515 */ 2516 if (cgroup_reclaim(sc) && !swappiness) { 2517 scan_balance = SCAN_FILE; 2518 goto out; 2519 } 2520 2521 /* Proactive reclaim initiated by userspace for anonymous memory only */ 2522 if (swappiness == SWAPPINESS_ANON_ONLY) { 2523 WARN_ON_ONCE(!sc->proactive); 2524 scan_balance = SCAN_ANON; 2525 goto out; 2526 } 2527 2528 /* 2529 * Do not apply any pressure balancing cleverness when the 2530 * system is close to OOM, scan both anon and file equally 2531 * (unless the swappiness setting disagrees with swapping). 2532 */ 2533 if (!sc->priority && swappiness) { 2534 scan_balance = SCAN_EQUAL; 2535 goto out; 2536 } 2537 2538 /* 2539 * If the system is almost out of file pages, force-scan anon. 2540 */ 2541 if (sc->file_is_tiny) { 2542 scan_balance = SCAN_ANON; 2543 goto out; 2544 } 2545 2546 /* 2547 * If there is enough inactive page cache, we do not reclaim 2548 * anything from the anonymous working right now to make sure 2549 * a streaming file access pattern doesn't cause swapping. 2550 */ 2551 if (sc->cache_trim_mode) { 2552 scan_balance = SCAN_FILE; 2553 goto out; 2554 } 2555 2556 scan_balance = SCAN_FRACT; 2557 calculate_pressure_balance(sc, swappiness, fraction, &denominator); 2558 2559 out: 2560 for_each_evictable_lru(lru) { 2561 bool file = is_file_lru(lru); 2562 unsigned long lruvec_size; 2563 unsigned long low, min; 2564 unsigned long scan; 2565 2566 lruvec_size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx); 2567 mem_cgroup_protection(sc->target_mem_cgroup, memcg, 2568 &min, &low); 2569 2570 if (min || low) { 2571 /* 2572 * Scale a cgroup's reclaim pressure by proportioning 2573 * its current usage to its memory.low or memory.min 2574 * setting. 2575 * 2576 * This is important, as otherwise scanning aggression 2577 * becomes extremely binary -- from nothing as we 2578 * approach the memory protection threshold, to totally 2579 * nominal as we exceed it. This results in requiring 2580 * setting extremely liberal protection thresholds. It 2581 * also means we simply get no protection at all if we 2582 * set it too low, which is not ideal. 2583 * 2584 * If there is any protection in place, we reduce scan 2585 * pressure by how much of the total memory used is 2586 * within protection thresholds. 2587 * 2588 * There is one special case: in the first reclaim pass, 2589 * we skip over all groups that are within their low 2590 * protection. If that fails to reclaim enough pages to 2591 * satisfy the reclaim goal, we come back and override 2592 * the best-effort low protection. However, we still 2593 * ideally want to honor how well-behaved groups are in 2594 * that case instead of simply punishing them all 2595 * equally. As such, we reclaim them based on how much 2596 * memory they are using, reducing the scan pressure 2597 * again by how much of the total memory used is under 2598 * hard protection. 2599 */ 2600 unsigned long cgroup_size = mem_cgroup_size(memcg); 2601 unsigned long protection; 2602 2603 /* memory.low scaling, make sure we retry before OOM */ 2604 if (!sc->memcg_low_reclaim && low > min) { 2605 protection = low; 2606 sc->memcg_low_skipped = 1; 2607 } else { 2608 protection = min; 2609 } 2610 2611 /* Avoid TOCTOU with earlier protection check */ 2612 cgroup_size = max(cgroup_size, protection); 2613 2614 scan = lruvec_size - lruvec_size * protection / 2615 (cgroup_size + 1); 2616 2617 /* 2618 * Minimally target SWAP_CLUSTER_MAX pages to keep 2619 * reclaim moving forwards, avoiding decrementing 2620 * sc->priority further than desirable. 2621 */ 2622 scan = max(scan, SWAP_CLUSTER_MAX); 2623 } else { 2624 scan = lruvec_size; 2625 } 2626 2627 scan >>= sc->priority; 2628 2629 /* 2630 * If the cgroup's already been deleted, make sure to 2631 * scrape out the remaining cache. 2632 */ 2633 if (!scan && !mem_cgroup_online(memcg)) 2634 scan = min(lruvec_size, SWAP_CLUSTER_MAX); 2635 2636 switch (scan_balance) { 2637 case SCAN_EQUAL: 2638 /* Scan lists relative to size */ 2639 break; 2640 case SCAN_FRACT: 2641 /* 2642 * Scan types proportional to swappiness and 2643 * their relative recent reclaim efficiency. 2644 * Make sure we don't miss the last page on 2645 * the offlined memory cgroups because of a 2646 * round-off error. 2647 */ 2648 scan = mem_cgroup_online(memcg) ? 2649 div64_u64(scan * fraction[file], denominator) : 2650 DIV64_U64_ROUND_UP(scan * fraction[file], 2651 denominator); 2652 break; 2653 case SCAN_FILE: 2654 case SCAN_ANON: 2655 /* Scan one type exclusively */ 2656 if ((scan_balance == SCAN_FILE) != file) 2657 scan = 0; 2658 break; 2659 default: 2660 /* Look ma, no brain */ 2661 BUG(); 2662 } 2663 2664 nr[lru] = scan; 2665 } 2666 } 2667 2668 /* 2669 * Anonymous LRU management is a waste if there is 2670 * ultimately no way to reclaim the memory. 2671 */ 2672 static bool can_age_anon_pages(struct lruvec *lruvec, 2673 struct scan_control *sc) 2674 { 2675 /* Aging the anon LRU is valuable if swap is present: */ 2676 if (total_swap_pages > 0) 2677 return true; 2678 2679 /* Also valuable if anon pages can be demoted: */ 2680 return can_demote(lruvec_pgdat(lruvec)->node_id, sc, 2681 lruvec_memcg(lruvec)); 2682 } 2683 2684 #ifdef CONFIG_LRU_GEN 2685 2686 #ifdef CONFIG_LRU_GEN_ENABLED 2687 DEFINE_STATIC_KEY_ARRAY_TRUE(lru_gen_caps, NR_LRU_GEN_CAPS); 2688 #define get_cap(cap) static_branch_likely(&lru_gen_caps[cap]) 2689 #else 2690 DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS); 2691 #define get_cap(cap) static_branch_unlikely(&lru_gen_caps[cap]) 2692 #endif 2693 2694 static bool should_walk_mmu(void) 2695 { 2696 return arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK); 2697 } 2698 2699 static bool should_clear_pmd_young(void) 2700 { 2701 return arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG); 2702 } 2703 2704 /****************************************************************************** 2705 * shorthand helpers 2706 ******************************************************************************/ 2707 2708 #define DEFINE_MAX_SEQ(lruvec) \ 2709 unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq) 2710 2711 #define DEFINE_MIN_SEQ(lruvec) \ 2712 unsigned long min_seq[ANON_AND_FILE] = { \ 2713 READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \ 2714 READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \ 2715 } 2716 2717 /* Get the min/max evictable type based on swappiness */ 2718 #define min_type(swappiness) (!(swappiness)) 2719 #define max_type(swappiness) ((swappiness) < SWAPPINESS_ANON_ONLY) 2720 2721 #define evictable_min_seq(min_seq, swappiness) \ 2722 min((min_seq)[min_type(swappiness)], (min_seq)[max_type(swappiness)]) 2723 2724 #define for_each_gen_type_zone(gen, type, zone) \ 2725 for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++) \ 2726 for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \ 2727 for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++) 2728 2729 #define for_each_evictable_type(type, swappiness) \ 2730 for ((type) = min_type(swappiness); (type) <= max_type(swappiness); (type)++) 2731 2732 #define get_memcg_gen(seq) ((seq) % MEMCG_NR_GENS) 2733 #define get_memcg_bin(bin) ((bin) % MEMCG_NR_BINS) 2734 2735 static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid) 2736 { 2737 struct pglist_data *pgdat = NODE_DATA(nid); 2738 2739 #ifdef CONFIG_MEMCG 2740 if (memcg) { 2741 struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec; 2742 2743 /* see the comment in mem_cgroup_lruvec() */ 2744 if (!lruvec->pgdat) 2745 lruvec->pgdat = pgdat; 2746 2747 return lruvec; 2748 } 2749 #endif 2750 VM_WARN_ON_ONCE(!mem_cgroup_disabled()); 2751 2752 return &pgdat->__lruvec; 2753 } 2754 2755 static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) 2756 { 2757 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 2758 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 2759 2760 if (!sc->may_swap) 2761 return 0; 2762 2763 if (!can_demote(pgdat->node_id, sc, memcg) && 2764 mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH) 2765 return 0; 2766 2767 return sc_swappiness(sc, memcg); 2768 } 2769 2770 static int get_nr_gens(struct lruvec *lruvec, int type) 2771 { 2772 return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1; 2773 } 2774 2775 static bool __maybe_unused seq_is_valid(struct lruvec *lruvec) 2776 { 2777 int type; 2778 2779 for (type = 0; type < ANON_AND_FILE; type++) { 2780 int n = get_nr_gens(lruvec, type); 2781 2782 if (n < MIN_NR_GENS || n > MAX_NR_GENS) 2783 return false; 2784 } 2785 2786 return true; 2787 } 2788 2789 /****************************************************************************** 2790 * Bloom filters 2791 ******************************************************************************/ 2792 2793 /* 2794 * Bloom filters with m=1<<15, k=2 and the false positive rates of ~1/5 when 2795 * n=10,000 and ~1/2 when n=20,000, where, conventionally, m is the number of 2796 * bits in a bitmap, k is the number of hash functions and n is the number of 2797 * inserted items. 2798 * 2799 * Page table walkers use one of the two filters to reduce their search space. 2800 * To get rid of non-leaf entries that no longer have enough leaf entries, the 2801 * aging uses the double-buffering technique to flip to the other filter each 2802 * time it produces a new generation. For non-leaf entries that have enough 2803 * leaf entries, the aging carries them over to the next generation in 2804 * walk_pmd_range(); the eviction also report them when walking the rmap 2805 * in lru_gen_look_around(). 2806 * 2807 * For future optimizations: 2808 * 1. It's not necessary to keep both filters all the time. The spare one can be 2809 * freed after the RCU grace period and reallocated if needed again. 2810 * 2. And when reallocating, it's worth scaling its size according to the number 2811 * of inserted entries in the other filter, to reduce the memory overhead on 2812 * small systems and false positives on large systems. 2813 * 3. Jenkins' hash function is an alternative to Knuth's. 2814 */ 2815 #define BLOOM_FILTER_SHIFT 15 2816 2817 static inline int filter_gen_from_seq(unsigned long seq) 2818 { 2819 return seq % NR_BLOOM_FILTERS; 2820 } 2821 2822 static void get_item_key(void *item, int *key) 2823 { 2824 u32 hash = hash_ptr(item, BLOOM_FILTER_SHIFT * 2); 2825 2826 BUILD_BUG_ON(BLOOM_FILTER_SHIFT * 2 > BITS_PER_TYPE(u32)); 2827 2828 key[0] = hash & (BIT(BLOOM_FILTER_SHIFT) - 1); 2829 key[1] = hash >> BLOOM_FILTER_SHIFT; 2830 } 2831 2832 static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq, 2833 void *item) 2834 { 2835 int key[2]; 2836 unsigned long *filter; 2837 int gen = filter_gen_from_seq(seq); 2838 2839 filter = READ_ONCE(mm_state->filters[gen]); 2840 if (!filter) 2841 return true; 2842 2843 get_item_key(item, key); 2844 2845 return test_bit(key[0], filter) && test_bit(key[1], filter); 2846 } 2847 2848 static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq, 2849 void *item) 2850 { 2851 int key[2]; 2852 unsigned long *filter; 2853 int gen = filter_gen_from_seq(seq); 2854 2855 filter = READ_ONCE(mm_state->filters[gen]); 2856 if (!filter) 2857 return; 2858 2859 get_item_key(item, key); 2860 2861 if (!test_bit(key[0], filter)) 2862 set_bit(key[0], filter); 2863 if (!test_bit(key[1], filter)) 2864 set_bit(key[1], filter); 2865 } 2866 2867 static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq) 2868 { 2869 unsigned long *filter; 2870 int gen = filter_gen_from_seq(seq); 2871 2872 filter = mm_state->filters[gen]; 2873 if (filter) { 2874 bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT)); 2875 return; 2876 } 2877 2878 filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT), 2879 __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); 2880 WRITE_ONCE(mm_state->filters[gen], filter); 2881 } 2882 2883 /****************************************************************************** 2884 * mm_struct list 2885 ******************************************************************************/ 2886 2887 #ifdef CONFIG_LRU_GEN_WALKS_MMU 2888 2889 static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg) 2890 { 2891 static struct lru_gen_mm_list mm_list = { 2892 .fifo = LIST_HEAD_INIT(mm_list.fifo), 2893 .lock = __SPIN_LOCK_UNLOCKED(mm_list.lock), 2894 }; 2895 2896 #ifdef CONFIG_MEMCG 2897 if (memcg) 2898 return &memcg->mm_list; 2899 #endif 2900 VM_WARN_ON_ONCE(!mem_cgroup_disabled()); 2901 2902 return &mm_list; 2903 } 2904 2905 static struct lru_gen_mm_state *get_mm_state(struct lruvec *lruvec) 2906 { 2907 return &lruvec->mm_state; 2908 } 2909 2910 static struct mm_struct *get_next_mm(struct lru_gen_mm_walk *walk) 2911 { 2912 int key; 2913 struct mm_struct *mm; 2914 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); 2915 struct lru_gen_mm_state *mm_state = get_mm_state(walk->lruvec); 2916 2917 mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list); 2918 key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap); 2919 2920 if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap)) 2921 return NULL; 2922 2923 clear_bit(key, &mm->lru_gen.bitmap); 2924 2925 return mmget_not_zero(mm) ? mm : NULL; 2926 } 2927 2928 void lru_gen_add_mm(struct mm_struct *mm) 2929 { 2930 int nid; 2931 struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); 2932 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 2933 2934 VM_WARN_ON_ONCE(!list_empty(&mm->lru_gen.list)); 2935 #ifdef CONFIG_MEMCG 2936 VM_WARN_ON_ONCE(mm->lru_gen.memcg); 2937 mm->lru_gen.memcg = memcg; 2938 #endif 2939 spin_lock(&mm_list->lock); 2940 2941 for_each_node_state(nid, N_MEMORY) { 2942 struct lruvec *lruvec = get_lruvec(memcg, nid); 2943 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 2944 2945 /* the first addition since the last iteration */ 2946 if (mm_state->tail == &mm_list->fifo) 2947 mm_state->tail = &mm->lru_gen.list; 2948 } 2949 2950 list_add_tail(&mm->lru_gen.list, &mm_list->fifo); 2951 2952 spin_unlock(&mm_list->lock); 2953 } 2954 2955 void lru_gen_del_mm(struct mm_struct *mm) 2956 { 2957 int nid; 2958 struct lru_gen_mm_list *mm_list; 2959 struct mem_cgroup *memcg = NULL; 2960 2961 if (list_empty(&mm->lru_gen.list)) 2962 return; 2963 2964 #ifdef CONFIG_MEMCG 2965 memcg = mm->lru_gen.memcg; 2966 #endif 2967 mm_list = get_mm_list(memcg); 2968 2969 spin_lock(&mm_list->lock); 2970 2971 for_each_node(nid) { 2972 struct lruvec *lruvec = get_lruvec(memcg, nid); 2973 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 2974 2975 /* where the current iteration continues after */ 2976 if (mm_state->head == &mm->lru_gen.list) 2977 mm_state->head = mm_state->head->prev; 2978 2979 /* where the last iteration ended before */ 2980 if (mm_state->tail == &mm->lru_gen.list) 2981 mm_state->tail = mm_state->tail->next; 2982 } 2983 2984 list_del_init(&mm->lru_gen.list); 2985 2986 spin_unlock(&mm_list->lock); 2987 2988 #ifdef CONFIG_MEMCG 2989 mem_cgroup_put(mm->lru_gen.memcg); 2990 mm->lru_gen.memcg = NULL; 2991 #endif 2992 } 2993 2994 #ifdef CONFIG_MEMCG 2995 void lru_gen_migrate_mm(struct mm_struct *mm) 2996 { 2997 struct mem_cgroup *memcg; 2998 struct task_struct *task = rcu_dereference_protected(mm->owner, true); 2999 3000 VM_WARN_ON_ONCE(task->mm != mm); 3001 lockdep_assert_held(&task->alloc_lock); 3002 3003 /* for mm_update_next_owner() */ 3004 if (mem_cgroup_disabled()) 3005 return; 3006 3007 /* migration can happen before addition */ 3008 if (!mm->lru_gen.memcg) 3009 return; 3010 3011 rcu_read_lock(); 3012 memcg = mem_cgroup_from_task(task); 3013 rcu_read_unlock(); 3014 if (memcg == mm->lru_gen.memcg) 3015 return; 3016 3017 VM_WARN_ON_ONCE(list_empty(&mm->lru_gen.list)); 3018 3019 lru_gen_del_mm(mm); 3020 lru_gen_add_mm(mm); 3021 } 3022 #endif 3023 3024 #else /* !CONFIG_LRU_GEN_WALKS_MMU */ 3025 3026 static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg) 3027 { 3028 return NULL; 3029 } 3030 3031 static struct lru_gen_mm_state *get_mm_state(struct lruvec *lruvec) 3032 { 3033 return NULL; 3034 } 3035 3036 static struct mm_struct *get_next_mm(struct lru_gen_mm_walk *walk) 3037 { 3038 return NULL; 3039 } 3040 3041 #endif 3042 3043 static void reset_mm_stats(struct lru_gen_mm_walk *walk, bool last) 3044 { 3045 int i; 3046 int hist; 3047 struct lruvec *lruvec = walk->lruvec; 3048 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 3049 3050 lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock); 3051 3052 hist = lru_hist_from_seq(walk->seq); 3053 3054 for (i = 0; i < NR_MM_STATS; i++) { 3055 WRITE_ONCE(mm_state->stats[hist][i], 3056 mm_state->stats[hist][i] + walk->mm_stats[i]); 3057 walk->mm_stats[i] = 0; 3058 } 3059 3060 if (NR_HIST_GENS > 1 && last) { 3061 hist = lru_hist_from_seq(walk->seq + 1); 3062 3063 for (i = 0; i < NR_MM_STATS; i++) 3064 WRITE_ONCE(mm_state->stats[hist][i], 0); 3065 } 3066 } 3067 3068 static bool iterate_mm_list(struct lru_gen_mm_walk *walk, struct mm_struct **iter) 3069 { 3070 bool first = false; 3071 bool last = false; 3072 struct mm_struct *mm = NULL; 3073 struct lruvec *lruvec = walk->lruvec; 3074 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 3075 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 3076 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 3077 3078 /* 3079 * mm_state->seq is incremented after each iteration of mm_list. There 3080 * are three interesting cases for this page table walker: 3081 * 1. It tries to start a new iteration with a stale max_seq: there is 3082 * nothing left to do. 3083 * 2. It started the next iteration: it needs to reset the Bloom filter 3084 * so that a fresh set of PTE tables can be recorded. 3085 * 3. It ended the current iteration: it needs to reset the mm stats 3086 * counters and tell its caller to increment max_seq. 3087 */ 3088 spin_lock(&mm_list->lock); 3089 3090 VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->seq); 3091 3092 if (walk->seq <= mm_state->seq) 3093 goto done; 3094 3095 if (!mm_state->head) 3096 mm_state->head = &mm_list->fifo; 3097 3098 if (mm_state->head == &mm_list->fifo) 3099 first = true; 3100 3101 do { 3102 mm_state->head = mm_state->head->next; 3103 if (mm_state->head == &mm_list->fifo) { 3104 WRITE_ONCE(mm_state->seq, mm_state->seq + 1); 3105 last = true; 3106 break; 3107 } 3108 3109 /* force scan for those added after the last iteration */ 3110 if (!mm_state->tail || mm_state->tail == mm_state->head) { 3111 mm_state->tail = mm_state->head->next; 3112 walk->force_scan = true; 3113 } 3114 } while (!(mm = get_next_mm(walk))); 3115 done: 3116 if (*iter || last) 3117 reset_mm_stats(walk, last); 3118 3119 spin_unlock(&mm_list->lock); 3120 3121 if (mm && first) 3122 reset_bloom_filter(mm_state, walk->seq + 1); 3123 3124 if (*iter) 3125 mmput_async(*iter); 3126 3127 *iter = mm; 3128 3129 return last; 3130 } 3131 3132 static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long seq) 3133 { 3134 bool success = false; 3135 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 3136 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 3137 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 3138 3139 spin_lock(&mm_list->lock); 3140 3141 VM_WARN_ON_ONCE(mm_state->seq + 1 < seq); 3142 3143 if (seq > mm_state->seq) { 3144 mm_state->head = NULL; 3145 mm_state->tail = NULL; 3146 WRITE_ONCE(mm_state->seq, mm_state->seq + 1); 3147 success = true; 3148 } 3149 3150 spin_unlock(&mm_list->lock); 3151 3152 return success; 3153 } 3154 3155 /****************************************************************************** 3156 * PID controller 3157 ******************************************************************************/ 3158 3159 /* 3160 * A feedback loop based on Proportional-Integral-Derivative (PID) controller. 3161 * 3162 * The P term is refaulted/(evicted+protected) from a tier in the generation 3163 * currently being evicted; the I term is the exponential moving average of the 3164 * P term over the generations previously evicted, using the smoothing factor 3165 * 1/2; the D term isn't supported. 3166 * 3167 * The setpoint (SP) is always the first tier of one type; the process variable 3168 * (PV) is either any tier of the other type or any other tier of the same 3169 * type. 3170 * 3171 * The error is the difference between the SP and the PV; the correction is to 3172 * turn off protection when SP>PV or turn on protection when SP<PV. 3173 * 3174 * For future optimizations: 3175 * 1. The D term may discount the other two terms over time so that long-lived 3176 * generations can resist stale information. 3177 */ 3178 struct ctrl_pos { 3179 unsigned long refaulted; 3180 unsigned long total; 3181 int gain; 3182 }; 3183 3184 static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain, 3185 struct ctrl_pos *pos) 3186 { 3187 int i; 3188 struct lru_gen_folio *lrugen = &lruvec->lrugen; 3189 int hist = lru_hist_from_seq(lrugen->min_seq[type]); 3190 3191 pos->gain = gain; 3192 pos->refaulted = pos->total = 0; 3193 3194 for (i = tier % MAX_NR_TIERS; i <= min(tier, MAX_NR_TIERS - 1); i++) { 3195 pos->refaulted += lrugen->avg_refaulted[type][i] + 3196 atomic_long_read(&lrugen->refaulted[hist][type][i]); 3197 pos->total += lrugen->avg_total[type][i] + 3198 lrugen->protected[hist][type][i] + 3199 atomic_long_read(&lrugen->evicted[hist][type][i]); 3200 } 3201 } 3202 3203 static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover) 3204 { 3205 int hist, tier; 3206 struct lru_gen_folio *lrugen = &lruvec->lrugen; 3207 bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1; 3208 unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1; 3209 3210 lockdep_assert_held(&lruvec->lru_lock); 3211 3212 if (!carryover && !clear) 3213 return; 3214 3215 hist = lru_hist_from_seq(seq); 3216 3217 for (tier = 0; tier < MAX_NR_TIERS; tier++) { 3218 if (carryover) { 3219 unsigned long sum; 3220 3221 sum = lrugen->avg_refaulted[type][tier] + 3222 atomic_long_read(&lrugen->refaulted[hist][type][tier]); 3223 WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2); 3224 3225 sum = lrugen->avg_total[type][tier] + 3226 lrugen->protected[hist][type][tier] + 3227 atomic_long_read(&lrugen->evicted[hist][type][tier]); 3228 WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2); 3229 } 3230 3231 if (clear) { 3232 atomic_long_set(&lrugen->refaulted[hist][type][tier], 0); 3233 atomic_long_set(&lrugen->evicted[hist][type][tier], 0); 3234 WRITE_ONCE(lrugen->protected[hist][type][tier], 0); 3235 } 3236 } 3237 } 3238 3239 static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) 3240 { 3241 /* 3242 * Return true if the PV has a limited number of refaults or a lower 3243 * refaulted/total than the SP. 3244 */ 3245 return pv->refaulted < MIN_LRU_BATCH || 3246 pv->refaulted * (sp->total + MIN_LRU_BATCH) * sp->gain <= 3247 (sp->refaulted + 1) * pv->total * pv->gain; 3248 } 3249 3250 /****************************************************************************** 3251 * the aging 3252 ******************************************************************************/ 3253 3254 /* promote pages accessed through page tables */ 3255 static int folio_update_gen(struct folio *folio, int gen) 3256 { 3257 unsigned long new_flags, old_flags = READ_ONCE(folio->flags); 3258 3259 VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); 3260 3261 /* see the comment on LRU_REFS_FLAGS */ 3262 if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { 3263 set_mask_bits(&folio->flags, LRU_REFS_MASK, BIT(PG_referenced)); 3264 return -1; 3265 } 3266 3267 do { 3268 /* lru_gen_del_folio() has isolated this page? */ 3269 if (!(old_flags & LRU_GEN_MASK)) 3270 return -1; 3271 3272 new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS); 3273 new_flags |= ((gen + 1UL) << LRU_GEN_PGOFF) | BIT(PG_workingset); 3274 } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); 3275 3276 return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; 3277 } 3278 3279 /* protect pages accessed multiple times through file descriptors */ 3280 static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming) 3281 { 3282 int type = folio_is_file_lru(folio); 3283 struct lru_gen_folio *lrugen = &lruvec->lrugen; 3284 int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); 3285 unsigned long new_flags, old_flags = READ_ONCE(folio->flags); 3286 3287 VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); 3288 3289 do { 3290 new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; 3291 /* folio_update_gen() has promoted this page? */ 3292 if (new_gen >= 0 && new_gen != old_gen) 3293 return new_gen; 3294 3295 new_gen = (old_gen + 1) % MAX_NR_GENS; 3296 3297 new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS); 3298 new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF; 3299 /* for folio_end_writeback() */ 3300 if (reclaiming) 3301 new_flags |= BIT(PG_reclaim); 3302 } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); 3303 3304 lru_gen_update_size(lruvec, folio, old_gen, new_gen); 3305 3306 return new_gen; 3307 } 3308 3309 static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio, 3310 int old_gen, int new_gen) 3311 { 3312 int type = folio_is_file_lru(folio); 3313 int zone = folio_zonenum(folio); 3314 int delta = folio_nr_pages(folio); 3315 3316 VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS); 3317 VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS); 3318 3319 walk->batched++; 3320 3321 walk->nr_pages[old_gen][type][zone] -= delta; 3322 walk->nr_pages[new_gen][type][zone] += delta; 3323 } 3324 3325 static void reset_batch_size(struct lru_gen_mm_walk *walk) 3326 { 3327 int gen, type, zone; 3328 struct lruvec *lruvec = walk->lruvec; 3329 struct lru_gen_folio *lrugen = &lruvec->lrugen; 3330 3331 walk->batched = 0; 3332 3333 for_each_gen_type_zone(gen, type, zone) { 3334 enum lru_list lru = type * LRU_INACTIVE_FILE; 3335 int delta = walk->nr_pages[gen][type][zone]; 3336 3337 if (!delta) 3338 continue; 3339 3340 walk->nr_pages[gen][type][zone] = 0; 3341 WRITE_ONCE(lrugen->nr_pages[gen][type][zone], 3342 lrugen->nr_pages[gen][type][zone] + delta); 3343 3344 if (lru_gen_is_active(lruvec, gen)) 3345 lru += LRU_ACTIVE; 3346 __update_lru_size(lruvec, lru, zone, delta); 3347 } 3348 } 3349 3350 static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *args) 3351 { 3352 struct address_space *mapping; 3353 struct vm_area_struct *vma = args->vma; 3354 struct lru_gen_mm_walk *walk = args->private; 3355 3356 if (!vma_is_accessible(vma)) 3357 return true; 3358 3359 if (is_vm_hugetlb_page(vma)) 3360 return true; 3361 3362 if (!vma_has_recency(vma)) 3363 return true; 3364 3365 if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) 3366 return true; 3367 3368 if (vma == get_gate_vma(vma->vm_mm)) 3369 return true; 3370 3371 if (vma_is_anonymous(vma)) 3372 return !walk->swappiness; 3373 3374 if (WARN_ON_ONCE(!vma->vm_file || !vma->vm_file->f_mapping)) 3375 return true; 3376 3377 mapping = vma->vm_file->f_mapping; 3378 if (mapping_unevictable(mapping)) 3379 return true; 3380 3381 if (shmem_mapping(mapping)) 3382 return !walk->swappiness; 3383 3384 if (walk->swappiness > MAX_SWAPPINESS) 3385 return true; 3386 3387 /* to exclude special mappings like dax, etc. */ 3388 return !mapping->a_ops->read_folio; 3389 } 3390 3391 /* 3392 * Some userspace memory allocators map many single-page VMAs. Instead of 3393 * returning back to the PGD table for each of such VMAs, finish an entire PMD 3394 * table to reduce zigzags and improve cache performance. 3395 */ 3396 static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk *args, 3397 unsigned long *vm_start, unsigned long *vm_end) 3398 { 3399 unsigned long start = round_up(*vm_end, size); 3400 unsigned long end = (start | ~mask) + 1; 3401 VMA_ITERATOR(vmi, args->mm, start); 3402 3403 VM_WARN_ON_ONCE(mask & size); 3404 VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask)); 3405 3406 for_each_vma(vmi, args->vma) { 3407 if (end && end <= args->vma->vm_start) 3408 return false; 3409 3410 if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) 3411 continue; 3412 3413 *vm_start = max(start, args->vma->vm_start); 3414 *vm_end = min(end - 1, args->vma->vm_end - 1) + 1; 3415 3416 return true; 3417 } 3418 3419 return false; 3420 } 3421 3422 static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr, 3423 struct pglist_data *pgdat) 3424 { 3425 unsigned long pfn = pte_pfn(pte); 3426 3427 VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); 3428 3429 if (!pte_present(pte) || is_zero_pfn(pfn)) 3430 return -1; 3431 3432 if (WARN_ON_ONCE(pte_devmap(pte) || pte_special(pte))) 3433 return -1; 3434 3435 if (!pte_young(pte) && !mm_has_notifiers(vma->vm_mm)) 3436 return -1; 3437 3438 if (WARN_ON_ONCE(!pfn_valid(pfn))) 3439 return -1; 3440 3441 if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) 3442 return -1; 3443 3444 return pfn; 3445 } 3446 3447 static unsigned long get_pmd_pfn(pmd_t pmd, struct vm_area_struct *vma, unsigned long addr, 3448 struct pglist_data *pgdat) 3449 { 3450 unsigned long pfn = pmd_pfn(pmd); 3451 3452 VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); 3453 3454 if (!pmd_present(pmd) || is_huge_zero_pmd(pmd)) 3455 return -1; 3456 3457 if (WARN_ON_ONCE(pmd_devmap(pmd))) 3458 return -1; 3459 3460 if (!pmd_young(pmd) && !mm_has_notifiers(vma->vm_mm)) 3461 return -1; 3462 3463 if (WARN_ON_ONCE(!pfn_valid(pfn))) 3464 return -1; 3465 3466 if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) 3467 return -1; 3468 3469 return pfn; 3470 } 3471 3472 static struct folio *get_pfn_folio(unsigned long pfn, struct mem_cgroup *memcg, 3473 struct pglist_data *pgdat) 3474 { 3475 struct folio *folio = pfn_folio(pfn); 3476 3477 if (folio_lru_gen(folio) < 0) 3478 return NULL; 3479 3480 if (folio_nid(folio) != pgdat->node_id) 3481 return NULL; 3482 3483 if (folio_memcg(folio) != memcg) 3484 return NULL; 3485 3486 return folio; 3487 } 3488 3489 static bool suitable_to_scan(int total, int young) 3490 { 3491 int n = clamp_t(int, cache_line_size() / sizeof(pte_t), 2, 8); 3492 3493 /* suitable if the average number of young PTEs per cacheline is >=1 */ 3494 return young * n >= total; 3495 } 3496 3497 static void walk_update_folio(struct lru_gen_mm_walk *walk, struct folio *folio, 3498 int new_gen, bool dirty) 3499 { 3500 int old_gen; 3501 3502 if (!folio) 3503 return; 3504 3505 if (dirty && !folio_test_dirty(folio) && 3506 !(folio_test_anon(folio) && folio_test_swapbacked(folio) && 3507 !folio_test_swapcache(folio))) 3508 folio_mark_dirty(folio); 3509 3510 if (walk) { 3511 old_gen = folio_update_gen(folio, new_gen); 3512 if (old_gen >= 0 && old_gen != new_gen) 3513 update_batch_size(walk, folio, old_gen, new_gen); 3514 } else if (lru_gen_set_refs(folio)) { 3515 old_gen = folio_lru_gen(folio); 3516 if (old_gen >= 0 && old_gen != new_gen) 3517 folio_activate(folio); 3518 } 3519 } 3520 3521 static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, 3522 struct mm_walk *args) 3523 { 3524 int i; 3525 bool dirty; 3526 pte_t *pte; 3527 spinlock_t *ptl; 3528 unsigned long addr; 3529 int total = 0; 3530 int young = 0; 3531 struct folio *last = NULL; 3532 struct lru_gen_mm_walk *walk = args->private; 3533 struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); 3534 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); 3535 DEFINE_MAX_SEQ(walk->lruvec); 3536 int gen = lru_gen_from_seq(max_seq); 3537 pmd_t pmdval; 3538 3539 pte = pte_offset_map_rw_nolock(args->mm, pmd, start & PMD_MASK, &pmdval, &ptl); 3540 if (!pte) 3541 return false; 3542 3543 if (!spin_trylock(ptl)) { 3544 pte_unmap(pte); 3545 return true; 3546 } 3547 3548 if (unlikely(!pmd_same(pmdval, pmdp_get_lockless(pmd)))) { 3549 pte_unmap_unlock(pte, ptl); 3550 return false; 3551 } 3552 3553 arch_enter_lazy_mmu_mode(); 3554 restart: 3555 for (i = pte_index(start), addr = start; addr != end; i++, addr += PAGE_SIZE) { 3556 unsigned long pfn; 3557 struct folio *folio; 3558 pte_t ptent = ptep_get(pte + i); 3559 3560 total++; 3561 walk->mm_stats[MM_LEAF_TOTAL]++; 3562 3563 pfn = get_pte_pfn(ptent, args->vma, addr, pgdat); 3564 if (pfn == -1) 3565 continue; 3566 3567 folio = get_pfn_folio(pfn, memcg, pgdat); 3568 if (!folio) 3569 continue; 3570 3571 if (!ptep_clear_young_notify(args->vma, addr, pte + i)) 3572 continue; 3573 3574 if (last != folio) { 3575 walk_update_folio(walk, last, gen, dirty); 3576 3577 last = folio; 3578 dirty = false; 3579 } 3580 3581 if (pte_dirty(ptent)) 3582 dirty = true; 3583 3584 young++; 3585 walk->mm_stats[MM_LEAF_YOUNG]++; 3586 } 3587 3588 walk_update_folio(walk, last, gen, dirty); 3589 last = NULL; 3590 3591 if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) 3592 goto restart; 3593 3594 arch_leave_lazy_mmu_mode(); 3595 pte_unmap_unlock(pte, ptl); 3596 3597 return suitable_to_scan(total, young); 3598 } 3599 3600 static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma, 3601 struct mm_walk *args, unsigned long *bitmap, unsigned long *first) 3602 { 3603 int i; 3604 bool dirty; 3605 pmd_t *pmd; 3606 spinlock_t *ptl; 3607 struct folio *last = NULL; 3608 struct lru_gen_mm_walk *walk = args->private; 3609 struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); 3610 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); 3611 DEFINE_MAX_SEQ(walk->lruvec); 3612 int gen = lru_gen_from_seq(max_seq); 3613 3614 VM_WARN_ON_ONCE(pud_leaf(*pud)); 3615 3616 /* try to batch at most 1+MIN_LRU_BATCH+1 entries */ 3617 if (*first == -1) { 3618 *first = addr; 3619 bitmap_zero(bitmap, MIN_LRU_BATCH); 3620 return; 3621 } 3622 3623 i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first); 3624 if (i && i <= MIN_LRU_BATCH) { 3625 __set_bit(i - 1, bitmap); 3626 return; 3627 } 3628 3629 pmd = pmd_offset(pud, *first); 3630 3631 ptl = pmd_lockptr(args->mm, pmd); 3632 if (!spin_trylock(ptl)) 3633 goto done; 3634 3635 arch_enter_lazy_mmu_mode(); 3636 3637 do { 3638 unsigned long pfn; 3639 struct folio *folio; 3640 3641 /* don't round down the first address */ 3642 addr = i ? (*first & PMD_MASK) + i * PMD_SIZE : *first; 3643 3644 if (!pmd_present(pmd[i])) 3645 goto next; 3646 3647 if (!pmd_trans_huge(pmd[i])) { 3648 if (!walk->force_scan && should_clear_pmd_young() && 3649 !mm_has_notifiers(args->mm)) 3650 pmdp_test_and_clear_young(vma, addr, pmd + i); 3651 goto next; 3652 } 3653 3654 pfn = get_pmd_pfn(pmd[i], vma, addr, pgdat); 3655 if (pfn == -1) 3656 goto next; 3657 3658 folio = get_pfn_folio(pfn, memcg, pgdat); 3659 if (!folio) 3660 goto next; 3661 3662 if (!pmdp_clear_young_notify(vma, addr, pmd + i)) 3663 goto next; 3664 3665 if (last != folio) { 3666 walk_update_folio(walk, last, gen, dirty); 3667 3668 last = folio; 3669 dirty = false; 3670 } 3671 3672 if (pmd_dirty(pmd[i])) 3673 dirty = true; 3674 3675 walk->mm_stats[MM_LEAF_YOUNG]++; 3676 next: 3677 i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; 3678 } while (i <= MIN_LRU_BATCH); 3679 3680 walk_update_folio(walk, last, gen, dirty); 3681 3682 arch_leave_lazy_mmu_mode(); 3683 spin_unlock(ptl); 3684 done: 3685 *first = -1; 3686 } 3687 3688 static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, 3689 struct mm_walk *args) 3690 { 3691 int i; 3692 pmd_t *pmd; 3693 unsigned long next; 3694 unsigned long addr; 3695 struct vm_area_struct *vma; 3696 DECLARE_BITMAP(bitmap, MIN_LRU_BATCH); 3697 unsigned long first = -1; 3698 struct lru_gen_mm_walk *walk = args->private; 3699 struct lru_gen_mm_state *mm_state = get_mm_state(walk->lruvec); 3700 3701 VM_WARN_ON_ONCE(pud_leaf(*pud)); 3702 3703 /* 3704 * Finish an entire PMD in two passes: the first only reaches to PTE 3705 * tables to avoid taking the PMD lock; the second, if necessary, takes 3706 * the PMD lock to clear the accessed bit in PMD entries. 3707 */ 3708 pmd = pmd_offset(pud, start & PUD_MASK); 3709 restart: 3710 /* walk_pte_range() may call get_next_vma() */ 3711 vma = args->vma; 3712 for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) { 3713 pmd_t val = pmdp_get_lockless(pmd + i); 3714 3715 next = pmd_addr_end(addr, end); 3716 3717 if (!pmd_present(val) || is_huge_zero_pmd(val)) { 3718 walk->mm_stats[MM_LEAF_TOTAL]++; 3719 continue; 3720 } 3721 3722 if (pmd_trans_huge(val)) { 3723 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); 3724 unsigned long pfn = get_pmd_pfn(val, vma, addr, pgdat); 3725 3726 walk->mm_stats[MM_LEAF_TOTAL]++; 3727 3728 if (pfn != -1) 3729 walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first); 3730 continue; 3731 } 3732 3733 if (!walk->force_scan && should_clear_pmd_young() && 3734 !mm_has_notifiers(args->mm)) { 3735 if (!pmd_young(val)) 3736 continue; 3737 3738 walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first); 3739 } 3740 3741 if (!walk->force_scan && !test_bloom_filter(mm_state, walk->seq, pmd + i)) 3742 continue; 3743 3744 walk->mm_stats[MM_NONLEAF_FOUND]++; 3745 3746 if (!walk_pte_range(&val, addr, next, args)) 3747 continue; 3748 3749 walk->mm_stats[MM_NONLEAF_ADDED]++; 3750 3751 /* carry over to the next generation */ 3752 update_bloom_filter(mm_state, walk->seq + 1, pmd + i); 3753 } 3754 3755 walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first); 3756 3757 if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) 3758 goto restart; 3759 } 3760 3761 static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, 3762 struct mm_walk *args) 3763 { 3764 int i; 3765 pud_t *pud; 3766 unsigned long addr; 3767 unsigned long next; 3768 struct lru_gen_mm_walk *walk = args->private; 3769 3770 VM_WARN_ON_ONCE(p4d_leaf(*p4d)); 3771 3772 pud = pud_offset(p4d, start & P4D_MASK); 3773 restart: 3774 for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { 3775 pud_t val = READ_ONCE(pud[i]); 3776 3777 next = pud_addr_end(addr, end); 3778 3779 if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) 3780 continue; 3781 3782 walk_pmd_range(&val, addr, next, args); 3783 3784 if (need_resched() || walk->batched >= MAX_LRU_BATCH) { 3785 end = (addr | ~PUD_MASK) + 1; 3786 goto done; 3787 } 3788 } 3789 3790 if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end)) 3791 goto restart; 3792 3793 end = round_up(end, P4D_SIZE); 3794 done: 3795 if (!end || !args->vma) 3796 return 1; 3797 3798 walk->next_addr = max(end, args->vma->vm_start); 3799 3800 return -EAGAIN; 3801 } 3802 3803 static void walk_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk) 3804 { 3805 static const struct mm_walk_ops mm_walk_ops = { 3806 .test_walk = should_skip_vma, 3807 .p4d_entry = walk_pud_range, 3808 .walk_lock = PGWALK_RDLOCK, 3809 }; 3810 int err; 3811 struct lruvec *lruvec = walk->lruvec; 3812 3813 walk->next_addr = FIRST_USER_ADDRESS; 3814 3815 do { 3816 DEFINE_MAX_SEQ(lruvec); 3817 3818 err = -EBUSY; 3819 3820 /* another thread might have called inc_max_seq() */ 3821 if (walk->seq != max_seq) 3822 break; 3823 3824 /* the caller might be holding the lock for write */ 3825 if (mmap_read_trylock(mm)) { 3826 err = walk_page_range(mm, walk->next_addr, ULONG_MAX, &mm_walk_ops, walk); 3827 3828 mmap_read_unlock(mm); 3829 } 3830 3831 if (walk->batched) { 3832 spin_lock_irq(&lruvec->lru_lock); 3833 reset_batch_size(walk); 3834 spin_unlock_irq(&lruvec->lru_lock); 3835 } 3836 3837 cond_resched(); 3838 } while (err == -EAGAIN); 3839 } 3840 3841 static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat, bool force_alloc) 3842 { 3843 struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; 3844 3845 if (pgdat && current_is_kswapd()) { 3846 VM_WARN_ON_ONCE(walk); 3847 3848 walk = &pgdat->mm_walk; 3849 } else if (!walk && force_alloc) { 3850 VM_WARN_ON_ONCE(current_is_kswapd()); 3851 3852 walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); 3853 } 3854 3855 current->reclaim_state->mm_walk = walk; 3856 3857 return walk; 3858 } 3859 3860 static void clear_mm_walk(void) 3861 { 3862 struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; 3863 3864 VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages))); 3865 VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats))); 3866 3867 current->reclaim_state->mm_walk = NULL; 3868 3869 if (!current_is_kswapd()) 3870 kfree(walk); 3871 } 3872 3873 static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) 3874 { 3875 int zone; 3876 int remaining = MAX_LRU_BATCH; 3877 struct lru_gen_folio *lrugen = &lruvec->lrugen; 3878 int hist = lru_hist_from_seq(lrugen->min_seq[type]); 3879 int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); 3880 3881 /* For file type, skip the check if swappiness is anon only */ 3882 if (type && (swappiness == SWAPPINESS_ANON_ONLY)) 3883 goto done; 3884 3885 /* For anon type, skip the check if swappiness is zero (file only) */ 3886 if (!type && !swappiness) 3887 goto done; 3888 3889 /* prevent cold/hot inversion if the type is evictable */ 3890 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 3891 struct list_head *head = &lrugen->folios[old_gen][type][zone]; 3892 3893 while (!list_empty(head)) { 3894 struct folio *folio = lru_to_folio(head); 3895 int refs = folio_lru_refs(folio); 3896 bool workingset = folio_test_workingset(folio); 3897 3898 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); 3899 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); 3900 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); 3901 VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); 3902 3903 new_gen = folio_inc_gen(lruvec, folio, false); 3904 list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]); 3905 3906 /* don't count the workingset being lazily promoted */ 3907 if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) { 3908 int tier = lru_tier_from_refs(refs, workingset); 3909 int delta = folio_nr_pages(folio); 3910 3911 WRITE_ONCE(lrugen->protected[hist][type][tier], 3912 lrugen->protected[hist][type][tier] + delta); 3913 } 3914 3915 if (!--remaining) 3916 return false; 3917 } 3918 } 3919 done: 3920 reset_ctrl_pos(lruvec, type, true); 3921 WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1); 3922 3923 return true; 3924 } 3925 3926 static bool try_to_inc_min_seq(struct lruvec *lruvec, int swappiness) 3927 { 3928 int gen, type, zone; 3929 bool success = false; 3930 struct lru_gen_folio *lrugen = &lruvec->lrugen; 3931 DEFINE_MIN_SEQ(lruvec); 3932 3933 VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); 3934 3935 /* find the oldest populated generation */ 3936 for_each_evictable_type(type, swappiness) { 3937 while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) { 3938 gen = lru_gen_from_seq(min_seq[type]); 3939 3940 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 3941 if (!list_empty(&lrugen->folios[gen][type][zone])) 3942 goto next; 3943 } 3944 3945 min_seq[type]++; 3946 } 3947 next: 3948 ; 3949 } 3950 3951 /* see the comment on lru_gen_folio */ 3952 if (swappiness && swappiness <= MAX_SWAPPINESS) { 3953 unsigned long seq = lrugen->max_seq - MIN_NR_GENS; 3954 3955 if (min_seq[LRU_GEN_ANON] > seq && min_seq[LRU_GEN_FILE] < seq) 3956 min_seq[LRU_GEN_ANON] = seq; 3957 else if (min_seq[LRU_GEN_FILE] > seq && min_seq[LRU_GEN_ANON] < seq) 3958 min_seq[LRU_GEN_FILE] = seq; 3959 } 3960 3961 for_each_evictable_type(type, swappiness) { 3962 if (min_seq[type] <= lrugen->min_seq[type]) 3963 continue; 3964 3965 reset_ctrl_pos(lruvec, type, true); 3966 WRITE_ONCE(lrugen->min_seq[type], min_seq[type]); 3967 success = true; 3968 } 3969 3970 return success; 3971 } 3972 3973 static bool inc_max_seq(struct lruvec *lruvec, unsigned long seq, int swappiness) 3974 { 3975 bool success; 3976 int prev, next; 3977 int type, zone; 3978 struct lru_gen_folio *lrugen = &lruvec->lrugen; 3979 restart: 3980 if (seq < READ_ONCE(lrugen->max_seq)) 3981 return false; 3982 3983 spin_lock_irq(&lruvec->lru_lock); 3984 3985 VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); 3986 3987 success = seq == lrugen->max_seq; 3988 if (!success) 3989 goto unlock; 3990 3991 for (type = 0; type < ANON_AND_FILE; type++) { 3992 if (get_nr_gens(lruvec, type) != MAX_NR_GENS) 3993 continue; 3994 3995 if (inc_min_seq(lruvec, type, swappiness)) 3996 continue; 3997 3998 spin_unlock_irq(&lruvec->lru_lock); 3999 cond_resched(); 4000 goto restart; 4001 } 4002 4003 /* 4004 * Update the active/inactive LRU sizes for compatibility. Both sides of 4005 * the current max_seq need to be covered, since max_seq+1 can overlap 4006 * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do 4007 * overlap, cold/hot inversion happens. 4008 */ 4009 prev = lru_gen_from_seq(lrugen->max_seq - 1); 4010 next = lru_gen_from_seq(lrugen->max_seq + 1); 4011 4012 for (type = 0; type < ANON_AND_FILE; type++) { 4013 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 4014 enum lru_list lru = type * LRU_INACTIVE_FILE; 4015 long delta = lrugen->nr_pages[prev][type][zone] - 4016 lrugen->nr_pages[next][type][zone]; 4017 4018 if (!delta) 4019 continue; 4020 4021 __update_lru_size(lruvec, lru, zone, delta); 4022 __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta); 4023 } 4024 } 4025 4026 for (type = 0; type < ANON_AND_FILE; type++) 4027 reset_ctrl_pos(lruvec, type, false); 4028 4029 WRITE_ONCE(lrugen->timestamps[next], jiffies); 4030 /* make sure preceding modifications appear */ 4031 smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1); 4032 unlock: 4033 spin_unlock_irq(&lruvec->lru_lock); 4034 4035 return success; 4036 } 4037 4038 static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long seq, 4039 int swappiness, bool force_scan) 4040 { 4041 bool success; 4042 struct lru_gen_mm_walk *walk; 4043 struct mm_struct *mm = NULL; 4044 struct lru_gen_folio *lrugen = &lruvec->lrugen; 4045 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 4046 4047 VM_WARN_ON_ONCE(seq > READ_ONCE(lrugen->max_seq)); 4048 4049 if (!mm_state) 4050 return inc_max_seq(lruvec, seq, swappiness); 4051 4052 /* see the comment in iterate_mm_list() */ 4053 if (seq <= READ_ONCE(mm_state->seq)) 4054 return false; 4055 4056 /* 4057 * If the hardware doesn't automatically set the accessed bit, fallback 4058 * to lru_gen_look_around(), which only clears the accessed bit in a 4059 * handful of PTEs. Spreading the work out over a period of time usually 4060 * is less efficient, but it avoids bursty page faults. 4061 */ 4062 if (!should_walk_mmu()) { 4063 success = iterate_mm_list_nowalk(lruvec, seq); 4064 goto done; 4065 } 4066 4067 walk = set_mm_walk(NULL, true); 4068 if (!walk) { 4069 success = iterate_mm_list_nowalk(lruvec, seq); 4070 goto done; 4071 } 4072 4073 walk->lruvec = lruvec; 4074 walk->seq = seq; 4075 walk->swappiness = swappiness; 4076 walk->force_scan = force_scan; 4077 4078 do { 4079 success = iterate_mm_list(walk, &mm); 4080 if (mm) 4081 walk_mm(mm, walk); 4082 } while (mm); 4083 done: 4084 if (success) { 4085 success = inc_max_seq(lruvec, seq, swappiness); 4086 WARN_ON_ONCE(!success); 4087 } 4088 4089 return success; 4090 } 4091 4092 /****************************************************************************** 4093 * working set protection 4094 ******************************************************************************/ 4095 4096 static void set_initial_priority(struct pglist_data *pgdat, struct scan_control *sc) 4097 { 4098 int priority; 4099 unsigned long reclaimable; 4100 4101 if (sc->priority != DEF_PRIORITY || sc->nr_to_reclaim < MIN_LRU_BATCH) 4102 return; 4103 /* 4104 * Determine the initial priority based on 4105 * (total >> priority) * reclaimed_to_scanned_ratio = nr_to_reclaim, 4106 * where reclaimed_to_scanned_ratio = inactive / total. 4107 */ 4108 reclaimable = node_page_state(pgdat, NR_INACTIVE_FILE); 4109 if (can_reclaim_anon_pages(NULL, pgdat->node_id, sc)) 4110 reclaimable += node_page_state(pgdat, NR_INACTIVE_ANON); 4111 4112 /* round down reclaimable and round up sc->nr_to_reclaim */ 4113 priority = fls_long(reclaimable) - 1 - fls_long(sc->nr_to_reclaim - 1); 4114 4115 /* 4116 * The estimation is based on LRU pages only, so cap it to prevent 4117 * overshoots of shrinker objects by large margins. 4118 */ 4119 sc->priority = clamp(priority, DEF_PRIORITY / 2, DEF_PRIORITY); 4120 } 4121 4122 static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc) 4123 { 4124 int gen, type, zone; 4125 unsigned long total = 0; 4126 int swappiness = get_swappiness(lruvec, sc); 4127 struct lru_gen_folio *lrugen = &lruvec->lrugen; 4128 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4129 DEFINE_MAX_SEQ(lruvec); 4130 DEFINE_MIN_SEQ(lruvec); 4131 4132 for_each_evictable_type(type, swappiness) { 4133 unsigned long seq; 4134 4135 for (seq = min_seq[type]; seq <= max_seq; seq++) { 4136 gen = lru_gen_from_seq(seq); 4137 4138 for (zone = 0; zone < MAX_NR_ZONES; zone++) 4139 total += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); 4140 } 4141 } 4142 4143 /* whether the size is big enough to be helpful */ 4144 return mem_cgroup_online(memcg) ? (total >> sc->priority) : total; 4145 } 4146 4147 static bool lruvec_is_reclaimable(struct lruvec *lruvec, struct scan_control *sc, 4148 unsigned long min_ttl) 4149 { 4150 int gen; 4151 unsigned long birth; 4152 int swappiness = get_swappiness(lruvec, sc); 4153 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4154 DEFINE_MIN_SEQ(lruvec); 4155 4156 if (mem_cgroup_below_min(NULL, memcg)) 4157 return false; 4158 4159 if (!lruvec_is_sizable(lruvec, sc)) 4160 return false; 4161 4162 gen = lru_gen_from_seq(evictable_min_seq(min_seq, swappiness)); 4163 birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); 4164 4165 return time_is_before_jiffies(birth + min_ttl); 4166 } 4167 4168 /* to protect the working set of the last N jiffies */ 4169 static unsigned long lru_gen_min_ttl __read_mostly; 4170 4171 static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) 4172 { 4173 struct mem_cgroup *memcg; 4174 unsigned long min_ttl = READ_ONCE(lru_gen_min_ttl); 4175 bool reclaimable = !min_ttl; 4176 4177 VM_WARN_ON_ONCE(!current_is_kswapd()); 4178 4179 set_initial_priority(pgdat, sc); 4180 4181 memcg = mem_cgroup_iter(NULL, NULL, NULL); 4182 do { 4183 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); 4184 4185 mem_cgroup_calculate_protection(NULL, memcg); 4186 4187 if (!reclaimable) 4188 reclaimable = lruvec_is_reclaimable(lruvec, sc, min_ttl); 4189 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); 4190 4191 /* 4192 * The main goal is to OOM kill if every generation from all memcgs is 4193 * younger than min_ttl. However, another possibility is all memcgs are 4194 * either too small or below min. 4195 */ 4196 if (!reclaimable && mutex_trylock(&oom_lock)) { 4197 struct oom_control oc = { 4198 .gfp_mask = sc->gfp_mask, 4199 }; 4200 4201 out_of_memory(&oc); 4202 4203 mutex_unlock(&oom_lock); 4204 } 4205 } 4206 4207 /****************************************************************************** 4208 * rmap/PT walk feedback 4209 ******************************************************************************/ 4210 4211 /* 4212 * This function exploits spatial locality when shrink_folio_list() walks the 4213 * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If 4214 * the scan was done cacheline efficiently, it adds the PMD entry pointing to 4215 * the PTE table to the Bloom filter. This forms a feedback loop between the 4216 * eviction and the aging. 4217 */ 4218 bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw) 4219 { 4220 int i; 4221 bool dirty; 4222 unsigned long start; 4223 unsigned long end; 4224 struct lru_gen_mm_walk *walk; 4225 struct folio *last = NULL; 4226 int young = 1; 4227 pte_t *pte = pvmw->pte; 4228 unsigned long addr = pvmw->address; 4229 struct vm_area_struct *vma = pvmw->vma; 4230 struct folio *folio = pfn_folio(pvmw->pfn); 4231 struct mem_cgroup *memcg = folio_memcg(folio); 4232 struct pglist_data *pgdat = folio_pgdat(folio); 4233 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); 4234 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 4235 DEFINE_MAX_SEQ(lruvec); 4236 int gen = lru_gen_from_seq(max_seq); 4237 4238 lockdep_assert_held(pvmw->ptl); 4239 VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio); 4240 4241 if (!ptep_clear_young_notify(vma, addr, pte)) 4242 return false; 4243 4244 if (spin_is_contended(pvmw->ptl)) 4245 return true; 4246 4247 /* exclude special VMAs containing anon pages from COW */ 4248 if (vma->vm_flags & VM_SPECIAL) 4249 return true; 4250 4251 /* avoid taking the LRU lock under the PTL when possible */ 4252 walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; 4253 4254 start = max(addr & PMD_MASK, vma->vm_start); 4255 end = min(addr | ~PMD_MASK, vma->vm_end - 1) + 1; 4256 4257 if (end - start == PAGE_SIZE) 4258 return true; 4259 4260 if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { 4261 if (addr - start < MIN_LRU_BATCH * PAGE_SIZE / 2) 4262 end = start + MIN_LRU_BATCH * PAGE_SIZE; 4263 else if (end - addr < MIN_LRU_BATCH * PAGE_SIZE / 2) 4264 start = end - MIN_LRU_BATCH * PAGE_SIZE; 4265 else { 4266 start = addr - MIN_LRU_BATCH * PAGE_SIZE / 2; 4267 end = addr + MIN_LRU_BATCH * PAGE_SIZE / 2; 4268 } 4269 } 4270 4271 arch_enter_lazy_mmu_mode(); 4272 4273 pte -= (addr - start) / PAGE_SIZE; 4274 4275 for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) { 4276 unsigned long pfn; 4277 pte_t ptent = ptep_get(pte + i); 4278 4279 pfn = get_pte_pfn(ptent, vma, addr, pgdat); 4280 if (pfn == -1) 4281 continue; 4282 4283 folio = get_pfn_folio(pfn, memcg, pgdat); 4284 if (!folio) 4285 continue; 4286 4287 if (!ptep_clear_young_notify(vma, addr, pte + i)) 4288 continue; 4289 4290 if (last != folio) { 4291 walk_update_folio(walk, last, gen, dirty); 4292 4293 last = folio; 4294 dirty = false; 4295 } 4296 4297 if (pte_dirty(ptent)) 4298 dirty = true; 4299 4300 young++; 4301 } 4302 4303 walk_update_folio(walk, last, gen, dirty); 4304 4305 arch_leave_lazy_mmu_mode(); 4306 4307 /* feedback from rmap walkers to page table walkers */ 4308 if (mm_state && suitable_to_scan(i, young)) 4309 update_bloom_filter(mm_state, max_seq, pvmw->pmd); 4310 4311 return true; 4312 } 4313 4314 /****************************************************************************** 4315 * memcg LRU 4316 ******************************************************************************/ 4317 4318 /* see the comment on MEMCG_NR_GENS */ 4319 enum { 4320 MEMCG_LRU_NOP, 4321 MEMCG_LRU_HEAD, 4322 MEMCG_LRU_TAIL, 4323 MEMCG_LRU_OLD, 4324 MEMCG_LRU_YOUNG, 4325 }; 4326 4327 static void lru_gen_rotate_memcg(struct lruvec *lruvec, int op) 4328 { 4329 int seg; 4330 int old, new; 4331 unsigned long flags; 4332 int bin = get_random_u32_below(MEMCG_NR_BINS); 4333 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 4334 4335 spin_lock_irqsave(&pgdat->memcg_lru.lock, flags); 4336 4337 VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list)); 4338 4339 seg = 0; 4340 new = old = lruvec->lrugen.gen; 4341 4342 /* see the comment on MEMCG_NR_GENS */ 4343 if (op == MEMCG_LRU_HEAD) 4344 seg = MEMCG_LRU_HEAD; 4345 else if (op == MEMCG_LRU_TAIL) 4346 seg = MEMCG_LRU_TAIL; 4347 else if (op == MEMCG_LRU_OLD) 4348 new = get_memcg_gen(pgdat->memcg_lru.seq); 4349 else if (op == MEMCG_LRU_YOUNG) 4350 new = get_memcg_gen(pgdat->memcg_lru.seq + 1); 4351 else 4352 VM_WARN_ON_ONCE(true); 4353 4354 WRITE_ONCE(lruvec->lrugen.seg, seg); 4355 WRITE_ONCE(lruvec->lrugen.gen, new); 4356 4357 hlist_nulls_del_rcu(&lruvec->lrugen.list); 4358 4359 if (op == MEMCG_LRU_HEAD || op == MEMCG_LRU_OLD) 4360 hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]); 4361 else 4362 hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]); 4363 4364 pgdat->memcg_lru.nr_memcgs[old]--; 4365 pgdat->memcg_lru.nr_memcgs[new]++; 4366 4367 if (!pgdat->memcg_lru.nr_memcgs[old] && old == get_memcg_gen(pgdat->memcg_lru.seq)) 4368 WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1); 4369 4370 spin_unlock_irqrestore(&pgdat->memcg_lru.lock, flags); 4371 } 4372 4373 #ifdef CONFIG_MEMCG 4374 4375 void lru_gen_online_memcg(struct mem_cgroup *memcg) 4376 { 4377 int gen; 4378 int nid; 4379 int bin = get_random_u32_below(MEMCG_NR_BINS); 4380 4381 for_each_node(nid) { 4382 struct pglist_data *pgdat = NODE_DATA(nid); 4383 struct lruvec *lruvec = get_lruvec(memcg, nid); 4384 4385 spin_lock_irq(&pgdat->memcg_lru.lock); 4386 4387 VM_WARN_ON_ONCE(!hlist_nulls_unhashed(&lruvec->lrugen.list)); 4388 4389 gen = get_memcg_gen(pgdat->memcg_lru.seq); 4390 4391 lruvec->lrugen.gen = gen; 4392 4393 hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[gen][bin]); 4394 pgdat->memcg_lru.nr_memcgs[gen]++; 4395 4396 spin_unlock_irq(&pgdat->memcg_lru.lock); 4397 } 4398 } 4399 4400 void lru_gen_offline_memcg(struct mem_cgroup *memcg) 4401 { 4402 int nid; 4403 4404 for_each_node(nid) { 4405 struct lruvec *lruvec = get_lruvec(memcg, nid); 4406 4407 lru_gen_rotate_memcg(lruvec, MEMCG_LRU_OLD); 4408 } 4409 } 4410 4411 void lru_gen_release_memcg(struct mem_cgroup *memcg) 4412 { 4413 int gen; 4414 int nid; 4415 4416 for_each_node(nid) { 4417 struct pglist_data *pgdat = NODE_DATA(nid); 4418 struct lruvec *lruvec = get_lruvec(memcg, nid); 4419 4420 spin_lock_irq(&pgdat->memcg_lru.lock); 4421 4422 if (hlist_nulls_unhashed(&lruvec->lrugen.list)) 4423 goto unlock; 4424 4425 gen = lruvec->lrugen.gen; 4426 4427 hlist_nulls_del_init_rcu(&lruvec->lrugen.list); 4428 pgdat->memcg_lru.nr_memcgs[gen]--; 4429 4430 if (!pgdat->memcg_lru.nr_memcgs[gen] && gen == get_memcg_gen(pgdat->memcg_lru.seq)) 4431 WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1); 4432 unlock: 4433 spin_unlock_irq(&pgdat->memcg_lru.lock); 4434 } 4435 } 4436 4437 void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid) 4438 { 4439 struct lruvec *lruvec = get_lruvec(memcg, nid); 4440 4441 /* see the comment on MEMCG_NR_GENS */ 4442 if (READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_HEAD) 4443 lru_gen_rotate_memcg(lruvec, MEMCG_LRU_HEAD); 4444 } 4445 4446 #endif /* CONFIG_MEMCG */ 4447 4448 /****************************************************************************** 4449 * the eviction 4450 ******************************************************************************/ 4451 4452 static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc, 4453 int tier_idx) 4454 { 4455 bool success; 4456 bool dirty, writeback; 4457 int gen = folio_lru_gen(folio); 4458 int type = folio_is_file_lru(folio); 4459 int zone = folio_zonenum(folio); 4460 int delta = folio_nr_pages(folio); 4461 int refs = folio_lru_refs(folio); 4462 bool workingset = folio_test_workingset(folio); 4463 int tier = lru_tier_from_refs(refs, workingset); 4464 struct lru_gen_folio *lrugen = &lruvec->lrugen; 4465 4466 VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio); 4467 4468 /* unevictable */ 4469 if (!folio_evictable(folio)) { 4470 success = lru_gen_del_folio(lruvec, folio, true); 4471 VM_WARN_ON_ONCE_FOLIO(!success, folio); 4472 folio_set_unevictable(folio); 4473 lruvec_add_folio(lruvec, folio); 4474 __count_vm_events(UNEVICTABLE_PGCULLED, delta); 4475 return true; 4476 } 4477 4478 /* promoted */ 4479 if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { 4480 list_move(&folio->lru, &lrugen->folios[gen][type][zone]); 4481 return true; 4482 } 4483 4484 /* protected */ 4485 if (tier > tier_idx || refs + workingset == BIT(LRU_REFS_WIDTH) + 1) { 4486 gen = folio_inc_gen(lruvec, folio, false); 4487 list_move(&folio->lru, &lrugen->folios[gen][type][zone]); 4488 4489 /* don't count the workingset being lazily promoted */ 4490 if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) { 4491 int hist = lru_hist_from_seq(lrugen->min_seq[type]); 4492 4493 WRITE_ONCE(lrugen->protected[hist][type][tier], 4494 lrugen->protected[hist][type][tier] + delta); 4495 } 4496 return true; 4497 } 4498 4499 /* ineligible */ 4500 if (!folio_test_lru(folio) || zone > sc->reclaim_idx) { 4501 gen = folio_inc_gen(lruvec, folio, false); 4502 list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]); 4503 return true; 4504 } 4505 4506 dirty = folio_test_dirty(folio); 4507 writeback = folio_test_writeback(folio); 4508 if (type == LRU_GEN_FILE && dirty) { 4509 sc->nr.file_taken += delta; 4510 if (!writeback) 4511 sc->nr.unqueued_dirty += delta; 4512 } 4513 4514 /* waiting for writeback */ 4515 if (writeback || (type == LRU_GEN_FILE && dirty)) { 4516 gen = folio_inc_gen(lruvec, folio, true); 4517 list_move(&folio->lru, &lrugen->folios[gen][type][zone]); 4518 return true; 4519 } 4520 4521 return false; 4522 } 4523 4524 static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc) 4525 { 4526 bool success; 4527 4528 /* swap constrained */ 4529 if (!(sc->gfp_mask & __GFP_IO) && 4530 (folio_test_dirty(folio) || 4531 (folio_test_anon(folio) && !folio_test_swapcache(folio)))) 4532 return false; 4533 4534 /* raced with release_pages() */ 4535 if (!folio_try_get(folio)) 4536 return false; 4537 4538 /* raced with another isolation */ 4539 if (!folio_test_clear_lru(folio)) { 4540 folio_put(folio); 4541 return false; 4542 } 4543 4544 /* see the comment on LRU_REFS_FLAGS */ 4545 if (!folio_test_referenced(folio)) 4546 set_mask_bits(&folio->flags, LRU_REFS_MASK, 0); 4547 4548 /* for shrink_folio_list() */ 4549 folio_clear_reclaim(folio); 4550 4551 success = lru_gen_del_folio(lruvec, folio, true); 4552 VM_WARN_ON_ONCE_FOLIO(!success, folio); 4553 4554 return true; 4555 } 4556 4557 static int scan_folios(struct lruvec *lruvec, struct scan_control *sc, 4558 int type, int tier, struct list_head *list) 4559 { 4560 int i; 4561 int gen; 4562 enum vm_event_item item; 4563 int sorted = 0; 4564 int scanned = 0; 4565 int isolated = 0; 4566 int skipped = 0; 4567 int remaining = MAX_LRU_BATCH; 4568 struct lru_gen_folio *lrugen = &lruvec->lrugen; 4569 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4570 4571 VM_WARN_ON_ONCE(!list_empty(list)); 4572 4573 if (get_nr_gens(lruvec, type) == MIN_NR_GENS) 4574 return 0; 4575 4576 gen = lru_gen_from_seq(lrugen->min_seq[type]); 4577 4578 for (i = MAX_NR_ZONES; i > 0; i--) { 4579 LIST_HEAD(moved); 4580 int skipped_zone = 0; 4581 int zone = (sc->reclaim_idx + i) % MAX_NR_ZONES; 4582 struct list_head *head = &lrugen->folios[gen][type][zone]; 4583 4584 while (!list_empty(head)) { 4585 struct folio *folio = lru_to_folio(head); 4586 int delta = folio_nr_pages(folio); 4587 4588 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); 4589 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); 4590 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); 4591 VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); 4592 4593 scanned += delta; 4594 4595 if (sort_folio(lruvec, folio, sc, tier)) 4596 sorted += delta; 4597 else if (isolate_folio(lruvec, folio, sc)) { 4598 list_add(&folio->lru, list); 4599 isolated += delta; 4600 } else { 4601 list_move(&folio->lru, &moved); 4602 skipped_zone += delta; 4603 } 4604 4605 if (!--remaining || max(isolated, skipped_zone) >= MIN_LRU_BATCH) 4606 break; 4607 } 4608 4609 if (skipped_zone) { 4610 list_splice(&moved, head); 4611 __count_zid_vm_events(PGSCAN_SKIP, zone, skipped_zone); 4612 skipped += skipped_zone; 4613 } 4614 4615 if (!remaining || isolated >= MIN_LRU_BATCH) 4616 break; 4617 } 4618 4619 item = PGSCAN_KSWAPD + reclaimer_offset(sc); 4620 if (!cgroup_reclaim(sc)) { 4621 __count_vm_events(item, isolated); 4622 __count_vm_events(PGREFILL, sorted); 4623 } 4624 count_memcg_events(memcg, item, isolated); 4625 count_memcg_events(memcg, PGREFILL, sorted); 4626 __count_vm_events(PGSCAN_ANON + type, isolated); 4627 trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, MAX_LRU_BATCH, 4628 scanned, skipped, isolated, 4629 type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON); 4630 if (type == LRU_GEN_FILE) 4631 sc->nr.file_taken += isolated; 4632 /* 4633 * There might not be eligible folios due to reclaim_idx. Check the 4634 * remaining to prevent livelock if it's not making progress. 4635 */ 4636 return isolated || !remaining ? scanned : 0; 4637 } 4638 4639 static int get_tier_idx(struct lruvec *lruvec, int type) 4640 { 4641 int tier; 4642 struct ctrl_pos sp, pv; 4643 4644 /* 4645 * To leave a margin for fluctuations, use a larger gain factor (2:3). 4646 * This value is chosen because any other tier would have at least twice 4647 * as many refaults as the first tier. 4648 */ 4649 read_ctrl_pos(lruvec, type, 0, 2, &sp); 4650 for (tier = 1; tier < MAX_NR_TIERS; tier++) { 4651 read_ctrl_pos(lruvec, type, tier, 3, &pv); 4652 if (!positive_ctrl_err(&sp, &pv)) 4653 break; 4654 } 4655 4656 return tier - 1; 4657 } 4658 4659 static int get_type_to_scan(struct lruvec *lruvec, int swappiness) 4660 { 4661 struct ctrl_pos sp, pv; 4662 4663 if (swappiness <= MIN_SWAPPINESS + 1) 4664 return LRU_GEN_FILE; 4665 4666 if (swappiness >= MAX_SWAPPINESS) 4667 return LRU_GEN_ANON; 4668 /* 4669 * Compare the sum of all tiers of anon with that of file to determine 4670 * which type to scan. 4671 */ 4672 read_ctrl_pos(lruvec, LRU_GEN_ANON, MAX_NR_TIERS, swappiness, &sp); 4673 read_ctrl_pos(lruvec, LRU_GEN_FILE, MAX_NR_TIERS, MAX_SWAPPINESS - swappiness, &pv); 4674 4675 return positive_ctrl_err(&sp, &pv); 4676 } 4677 4678 static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, 4679 int *type_scanned, struct list_head *list) 4680 { 4681 int i; 4682 int type = get_type_to_scan(lruvec, swappiness); 4683 4684 for_each_evictable_type(i, swappiness) { 4685 int scanned; 4686 int tier = get_tier_idx(lruvec, type); 4687 4688 *type_scanned = type; 4689 4690 scanned = scan_folios(lruvec, sc, type, tier, list); 4691 if (scanned) 4692 return scanned; 4693 4694 type = !type; 4695 } 4696 4697 return 0; 4698 } 4699 4700 static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness) 4701 { 4702 int type; 4703 int scanned; 4704 int reclaimed; 4705 LIST_HEAD(list); 4706 LIST_HEAD(clean); 4707 struct folio *folio; 4708 struct folio *next; 4709 enum vm_event_item item; 4710 struct reclaim_stat stat; 4711 struct lru_gen_mm_walk *walk; 4712 bool skip_retry = false; 4713 struct lru_gen_folio *lrugen = &lruvec->lrugen; 4714 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4715 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 4716 4717 spin_lock_irq(&lruvec->lru_lock); 4718 4719 scanned = isolate_folios(lruvec, sc, swappiness, &type, &list); 4720 4721 scanned += try_to_inc_min_seq(lruvec, swappiness); 4722 4723 if (evictable_min_seq(lrugen->min_seq, swappiness) + MIN_NR_GENS > lrugen->max_seq) 4724 scanned = 0; 4725 4726 spin_unlock_irq(&lruvec->lru_lock); 4727 4728 if (list_empty(&list)) 4729 return scanned; 4730 retry: 4731 reclaimed = shrink_folio_list(&list, pgdat, sc, &stat, false, memcg); 4732 sc->nr.unqueued_dirty += stat.nr_unqueued_dirty; 4733 sc->nr_reclaimed += reclaimed; 4734 trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id, 4735 scanned, reclaimed, &stat, sc->priority, 4736 type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON); 4737 4738 list_for_each_entry_safe_reverse(folio, next, &list, lru) { 4739 DEFINE_MIN_SEQ(lruvec); 4740 4741 if (!folio_evictable(folio)) { 4742 list_del(&folio->lru); 4743 folio_putback_lru(folio); 4744 continue; 4745 } 4746 4747 /* retry folios that may have missed folio_rotate_reclaimable() */ 4748 if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) && 4749 !folio_test_dirty(folio) && !folio_test_writeback(folio)) { 4750 list_move(&folio->lru, &clean); 4751 continue; 4752 } 4753 4754 /* don't add rejected folios to the oldest generation */ 4755 if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) 4756 set_mask_bits(&folio->flags, LRU_REFS_FLAGS, BIT(PG_active)); 4757 } 4758 4759 spin_lock_irq(&lruvec->lru_lock); 4760 4761 move_folios_to_lru(lruvec, &list); 4762 4763 walk = current->reclaim_state->mm_walk; 4764 if (walk && walk->batched) { 4765 walk->lruvec = lruvec; 4766 reset_batch_size(walk); 4767 } 4768 4769 __mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc), 4770 stat.nr_demoted); 4771 4772 item = PGSTEAL_KSWAPD + reclaimer_offset(sc); 4773 if (!cgroup_reclaim(sc)) 4774 __count_vm_events(item, reclaimed); 4775 count_memcg_events(memcg, item, reclaimed); 4776 __count_vm_events(PGSTEAL_ANON + type, reclaimed); 4777 4778 spin_unlock_irq(&lruvec->lru_lock); 4779 4780 list_splice_init(&clean, &list); 4781 4782 if (!list_empty(&list)) { 4783 skip_retry = true; 4784 goto retry; 4785 } 4786 4787 return scanned; 4788 } 4789 4790 static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, 4791 int swappiness, unsigned long *nr_to_scan) 4792 { 4793 int gen, type, zone; 4794 unsigned long size = 0; 4795 struct lru_gen_folio *lrugen = &lruvec->lrugen; 4796 DEFINE_MIN_SEQ(lruvec); 4797 4798 *nr_to_scan = 0; 4799 /* have to run aging, since eviction is not possible anymore */ 4800 if (evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS > max_seq) 4801 return true; 4802 4803 for_each_evictable_type(type, swappiness) { 4804 unsigned long seq; 4805 4806 for (seq = min_seq[type]; seq <= max_seq; seq++) { 4807 gen = lru_gen_from_seq(seq); 4808 4809 for (zone = 0; zone < MAX_NR_ZONES; zone++) 4810 size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); 4811 } 4812 } 4813 4814 *nr_to_scan = size; 4815 /* better to run aging even though eviction is still possible */ 4816 return evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS == max_seq; 4817 } 4818 4819 /* 4820 * For future optimizations: 4821 * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg 4822 * reclaim. 4823 */ 4824 static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, int swappiness) 4825 { 4826 bool success; 4827 unsigned long nr_to_scan; 4828 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4829 DEFINE_MAX_SEQ(lruvec); 4830 4831 if (mem_cgroup_below_min(sc->target_mem_cgroup, memcg)) 4832 return -1; 4833 4834 success = should_run_aging(lruvec, max_seq, swappiness, &nr_to_scan); 4835 4836 /* try to scrape all its memory if this memcg was deleted */ 4837 if (nr_to_scan && !mem_cgroup_online(memcg)) 4838 return nr_to_scan; 4839 4840 /* try to get away with not aging at the default priority */ 4841 if (!success || sc->priority == DEF_PRIORITY) 4842 return nr_to_scan >> sc->priority; 4843 4844 /* stop scanning this lruvec as it's low on cold folios */ 4845 return try_to_inc_max_seq(lruvec, max_seq, swappiness, false) ? -1 : 0; 4846 } 4847 4848 static bool should_abort_scan(struct lruvec *lruvec, struct scan_control *sc) 4849 { 4850 int i; 4851 enum zone_watermarks mark; 4852 4853 /* don't abort memcg reclaim to ensure fairness */ 4854 if (!root_reclaim(sc)) 4855 return false; 4856 4857 if (sc->nr_reclaimed >= max(sc->nr_to_reclaim, compact_gap(sc->order))) 4858 return true; 4859 4860 /* check the order to exclude compaction-induced reclaim */ 4861 if (!current_is_kswapd() || sc->order) 4862 return false; 4863 4864 mark = sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING ? 4865 WMARK_PROMO : WMARK_HIGH; 4866 4867 for (i = 0; i <= sc->reclaim_idx; i++) { 4868 struct zone *zone = lruvec_pgdat(lruvec)->node_zones + i; 4869 unsigned long size = wmark_pages(zone, mark) + MIN_LRU_BATCH; 4870 4871 if (managed_zone(zone) && !zone_watermark_ok(zone, 0, size, sc->reclaim_idx, 0)) 4872 return false; 4873 } 4874 4875 /* kswapd should abort if all eligible zones are safe */ 4876 return true; 4877 } 4878 4879 static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) 4880 { 4881 long nr_to_scan; 4882 unsigned long scanned = 0; 4883 int swappiness = get_swappiness(lruvec, sc); 4884 4885 while (true) { 4886 int delta; 4887 4888 nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness); 4889 if (nr_to_scan <= 0) 4890 break; 4891 4892 delta = evict_folios(lruvec, sc, swappiness); 4893 if (!delta) 4894 break; 4895 4896 scanned += delta; 4897 if (scanned >= nr_to_scan) 4898 break; 4899 4900 if (should_abort_scan(lruvec, sc)) 4901 break; 4902 4903 cond_resched(); 4904 } 4905 4906 /* 4907 * If too many file cache in the coldest generation can't be evicted 4908 * due to being dirty, wake up the flusher. 4909 */ 4910 if (sc->nr.unqueued_dirty && sc->nr.unqueued_dirty == sc->nr.file_taken) 4911 wakeup_flusher_threads(WB_REASON_VMSCAN); 4912 4913 /* whether this lruvec should be rotated */ 4914 return nr_to_scan < 0; 4915 } 4916 4917 static int shrink_one(struct lruvec *lruvec, struct scan_control *sc) 4918 { 4919 bool success; 4920 unsigned long scanned = sc->nr_scanned; 4921 unsigned long reclaimed = sc->nr_reclaimed; 4922 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4923 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 4924 4925 /* lru_gen_age_node() called mem_cgroup_calculate_protection() */ 4926 if (mem_cgroup_below_min(NULL, memcg)) 4927 return MEMCG_LRU_YOUNG; 4928 4929 if (mem_cgroup_below_low(NULL, memcg)) { 4930 /* see the comment on MEMCG_NR_GENS */ 4931 if (READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_TAIL) 4932 return MEMCG_LRU_TAIL; 4933 4934 memcg_memory_event(memcg, MEMCG_LOW); 4935 } 4936 4937 success = try_to_shrink_lruvec(lruvec, sc); 4938 4939 shrink_slab(sc->gfp_mask, pgdat->node_id, memcg, sc->priority); 4940 4941 if (!sc->proactive) 4942 vmpressure(sc->gfp_mask, memcg, false, sc->nr_scanned - scanned, 4943 sc->nr_reclaimed - reclaimed); 4944 4945 flush_reclaim_state(sc); 4946 4947 if (success && mem_cgroup_online(memcg)) 4948 return MEMCG_LRU_YOUNG; 4949 4950 if (!success && lruvec_is_sizable(lruvec, sc)) 4951 return 0; 4952 4953 /* one retry if offlined or too small */ 4954 return READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_TAIL ? 4955 MEMCG_LRU_TAIL : MEMCG_LRU_YOUNG; 4956 } 4957 4958 static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc) 4959 { 4960 int op; 4961 int gen; 4962 int bin; 4963 int first_bin; 4964 struct lruvec *lruvec; 4965 struct lru_gen_folio *lrugen; 4966 struct mem_cgroup *memcg; 4967 struct hlist_nulls_node *pos; 4968 4969 gen = get_memcg_gen(READ_ONCE(pgdat->memcg_lru.seq)); 4970 bin = first_bin = get_random_u32_below(MEMCG_NR_BINS); 4971 restart: 4972 op = 0; 4973 memcg = NULL; 4974 4975 rcu_read_lock(); 4976 4977 hlist_nulls_for_each_entry_rcu(lrugen, pos, &pgdat->memcg_lru.fifo[gen][bin], list) { 4978 if (op) { 4979 lru_gen_rotate_memcg(lruvec, op); 4980 op = 0; 4981 } 4982 4983 mem_cgroup_put(memcg); 4984 memcg = NULL; 4985 4986 if (gen != READ_ONCE(lrugen->gen)) 4987 continue; 4988 4989 lruvec = container_of(lrugen, struct lruvec, lrugen); 4990 memcg = lruvec_memcg(lruvec); 4991 4992 if (!mem_cgroup_tryget(memcg)) { 4993 lru_gen_release_memcg(memcg); 4994 memcg = NULL; 4995 continue; 4996 } 4997 4998 rcu_read_unlock(); 4999 5000 op = shrink_one(lruvec, sc); 5001 5002 rcu_read_lock(); 5003 5004 if (should_abort_scan(lruvec, sc)) 5005 break; 5006 } 5007 5008 rcu_read_unlock(); 5009 5010 if (op) 5011 lru_gen_rotate_memcg(lruvec, op); 5012 5013 mem_cgroup_put(memcg); 5014 5015 if (!is_a_nulls(pos)) 5016 return; 5017 5018 /* restart if raced with lru_gen_rotate_memcg() */ 5019 if (gen != get_nulls_value(pos)) 5020 goto restart; 5021 5022 /* try the rest of the bins of the current generation */ 5023 bin = get_memcg_bin(bin + 1); 5024 if (bin != first_bin) 5025 goto restart; 5026 } 5027 5028 static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) 5029 { 5030 struct blk_plug plug; 5031 5032 VM_WARN_ON_ONCE(root_reclaim(sc)); 5033 VM_WARN_ON_ONCE(!sc->may_writepage || !sc->may_unmap); 5034 5035 lru_add_drain(); 5036 5037 blk_start_plug(&plug); 5038 5039 set_mm_walk(NULL, sc->proactive); 5040 5041 if (try_to_shrink_lruvec(lruvec, sc)) 5042 lru_gen_rotate_memcg(lruvec, MEMCG_LRU_YOUNG); 5043 5044 clear_mm_walk(); 5045 5046 blk_finish_plug(&plug); 5047 } 5048 5049 static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *sc) 5050 { 5051 struct blk_plug plug; 5052 unsigned long reclaimed = sc->nr_reclaimed; 5053 5054 VM_WARN_ON_ONCE(!root_reclaim(sc)); 5055 5056 /* 5057 * Unmapped clean folios are already prioritized. Scanning for more of 5058 * them is likely futile and can cause high reclaim latency when there 5059 * is a large number of memcgs. 5060 */ 5061 if (!sc->may_writepage || !sc->may_unmap) 5062 goto done; 5063 5064 lru_add_drain(); 5065 5066 blk_start_plug(&plug); 5067 5068 set_mm_walk(pgdat, sc->proactive); 5069 5070 set_initial_priority(pgdat, sc); 5071 5072 if (current_is_kswapd()) 5073 sc->nr_reclaimed = 0; 5074 5075 if (mem_cgroup_disabled()) 5076 shrink_one(&pgdat->__lruvec, sc); 5077 else 5078 shrink_many(pgdat, sc); 5079 5080 if (current_is_kswapd()) 5081 sc->nr_reclaimed += reclaimed; 5082 5083 clear_mm_walk(); 5084 5085 blk_finish_plug(&plug); 5086 done: 5087 if (sc->nr_reclaimed > reclaimed) 5088 pgdat->kswapd_failures = 0; 5089 } 5090 5091 /****************************************************************************** 5092 * state change 5093 ******************************************************************************/ 5094 5095 static bool __maybe_unused state_is_valid(struct lruvec *lruvec) 5096 { 5097 struct lru_gen_folio *lrugen = &lruvec->lrugen; 5098 5099 if (lrugen->enabled) { 5100 enum lru_list lru; 5101 5102 for_each_evictable_lru(lru) { 5103 if (!list_empty(&lruvec->lists[lru])) 5104 return false; 5105 } 5106 } else { 5107 int gen, type, zone; 5108 5109 for_each_gen_type_zone(gen, type, zone) { 5110 if (!list_empty(&lrugen->folios[gen][type][zone])) 5111 return false; 5112 } 5113 } 5114 5115 return true; 5116 } 5117 5118 static bool fill_evictable(struct lruvec *lruvec) 5119 { 5120 enum lru_list lru; 5121 int remaining = MAX_LRU_BATCH; 5122 5123 for_each_evictable_lru(lru) { 5124 int type = is_file_lru(lru); 5125 bool active = is_active_lru(lru); 5126 struct list_head *head = &lruvec->lists[lru]; 5127 5128 while (!list_empty(head)) { 5129 bool success; 5130 struct folio *folio = lru_to_folio(head); 5131 5132 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); 5133 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio); 5134 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); 5135 VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio); 5136 5137 lruvec_del_folio(lruvec, folio); 5138 success = lru_gen_add_folio(lruvec, folio, false); 5139 VM_WARN_ON_ONCE(!success); 5140 5141 if (!--remaining) 5142 return false; 5143 } 5144 } 5145 5146 return true; 5147 } 5148 5149 static bool drain_evictable(struct lruvec *lruvec) 5150 { 5151 int gen, type, zone; 5152 int remaining = MAX_LRU_BATCH; 5153 5154 for_each_gen_type_zone(gen, type, zone) { 5155 struct list_head *head = &lruvec->lrugen.folios[gen][type][zone]; 5156 5157 while (!list_empty(head)) { 5158 bool success; 5159 struct folio *folio = lru_to_folio(head); 5160 5161 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); 5162 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); 5163 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); 5164 VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); 5165 5166 success = lru_gen_del_folio(lruvec, folio, false); 5167 VM_WARN_ON_ONCE(!success); 5168 lruvec_add_folio(lruvec, folio); 5169 5170 if (!--remaining) 5171 return false; 5172 } 5173 } 5174 5175 return true; 5176 } 5177 5178 static void lru_gen_change_state(bool enabled) 5179 { 5180 static DEFINE_MUTEX(state_mutex); 5181 5182 struct mem_cgroup *memcg; 5183 5184 cgroup_lock(); 5185 cpus_read_lock(); 5186 get_online_mems(); 5187 mutex_lock(&state_mutex); 5188 5189 if (enabled == lru_gen_enabled()) 5190 goto unlock; 5191 5192 if (enabled) 5193 static_branch_enable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); 5194 else 5195 static_branch_disable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); 5196 5197 memcg = mem_cgroup_iter(NULL, NULL, NULL); 5198 do { 5199 int nid; 5200 5201 for_each_node(nid) { 5202 struct lruvec *lruvec = get_lruvec(memcg, nid); 5203 5204 spin_lock_irq(&lruvec->lru_lock); 5205 5206 VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); 5207 VM_WARN_ON_ONCE(!state_is_valid(lruvec)); 5208 5209 lruvec->lrugen.enabled = enabled; 5210 5211 while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) { 5212 spin_unlock_irq(&lruvec->lru_lock); 5213 cond_resched(); 5214 spin_lock_irq(&lruvec->lru_lock); 5215 } 5216 5217 spin_unlock_irq(&lruvec->lru_lock); 5218 } 5219 5220 cond_resched(); 5221 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); 5222 unlock: 5223 mutex_unlock(&state_mutex); 5224 put_online_mems(); 5225 cpus_read_unlock(); 5226 cgroup_unlock(); 5227 } 5228 5229 /****************************************************************************** 5230 * sysfs interface 5231 ******************************************************************************/ 5232 5233 static ssize_t min_ttl_ms_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) 5234 { 5235 return sysfs_emit(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl))); 5236 } 5237 5238 /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ 5239 static ssize_t min_ttl_ms_store(struct kobject *kobj, struct kobj_attribute *attr, 5240 const char *buf, size_t len) 5241 { 5242 unsigned int msecs; 5243 5244 if (kstrtouint(buf, 0, &msecs)) 5245 return -EINVAL; 5246 5247 WRITE_ONCE(lru_gen_min_ttl, msecs_to_jiffies(msecs)); 5248 5249 return len; 5250 } 5251 5252 static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR_RW(min_ttl_ms); 5253 5254 static ssize_t enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) 5255 { 5256 unsigned int caps = 0; 5257 5258 if (get_cap(LRU_GEN_CORE)) 5259 caps |= BIT(LRU_GEN_CORE); 5260 5261 if (should_walk_mmu()) 5262 caps |= BIT(LRU_GEN_MM_WALK); 5263 5264 if (should_clear_pmd_young()) 5265 caps |= BIT(LRU_GEN_NONLEAF_YOUNG); 5266 5267 return sysfs_emit(buf, "0x%04x\n", caps); 5268 } 5269 5270 /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ 5271 static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, 5272 const char *buf, size_t len) 5273 { 5274 int i; 5275 unsigned int caps; 5276 5277 if (tolower(*buf) == 'n') 5278 caps = 0; 5279 else if (tolower(*buf) == 'y') 5280 caps = -1; 5281 else if (kstrtouint(buf, 0, &caps)) 5282 return -EINVAL; 5283 5284 for (i = 0; i < NR_LRU_GEN_CAPS; i++) { 5285 bool enabled = caps & BIT(i); 5286 5287 if (i == LRU_GEN_CORE) 5288 lru_gen_change_state(enabled); 5289 else if (enabled) 5290 static_branch_enable(&lru_gen_caps[i]); 5291 else 5292 static_branch_disable(&lru_gen_caps[i]); 5293 } 5294 5295 return len; 5296 } 5297 5298 static struct kobj_attribute lru_gen_enabled_attr = __ATTR_RW(enabled); 5299 5300 static struct attribute *lru_gen_attrs[] = { 5301 &lru_gen_min_ttl_attr.attr, 5302 &lru_gen_enabled_attr.attr, 5303 NULL 5304 }; 5305 5306 static const struct attribute_group lru_gen_attr_group = { 5307 .name = "lru_gen", 5308 .attrs = lru_gen_attrs, 5309 }; 5310 5311 /****************************************************************************** 5312 * debugfs interface 5313 ******************************************************************************/ 5314 5315 static void *lru_gen_seq_start(struct seq_file *m, loff_t *pos) 5316 { 5317 struct mem_cgroup *memcg; 5318 loff_t nr_to_skip = *pos; 5319 5320 m->private = kvmalloc(PATH_MAX, GFP_KERNEL); 5321 if (!m->private) 5322 return ERR_PTR(-ENOMEM); 5323 5324 memcg = mem_cgroup_iter(NULL, NULL, NULL); 5325 do { 5326 int nid; 5327 5328 for_each_node_state(nid, N_MEMORY) { 5329 if (!nr_to_skip--) 5330 return get_lruvec(memcg, nid); 5331 } 5332 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); 5333 5334 return NULL; 5335 } 5336 5337 static void lru_gen_seq_stop(struct seq_file *m, void *v) 5338 { 5339 if (!IS_ERR_OR_NULL(v)) 5340 mem_cgroup_iter_break(NULL, lruvec_memcg(v)); 5341 5342 kvfree(m->private); 5343 m->private = NULL; 5344 } 5345 5346 static void *lru_gen_seq_next(struct seq_file *m, void *v, loff_t *pos) 5347 { 5348 int nid = lruvec_pgdat(v)->node_id; 5349 struct mem_cgroup *memcg = lruvec_memcg(v); 5350 5351 ++*pos; 5352 5353 nid = next_memory_node(nid); 5354 if (nid == MAX_NUMNODES) { 5355 memcg = mem_cgroup_iter(NULL, memcg, NULL); 5356 if (!memcg) 5357 return NULL; 5358 5359 nid = first_memory_node; 5360 } 5361 5362 return get_lruvec(memcg, nid); 5363 } 5364 5365 static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec, 5366 unsigned long max_seq, unsigned long *min_seq, 5367 unsigned long seq) 5368 { 5369 int i; 5370 int type, tier; 5371 int hist = lru_hist_from_seq(seq); 5372 struct lru_gen_folio *lrugen = &lruvec->lrugen; 5373 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 5374 5375 for (tier = 0; tier < MAX_NR_TIERS; tier++) { 5376 seq_printf(m, " %10d", tier); 5377 for (type = 0; type < ANON_AND_FILE; type++) { 5378 const char *s = "xxx"; 5379 unsigned long n[3] = {}; 5380 5381 if (seq == max_seq) { 5382 s = "RTx"; 5383 n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]); 5384 n[1] = READ_ONCE(lrugen->avg_total[type][tier]); 5385 } else if (seq == min_seq[type] || NR_HIST_GENS > 1) { 5386 s = "rep"; 5387 n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]); 5388 n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]); 5389 n[2] = READ_ONCE(lrugen->protected[hist][type][tier]); 5390 } 5391 5392 for (i = 0; i < 3; i++) 5393 seq_printf(m, " %10lu%c", n[i], s[i]); 5394 } 5395 seq_putc(m, '\n'); 5396 } 5397 5398 if (!mm_state) 5399 return; 5400 5401 seq_puts(m, " "); 5402 for (i = 0; i < NR_MM_STATS; i++) { 5403 const char *s = "xxxx"; 5404 unsigned long n = 0; 5405 5406 if (seq == max_seq && NR_HIST_GENS == 1) { 5407 s = "TYFA"; 5408 n = READ_ONCE(mm_state->stats[hist][i]); 5409 } else if (seq != max_seq && NR_HIST_GENS > 1) { 5410 s = "tyfa"; 5411 n = READ_ONCE(mm_state->stats[hist][i]); 5412 } 5413 5414 seq_printf(m, " %10lu%c", n, s[i]); 5415 } 5416 seq_putc(m, '\n'); 5417 } 5418 5419 /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ 5420 static int lru_gen_seq_show(struct seq_file *m, void *v) 5421 { 5422 unsigned long seq; 5423 bool full = !debugfs_real_fops(m->file)->write; 5424 struct lruvec *lruvec = v; 5425 struct lru_gen_folio *lrugen = &lruvec->lrugen; 5426 int nid = lruvec_pgdat(lruvec)->node_id; 5427 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 5428 DEFINE_MAX_SEQ(lruvec); 5429 DEFINE_MIN_SEQ(lruvec); 5430 5431 if (nid == first_memory_node) { 5432 const char *path = memcg ? m->private : ""; 5433 5434 #ifdef CONFIG_MEMCG 5435 if (memcg) 5436 cgroup_path(memcg->css.cgroup, m->private, PATH_MAX); 5437 #endif 5438 seq_printf(m, "memcg %5hu %s\n", mem_cgroup_id(memcg), path); 5439 } 5440 5441 seq_printf(m, " node %5d\n", nid); 5442 5443 if (!full) 5444 seq = evictable_min_seq(min_seq, MAX_SWAPPINESS / 2); 5445 else if (max_seq >= MAX_NR_GENS) 5446 seq = max_seq - MAX_NR_GENS + 1; 5447 else 5448 seq = 0; 5449 5450 for (; seq <= max_seq; seq++) { 5451 int type, zone; 5452 int gen = lru_gen_from_seq(seq); 5453 unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); 5454 5455 seq_printf(m, " %10lu %10u", seq, jiffies_to_msecs(jiffies - birth)); 5456 5457 for (type = 0; type < ANON_AND_FILE; type++) { 5458 unsigned long size = 0; 5459 char mark = full && seq < min_seq[type] ? 'x' : ' '; 5460 5461 for (zone = 0; zone < MAX_NR_ZONES; zone++) 5462 size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); 5463 5464 seq_printf(m, " %10lu%c", size, mark); 5465 } 5466 5467 seq_putc(m, '\n'); 5468 5469 if (full) 5470 lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq); 5471 } 5472 5473 return 0; 5474 } 5475 5476 static const struct seq_operations lru_gen_seq_ops = { 5477 .start = lru_gen_seq_start, 5478 .stop = lru_gen_seq_stop, 5479 .next = lru_gen_seq_next, 5480 .show = lru_gen_seq_show, 5481 }; 5482 5483 static int run_aging(struct lruvec *lruvec, unsigned long seq, 5484 int swappiness, bool force_scan) 5485 { 5486 DEFINE_MAX_SEQ(lruvec); 5487 5488 if (seq > max_seq) 5489 return -EINVAL; 5490 5491 return try_to_inc_max_seq(lruvec, max_seq, swappiness, force_scan) ? 0 : -EEXIST; 5492 } 5493 5494 static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, 5495 int swappiness, unsigned long nr_to_reclaim) 5496 { 5497 DEFINE_MAX_SEQ(lruvec); 5498 5499 if (seq + MIN_NR_GENS > max_seq) 5500 return -EINVAL; 5501 5502 sc->nr_reclaimed = 0; 5503 5504 while (!signal_pending(current)) { 5505 DEFINE_MIN_SEQ(lruvec); 5506 5507 if (seq < evictable_min_seq(min_seq, swappiness)) 5508 return 0; 5509 5510 if (sc->nr_reclaimed >= nr_to_reclaim) 5511 return 0; 5512 5513 if (!evict_folios(lruvec, sc, swappiness)) 5514 return 0; 5515 5516 cond_resched(); 5517 } 5518 5519 return -EINTR; 5520 } 5521 5522 static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq, 5523 struct scan_control *sc, int swappiness, unsigned long opt) 5524 { 5525 struct lruvec *lruvec; 5526 int err = -EINVAL; 5527 struct mem_cgroup *memcg = NULL; 5528 5529 if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY)) 5530 return -EINVAL; 5531 5532 if (!mem_cgroup_disabled()) { 5533 rcu_read_lock(); 5534 5535 memcg = mem_cgroup_from_id(memcg_id); 5536 if (!mem_cgroup_tryget(memcg)) 5537 memcg = NULL; 5538 5539 rcu_read_unlock(); 5540 5541 if (!memcg) 5542 return -EINVAL; 5543 } 5544 5545 if (memcg_id != mem_cgroup_id(memcg)) 5546 goto done; 5547 5548 lruvec = get_lruvec(memcg, nid); 5549 5550 if (swappiness < MIN_SWAPPINESS) 5551 swappiness = get_swappiness(lruvec, sc); 5552 else if (swappiness > SWAPPINESS_ANON_ONLY) 5553 goto done; 5554 5555 switch (cmd) { 5556 case '+': 5557 err = run_aging(lruvec, seq, swappiness, opt); 5558 break; 5559 case '-': 5560 err = run_eviction(lruvec, seq, sc, swappiness, opt); 5561 break; 5562 } 5563 done: 5564 mem_cgroup_put(memcg); 5565 5566 return err; 5567 } 5568 5569 /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ 5570 static ssize_t lru_gen_seq_write(struct file *file, const char __user *src, 5571 size_t len, loff_t *pos) 5572 { 5573 void *buf; 5574 char *cur, *next; 5575 unsigned int flags; 5576 struct blk_plug plug; 5577 int err = -EINVAL; 5578 struct scan_control sc = { 5579 .may_writepage = true, 5580 .may_unmap = true, 5581 .may_swap = true, 5582 .reclaim_idx = MAX_NR_ZONES - 1, 5583 .gfp_mask = GFP_KERNEL, 5584 }; 5585 5586 buf = kvmalloc(len + 1, GFP_KERNEL); 5587 if (!buf) 5588 return -ENOMEM; 5589 5590 if (copy_from_user(buf, src, len)) { 5591 kvfree(buf); 5592 return -EFAULT; 5593 } 5594 5595 set_task_reclaim_state(current, &sc.reclaim_state); 5596 flags = memalloc_noreclaim_save(); 5597 blk_start_plug(&plug); 5598 if (!set_mm_walk(NULL, true)) { 5599 err = -ENOMEM; 5600 goto done; 5601 } 5602 5603 next = buf; 5604 next[len] = '\0'; 5605 5606 while ((cur = strsep(&next, ",;\n"))) { 5607 int n; 5608 int end; 5609 char cmd, swap_string[5]; 5610 unsigned int memcg_id; 5611 unsigned int nid; 5612 unsigned long seq; 5613 unsigned int swappiness; 5614 unsigned long opt = -1; 5615 5616 cur = skip_spaces(cur); 5617 if (!*cur) 5618 continue; 5619 5620 n = sscanf(cur, "%c %u %u %lu %n %4s %n %lu %n", &cmd, &memcg_id, &nid, 5621 &seq, &end, swap_string, &end, &opt, &end); 5622 if (n < 4 || cur[end]) { 5623 err = -EINVAL; 5624 break; 5625 } 5626 5627 if (n == 4) { 5628 swappiness = -1; 5629 } else if (!strcmp("max", swap_string)) { 5630 /* set by userspace for anonymous memory only */ 5631 swappiness = SWAPPINESS_ANON_ONLY; 5632 } else { 5633 err = kstrtouint(swap_string, 0, &swappiness); 5634 if (err) 5635 break; 5636 } 5637 5638 err = run_cmd(cmd, memcg_id, nid, seq, &sc, swappiness, opt); 5639 if (err) 5640 break; 5641 } 5642 done: 5643 clear_mm_walk(); 5644 blk_finish_plug(&plug); 5645 memalloc_noreclaim_restore(flags); 5646 set_task_reclaim_state(current, NULL); 5647 5648 kvfree(buf); 5649 5650 return err ? : len; 5651 } 5652 5653 static int lru_gen_seq_open(struct inode *inode, struct file *file) 5654 { 5655 return seq_open(file, &lru_gen_seq_ops); 5656 } 5657 5658 static const struct file_operations lru_gen_rw_fops = { 5659 .open = lru_gen_seq_open, 5660 .read = seq_read, 5661 .write = lru_gen_seq_write, 5662 .llseek = seq_lseek, 5663 .release = seq_release, 5664 }; 5665 5666 static const struct file_operations lru_gen_ro_fops = { 5667 .open = lru_gen_seq_open, 5668 .read = seq_read, 5669 .llseek = seq_lseek, 5670 .release = seq_release, 5671 }; 5672 5673 /****************************************************************************** 5674 * initialization 5675 ******************************************************************************/ 5676 5677 void lru_gen_init_pgdat(struct pglist_data *pgdat) 5678 { 5679 int i, j; 5680 5681 spin_lock_init(&pgdat->memcg_lru.lock); 5682 5683 for (i = 0; i < MEMCG_NR_GENS; i++) { 5684 for (j = 0; j < MEMCG_NR_BINS; j++) 5685 INIT_HLIST_NULLS_HEAD(&pgdat->memcg_lru.fifo[i][j], i); 5686 } 5687 } 5688 5689 void lru_gen_init_lruvec(struct lruvec *lruvec) 5690 { 5691 int i; 5692 int gen, type, zone; 5693 struct lru_gen_folio *lrugen = &lruvec->lrugen; 5694 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 5695 5696 lrugen->max_seq = MIN_NR_GENS + 1; 5697 lrugen->enabled = lru_gen_enabled(); 5698 5699 for (i = 0; i <= MIN_NR_GENS + 1; i++) 5700 lrugen->timestamps[i] = jiffies; 5701 5702 for_each_gen_type_zone(gen, type, zone) 5703 INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]); 5704 5705 if (mm_state) 5706 mm_state->seq = MIN_NR_GENS; 5707 } 5708 5709 #ifdef CONFIG_MEMCG 5710 5711 void lru_gen_init_memcg(struct mem_cgroup *memcg) 5712 { 5713 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 5714 5715 if (!mm_list) 5716 return; 5717 5718 INIT_LIST_HEAD(&mm_list->fifo); 5719 spin_lock_init(&mm_list->lock); 5720 } 5721 5722 void lru_gen_exit_memcg(struct mem_cgroup *memcg) 5723 { 5724 int i; 5725 int nid; 5726 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 5727 5728 VM_WARN_ON_ONCE(mm_list && !list_empty(&mm_list->fifo)); 5729 5730 for_each_node(nid) { 5731 struct lruvec *lruvec = get_lruvec(memcg, nid); 5732 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 5733 5734 VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0, 5735 sizeof(lruvec->lrugen.nr_pages))); 5736 5737 lruvec->lrugen.list.next = LIST_POISON1; 5738 5739 if (!mm_state) 5740 continue; 5741 5742 for (i = 0; i < NR_BLOOM_FILTERS; i++) { 5743 bitmap_free(mm_state->filters[i]); 5744 mm_state->filters[i] = NULL; 5745 } 5746 } 5747 } 5748 5749 #endif /* CONFIG_MEMCG */ 5750 5751 static int __init init_lru_gen(void) 5752 { 5753 BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS); 5754 BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS); 5755 5756 if (sysfs_create_group(mm_kobj, &lru_gen_attr_group)) 5757 pr_err("lru_gen: failed to create sysfs group\n"); 5758 5759 debugfs_create_file("lru_gen", 0644, NULL, NULL, &lru_gen_rw_fops); 5760 debugfs_create_file("lru_gen_full", 0444, NULL, NULL, &lru_gen_ro_fops); 5761 5762 return 0; 5763 }; 5764 late_initcall(init_lru_gen); 5765 5766 #else /* !CONFIG_LRU_GEN */ 5767 5768 static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) 5769 { 5770 BUILD_BUG(); 5771 } 5772 5773 static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) 5774 { 5775 BUILD_BUG(); 5776 } 5777 5778 static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *sc) 5779 { 5780 BUILD_BUG(); 5781 } 5782 5783 #endif /* CONFIG_LRU_GEN */ 5784 5785 static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) 5786 { 5787 unsigned long nr[NR_LRU_LISTS]; 5788 unsigned long targets[NR_LRU_LISTS]; 5789 unsigned long nr_to_scan; 5790 enum lru_list lru; 5791 unsigned long nr_reclaimed = 0; 5792 unsigned long nr_to_reclaim = sc->nr_to_reclaim; 5793 bool proportional_reclaim; 5794 struct blk_plug plug; 5795 5796 if (lru_gen_enabled() && !root_reclaim(sc)) { 5797 lru_gen_shrink_lruvec(lruvec, sc); 5798 return; 5799 } 5800 5801 get_scan_count(lruvec, sc, nr); 5802 5803 /* Record the original scan target for proportional adjustments later */ 5804 memcpy(targets, nr, sizeof(nr)); 5805 5806 /* 5807 * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal 5808 * event that can occur when there is little memory pressure e.g. 5809 * multiple streaming readers/writers. Hence, we do not abort scanning 5810 * when the requested number of pages are reclaimed when scanning at 5811 * DEF_PRIORITY on the assumption that the fact we are direct 5812 * reclaiming implies that kswapd is not keeping up and it is best to 5813 * do a batch of work at once. For memcg reclaim one check is made to 5814 * abort proportional reclaim if either the file or anon lru has already 5815 * dropped to zero at the first pass. 5816 */ 5817 proportional_reclaim = (!cgroup_reclaim(sc) && !current_is_kswapd() && 5818 sc->priority == DEF_PRIORITY); 5819 5820 blk_start_plug(&plug); 5821 while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || 5822 nr[LRU_INACTIVE_FILE]) { 5823 unsigned long nr_anon, nr_file, percentage; 5824 unsigned long nr_scanned; 5825 5826 for_each_evictable_lru(lru) { 5827 if (nr[lru]) { 5828 nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); 5829 nr[lru] -= nr_to_scan; 5830 5831 nr_reclaimed += shrink_list(lru, nr_to_scan, 5832 lruvec, sc); 5833 } 5834 } 5835 5836 cond_resched(); 5837 5838 if (nr_reclaimed < nr_to_reclaim || proportional_reclaim) 5839 continue; 5840 5841 /* 5842 * For kswapd and memcg, reclaim at least the number of pages 5843 * requested. Ensure that the anon and file LRUs are scanned 5844 * proportionally what was requested by get_scan_count(). We 5845 * stop reclaiming one LRU and reduce the amount scanning 5846 * proportional to the original scan target. 5847 */ 5848 nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; 5849 nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; 5850 5851 /* 5852 * It's just vindictive to attack the larger once the smaller 5853 * has gone to zero. And given the way we stop scanning the 5854 * smaller below, this makes sure that we only make one nudge 5855 * towards proportionality once we've got nr_to_reclaim. 5856 */ 5857 if (!nr_file || !nr_anon) 5858 break; 5859 5860 if (nr_file > nr_anon) { 5861 unsigned long scan_target = targets[LRU_INACTIVE_ANON] + 5862 targets[LRU_ACTIVE_ANON] + 1; 5863 lru = LRU_BASE; 5864 percentage = nr_anon * 100 / scan_target; 5865 } else { 5866 unsigned long scan_target = targets[LRU_INACTIVE_FILE] + 5867 targets[LRU_ACTIVE_FILE] + 1; 5868 lru = LRU_FILE; 5869 percentage = nr_file * 100 / scan_target; 5870 } 5871 5872 /* Stop scanning the smaller of the LRU */ 5873 nr[lru] = 0; 5874 nr[lru + LRU_ACTIVE] = 0; 5875 5876 /* 5877 * Recalculate the other LRU scan count based on its original 5878 * scan target and the percentage scanning already complete 5879 */ 5880 lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; 5881 nr_scanned = targets[lru] - nr[lru]; 5882 nr[lru] = targets[lru] * (100 - percentage) / 100; 5883 nr[lru] -= min(nr[lru], nr_scanned); 5884 5885 lru += LRU_ACTIVE; 5886 nr_scanned = targets[lru] - nr[lru]; 5887 nr[lru] = targets[lru] * (100 - percentage) / 100; 5888 nr[lru] -= min(nr[lru], nr_scanned); 5889 } 5890 blk_finish_plug(&plug); 5891 sc->nr_reclaimed += nr_reclaimed; 5892 5893 /* 5894 * Even if we did not try to evict anon pages at all, we want to 5895 * rebalance the anon lru active/inactive ratio. 5896 */ 5897 if (can_age_anon_pages(lruvec, sc) && 5898 inactive_is_low(lruvec, LRU_INACTIVE_ANON)) 5899 shrink_active_list(SWAP_CLUSTER_MAX, lruvec, 5900 sc, LRU_ACTIVE_ANON); 5901 } 5902 5903 /* Use reclaim/compaction for costly allocs or under memory pressure */ 5904 static bool in_reclaim_compaction(struct scan_control *sc) 5905 { 5906 if (gfp_compaction_allowed(sc->gfp_mask) && sc->order && 5907 (sc->order > PAGE_ALLOC_COSTLY_ORDER || 5908 sc->priority < DEF_PRIORITY - 2)) 5909 return true; 5910 5911 return false; 5912 } 5913 5914 /* 5915 * Reclaim/compaction is used for high-order allocation requests. It reclaims 5916 * order-0 pages before compacting the zone. should_continue_reclaim() returns 5917 * true if more pages should be reclaimed such that when the page allocator 5918 * calls try_to_compact_pages() that it will have enough free pages to succeed. 5919 * It will give up earlier than that if there is difficulty reclaiming pages. 5920 */ 5921 static inline bool should_continue_reclaim(struct pglist_data *pgdat, 5922 unsigned long nr_reclaimed, 5923 struct scan_control *sc) 5924 { 5925 unsigned long pages_for_compaction; 5926 unsigned long inactive_lru_pages; 5927 int z; 5928 struct zone *zone; 5929 5930 /* If not in reclaim/compaction mode, stop */ 5931 if (!in_reclaim_compaction(sc)) 5932 return false; 5933 5934 /* 5935 * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX 5936 * number of pages that were scanned. This will return to the caller 5937 * with the risk reclaim/compaction and the resulting allocation attempt 5938 * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL 5939 * allocations through requiring that the full LRU list has been scanned 5940 * first, by assuming that zero delta of sc->nr_scanned means full LRU 5941 * scan, but that approximation was wrong, and there were corner cases 5942 * where always a non-zero amount of pages were scanned. 5943 */ 5944 if (!nr_reclaimed) 5945 return false; 5946 5947 /* If compaction would go ahead or the allocation would succeed, stop */ 5948 for_each_managed_zone_pgdat(zone, pgdat, z, sc->reclaim_idx) { 5949 unsigned long watermark = min_wmark_pages(zone); 5950 5951 /* Allocation can already succeed, nothing to do */ 5952 if (zone_watermark_ok(zone, sc->order, watermark, 5953 sc->reclaim_idx, 0)) 5954 return false; 5955 5956 if (compaction_suitable(zone, sc->order, watermark, 5957 sc->reclaim_idx)) 5958 return false; 5959 } 5960 5961 /* 5962 * If we have not reclaimed enough pages for compaction and the 5963 * inactive lists are large enough, continue reclaiming 5964 */ 5965 pages_for_compaction = compact_gap(sc->order); 5966 inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE); 5967 if (can_reclaim_anon_pages(NULL, pgdat->node_id, sc)) 5968 inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON); 5969 5970 return inactive_lru_pages > pages_for_compaction; 5971 } 5972 5973 static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc) 5974 { 5975 struct mem_cgroup *target_memcg = sc->target_mem_cgroup; 5976 struct mem_cgroup_reclaim_cookie reclaim = { 5977 .pgdat = pgdat, 5978 }; 5979 struct mem_cgroup_reclaim_cookie *partial = &reclaim; 5980 struct mem_cgroup *memcg; 5981 5982 /* 5983 * In most cases, direct reclaimers can do partial walks 5984 * through the cgroup tree, using an iterator state that 5985 * persists across invocations. This strikes a balance between 5986 * fairness and allocation latency. 5987 * 5988 * For kswapd, reliable forward progress is more important 5989 * than a quick return to idle. Always do full walks. 5990 */ 5991 if (current_is_kswapd() || sc->memcg_full_walk) 5992 partial = NULL; 5993 5994 memcg = mem_cgroup_iter(target_memcg, NULL, partial); 5995 do { 5996 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); 5997 unsigned long reclaimed; 5998 unsigned long scanned; 5999 6000 /* 6001 * This loop can become CPU-bound when target memcgs 6002 * aren't eligible for reclaim - either because they 6003 * don't have any reclaimable pages, or because their 6004 * memory is explicitly protected. Avoid soft lockups. 6005 */ 6006 cond_resched(); 6007 6008 mem_cgroup_calculate_protection(target_memcg, memcg); 6009 6010 if (mem_cgroup_below_min(target_memcg, memcg)) { 6011 /* 6012 * Hard protection. 6013 * If there is no reclaimable memory, OOM. 6014 */ 6015 continue; 6016 } else if (mem_cgroup_below_low(target_memcg, memcg)) { 6017 /* 6018 * Soft protection. 6019 * Respect the protection only as long as 6020 * there is an unprotected supply 6021 * of reclaimable memory from other cgroups. 6022 */ 6023 if (!sc->memcg_low_reclaim) { 6024 sc->memcg_low_skipped = 1; 6025 continue; 6026 } 6027 memcg_memory_event(memcg, MEMCG_LOW); 6028 } 6029 6030 reclaimed = sc->nr_reclaimed; 6031 scanned = sc->nr_scanned; 6032 6033 shrink_lruvec(lruvec, sc); 6034 6035 shrink_slab(sc->gfp_mask, pgdat->node_id, memcg, 6036 sc->priority); 6037 6038 /* Record the group's reclaim efficiency */ 6039 if (!sc->proactive) 6040 vmpressure(sc->gfp_mask, memcg, false, 6041 sc->nr_scanned - scanned, 6042 sc->nr_reclaimed - reclaimed); 6043 6044 /* If partial walks are allowed, bail once goal is reached */ 6045 if (partial && sc->nr_reclaimed >= sc->nr_to_reclaim) { 6046 mem_cgroup_iter_break(target_memcg, memcg); 6047 break; 6048 } 6049 } while ((memcg = mem_cgroup_iter(target_memcg, memcg, partial))); 6050 } 6051 6052 static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) 6053 { 6054 unsigned long nr_reclaimed, nr_scanned, nr_node_reclaimed; 6055 struct lruvec *target_lruvec; 6056 bool reclaimable = false; 6057 6058 if (lru_gen_enabled() && root_reclaim(sc)) { 6059 memset(&sc->nr, 0, sizeof(sc->nr)); 6060 lru_gen_shrink_node(pgdat, sc); 6061 return; 6062 } 6063 6064 target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); 6065 6066 again: 6067 memset(&sc->nr, 0, sizeof(sc->nr)); 6068 6069 nr_reclaimed = sc->nr_reclaimed; 6070 nr_scanned = sc->nr_scanned; 6071 6072 prepare_scan_control(pgdat, sc); 6073 6074 shrink_node_memcgs(pgdat, sc); 6075 6076 flush_reclaim_state(sc); 6077 6078 nr_node_reclaimed = sc->nr_reclaimed - nr_reclaimed; 6079 6080 /* Record the subtree's reclaim efficiency */ 6081 if (!sc->proactive) 6082 vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true, 6083 sc->nr_scanned - nr_scanned, nr_node_reclaimed); 6084 6085 if (nr_node_reclaimed) 6086 reclaimable = true; 6087 6088 if (current_is_kswapd()) { 6089 /* 6090 * If reclaim is isolating dirty pages under writeback, 6091 * it implies that the long-lived page allocation rate 6092 * is exceeding the page laundering rate. Either the 6093 * global limits are not being effective at throttling 6094 * processes due to the page distribution throughout 6095 * zones or there is heavy usage of a slow backing 6096 * device. The only option is to throttle from reclaim 6097 * context which is not ideal as there is no guarantee 6098 * the dirtying process is throttled in the same way 6099 * balance_dirty_pages() manages. 6100 * 6101 * Once a node is flagged PGDAT_WRITEBACK, kswapd will 6102 * count the number of pages under pages flagged for 6103 * immediate reclaim and stall if any are encountered 6104 * in the nr_immediate check below. 6105 */ 6106 if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken) 6107 set_bit(PGDAT_WRITEBACK, &pgdat->flags); 6108 6109 /* Allow kswapd to start writing pages during reclaim.*/ 6110 if (sc->nr.unqueued_dirty && 6111 sc->nr.unqueued_dirty == sc->nr.file_taken) 6112 set_bit(PGDAT_DIRTY, &pgdat->flags); 6113 6114 /* 6115 * If kswapd scans pages marked for immediate 6116 * reclaim and under writeback (nr_immediate), it 6117 * implies that pages are cycling through the LRU 6118 * faster than they are written so forcibly stall 6119 * until some pages complete writeback. 6120 */ 6121 if (sc->nr.immediate) 6122 reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK); 6123 } 6124 6125 /* 6126 * Tag a node/memcg as congested if all the dirty pages were marked 6127 * for writeback and immediate reclaim (counted in nr.congested). 6128 * 6129 * Legacy memcg will stall in page writeback so avoid forcibly 6130 * stalling in reclaim_throttle(). 6131 */ 6132 if (sc->nr.dirty && sc->nr.dirty == sc->nr.congested) { 6133 if (cgroup_reclaim(sc) && writeback_throttling_sane(sc)) 6134 set_bit(LRUVEC_CGROUP_CONGESTED, &target_lruvec->flags); 6135 6136 if (current_is_kswapd()) 6137 set_bit(LRUVEC_NODE_CONGESTED, &target_lruvec->flags); 6138 } 6139 6140 /* 6141 * Stall direct reclaim for IO completions if the lruvec is 6142 * node is congested. Allow kswapd to continue until it 6143 * starts encountering unqueued dirty pages or cycling through 6144 * the LRU too quickly. 6145 */ 6146 if (!current_is_kswapd() && current_may_throttle() && 6147 !sc->hibernation_mode && 6148 (test_bit(LRUVEC_CGROUP_CONGESTED, &target_lruvec->flags) || 6149 test_bit(LRUVEC_NODE_CONGESTED, &target_lruvec->flags))) 6150 reclaim_throttle(pgdat, VMSCAN_THROTTLE_CONGESTED); 6151 6152 if (should_continue_reclaim(pgdat, nr_node_reclaimed, sc)) 6153 goto again; 6154 6155 /* 6156 * Kswapd gives up on balancing particular nodes after too 6157 * many failures to reclaim anything from them and goes to 6158 * sleep. On reclaim progress, reset the failure counter. A 6159 * successful direct reclaim run will revive a dormant kswapd. 6160 */ 6161 if (reclaimable) 6162 pgdat->kswapd_failures = 0; 6163 else if (sc->cache_trim_mode) 6164 sc->cache_trim_mode_failed = 1; 6165 } 6166 6167 /* 6168 * Returns true if compaction should go ahead for a costly-order request, or 6169 * the allocation would already succeed without compaction. Return false if we 6170 * should reclaim first. 6171 */ 6172 static inline bool compaction_ready(struct zone *zone, struct scan_control *sc) 6173 { 6174 unsigned long watermark; 6175 6176 if (!gfp_compaction_allowed(sc->gfp_mask)) 6177 return false; 6178 6179 /* Allocation can already succeed, nothing to do */ 6180 if (zone_watermark_ok(zone, sc->order, min_wmark_pages(zone), 6181 sc->reclaim_idx, 0)) 6182 return true; 6183 6184 /* 6185 * Direct reclaim usually targets the min watermark, but compaction 6186 * takes time to run and there are potentially other callers using the 6187 * pages just freed. So target a higher buffer to give compaction a 6188 * reasonable chance of completing and allocating the pages. 6189 * 6190 * Note that we won't actually reclaim the whole buffer in one attempt 6191 * as the target watermark in should_continue_reclaim() is lower. But if 6192 * we are already above the high+gap watermark, don't reclaim at all. 6193 */ 6194 watermark = high_wmark_pages(zone); 6195 if (compaction_suitable(zone, sc->order, watermark, sc->reclaim_idx)) 6196 return true; 6197 6198 return false; 6199 } 6200 6201 static void consider_reclaim_throttle(pg_data_t *pgdat, struct scan_control *sc) 6202 { 6203 /* 6204 * If reclaim is making progress greater than 12% efficiency then 6205 * wake all the NOPROGRESS throttled tasks. 6206 */ 6207 if (sc->nr_reclaimed > (sc->nr_scanned >> 3)) { 6208 wait_queue_head_t *wqh; 6209 6210 wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_NOPROGRESS]; 6211 if (waitqueue_active(wqh)) 6212 wake_up(wqh); 6213 6214 return; 6215 } 6216 6217 /* 6218 * Do not throttle kswapd or cgroup reclaim on NOPROGRESS as it will 6219 * throttle on VMSCAN_THROTTLE_WRITEBACK if there are too many pages 6220 * under writeback and marked for immediate reclaim at the tail of the 6221 * LRU. 6222 */ 6223 if (current_is_kswapd() || cgroup_reclaim(sc)) 6224 return; 6225 6226 /* Throttle if making no progress at high prioities. */ 6227 if (sc->priority == 1 && !sc->nr_reclaimed) 6228 reclaim_throttle(pgdat, VMSCAN_THROTTLE_NOPROGRESS); 6229 } 6230 6231 /* 6232 * This is the direct reclaim path, for page-allocating processes. We only 6233 * try to reclaim pages from zones which will satisfy the caller's allocation 6234 * request. 6235 * 6236 * If a zone is deemed to be full of pinned pages then just give it a light 6237 * scan then give up on it. 6238 */ 6239 static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc) 6240 { 6241 struct zoneref *z; 6242 struct zone *zone; 6243 unsigned long nr_soft_reclaimed; 6244 unsigned long nr_soft_scanned; 6245 gfp_t orig_mask; 6246 pg_data_t *last_pgdat = NULL; 6247 pg_data_t *first_pgdat = NULL; 6248 6249 /* 6250 * If the number of buffer_heads in the machine exceeds the maximum 6251 * allowed level, force direct reclaim to scan the highmem zone as 6252 * highmem pages could be pinning lowmem pages storing buffer_heads 6253 */ 6254 orig_mask = sc->gfp_mask; 6255 if (buffer_heads_over_limit) { 6256 sc->gfp_mask |= __GFP_HIGHMEM; 6257 sc->reclaim_idx = gfp_zone(sc->gfp_mask); 6258 } 6259 6260 for_each_zone_zonelist_nodemask(zone, z, zonelist, 6261 sc->reclaim_idx, sc->nodemask) { 6262 /* 6263 * Take care memory controller reclaiming has small influence 6264 * to global LRU. 6265 */ 6266 if (!cgroup_reclaim(sc)) { 6267 if (!cpuset_zone_allowed(zone, 6268 GFP_KERNEL | __GFP_HARDWALL)) 6269 continue; 6270 6271 /* 6272 * If we already have plenty of memory free for 6273 * compaction in this zone, don't free any more. 6274 * Even though compaction is invoked for any 6275 * non-zero order, only frequent costly order 6276 * reclamation is disruptive enough to become a 6277 * noticeable problem, like transparent huge 6278 * page allocations. 6279 */ 6280 if (IS_ENABLED(CONFIG_COMPACTION) && 6281 sc->order > PAGE_ALLOC_COSTLY_ORDER && 6282 compaction_ready(zone, sc)) { 6283 sc->compaction_ready = true; 6284 continue; 6285 } 6286 6287 /* 6288 * Shrink each node in the zonelist once. If the 6289 * zonelist is ordered by zone (not the default) then a 6290 * node may be shrunk multiple times but in that case 6291 * the user prefers lower zones being preserved. 6292 */ 6293 if (zone->zone_pgdat == last_pgdat) 6294 continue; 6295 6296 /* 6297 * This steals pages from memory cgroups over softlimit 6298 * and returns the number of reclaimed pages and 6299 * scanned pages. This works for global memory pressure 6300 * and balancing, not for a memcg's limit. 6301 */ 6302 nr_soft_scanned = 0; 6303 nr_soft_reclaimed = memcg1_soft_limit_reclaim(zone->zone_pgdat, 6304 sc->order, sc->gfp_mask, 6305 &nr_soft_scanned); 6306 sc->nr_reclaimed += nr_soft_reclaimed; 6307 sc->nr_scanned += nr_soft_scanned; 6308 /* need some check for avoid more shrink_zone() */ 6309 } 6310 6311 if (!first_pgdat) 6312 first_pgdat = zone->zone_pgdat; 6313 6314 /* See comment about same check for global reclaim above */ 6315 if (zone->zone_pgdat == last_pgdat) 6316 continue; 6317 last_pgdat = zone->zone_pgdat; 6318 shrink_node(zone->zone_pgdat, sc); 6319 } 6320 6321 if (first_pgdat) 6322 consider_reclaim_throttle(first_pgdat, sc); 6323 6324 /* 6325 * Restore to original mask to avoid the impact on the caller if we 6326 * promoted it to __GFP_HIGHMEM. 6327 */ 6328 sc->gfp_mask = orig_mask; 6329 } 6330 6331 static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat) 6332 { 6333 struct lruvec *target_lruvec; 6334 unsigned long refaults; 6335 6336 if (lru_gen_enabled()) 6337 return; 6338 6339 target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat); 6340 refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON); 6341 target_lruvec->refaults[WORKINGSET_ANON] = refaults; 6342 refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE); 6343 target_lruvec->refaults[WORKINGSET_FILE] = refaults; 6344 } 6345 6346 /* 6347 * This is the main entry point to direct page reclaim. 6348 * 6349 * If a full scan of the inactive list fails to free enough memory then we 6350 * are "out of memory" and something needs to be killed. 6351 * 6352 * If the caller is !__GFP_FS then the probability of a failure is reasonably 6353 * high - the zone may be full of dirty or under-writeback pages, which this 6354 * caller can't do much about. We kick the writeback threads and take explicit 6355 * naps in the hope that some of these pages can be written. But if the 6356 * allocating task holds filesystem locks which prevent writeout this might not 6357 * work, and the allocation attempt will fail. 6358 * 6359 * returns: 0, if no pages reclaimed 6360 * else, the number of pages reclaimed 6361 */ 6362 static unsigned long do_try_to_free_pages(struct zonelist *zonelist, 6363 struct scan_control *sc) 6364 { 6365 int initial_priority = sc->priority; 6366 pg_data_t *last_pgdat; 6367 struct zoneref *z; 6368 struct zone *zone; 6369 retry: 6370 delayacct_freepages_start(); 6371 6372 if (!cgroup_reclaim(sc)) 6373 __count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1); 6374 6375 do { 6376 if (!sc->proactive) 6377 vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup, 6378 sc->priority); 6379 sc->nr_scanned = 0; 6380 shrink_zones(zonelist, sc); 6381 6382 if (sc->nr_reclaimed >= sc->nr_to_reclaim) 6383 break; 6384 6385 if (sc->compaction_ready) 6386 break; 6387 6388 /* 6389 * If we're getting trouble reclaiming, start doing 6390 * writepage even in laptop mode. 6391 */ 6392 if (sc->priority < DEF_PRIORITY - 2) 6393 sc->may_writepage = 1; 6394 } while (--sc->priority >= 0); 6395 6396 last_pgdat = NULL; 6397 for_each_zone_zonelist_nodemask(zone, z, zonelist, sc->reclaim_idx, 6398 sc->nodemask) { 6399 if (zone->zone_pgdat == last_pgdat) 6400 continue; 6401 last_pgdat = zone->zone_pgdat; 6402 6403 snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat); 6404 6405 if (cgroup_reclaim(sc)) { 6406 struct lruvec *lruvec; 6407 6408 lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, 6409 zone->zone_pgdat); 6410 clear_bit(LRUVEC_CGROUP_CONGESTED, &lruvec->flags); 6411 } 6412 } 6413 6414 delayacct_freepages_end(); 6415 6416 if (sc->nr_reclaimed) 6417 return sc->nr_reclaimed; 6418 6419 /* Aborted reclaim to try compaction? don't OOM, then */ 6420 if (sc->compaction_ready) 6421 return 1; 6422 6423 /* 6424 * In most cases, direct reclaimers can do partial walks 6425 * through the cgroup tree to meet the reclaim goal while 6426 * keeping latency low. Since the iterator state is shared 6427 * among all direct reclaim invocations (to retain fairness 6428 * among cgroups), though, high concurrency can result in 6429 * individual threads not seeing enough cgroups to make 6430 * meaningful forward progress. Avoid false OOMs in this case. 6431 */ 6432 if (!sc->memcg_full_walk) { 6433 sc->priority = initial_priority; 6434 sc->memcg_full_walk = 1; 6435 goto retry; 6436 } 6437 6438 /* 6439 * We make inactive:active ratio decisions based on the node's 6440 * composition of memory, but a restrictive reclaim_idx or a 6441 * memory.low cgroup setting can exempt large amounts of 6442 * memory from reclaim. Neither of which are very common, so 6443 * instead of doing costly eligibility calculations of the 6444 * entire cgroup subtree up front, we assume the estimates are 6445 * good, and retry with forcible deactivation if that fails. 6446 */ 6447 if (sc->skipped_deactivate) { 6448 sc->priority = initial_priority; 6449 sc->force_deactivate = 1; 6450 sc->skipped_deactivate = 0; 6451 goto retry; 6452 } 6453 6454 /* Untapped cgroup reserves? Don't OOM, retry. */ 6455 if (sc->memcg_low_skipped) { 6456 sc->priority = initial_priority; 6457 sc->force_deactivate = 0; 6458 sc->memcg_low_reclaim = 1; 6459 sc->memcg_low_skipped = 0; 6460 goto retry; 6461 } 6462 6463 return 0; 6464 } 6465 6466 static bool allow_direct_reclaim(pg_data_t *pgdat) 6467 { 6468 struct zone *zone; 6469 unsigned long pfmemalloc_reserve = 0; 6470 unsigned long free_pages = 0; 6471 int i; 6472 bool wmark_ok; 6473 6474 if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES) 6475 return true; 6476 6477 for_each_managed_zone_pgdat(zone, pgdat, i, ZONE_NORMAL) { 6478 if (!zone_reclaimable_pages(zone)) 6479 continue; 6480 6481 pfmemalloc_reserve += min_wmark_pages(zone); 6482 free_pages += zone_page_state_snapshot(zone, NR_FREE_PAGES); 6483 } 6484 6485 /* If there are no reserves (unexpected config) then do not throttle */ 6486 if (!pfmemalloc_reserve) 6487 return true; 6488 6489 wmark_ok = free_pages > pfmemalloc_reserve / 2; 6490 6491 /* kswapd must be awake if processes are being throttled */ 6492 if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) { 6493 if (READ_ONCE(pgdat->kswapd_highest_zoneidx) > ZONE_NORMAL) 6494 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, ZONE_NORMAL); 6495 6496 wake_up_interruptible(&pgdat->kswapd_wait); 6497 } 6498 6499 return wmark_ok; 6500 } 6501 6502 /* 6503 * Throttle direct reclaimers if backing storage is backed by the network 6504 * and the PFMEMALLOC reserve for the preferred node is getting dangerously 6505 * depleted. kswapd will continue to make progress and wake the processes 6506 * when the low watermark is reached. 6507 * 6508 * Returns true if a fatal signal was delivered during throttling. If this 6509 * happens, the page allocator should not consider triggering the OOM killer. 6510 */ 6511 static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist, 6512 nodemask_t *nodemask) 6513 { 6514 struct zoneref *z; 6515 struct zone *zone; 6516 pg_data_t *pgdat = NULL; 6517 6518 /* 6519 * Kernel threads should not be throttled as they may be indirectly 6520 * responsible for cleaning pages necessary for reclaim to make forward 6521 * progress. kjournald for example may enter direct reclaim while 6522 * committing a transaction where throttling it could forcing other 6523 * processes to block on log_wait_commit(). 6524 */ 6525 if (current->flags & PF_KTHREAD) 6526 goto out; 6527 6528 /* 6529 * If a fatal signal is pending, this process should not throttle. 6530 * It should return quickly so it can exit and free its memory 6531 */ 6532 if (fatal_signal_pending(current)) 6533 goto out; 6534 6535 /* 6536 * Check if the pfmemalloc reserves are ok by finding the first node 6537 * with a usable ZONE_NORMAL or lower zone. The expectation is that 6538 * GFP_KERNEL will be required for allocating network buffers when 6539 * swapping over the network so ZONE_HIGHMEM is unusable. 6540 * 6541 * Throttling is based on the first usable node and throttled processes 6542 * wait on a queue until kswapd makes progress and wakes them. There 6543 * is an affinity then between processes waking up and where reclaim 6544 * progress has been made assuming the process wakes on the same node. 6545 * More importantly, processes running on remote nodes will not compete 6546 * for remote pfmemalloc reserves and processes on different nodes 6547 * should make reasonable progress. 6548 */ 6549 for_each_zone_zonelist_nodemask(zone, z, zonelist, 6550 gfp_zone(gfp_mask), nodemask) { 6551 if (zone_idx(zone) > ZONE_NORMAL) 6552 continue; 6553 6554 /* Throttle based on the first usable node */ 6555 pgdat = zone->zone_pgdat; 6556 if (allow_direct_reclaim(pgdat)) 6557 goto out; 6558 break; 6559 } 6560 6561 /* If no zone was usable by the allocation flags then do not throttle */ 6562 if (!pgdat) 6563 goto out; 6564 6565 /* Account for the throttling */ 6566 count_vm_event(PGSCAN_DIRECT_THROTTLE); 6567 6568 /* 6569 * If the caller cannot enter the filesystem, it's possible that it 6570 * is due to the caller holding an FS lock or performing a journal 6571 * transaction in the case of a filesystem like ext[3|4]. In this case, 6572 * it is not safe to block on pfmemalloc_wait as kswapd could be 6573 * blocked waiting on the same lock. Instead, throttle for up to a 6574 * second before continuing. 6575 */ 6576 if (!(gfp_mask & __GFP_FS)) 6577 wait_event_interruptible_timeout(pgdat->pfmemalloc_wait, 6578 allow_direct_reclaim(pgdat), HZ); 6579 else 6580 /* Throttle until kswapd wakes the process */ 6581 wait_event_killable(zone->zone_pgdat->pfmemalloc_wait, 6582 allow_direct_reclaim(pgdat)); 6583 6584 if (fatal_signal_pending(current)) 6585 return true; 6586 6587 out: 6588 return false; 6589 } 6590 6591 unsigned long try_to_free_pages(struct zonelist *zonelist, int order, 6592 gfp_t gfp_mask, nodemask_t *nodemask) 6593 { 6594 unsigned long nr_reclaimed; 6595 struct scan_control sc = { 6596 .nr_to_reclaim = SWAP_CLUSTER_MAX, 6597 .gfp_mask = current_gfp_context(gfp_mask), 6598 .reclaim_idx = gfp_zone(gfp_mask), 6599 .order = order, 6600 .nodemask = nodemask, 6601 .priority = DEF_PRIORITY, 6602 .may_writepage = !laptop_mode, 6603 .may_unmap = 1, 6604 .may_swap = 1, 6605 }; 6606 6607 /* 6608 * scan_control uses s8 fields for order, priority, and reclaim_idx. 6609 * Confirm they are large enough for max values. 6610 */ 6611 BUILD_BUG_ON(MAX_PAGE_ORDER >= S8_MAX); 6612 BUILD_BUG_ON(DEF_PRIORITY > S8_MAX); 6613 BUILD_BUG_ON(MAX_NR_ZONES > S8_MAX); 6614 6615 /* 6616 * Do not enter reclaim if fatal signal was delivered while throttled. 6617 * 1 is returned so that the page allocator does not OOM kill at this 6618 * point. 6619 */ 6620 if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask)) 6621 return 1; 6622 6623 set_task_reclaim_state(current, &sc.reclaim_state); 6624 trace_mm_vmscan_direct_reclaim_begin(order, sc.gfp_mask); 6625 6626 nr_reclaimed = do_try_to_free_pages(zonelist, &sc); 6627 6628 trace_mm_vmscan_direct_reclaim_end(nr_reclaimed); 6629 set_task_reclaim_state(current, NULL); 6630 6631 return nr_reclaimed; 6632 } 6633 6634 #ifdef CONFIG_MEMCG 6635 6636 /* Only used by soft limit reclaim. Do not reuse for anything else. */ 6637 unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, 6638 gfp_t gfp_mask, bool noswap, 6639 pg_data_t *pgdat, 6640 unsigned long *nr_scanned) 6641 { 6642 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); 6643 struct scan_control sc = { 6644 .nr_to_reclaim = SWAP_CLUSTER_MAX, 6645 .target_mem_cgroup = memcg, 6646 .may_writepage = !laptop_mode, 6647 .may_unmap = 1, 6648 .reclaim_idx = MAX_NR_ZONES - 1, 6649 .may_swap = !noswap, 6650 }; 6651 6652 WARN_ON_ONCE(!current->reclaim_state); 6653 6654 sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) | 6655 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK); 6656 6657 trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order, 6658 sc.gfp_mask); 6659 6660 /* 6661 * NOTE: Although we can get the priority field, using it 6662 * here is not a good idea, since it limits the pages we can scan. 6663 * if we don't reclaim here, the shrink_node from balance_pgdat 6664 * will pick up pages from other mem cgroup's as well. We hack 6665 * the priority and make it zero. 6666 */ 6667 shrink_lruvec(lruvec, &sc); 6668 6669 trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed); 6670 6671 *nr_scanned = sc.nr_scanned; 6672 6673 return sc.nr_reclaimed; 6674 } 6675 6676 unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, 6677 unsigned long nr_pages, 6678 gfp_t gfp_mask, 6679 unsigned int reclaim_options, 6680 int *swappiness) 6681 { 6682 unsigned long nr_reclaimed; 6683 unsigned int noreclaim_flag; 6684 struct scan_control sc = { 6685 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), 6686 .proactive_swappiness = swappiness, 6687 .gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) | 6688 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK), 6689 .reclaim_idx = MAX_NR_ZONES - 1, 6690 .target_mem_cgroup = memcg, 6691 .priority = DEF_PRIORITY, 6692 .may_writepage = !laptop_mode, 6693 .may_unmap = 1, 6694 .may_swap = !!(reclaim_options & MEMCG_RECLAIM_MAY_SWAP), 6695 .proactive = !!(reclaim_options & MEMCG_RECLAIM_PROACTIVE), 6696 }; 6697 /* 6698 * Traverse the ZONELIST_FALLBACK zonelist of the current node to put 6699 * equal pressure on all the nodes. This is based on the assumption that 6700 * the reclaim does not bail out early. 6701 */ 6702 struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); 6703 6704 set_task_reclaim_state(current, &sc.reclaim_state); 6705 trace_mm_vmscan_memcg_reclaim_begin(0, sc.gfp_mask); 6706 noreclaim_flag = memalloc_noreclaim_save(); 6707 6708 nr_reclaimed = do_try_to_free_pages(zonelist, &sc); 6709 6710 memalloc_noreclaim_restore(noreclaim_flag); 6711 trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed); 6712 set_task_reclaim_state(current, NULL); 6713 6714 return nr_reclaimed; 6715 } 6716 #endif 6717 6718 static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc) 6719 { 6720 struct mem_cgroup *memcg; 6721 struct lruvec *lruvec; 6722 6723 if (lru_gen_enabled()) { 6724 lru_gen_age_node(pgdat, sc); 6725 return; 6726 } 6727 6728 lruvec = mem_cgroup_lruvec(NULL, pgdat); 6729 if (!can_age_anon_pages(lruvec, sc)) 6730 return; 6731 6732 if (!inactive_is_low(lruvec, LRU_INACTIVE_ANON)) 6733 return; 6734 6735 memcg = mem_cgroup_iter(NULL, NULL, NULL); 6736 do { 6737 lruvec = mem_cgroup_lruvec(memcg, pgdat); 6738 shrink_active_list(SWAP_CLUSTER_MAX, lruvec, 6739 sc, LRU_ACTIVE_ANON); 6740 memcg = mem_cgroup_iter(NULL, memcg, NULL); 6741 } while (memcg); 6742 } 6743 6744 static bool pgdat_watermark_boosted(pg_data_t *pgdat, int highest_zoneidx) 6745 { 6746 int i; 6747 struct zone *zone; 6748 6749 /* 6750 * Check for watermark boosts top-down as the higher zones 6751 * are more likely to be boosted. Both watermarks and boosts 6752 * should not be checked at the same time as reclaim would 6753 * start prematurely when there is no boosting and a lower 6754 * zone is balanced. 6755 */ 6756 for (i = highest_zoneidx; i >= 0; i--) { 6757 zone = pgdat->node_zones + i; 6758 if (!managed_zone(zone)) 6759 continue; 6760 6761 if (zone->watermark_boost) 6762 return true; 6763 } 6764 6765 return false; 6766 } 6767 6768 /* 6769 * Returns true if there is an eligible zone balanced for the request order 6770 * and highest_zoneidx 6771 */ 6772 static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx) 6773 { 6774 int i; 6775 unsigned long mark = -1; 6776 struct zone *zone; 6777 6778 /* 6779 * Check watermarks bottom-up as lower zones are more likely to 6780 * meet watermarks. 6781 */ 6782 for_each_managed_zone_pgdat(zone, pgdat, i, highest_zoneidx) { 6783 enum zone_stat_item item; 6784 unsigned long free_pages; 6785 6786 if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) 6787 mark = promo_wmark_pages(zone); 6788 else 6789 mark = high_wmark_pages(zone); 6790 6791 /* 6792 * In defrag_mode, watermarks must be met in whole 6793 * blocks to avoid polluting allocator fallbacks. 6794 * 6795 * However, kswapd usually cannot accomplish this on 6796 * its own and needs kcompactd support. Once it's 6797 * reclaimed a compaction gap, and kswapd_shrink_node 6798 * has dropped order, simply ensure there are enough 6799 * base pages for compaction, wake kcompactd & sleep. 6800 */ 6801 if (defrag_mode && order) 6802 item = NR_FREE_PAGES_BLOCKS; 6803 else 6804 item = NR_FREE_PAGES; 6805 6806 /* 6807 * When there is a high number of CPUs in the system, 6808 * the cumulative error from the vmstat per-cpu cache 6809 * can blur the line between the watermarks. In that 6810 * case, be safe and get an accurate snapshot. 6811 * 6812 * TODO: NR_FREE_PAGES_BLOCKS moves in steps of 6813 * pageblock_nr_pages, while the vmstat pcp threshold 6814 * is limited to 125. On many configurations that 6815 * counter won't actually be per-cpu cached. But keep 6816 * things simple for now; revisit when somebody cares. 6817 */ 6818 free_pages = zone_page_state(zone, item); 6819 if (zone->percpu_drift_mark && free_pages < zone->percpu_drift_mark) 6820 free_pages = zone_page_state_snapshot(zone, item); 6821 6822 if (__zone_watermark_ok(zone, order, mark, highest_zoneidx, 6823 0, free_pages)) 6824 return true; 6825 } 6826 6827 /* 6828 * If a node has no managed zone within highest_zoneidx, it does not 6829 * need balancing by definition. This can happen if a zone-restricted 6830 * allocation tries to wake a remote kswapd. 6831 */ 6832 if (mark == -1) 6833 return true; 6834 6835 return false; 6836 } 6837 6838 /* Clear pgdat state for congested, dirty or under writeback. */ 6839 static void clear_pgdat_congested(pg_data_t *pgdat) 6840 { 6841 struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat); 6842 6843 clear_bit(LRUVEC_NODE_CONGESTED, &lruvec->flags); 6844 clear_bit(LRUVEC_CGROUP_CONGESTED, &lruvec->flags); 6845 clear_bit(PGDAT_DIRTY, &pgdat->flags); 6846 clear_bit(PGDAT_WRITEBACK, &pgdat->flags); 6847 } 6848 6849 /* 6850 * Prepare kswapd for sleeping. This verifies that there are no processes 6851 * waiting in throttle_direct_reclaim() and that watermarks have been met. 6852 * 6853 * Returns true if kswapd is ready to sleep 6854 */ 6855 static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, 6856 int highest_zoneidx) 6857 { 6858 /* 6859 * The throttled processes are normally woken up in balance_pgdat() as 6860 * soon as allow_direct_reclaim() is true. But there is a potential 6861 * race between when kswapd checks the watermarks and a process gets 6862 * throttled. There is also a potential race if processes get 6863 * throttled, kswapd wakes, a large process exits thereby balancing the 6864 * zones, which causes kswapd to exit balance_pgdat() before reaching 6865 * the wake up checks. If kswapd is going to sleep, no process should 6866 * be sleeping on pfmemalloc_wait, so wake them now if necessary. If 6867 * the wake up is premature, processes will wake kswapd and get 6868 * throttled again. The difference from wake ups in balance_pgdat() is 6869 * that here we are under prepare_to_wait(). 6870 */ 6871 if (waitqueue_active(&pgdat->pfmemalloc_wait)) 6872 wake_up_all(&pgdat->pfmemalloc_wait); 6873 6874 /* Hopeless node, leave it to direct reclaim */ 6875 if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES) 6876 return true; 6877 6878 if (pgdat_balanced(pgdat, order, highest_zoneidx)) { 6879 clear_pgdat_congested(pgdat); 6880 return true; 6881 } 6882 6883 return false; 6884 } 6885 6886 /* 6887 * kswapd shrinks a node of pages that are at or below the highest usable 6888 * zone that is currently unbalanced. 6889 * 6890 * Returns true if kswapd scanned at least the requested number of pages to 6891 * reclaim or if the lack of progress was due to pages under writeback. 6892 * This is used to determine if the scanning priority needs to be raised. 6893 */ 6894 static bool kswapd_shrink_node(pg_data_t *pgdat, 6895 struct scan_control *sc) 6896 { 6897 struct zone *zone; 6898 int z; 6899 unsigned long nr_reclaimed = sc->nr_reclaimed; 6900 6901 /* Reclaim a number of pages proportional to the number of zones */ 6902 sc->nr_to_reclaim = 0; 6903 for_each_managed_zone_pgdat(zone, pgdat, z, sc->reclaim_idx) { 6904 sc->nr_to_reclaim += max(high_wmark_pages(zone), SWAP_CLUSTER_MAX); 6905 } 6906 6907 /* 6908 * Historically care was taken to put equal pressure on all zones but 6909 * now pressure is applied based on node LRU order. 6910 */ 6911 shrink_node(pgdat, sc); 6912 6913 /* 6914 * Fragmentation may mean that the system cannot be rebalanced for 6915 * high-order allocations. If twice the allocation size has been 6916 * reclaimed then recheck watermarks only at order-0 to prevent 6917 * excessive reclaim. Assume that a process requested a high-order 6918 * can direct reclaim/compact. 6919 */ 6920 if (sc->order && sc->nr_reclaimed >= compact_gap(sc->order)) 6921 sc->order = 0; 6922 6923 /* account for progress from mm_account_reclaimed_pages() */ 6924 return max(sc->nr_scanned, sc->nr_reclaimed - nr_reclaimed) >= sc->nr_to_reclaim; 6925 } 6926 6927 /* Page allocator PCP high watermark is lowered if reclaim is active. */ 6928 static inline void 6929 update_reclaim_active(pg_data_t *pgdat, int highest_zoneidx, bool active) 6930 { 6931 int i; 6932 struct zone *zone; 6933 6934 for_each_managed_zone_pgdat(zone, pgdat, i, highest_zoneidx) { 6935 if (active) 6936 set_bit(ZONE_RECLAIM_ACTIVE, &zone->flags); 6937 else 6938 clear_bit(ZONE_RECLAIM_ACTIVE, &zone->flags); 6939 } 6940 } 6941 6942 static inline void 6943 set_reclaim_active(pg_data_t *pgdat, int highest_zoneidx) 6944 { 6945 update_reclaim_active(pgdat, highest_zoneidx, true); 6946 } 6947 6948 static inline void 6949 clear_reclaim_active(pg_data_t *pgdat, int highest_zoneidx) 6950 { 6951 update_reclaim_active(pgdat, highest_zoneidx, false); 6952 } 6953 6954 /* 6955 * For kswapd, balance_pgdat() will reclaim pages across a node from zones 6956 * that are eligible for use by the caller until at least one zone is 6957 * balanced. 6958 * 6959 * Returns the order kswapd finished reclaiming at. 6960 * 6961 * kswapd scans the zones in the highmem->normal->dma direction. It skips 6962 * zones which have free_pages > high_wmark_pages(zone), but once a zone is 6963 * found to have free_pages <= high_wmark_pages(zone), any page in that zone 6964 * or lower is eligible for reclaim until at least one usable zone is 6965 * balanced. 6966 */ 6967 static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx) 6968 { 6969 int i; 6970 unsigned long nr_soft_reclaimed; 6971 unsigned long nr_soft_scanned; 6972 unsigned long pflags; 6973 unsigned long nr_boost_reclaim; 6974 unsigned long zone_boosts[MAX_NR_ZONES] = { 0, }; 6975 bool boosted; 6976 struct zone *zone; 6977 struct scan_control sc = { 6978 .gfp_mask = GFP_KERNEL, 6979 .order = order, 6980 .may_unmap = 1, 6981 }; 6982 6983 set_task_reclaim_state(current, &sc.reclaim_state); 6984 psi_memstall_enter(&pflags); 6985 __fs_reclaim_acquire(_THIS_IP_); 6986 6987 count_vm_event(PAGEOUTRUN); 6988 6989 /* 6990 * Account for the reclaim boost. Note that the zone boost is left in 6991 * place so that parallel allocations that are near the watermark will 6992 * stall or direct reclaim until kswapd is finished. 6993 */ 6994 nr_boost_reclaim = 0; 6995 for_each_managed_zone_pgdat(zone, pgdat, i, highest_zoneidx) { 6996 nr_boost_reclaim += zone->watermark_boost; 6997 zone_boosts[i] = zone->watermark_boost; 6998 } 6999 boosted = nr_boost_reclaim; 7000 7001 restart: 7002 set_reclaim_active(pgdat, highest_zoneidx); 7003 sc.priority = DEF_PRIORITY; 7004 do { 7005 unsigned long nr_reclaimed = sc.nr_reclaimed; 7006 bool raise_priority = true; 7007 bool balanced; 7008 bool ret; 7009 bool was_frozen; 7010 7011 sc.reclaim_idx = highest_zoneidx; 7012 7013 /* 7014 * If the number of buffer_heads exceeds the maximum allowed 7015 * then consider reclaiming from all zones. This has a dual 7016 * purpose -- on 64-bit systems it is expected that 7017 * buffer_heads are stripped during active rotation. On 32-bit 7018 * systems, highmem pages can pin lowmem memory and shrinking 7019 * buffers can relieve lowmem pressure. Reclaim may still not 7020 * go ahead if all eligible zones for the original allocation 7021 * request are balanced to avoid excessive reclaim from kswapd. 7022 */ 7023 if (buffer_heads_over_limit) { 7024 for (i = MAX_NR_ZONES - 1; i >= 0; i--) { 7025 zone = pgdat->node_zones + i; 7026 if (!managed_zone(zone)) 7027 continue; 7028 7029 sc.reclaim_idx = i; 7030 break; 7031 } 7032 } 7033 7034 /* 7035 * If the pgdat is imbalanced then ignore boosting and preserve 7036 * the watermarks for a later time and restart. Note that the 7037 * zone watermarks will be still reset at the end of balancing 7038 * on the grounds that the normal reclaim should be enough to 7039 * re-evaluate if boosting is required when kswapd next wakes. 7040 */ 7041 balanced = pgdat_balanced(pgdat, sc.order, highest_zoneidx); 7042 if (!balanced && nr_boost_reclaim) { 7043 nr_boost_reclaim = 0; 7044 goto restart; 7045 } 7046 7047 /* 7048 * If boosting is not active then only reclaim if there are no 7049 * eligible zones. Note that sc.reclaim_idx is not used as 7050 * buffer_heads_over_limit may have adjusted it. 7051 */ 7052 if (!nr_boost_reclaim && balanced) 7053 goto out; 7054 7055 /* Limit the priority of boosting to avoid reclaim writeback */ 7056 if (nr_boost_reclaim && sc.priority == DEF_PRIORITY - 2) 7057 raise_priority = false; 7058 7059 /* 7060 * Do not writeback or swap pages for boosted reclaim. The 7061 * intent is to relieve pressure not issue sub-optimal IO 7062 * from reclaim context. If no pages are reclaimed, the 7063 * reclaim will be aborted. 7064 */ 7065 sc.may_writepage = !laptop_mode && !nr_boost_reclaim; 7066 sc.may_swap = !nr_boost_reclaim; 7067 7068 /* 7069 * Do some background aging, to give pages a chance to be 7070 * referenced before reclaiming. All pages are rotated 7071 * regardless of classzone as this is about consistent aging. 7072 */ 7073 kswapd_age_node(pgdat, &sc); 7074 7075 /* 7076 * If we're getting trouble reclaiming, start doing writepage 7077 * even in laptop mode. 7078 */ 7079 if (sc.priority < DEF_PRIORITY - 2) 7080 sc.may_writepage = 1; 7081 7082 /* Call soft limit reclaim before calling shrink_node. */ 7083 sc.nr_scanned = 0; 7084 nr_soft_scanned = 0; 7085 nr_soft_reclaimed = memcg1_soft_limit_reclaim(pgdat, sc.order, 7086 sc.gfp_mask, &nr_soft_scanned); 7087 sc.nr_reclaimed += nr_soft_reclaimed; 7088 7089 /* 7090 * There should be no need to raise the scanning priority if 7091 * enough pages are already being scanned that that high 7092 * watermark would be met at 100% efficiency. 7093 */ 7094 if (kswapd_shrink_node(pgdat, &sc)) 7095 raise_priority = false; 7096 7097 /* 7098 * If the low watermark is met there is no need for processes 7099 * to be throttled on pfmemalloc_wait as they should not be 7100 * able to safely make forward progress. Wake them 7101 */ 7102 if (waitqueue_active(&pgdat->pfmemalloc_wait) && 7103 allow_direct_reclaim(pgdat)) 7104 wake_up_all(&pgdat->pfmemalloc_wait); 7105 7106 /* Check if kswapd should be suspending */ 7107 __fs_reclaim_release(_THIS_IP_); 7108 ret = kthread_freezable_should_stop(&was_frozen); 7109 __fs_reclaim_acquire(_THIS_IP_); 7110 if (was_frozen || ret) 7111 break; 7112 7113 /* 7114 * Raise priority if scanning rate is too low or there was no 7115 * progress in reclaiming pages 7116 */ 7117 nr_reclaimed = sc.nr_reclaimed - nr_reclaimed; 7118 nr_boost_reclaim -= min(nr_boost_reclaim, nr_reclaimed); 7119 7120 /* 7121 * If reclaim made no progress for a boost, stop reclaim as 7122 * IO cannot be queued and it could be an infinite loop in 7123 * extreme circumstances. 7124 */ 7125 if (nr_boost_reclaim && !nr_reclaimed) 7126 break; 7127 7128 if (raise_priority || !nr_reclaimed) 7129 sc.priority--; 7130 } while (sc.priority >= 1); 7131 7132 /* 7133 * Restart only if it went through the priority loop all the way, 7134 * but cache_trim_mode didn't work. 7135 */ 7136 if (!sc.nr_reclaimed && sc.priority < 1 && 7137 !sc.no_cache_trim_mode && sc.cache_trim_mode_failed) { 7138 sc.no_cache_trim_mode = 1; 7139 goto restart; 7140 } 7141 7142 if (!sc.nr_reclaimed) 7143 pgdat->kswapd_failures++; 7144 7145 out: 7146 clear_reclaim_active(pgdat, highest_zoneidx); 7147 7148 /* If reclaim was boosted, account for the reclaim done in this pass */ 7149 if (boosted) { 7150 unsigned long flags; 7151 7152 for (i = 0; i <= highest_zoneidx; i++) { 7153 if (!zone_boosts[i]) 7154 continue; 7155 7156 /* Increments are under the zone lock */ 7157 zone = pgdat->node_zones + i; 7158 spin_lock_irqsave(&zone->lock, flags); 7159 zone->watermark_boost -= min(zone->watermark_boost, zone_boosts[i]); 7160 spin_unlock_irqrestore(&zone->lock, flags); 7161 } 7162 7163 /* 7164 * As there is now likely space, wakeup kcompact to defragment 7165 * pageblocks. 7166 */ 7167 wakeup_kcompactd(pgdat, pageblock_order, highest_zoneidx); 7168 } 7169 7170 snapshot_refaults(NULL, pgdat); 7171 __fs_reclaim_release(_THIS_IP_); 7172 psi_memstall_leave(&pflags); 7173 set_task_reclaim_state(current, NULL); 7174 7175 /* 7176 * Return the order kswapd stopped reclaiming at as 7177 * prepare_kswapd_sleep() takes it into account. If another caller 7178 * entered the allocator slow path while kswapd was awake, order will 7179 * remain at the higher level. 7180 */ 7181 return sc.order; 7182 } 7183 7184 /* 7185 * The pgdat->kswapd_highest_zoneidx is used to pass the highest zone index to 7186 * be reclaimed by kswapd from the waker. If the value is MAX_NR_ZONES which is 7187 * not a valid index then either kswapd runs for first time or kswapd couldn't 7188 * sleep after previous reclaim attempt (node is still unbalanced). In that 7189 * case return the zone index of the previous kswapd reclaim cycle. 7190 */ 7191 static enum zone_type kswapd_highest_zoneidx(pg_data_t *pgdat, 7192 enum zone_type prev_highest_zoneidx) 7193 { 7194 enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx); 7195 7196 return curr_idx == MAX_NR_ZONES ? prev_highest_zoneidx : curr_idx; 7197 } 7198 7199 static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order, 7200 unsigned int highest_zoneidx) 7201 { 7202 long remaining = 0; 7203 DEFINE_WAIT(wait); 7204 7205 if (freezing(current) || kthread_should_stop()) 7206 return; 7207 7208 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE); 7209 7210 /* 7211 * Try to sleep for a short interval. Note that kcompactd will only be 7212 * woken if it is possible to sleep for a short interval. This is 7213 * deliberate on the assumption that if reclaim cannot keep an 7214 * eligible zone balanced that it's also unlikely that compaction will 7215 * succeed. 7216 */ 7217 if (prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) { 7218 /* 7219 * Compaction records what page blocks it recently failed to 7220 * isolate pages from and skips them in the future scanning. 7221 * When kswapd is going to sleep, it is reasonable to assume 7222 * that pages and compaction may succeed so reset the cache. 7223 */ 7224 reset_isolation_suitable(pgdat); 7225 7226 /* 7227 * We have freed the memory, now we should compact it to make 7228 * allocation of the requested order possible. 7229 */ 7230 wakeup_kcompactd(pgdat, alloc_order, highest_zoneidx); 7231 7232 remaining = schedule_timeout(HZ/10); 7233 7234 /* 7235 * If woken prematurely then reset kswapd_highest_zoneidx and 7236 * order. The values will either be from a wakeup request or 7237 * the previous request that slept prematurely. 7238 */ 7239 if (remaining) { 7240 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, 7241 kswapd_highest_zoneidx(pgdat, 7242 highest_zoneidx)); 7243 7244 if (READ_ONCE(pgdat->kswapd_order) < reclaim_order) 7245 WRITE_ONCE(pgdat->kswapd_order, reclaim_order); 7246 } 7247 7248 finish_wait(&pgdat->kswapd_wait, &wait); 7249 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE); 7250 } 7251 7252 /* 7253 * After a short sleep, check if it was a premature sleep. If not, then 7254 * go fully to sleep until explicitly woken up. 7255 */ 7256 if (!remaining && 7257 prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) { 7258 trace_mm_vmscan_kswapd_sleep(pgdat->node_id); 7259 7260 /* 7261 * vmstat counters are not perfectly accurate and the estimated 7262 * value for counters such as NR_FREE_PAGES can deviate from the 7263 * true value by nr_online_cpus * threshold. To avoid the zone 7264 * watermarks being breached while under pressure, we reduce the 7265 * per-cpu vmstat threshold while kswapd is awake and restore 7266 * them before going back to sleep. 7267 */ 7268 set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold); 7269 7270 if (!kthread_should_stop()) 7271 schedule(); 7272 7273 set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold); 7274 } else { 7275 if (remaining) 7276 count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY); 7277 else 7278 count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY); 7279 } 7280 finish_wait(&pgdat->kswapd_wait, &wait); 7281 } 7282 7283 /* 7284 * The background pageout daemon, started as a kernel thread 7285 * from the init process. 7286 * 7287 * This basically trickles out pages so that we have _some_ 7288 * free memory available even if there is no other activity 7289 * that frees anything up. This is needed for things like routing 7290 * etc, where we otherwise might have all activity going on in 7291 * asynchronous contexts that cannot page things out. 7292 * 7293 * If there are applications that are active memory-allocators 7294 * (most normal use), this basically shouldn't matter. 7295 */ 7296 static int kswapd(void *p) 7297 { 7298 unsigned int alloc_order, reclaim_order; 7299 unsigned int highest_zoneidx = MAX_NR_ZONES - 1; 7300 pg_data_t *pgdat = (pg_data_t *)p; 7301 struct task_struct *tsk = current; 7302 7303 /* 7304 * Tell the memory management that we're a "memory allocator", 7305 * and that if we need more memory we should get access to it 7306 * regardless (see "__alloc_pages()"). "kswapd" should 7307 * never get caught in the normal page freeing logic. 7308 * 7309 * (Kswapd normally doesn't need memory anyway, but sometimes 7310 * you need a small amount of memory in order to be able to 7311 * page out something else, and this flag essentially protects 7312 * us from recursively trying to free more memory as we're 7313 * trying to free the first piece of memory in the first place). 7314 */ 7315 tsk->flags |= PF_MEMALLOC | PF_KSWAPD; 7316 set_freezable(); 7317 7318 WRITE_ONCE(pgdat->kswapd_order, 0); 7319 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES); 7320 atomic_set(&pgdat->nr_writeback_throttled, 0); 7321 for ( ; ; ) { 7322 bool was_frozen; 7323 7324 alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order); 7325 highest_zoneidx = kswapd_highest_zoneidx(pgdat, 7326 highest_zoneidx); 7327 7328 kswapd_try_sleep: 7329 kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order, 7330 highest_zoneidx); 7331 7332 /* Read the new order and highest_zoneidx */ 7333 alloc_order = READ_ONCE(pgdat->kswapd_order); 7334 highest_zoneidx = kswapd_highest_zoneidx(pgdat, 7335 highest_zoneidx); 7336 WRITE_ONCE(pgdat->kswapd_order, 0); 7337 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES); 7338 7339 if (kthread_freezable_should_stop(&was_frozen)) 7340 break; 7341 7342 /* 7343 * We can speed up thawing tasks if we don't call balance_pgdat 7344 * after returning from the refrigerator 7345 */ 7346 if (was_frozen) 7347 continue; 7348 7349 /* 7350 * Reclaim begins at the requested order but if a high-order 7351 * reclaim fails then kswapd falls back to reclaiming for 7352 * order-0. If that happens, kswapd will consider sleeping 7353 * for the order it finished reclaiming at (reclaim_order) 7354 * but kcompactd is woken to compact for the original 7355 * request (alloc_order). 7356 */ 7357 trace_mm_vmscan_kswapd_wake(pgdat->node_id, highest_zoneidx, 7358 alloc_order); 7359 reclaim_order = balance_pgdat(pgdat, alloc_order, 7360 highest_zoneidx); 7361 if (reclaim_order < alloc_order) 7362 goto kswapd_try_sleep; 7363 } 7364 7365 tsk->flags &= ~(PF_MEMALLOC | PF_KSWAPD); 7366 7367 return 0; 7368 } 7369 7370 /* 7371 * A zone is low on free memory or too fragmented for high-order memory. If 7372 * kswapd should reclaim (direct reclaim is deferred), wake it up for the zone's 7373 * pgdat. It will wake up kcompactd after reclaiming memory. If kswapd reclaim 7374 * has failed or is not needed, still wake up kcompactd if only compaction is 7375 * needed. 7376 */ 7377 void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order, 7378 enum zone_type highest_zoneidx) 7379 { 7380 pg_data_t *pgdat; 7381 enum zone_type curr_idx; 7382 7383 if (!managed_zone(zone)) 7384 return; 7385 7386 if (!cpuset_zone_allowed(zone, gfp_flags)) 7387 return; 7388 7389 pgdat = zone->zone_pgdat; 7390 curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx); 7391 7392 if (curr_idx == MAX_NR_ZONES || curr_idx < highest_zoneidx) 7393 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, highest_zoneidx); 7394 7395 if (READ_ONCE(pgdat->kswapd_order) < order) 7396 WRITE_ONCE(pgdat->kswapd_order, order); 7397 7398 if (!waitqueue_active(&pgdat->kswapd_wait)) 7399 return; 7400 7401 /* Hopeless node, leave it to direct reclaim if possible */ 7402 if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES || 7403 (pgdat_balanced(pgdat, order, highest_zoneidx) && 7404 !pgdat_watermark_boosted(pgdat, highest_zoneidx))) { 7405 /* 7406 * There may be plenty of free memory available, but it's too 7407 * fragmented for high-order allocations. Wake up kcompactd 7408 * and rely on compaction_suitable() to determine if it's 7409 * needed. If it fails, it will defer subsequent attempts to 7410 * ratelimit its work. 7411 */ 7412 if (!(gfp_flags & __GFP_DIRECT_RECLAIM)) 7413 wakeup_kcompactd(pgdat, order, highest_zoneidx); 7414 return; 7415 } 7416 7417 trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, highest_zoneidx, order, 7418 gfp_flags); 7419 wake_up_interruptible(&pgdat->kswapd_wait); 7420 } 7421 7422 #ifdef CONFIG_HIBERNATION 7423 /* 7424 * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of 7425 * freed pages. 7426 * 7427 * Rather than trying to age LRUs the aim is to preserve the overall 7428 * LRU order by reclaiming preferentially 7429 * inactive > active > active referenced > active mapped 7430 */ 7431 unsigned long shrink_all_memory(unsigned long nr_to_reclaim) 7432 { 7433 struct scan_control sc = { 7434 .nr_to_reclaim = nr_to_reclaim, 7435 .gfp_mask = GFP_HIGHUSER_MOVABLE, 7436 .reclaim_idx = MAX_NR_ZONES - 1, 7437 .priority = DEF_PRIORITY, 7438 .may_writepage = 1, 7439 .may_unmap = 1, 7440 .may_swap = 1, 7441 .hibernation_mode = 1, 7442 }; 7443 struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); 7444 unsigned long nr_reclaimed; 7445 unsigned int noreclaim_flag; 7446 7447 fs_reclaim_acquire(sc.gfp_mask); 7448 noreclaim_flag = memalloc_noreclaim_save(); 7449 set_task_reclaim_state(current, &sc.reclaim_state); 7450 7451 nr_reclaimed = do_try_to_free_pages(zonelist, &sc); 7452 7453 set_task_reclaim_state(current, NULL); 7454 memalloc_noreclaim_restore(noreclaim_flag); 7455 fs_reclaim_release(sc.gfp_mask); 7456 7457 return nr_reclaimed; 7458 } 7459 #endif /* CONFIG_HIBERNATION */ 7460 7461 /* 7462 * This kswapd start function will be called by init and node-hot-add. 7463 */ 7464 void __meminit kswapd_run(int nid) 7465 { 7466 pg_data_t *pgdat = NODE_DATA(nid); 7467 7468 pgdat_kswapd_lock(pgdat); 7469 if (!pgdat->kswapd) { 7470 pgdat->kswapd = kthread_create_on_node(kswapd, pgdat, nid, "kswapd%d", nid); 7471 if (IS_ERR(pgdat->kswapd)) { 7472 /* failure at boot is fatal */ 7473 pr_err("Failed to start kswapd on node %d,ret=%ld\n", 7474 nid, PTR_ERR(pgdat->kswapd)); 7475 BUG_ON(system_state < SYSTEM_RUNNING); 7476 pgdat->kswapd = NULL; 7477 } else { 7478 wake_up_process(pgdat->kswapd); 7479 } 7480 } 7481 pgdat_kswapd_unlock(pgdat); 7482 } 7483 7484 /* 7485 * Called by memory hotplug when all memory in a node is offlined. Caller must 7486 * be holding mem_hotplug_begin/done(). 7487 */ 7488 void __meminit kswapd_stop(int nid) 7489 { 7490 pg_data_t *pgdat = NODE_DATA(nid); 7491 struct task_struct *kswapd; 7492 7493 pgdat_kswapd_lock(pgdat); 7494 kswapd = pgdat->kswapd; 7495 if (kswapd) { 7496 kthread_stop(kswapd); 7497 pgdat->kswapd = NULL; 7498 } 7499 pgdat_kswapd_unlock(pgdat); 7500 } 7501 7502 static const struct ctl_table vmscan_sysctl_table[] = { 7503 { 7504 .procname = "swappiness", 7505 .data = &vm_swappiness, 7506 .maxlen = sizeof(vm_swappiness), 7507 .mode = 0644, 7508 .proc_handler = proc_dointvec_minmax, 7509 .extra1 = SYSCTL_ZERO, 7510 .extra2 = SYSCTL_TWO_HUNDRED, 7511 }, 7512 #ifdef CONFIG_NUMA 7513 { 7514 .procname = "zone_reclaim_mode", 7515 .data = &node_reclaim_mode, 7516 .maxlen = sizeof(node_reclaim_mode), 7517 .mode = 0644, 7518 .proc_handler = proc_dointvec_minmax, 7519 .extra1 = SYSCTL_ZERO, 7520 } 7521 #endif 7522 }; 7523 7524 static int __init kswapd_init(void) 7525 { 7526 int nid; 7527 7528 swap_setup(); 7529 for_each_node_state(nid, N_MEMORY) 7530 kswapd_run(nid); 7531 register_sysctl_init("vm", vmscan_sysctl_table); 7532 return 0; 7533 } 7534 7535 module_init(kswapd_init) 7536 7537 #ifdef CONFIG_NUMA 7538 /* 7539 * Node reclaim mode 7540 * 7541 * If non-zero call node_reclaim when the number of free pages falls below 7542 * the watermarks. 7543 */ 7544 int node_reclaim_mode __read_mostly; 7545 7546 /* 7547 * Priority for NODE_RECLAIM. This determines the fraction of pages 7548 * of a node considered for each zone_reclaim. 4 scans 1/16th of 7549 * a zone. 7550 */ 7551 #define NODE_RECLAIM_PRIORITY 4 7552 7553 /* 7554 * Percentage of pages in a zone that must be unmapped for node_reclaim to 7555 * occur. 7556 */ 7557 int sysctl_min_unmapped_ratio = 1; 7558 7559 /* 7560 * If the number of slab pages in a zone grows beyond this percentage then 7561 * slab reclaim needs to occur. 7562 */ 7563 int sysctl_min_slab_ratio = 5; 7564 7565 static inline unsigned long node_unmapped_file_pages(struct pglist_data *pgdat) 7566 { 7567 unsigned long file_mapped = node_page_state(pgdat, NR_FILE_MAPPED); 7568 unsigned long file_lru = node_page_state(pgdat, NR_INACTIVE_FILE) + 7569 node_page_state(pgdat, NR_ACTIVE_FILE); 7570 7571 /* 7572 * It's possible for there to be more file mapped pages than 7573 * accounted for by the pages on the file LRU lists because 7574 * tmpfs pages accounted for as ANON can also be FILE_MAPPED 7575 */ 7576 return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0; 7577 } 7578 7579 /* Work out how many page cache pages we can reclaim in this reclaim_mode */ 7580 static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat) 7581 { 7582 unsigned long nr_pagecache_reclaimable; 7583 unsigned long delta = 0; 7584 7585 /* 7586 * If RECLAIM_UNMAP is set, then all file pages are considered 7587 * potentially reclaimable. Otherwise, we have to worry about 7588 * pages like swapcache and node_unmapped_file_pages() provides 7589 * a better estimate 7590 */ 7591 if (node_reclaim_mode & RECLAIM_UNMAP) 7592 nr_pagecache_reclaimable = node_page_state(pgdat, NR_FILE_PAGES); 7593 else 7594 nr_pagecache_reclaimable = node_unmapped_file_pages(pgdat); 7595 7596 /* If we can't clean pages, remove dirty pages from consideration */ 7597 if (!(node_reclaim_mode & RECLAIM_WRITE)) 7598 delta += node_page_state(pgdat, NR_FILE_DIRTY); 7599 7600 /* Watch for any possible underflows due to delta */ 7601 if (unlikely(delta > nr_pagecache_reclaimable)) 7602 delta = nr_pagecache_reclaimable; 7603 7604 return nr_pagecache_reclaimable - delta; 7605 } 7606 7607 /* 7608 * Try to free up some pages from this node through reclaim. 7609 */ 7610 static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) 7611 { 7612 /* Minimum pages needed in order to stay on node */ 7613 const unsigned long nr_pages = 1 << order; 7614 struct task_struct *p = current; 7615 unsigned int noreclaim_flag; 7616 struct scan_control sc = { 7617 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), 7618 .gfp_mask = current_gfp_context(gfp_mask), 7619 .order = order, 7620 .priority = NODE_RECLAIM_PRIORITY, 7621 .may_writepage = !!(node_reclaim_mode & RECLAIM_WRITE), 7622 .may_unmap = !!(node_reclaim_mode & RECLAIM_UNMAP), 7623 .may_swap = 1, 7624 .reclaim_idx = gfp_zone(gfp_mask), 7625 }; 7626 unsigned long pflags; 7627 7628 trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, order, 7629 sc.gfp_mask); 7630 7631 cond_resched(); 7632 psi_memstall_enter(&pflags); 7633 delayacct_freepages_start(); 7634 fs_reclaim_acquire(sc.gfp_mask); 7635 /* 7636 * We need to be able to allocate from the reserves for RECLAIM_UNMAP 7637 */ 7638 noreclaim_flag = memalloc_noreclaim_save(); 7639 set_task_reclaim_state(p, &sc.reclaim_state); 7640 7641 if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages || 7642 node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) > pgdat->min_slab_pages) { 7643 /* 7644 * Free memory by calling shrink node with increasing 7645 * priorities until we have enough memory freed. 7646 */ 7647 do { 7648 shrink_node(pgdat, &sc); 7649 } while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0); 7650 } 7651 7652 set_task_reclaim_state(p, NULL); 7653 memalloc_noreclaim_restore(noreclaim_flag); 7654 fs_reclaim_release(sc.gfp_mask); 7655 psi_memstall_leave(&pflags); 7656 delayacct_freepages_end(); 7657 7658 trace_mm_vmscan_node_reclaim_end(sc.nr_reclaimed); 7659 7660 return sc.nr_reclaimed >= nr_pages; 7661 } 7662 7663 int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) 7664 { 7665 int ret; 7666 7667 /* 7668 * Node reclaim reclaims unmapped file backed pages and 7669 * slab pages if we are over the defined limits. 7670 * 7671 * A small portion of unmapped file backed pages is needed for 7672 * file I/O otherwise pages read by file I/O will be immediately 7673 * thrown out if the node is overallocated. So we do not reclaim 7674 * if less than a specified percentage of the node is used by 7675 * unmapped file backed pages. 7676 */ 7677 if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages && 7678 node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <= 7679 pgdat->min_slab_pages) 7680 return NODE_RECLAIM_FULL; 7681 7682 /* 7683 * Do not scan if the allocation should not be delayed. 7684 */ 7685 if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC)) 7686 return NODE_RECLAIM_NOSCAN; 7687 7688 /* 7689 * Only run node reclaim on the local node or on nodes that do not 7690 * have associated processors. This will favor the local processor 7691 * over remote processors and spread off node memory allocations 7692 * as wide as possible. 7693 */ 7694 if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id()) 7695 return NODE_RECLAIM_NOSCAN; 7696 7697 if (test_and_set_bit_lock(PGDAT_RECLAIM_LOCKED, &pgdat->flags)) 7698 return NODE_RECLAIM_NOSCAN; 7699 7700 ret = __node_reclaim(pgdat, gfp_mask, order); 7701 clear_bit_unlock(PGDAT_RECLAIM_LOCKED, &pgdat->flags); 7702 7703 if (ret) 7704 count_vm_event(PGSCAN_ZONE_RECLAIM_SUCCESS); 7705 else 7706 count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED); 7707 7708 return ret; 7709 } 7710 #endif 7711 7712 /** 7713 * check_move_unevictable_folios - Move evictable folios to appropriate zone 7714 * lru list 7715 * @fbatch: Batch of lru folios to check. 7716 * 7717 * Checks folios for evictability, if an evictable folio is in the unevictable 7718 * lru list, moves it to the appropriate evictable lru list. This function 7719 * should be only used for lru folios. 7720 */ 7721 void check_move_unevictable_folios(struct folio_batch *fbatch) 7722 { 7723 struct lruvec *lruvec = NULL; 7724 int pgscanned = 0; 7725 int pgrescued = 0; 7726 int i; 7727 7728 for (i = 0; i < fbatch->nr; i++) { 7729 struct folio *folio = fbatch->folios[i]; 7730 int nr_pages = folio_nr_pages(folio); 7731 7732 pgscanned += nr_pages; 7733 7734 /* block memcg migration while the folio moves between lrus */ 7735 if (!folio_test_clear_lru(folio)) 7736 continue; 7737 7738 lruvec = folio_lruvec_relock_irq(folio, lruvec); 7739 if (folio_evictable(folio) && folio_test_unevictable(folio)) { 7740 lruvec_del_folio(lruvec, folio); 7741 folio_clear_unevictable(folio); 7742 lruvec_add_folio(lruvec, folio); 7743 pgrescued += nr_pages; 7744 } 7745 folio_set_lru(folio); 7746 } 7747 7748 if (lruvec) { 7749 __count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued); 7750 __count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned); 7751 unlock_page_lruvec_irq(lruvec); 7752 } else if (pgscanned) { 7753 count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned); 7754 } 7755 } 7756 EXPORT_SYMBOL_GPL(check_move_unevictable_folios); 7757