1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * linux/mm/swap_state.c 4 * 5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 6 * Swap reorganised 29.12.95, Stephen Tweedie 7 * 8 * Rewritten to use page cache, (C) 1998 Stephen Tweedie 9 */ 10 #include <linux/mm.h> 11 #include <linux/gfp.h> 12 #include <linux/kernel_stat.h> 13 #include <linux/mempolicy.h> 14 #include <linux/swap.h> 15 #include <linux/leafops.h> 16 #include <linux/init.h> 17 #include <linux/pagemap.h> 18 #include <linux/pagevec.h> 19 #include <linux/backing-dev.h> 20 #include <linux/blkdev.h> 21 #include <linux/migrate.h> 22 #include <linux/vmalloc.h> 23 #include <linux/huge_mm.h> 24 #include <linux/shmem_fs.h> 25 #include "internal.h" 26 #include "swap_table.h" 27 #include "swap.h" 28 29 /* 30 * swapper_space is a fiction, retained to simplify the path through 31 * vmscan's shrink_folio_list. 32 */ 33 static const struct address_space_operations swap_aops = { 34 .dirty_folio = noop_dirty_folio, 35 #ifdef CONFIG_MIGRATION 36 .migrate_folio = migrate_folio, 37 #endif 38 }; 39 40 /* Set swap_space as read only as swap cache is handled by swap table */ 41 struct address_space swap_space __ro_after_init = { 42 .a_ops = &swap_aops, 43 }; 44 45 static bool enable_vma_readahead __read_mostly = true; 46 47 #define SWAP_RA_ORDER_CEILING 5 48 49 #define SWAP_RA_WIN_SHIFT (PAGE_SHIFT / 2) 50 #define SWAP_RA_HITS_MASK ((1UL << SWAP_RA_WIN_SHIFT) - 1) 51 #define SWAP_RA_HITS_MAX SWAP_RA_HITS_MASK 52 #define SWAP_RA_WIN_MASK (~PAGE_MASK & ~SWAP_RA_HITS_MASK) 53 54 #define SWAP_RA_HITS(v) ((v) & SWAP_RA_HITS_MASK) 55 #define SWAP_RA_WIN(v) (((v) & SWAP_RA_WIN_MASK) >> SWAP_RA_WIN_SHIFT) 56 #define SWAP_RA_ADDR(v) ((v) & PAGE_MASK) 57 58 #define SWAP_RA_VAL(addr, win, hits) \ 59 (((addr) & PAGE_MASK) | \ 60 (((win) << SWAP_RA_WIN_SHIFT) & SWAP_RA_WIN_MASK) | \ 61 ((hits) & SWAP_RA_HITS_MASK)) 62 63 /* Initial readahead hits is 4 to start up with a small window */ 64 #define GET_SWAP_RA_VAL(vma) \ 65 (atomic_long_read(&(vma)->swap_readahead_info) ? : 4) 66 67 static atomic_t swapin_readahead_hits = ATOMIC_INIT(4); 68 69 void show_swap_cache_info(void) 70 { 71 printk("%lu pages in swap cache\n", total_swapcache_pages()); 72 printk("Free swap = %ldkB\n", K(get_nr_swap_pages())); 73 printk("Total swap = %lukB\n", K(total_swap_pages)); 74 } 75 76 /** 77 * swap_cache_get_folio - Looks up a folio in the swap cache. 78 * @entry: swap entry used for the lookup. 79 * 80 * A found folio will be returned unlocked and with its refcount increased. 81 * 82 * Context: Caller must ensure @entry is valid and protect the swap device 83 * with reference count or locks. 84 * Return: Returns the found folio on success, NULL otherwise. The caller 85 * must lock nd check if the folio still matches the swap entry before 86 * use (e.g., folio_matches_swap_entry). 87 */ 88 struct folio *swap_cache_get_folio(swp_entry_t entry) 89 { 90 unsigned long swp_tb; 91 struct folio *folio; 92 93 for (;;) { 94 swp_tb = swap_table_get(__swap_entry_to_cluster(entry), 95 swp_cluster_offset(entry)); 96 if (!swp_tb_is_folio(swp_tb)) 97 return NULL; 98 folio = swp_tb_to_folio(swp_tb); 99 if (likely(folio_try_get(folio))) 100 return folio; 101 } 102 103 return NULL; 104 } 105 106 /** 107 * swap_cache_get_shadow - Looks up a shadow in the swap cache. 108 * @entry: swap entry used for the lookup. 109 * 110 * Context: Caller must ensure @entry is valid and protect the swap device 111 * with reference count or locks. 112 * Return: Returns either NULL or an XA_VALUE (shadow). 113 */ 114 void *swap_cache_get_shadow(swp_entry_t entry) 115 { 116 unsigned long swp_tb; 117 118 swp_tb = swap_table_get(__swap_entry_to_cluster(entry), 119 swp_cluster_offset(entry)); 120 if (swp_tb_is_shadow(swp_tb)) 121 return swp_tb_to_shadow(swp_tb); 122 return NULL; 123 } 124 125 /** 126 * swap_cache_add_folio - Add a folio into the swap cache. 127 * @folio: The folio to be added. 128 * @entry: The swap entry corresponding to the folio. 129 * @gfp: gfp_mask for XArray node allocation. 130 * @shadowp: If a shadow is found, return the shadow. 131 * 132 * Context: Caller must ensure @entry is valid and protect the swap device 133 * with reference count or locks. 134 * The caller also needs to update the corresponding swap_map slots with 135 * SWAP_HAS_CACHE bit to avoid race or conflict. 136 */ 137 void swap_cache_add_folio(struct folio *folio, swp_entry_t entry, void **shadowp) 138 { 139 void *shadow = NULL; 140 unsigned long old_tb, new_tb; 141 struct swap_cluster_info *ci; 142 unsigned int ci_start, ci_off, ci_end; 143 unsigned long nr_pages = folio_nr_pages(folio); 144 145 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); 146 VM_WARN_ON_ONCE_FOLIO(folio_test_swapcache(folio), folio); 147 VM_WARN_ON_ONCE_FOLIO(!folio_test_swapbacked(folio), folio); 148 149 new_tb = folio_to_swp_tb(folio); 150 ci_start = swp_cluster_offset(entry); 151 ci_end = ci_start + nr_pages; 152 ci_off = ci_start; 153 ci = swap_cluster_lock(__swap_entry_to_info(entry), swp_offset(entry)); 154 do { 155 old_tb = __swap_table_xchg(ci, ci_off, new_tb); 156 WARN_ON_ONCE(swp_tb_is_folio(old_tb)); 157 if (swp_tb_is_shadow(old_tb)) 158 shadow = swp_tb_to_shadow(old_tb); 159 } while (++ci_off < ci_end); 160 161 folio_ref_add(folio, nr_pages); 162 folio_set_swapcache(folio); 163 folio->swap = entry; 164 swap_cluster_unlock(ci); 165 166 node_stat_mod_folio(folio, NR_FILE_PAGES, nr_pages); 167 lruvec_stat_mod_folio(folio, NR_SWAPCACHE, nr_pages); 168 169 if (shadowp) 170 *shadowp = shadow; 171 } 172 173 /** 174 * __swap_cache_del_folio - Removes a folio from the swap cache. 175 * @ci: The locked swap cluster. 176 * @folio: The folio. 177 * @entry: The first swap entry that the folio corresponds to. 178 * @shadow: shadow value to be filled in the swap cache. 179 * 180 * Removes a folio from the swap cache and fills a shadow in place. 181 * This won't put the folio's refcount. The caller has to do that. 182 * 183 * Context: Caller must ensure the folio is locked and in the swap cache 184 * using the index of @entry, and lock the cluster that holds the entries. 185 */ 186 void __swap_cache_del_folio(struct swap_cluster_info *ci, struct folio *folio, 187 swp_entry_t entry, void *shadow) 188 { 189 unsigned long old_tb, new_tb; 190 unsigned int ci_start, ci_off, ci_end; 191 unsigned long nr_pages = folio_nr_pages(folio); 192 193 VM_WARN_ON_ONCE(__swap_entry_to_cluster(entry) != ci); 194 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); 195 VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio); 196 VM_WARN_ON_ONCE_FOLIO(folio_test_writeback(folio), folio); 197 198 new_tb = shadow_swp_to_tb(shadow); 199 ci_start = swp_cluster_offset(entry); 200 ci_end = ci_start + nr_pages; 201 ci_off = ci_start; 202 do { 203 /* If shadow is NULL, we sets an empty shadow */ 204 old_tb = __swap_table_xchg(ci, ci_off, new_tb); 205 WARN_ON_ONCE(!swp_tb_is_folio(old_tb) || 206 swp_tb_to_folio(old_tb) != folio); 207 } while (++ci_off < ci_end); 208 209 folio->swap.val = 0; 210 folio_clear_swapcache(folio); 211 node_stat_mod_folio(folio, NR_FILE_PAGES, -nr_pages); 212 lruvec_stat_mod_folio(folio, NR_SWAPCACHE, -nr_pages); 213 } 214 215 /** 216 * swap_cache_del_folio - Removes a folio from the swap cache. 217 * @folio: The folio. 218 * 219 * Same as __swap_cache_del_folio, but handles lock and refcount. The 220 * caller must ensure the folio is either clean or has a swap count 221 * equal to zero, or it may cause data loss. 222 * 223 * Context: Caller must ensure the folio is locked and in the swap cache. 224 */ 225 void swap_cache_del_folio(struct folio *folio) 226 { 227 struct swap_cluster_info *ci; 228 swp_entry_t entry = folio->swap; 229 230 ci = swap_cluster_lock(__swap_entry_to_info(entry), swp_offset(entry)); 231 __swap_cache_del_folio(ci, folio, entry, NULL); 232 swap_cluster_unlock(ci); 233 234 put_swap_folio(folio, entry); 235 folio_ref_sub(folio, folio_nr_pages(folio)); 236 } 237 238 /** 239 * __swap_cache_replace_folio - Replace a folio in the swap cache. 240 * @ci: The locked swap cluster. 241 * @old: The old folio to be replaced. 242 * @new: The new folio. 243 * 244 * Replace an existing folio in the swap cache with a new folio. The 245 * caller is responsible for setting up the new folio's flag and swap 246 * entries. Replacement will take the new folio's swap entry value as 247 * the starting offset to override all slots covered by the new folio. 248 * 249 * Context: Caller must ensure both folios are locked, and lock the 250 * cluster that holds the old folio to be replaced. 251 */ 252 void __swap_cache_replace_folio(struct swap_cluster_info *ci, 253 struct folio *old, struct folio *new) 254 { 255 swp_entry_t entry = new->swap; 256 unsigned long nr_pages = folio_nr_pages(new); 257 unsigned int ci_off = swp_cluster_offset(entry); 258 unsigned int ci_end = ci_off + nr_pages; 259 unsigned long old_tb, new_tb; 260 261 VM_WARN_ON_ONCE(!folio_test_swapcache(old) || !folio_test_swapcache(new)); 262 VM_WARN_ON_ONCE(!folio_test_locked(old) || !folio_test_locked(new)); 263 VM_WARN_ON_ONCE(!entry.val); 264 265 /* Swap cache still stores N entries instead of a high-order entry */ 266 new_tb = folio_to_swp_tb(new); 267 do { 268 old_tb = __swap_table_xchg(ci, ci_off, new_tb); 269 WARN_ON_ONCE(!swp_tb_is_folio(old_tb) || swp_tb_to_folio(old_tb) != old); 270 } while (++ci_off < ci_end); 271 272 /* 273 * If the old folio is partially replaced (e.g., splitting a large 274 * folio, the old folio is shrunk, and new split sub folios replace 275 * the shrunk part), ensure the new folio doesn't overlap it. 276 */ 277 if (IS_ENABLED(CONFIG_DEBUG_VM) && 278 folio_order(old) != folio_order(new)) { 279 ci_off = swp_cluster_offset(old->swap); 280 ci_end = ci_off + folio_nr_pages(old); 281 while (ci_off++ < ci_end) 282 WARN_ON_ONCE(swp_tb_to_folio(__swap_table_get(ci, ci_off)) != old); 283 } 284 } 285 286 /** 287 * swap_cache_clear_shadow - Clears a set of shadows in the swap cache. 288 * @entry: The starting index entry. 289 * @nr_ents: How many slots need to be cleared. 290 * 291 * Context: Caller must ensure the range is valid, all in one single cluster, 292 * not occupied by any folio, and lock the cluster. 293 */ 294 void __swap_cache_clear_shadow(swp_entry_t entry, int nr_ents) 295 { 296 struct swap_cluster_info *ci = __swap_entry_to_cluster(entry); 297 unsigned int ci_off = swp_cluster_offset(entry), ci_end; 298 unsigned long old; 299 300 ci_end = ci_off + nr_ents; 301 do { 302 old = __swap_table_xchg(ci, ci_off, null_to_swp_tb()); 303 WARN_ON_ONCE(swp_tb_is_folio(old)); 304 } while (++ci_off < ci_end); 305 } 306 307 /* 308 * If we are the only user, then try to free up the swap cache. 309 * 310 * Its ok to check the swapcache flag without the folio lock 311 * here because we are going to recheck again inside 312 * folio_free_swap() _with_ the lock. 313 * - Marcelo 314 */ 315 void free_swap_cache(struct folio *folio) 316 { 317 if (folio_test_swapcache(folio) && !folio_mapped(folio) && 318 folio_trylock(folio)) { 319 folio_free_swap(folio); 320 folio_unlock(folio); 321 } 322 } 323 324 /* 325 * Freeing a folio and also freeing any swap cache associated with 326 * this folio if it is the last user. 327 */ 328 void free_folio_and_swap_cache(struct folio *folio) 329 { 330 free_swap_cache(folio); 331 if (!is_huge_zero_folio(folio)) 332 folio_put(folio); 333 } 334 335 /* 336 * Passed an array of pages, drop them all from swapcache and then release 337 * them. They are removed from the LRU and freed if this is their last use. 338 */ 339 void free_pages_and_swap_cache(struct encoded_page **pages, int nr) 340 { 341 struct folio_batch folios; 342 unsigned int refs[PAGEVEC_SIZE]; 343 344 folio_batch_init(&folios); 345 for (int i = 0; i < nr; i++) { 346 struct folio *folio = page_folio(encoded_page_ptr(pages[i])); 347 348 free_swap_cache(folio); 349 refs[folios.nr] = 1; 350 if (unlikely(encoded_page_flags(pages[i]) & 351 ENCODED_PAGE_BIT_NR_PAGES_NEXT)) 352 refs[folios.nr] = encoded_nr_pages(pages[++i]); 353 354 if (folio_batch_add(&folios, folio) == 0) 355 folios_put_refs(&folios, refs); 356 } 357 if (folios.nr) 358 folios_put_refs(&folios, refs); 359 } 360 361 static inline bool swap_use_vma_readahead(void) 362 { 363 return READ_ONCE(enable_vma_readahead) && !atomic_read(&nr_rotate_swap); 364 } 365 366 /** 367 * swap_update_readahead - Update the readahead statistics of VMA or globally. 368 * @folio: the swap cache folio that just got hit. 369 * @vma: the VMA that should be updated, could be NULL for global update. 370 * @addr: the addr that triggered the swapin, ignored if @vma is NULL. 371 */ 372 void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma, 373 unsigned long addr) 374 { 375 bool readahead, vma_ra = swap_use_vma_readahead(); 376 377 /* 378 * At the moment, we don't support PG_readahead for anon THP 379 * so let's bail out rather than confusing the readahead stat. 380 */ 381 if (unlikely(folio_test_large(folio))) 382 return; 383 384 readahead = folio_test_clear_readahead(folio); 385 if (vma && vma_ra) { 386 unsigned long ra_val; 387 int win, hits; 388 389 ra_val = GET_SWAP_RA_VAL(vma); 390 win = SWAP_RA_WIN(ra_val); 391 hits = SWAP_RA_HITS(ra_val); 392 if (readahead) 393 hits = min_t(int, hits + 1, SWAP_RA_HITS_MAX); 394 atomic_long_set(&vma->swap_readahead_info, 395 SWAP_RA_VAL(addr, win, hits)); 396 } 397 398 if (readahead) { 399 count_vm_event(SWAP_RA_HIT); 400 if (!vma || !vma_ra) 401 atomic_inc(&swapin_readahead_hits); 402 } 403 } 404 405 struct folio *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, 406 struct mempolicy *mpol, pgoff_t ilx, bool *new_page_allocated, 407 bool skip_if_exists) 408 { 409 struct swap_info_struct *si = __swap_entry_to_info(entry); 410 struct folio *folio; 411 struct folio *new_folio = NULL; 412 struct folio *result = NULL; 413 void *shadow = NULL; 414 415 *new_page_allocated = false; 416 for (;;) { 417 int err; 418 419 /* 420 * Check the swap cache first, if a cached folio is found, 421 * return it unlocked. The caller will lock and check it. 422 */ 423 folio = swap_cache_get_folio(entry); 424 if (folio) 425 goto got_folio; 426 427 /* 428 * Just skip read ahead for unused swap slot. 429 */ 430 if (!swap_entry_swapped(si, entry)) 431 goto put_and_return; 432 433 /* 434 * Get a new folio to read into from swap. Allocate it now if 435 * new_folio not exist, before marking swap_map SWAP_HAS_CACHE, 436 * when -EEXIST will cause any racers to loop around until we 437 * add it to cache. 438 */ 439 if (!new_folio) { 440 new_folio = folio_alloc_mpol(gfp_mask, 0, mpol, ilx, numa_node_id()); 441 if (!new_folio) 442 goto put_and_return; 443 } 444 445 /* 446 * Swap entry may have been freed since our caller observed it. 447 */ 448 err = swapcache_prepare(entry, 1); 449 if (!err) 450 break; 451 else if (err != -EEXIST) 452 goto put_and_return; 453 454 /* 455 * Protect against a recursive call to __read_swap_cache_async() 456 * on the same entry waiting forever here because SWAP_HAS_CACHE 457 * is set but the folio is not the swap cache yet. This can 458 * happen today if mem_cgroup_swapin_charge_folio() below 459 * triggers reclaim through zswap, which may call 460 * __read_swap_cache_async() in the writeback path. 461 */ 462 if (skip_if_exists) 463 goto put_and_return; 464 465 /* 466 * We might race against __swap_cache_del_folio(), and 467 * stumble across a swap_map entry whose SWAP_HAS_CACHE 468 * has not yet been cleared. Or race against another 469 * __read_swap_cache_async(), which has set SWAP_HAS_CACHE 470 * in swap_map, but not yet added its folio to swap cache. 471 */ 472 schedule_timeout_uninterruptible(1); 473 } 474 475 /* 476 * The swap entry is ours to swap in. Prepare the new folio. 477 */ 478 __folio_set_locked(new_folio); 479 __folio_set_swapbacked(new_folio); 480 481 if (mem_cgroup_swapin_charge_folio(new_folio, NULL, gfp_mask, entry)) 482 goto fail_unlock; 483 484 swap_cache_add_folio(new_folio, entry, &shadow); 485 memcg1_swapin(entry, 1); 486 487 if (shadow) 488 workingset_refault(new_folio, shadow); 489 490 /* Caller will initiate read into locked new_folio */ 491 folio_add_lru(new_folio); 492 *new_page_allocated = true; 493 folio = new_folio; 494 got_folio: 495 result = folio; 496 goto put_and_return; 497 498 fail_unlock: 499 put_swap_folio(new_folio, entry); 500 folio_unlock(new_folio); 501 put_and_return: 502 if (!(*new_page_allocated) && new_folio) 503 folio_put(new_folio); 504 return result; 505 } 506 507 /* 508 * Locate a page of swap in physical memory, reserving swap cache space 509 * and reading the disk if it is not already cached. 510 * A failure return means that either the page allocation failed or that 511 * the swap entry is no longer in use. 512 */ 513 struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, 514 struct vm_area_struct *vma, unsigned long addr, 515 struct swap_iocb **plug) 516 { 517 struct swap_info_struct *si; 518 bool page_allocated; 519 struct mempolicy *mpol; 520 pgoff_t ilx; 521 struct folio *folio; 522 523 si = get_swap_device(entry); 524 if (!si) 525 return NULL; 526 527 mpol = get_vma_policy(vma, addr, 0, &ilx); 528 folio = __read_swap_cache_async(entry, gfp_mask, mpol, ilx, 529 &page_allocated, false); 530 mpol_cond_put(mpol); 531 532 if (page_allocated) 533 swap_read_folio(folio, plug); 534 535 put_swap_device(si); 536 return folio; 537 } 538 539 static unsigned int __swapin_nr_pages(unsigned long prev_offset, 540 unsigned long offset, 541 int hits, 542 int max_pages, 543 int prev_win) 544 { 545 unsigned int pages, last_ra; 546 547 /* 548 * This heuristic has been found to work well on both sequential and 549 * random loads, swapping to hard disk or to SSD: please don't ask 550 * what the "+ 2" means, it just happens to work well, that's all. 551 */ 552 pages = hits + 2; 553 if (pages == 2) { 554 /* 555 * We can have no readahead hits to judge by: but must not get 556 * stuck here forever, so check for an adjacent offset instead 557 * (and don't even bother to check whether swap type is same). 558 */ 559 if (offset != prev_offset + 1 && offset != prev_offset - 1) 560 pages = 1; 561 } else { 562 unsigned int roundup = 4; 563 while (roundup < pages) 564 roundup <<= 1; 565 pages = roundup; 566 } 567 568 if (pages > max_pages) 569 pages = max_pages; 570 571 /* Don't shrink readahead too fast */ 572 last_ra = prev_win / 2; 573 if (pages < last_ra) 574 pages = last_ra; 575 576 return pages; 577 } 578 579 static unsigned long swapin_nr_pages(unsigned long offset) 580 { 581 static unsigned long prev_offset; 582 unsigned int hits, pages, max_pages; 583 static atomic_t last_readahead_pages; 584 585 max_pages = 1 << READ_ONCE(page_cluster); 586 if (max_pages <= 1) 587 return 1; 588 589 hits = atomic_xchg(&swapin_readahead_hits, 0); 590 pages = __swapin_nr_pages(READ_ONCE(prev_offset), offset, hits, 591 max_pages, 592 atomic_read(&last_readahead_pages)); 593 if (!hits) 594 WRITE_ONCE(prev_offset, offset); 595 atomic_set(&last_readahead_pages, pages); 596 597 return pages; 598 } 599 600 /** 601 * swap_cluster_readahead - swap in pages in hope we need them soon 602 * @entry: swap entry of this memory 603 * @gfp_mask: memory allocation flags 604 * @mpol: NUMA memory allocation policy to be applied 605 * @ilx: NUMA interleave index, for use only when MPOL_INTERLEAVE 606 * 607 * Returns the struct folio for entry and addr, after queueing swapin. 608 * 609 * Primitive swap readahead code. We simply read an aligned block of 610 * (1 << page_cluster) entries in the swap area. This method is chosen 611 * because it doesn't cost us any seek time. We also make sure to queue 612 * the 'original' request together with the readahead ones... 613 * 614 * Note: it is intentional that the same NUMA policy and interleave index 615 * are used for every page of the readahead: neighbouring pages on swap 616 * are fairly likely to have been swapped out from the same node. 617 */ 618 struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask, 619 struct mempolicy *mpol, pgoff_t ilx) 620 { 621 struct folio *folio; 622 unsigned long entry_offset = swp_offset(entry); 623 unsigned long offset = entry_offset; 624 unsigned long start_offset, end_offset; 625 unsigned long mask; 626 struct swap_info_struct *si = __swap_entry_to_info(entry); 627 struct blk_plug plug; 628 struct swap_iocb *splug = NULL; 629 bool page_allocated; 630 631 mask = swapin_nr_pages(offset) - 1; 632 if (!mask) 633 goto skip; 634 635 /* Read a page_cluster sized and aligned cluster around offset. */ 636 start_offset = offset & ~mask; 637 end_offset = offset | mask; 638 if (!start_offset) /* First page is swap header. */ 639 start_offset++; 640 if (end_offset >= si->max) 641 end_offset = si->max - 1; 642 643 blk_start_plug(&plug); 644 for (offset = start_offset; offset <= end_offset ; offset++) { 645 /* Ok, do the async read-ahead now */ 646 folio = __read_swap_cache_async( 647 swp_entry(swp_type(entry), offset), 648 gfp_mask, mpol, ilx, &page_allocated, false); 649 if (!folio) 650 continue; 651 if (page_allocated) { 652 swap_read_folio(folio, &splug); 653 if (offset != entry_offset) { 654 folio_set_readahead(folio); 655 count_vm_event(SWAP_RA); 656 } 657 } 658 folio_put(folio); 659 } 660 blk_finish_plug(&plug); 661 swap_read_unplug(splug); 662 lru_add_drain(); /* Push any new pages onto the LRU now */ 663 skip: 664 /* The page was likely read above, so no need for plugging here */ 665 folio = __read_swap_cache_async(entry, gfp_mask, mpol, ilx, 666 &page_allocated, false); 667 if (unlikely(page_allocated)) 668 swap_read_folio(folio, NULL); 669 return folio; 670 } 671 672 static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start, 673 unsigned long *end) 674 { 675 struct vm_area_struct *vma = vmf->vma; 676 unsigned long ra_val; 677 unsigned long faddr, prev_faddr, left, right; 678 unsigned int max_win, hits, prev_win, win; 679 680 max_win = 1 << min(READ_ONCE(page_cluster), SWAP_RA_ORDER_CEILING); 681 if (max_win == 1) 682 return 1; 683 684 faddr = vmf->address; 685 ra_val = GET_SWAP_RA_VAL(vma); 686 prev_faddr = SWAP_RA_ADDR(ra_val); 687 prev_win = SWAP_RA_WIN(ra_val); 688 hits = SWAP_RA_HITS(ra_val); 689 win = __swapin_nr_pages(PFN_DOWN(prev_faddr), PFN_DOWN(faddr), hits, 690 max_win, prev_win); 691 atomic_long_set(&vma->swap_readahead_info, SWAP_RA_VAL(faddr, win, 0)); 692 if (win == 1) 693 return 1; 694 695 if (faddr == prev_faddr + PAGE_SIZE) 696 left = faddr; 697 else if (prev_faddr == faddr + PAGE_SIZE) 698 left = faddr - (win << PAGE_SHIFT) + PAGE_SIZE; 699 else 700 left = faddr - (((win - 1) / 2) << PAGE_SHIFT); 701 right = left + (win << PAGE_SHIFT); 702 if ((long)left < 0) 703 left = 0; 704 *start = max3(left, vma->vm_start, faddr & PMD_MASK); 705 *end = min3(right, vma->vm_end, (faddr & PMD_MASK) + PMD_SIZE); 706 707 return win; 708 } 709 710 /** 711 * swap_vma_readahead - swap in pages in hope we need them soon 712 * @targ_entry: swap entry of the targeted memory 713 * @gfp_mask: memory allocation flags 714 * @mpol: NUMA memory allocation policy to be applied 715 * @targ_ilx: NUMA interleave index, for use only when MPOL_INTERLEAVE 716 * @vmf: fault information 717 * 718 * Returns the struct folio for entry and addr, after queueing swapin. 719 * 720 * Primitive swap readahead code. We simply read in a few pages whose 721 * virtual addresses are around the fault address in the same vma. 722 * 723 * Caller must hold read mmap_lock if vmf->vma is not NULL. 724 * 725 */ 726 static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask, 727 struct mempolicy *mpol, pgoff_t targ_ilx, struct vm_fault *vmf) 728 { 729 struct blk_plug plug; 730 struct swap_iocb *splug = NULL; 731 struct folio *folio; 732 pte_t *pte = NULL, pentry; 733 int win; 734 unsigned long start, end, addr; 735 pgoff_t ilx; 736 bool page_allocated; 737 738 win = swap_vma_ra_win(vmf, &start, &end); 739 if (win == 1) 740 goto skip; 741 742 ilx = targ_ilx - PFN_DOWN(vmf->address - start); 743 744 blk_start_plug(&plug); 745 for (addr = start; addr < end; ilx++, addr += PAGE_SIZE) { 746 struct swap_info_struct *si = NULL; 747 softleaf_t entry; 748 749 if (!pte++) { 750 pte = pte_offset_map(vmf->pmd, addr); 751 if (!pte) 752 break; 753 } 754 pentry = ptep_get_lockless(pte); 755 entry = softleaf_from_pte(pentry); 756 757 if (!softleaf_is_swap(entry)) 758 continue; 759 pte_unmap(pte); 760 pte = NULL; 761 /* 762 * Readahead entry may come from a device that we are not 763 * holding a reference to, try to grab a reference, or skip. 764 */ 765 if (swp_type(entry) != swp_type(targ_entry)) { 766 si = get_swap_device(entry); 767 if (!si) 768 continue; 769 } 770 folio = __read_swap_cache_async(entry, gfp_mask, mpol, ilx, 771 &page_allocated, false); 772 if (si) 773 put_swap_device(si); 774 if (!folio) 775 continue; 776 if (page_allocated) { 777 swap_read_folio(folio, &splug); 778 if (addr != vmf->address) { 779 folio_set_readahead(folio); 780 count_vm_event(SWAP_RA); 781 } 782 } 783 folio_put(folio); 784 } 785 if (pte) 786 pte_unmap(pte); 787 blk_finish_plug(&plug); 788 swap_read_unplug(splug); 789 lru_add_drain(); 790 skip: 791 /* The folio was likely read above, so no need for plugging here */ 792 folio = __read_swap_cache_async(targ_entry, gfp_mask, mpol, targ_ilx, 793 &page_allocated, false); 794 if (unlikely(page_allocated)) 795 swap_read_folio(folio, NULL); 796 return folio; 797 } 798 799 /** 800 * swapin_readahead - swap in pages in hope we need them soon 801 * @entry: swap entry of this memory 802 * @gfp_mask: memory allocation flags 803 * @vmf: fault information 804 * 805 * Returns the struct folio for entry and addr, after queueing swapin. 806 * 807 * It's a main entry function for swap readahead. By the configuration, 808 * it will read ahead blocks by cluster-based(ie, physical disk based) 809 * or vma-based(ie, virtual address based on faulty address) readahead. 810 */ 811 struct folio *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask, 812 struct vm_fault *vmf) 813 { 814 struct mempolicy *mpol; 815 pgoff_t ilx; 816 struct folio *folio; 817 818 mpol = get_vma_policy(vmf->vma, vmf->address, 0, &ilx); 819 folio = swap_use_vma_readahead() ? 820 swap_vma_readahead(entry, gfp_mask, mpol, ilx, vmf) : 821 swap_cluster_readahead(entry, gfp_mask, mpol, ilx); 822 mpol_cond_put(mpol); 823 824 return folio; 825 } 826 827 #ifdef CONFIG_SYSFS 828 static ssize_t vma_ra_enabled_show(struct kobject *kobj, 829 struct kobj_attribute *attr, char *buf) 830 { 831 return sysfs_emit(buf, "%s\n", str_true_false(enable_vma_readahead)); 832 } 833 static ssize_t vma_ra_enabled_store(struct kobject *kobj, 834 struct kobj_attribute *attr, 835 const char *buf, size_t count) 836 { 837 ssize_t ret; 838 839 ret = kstrtobool(buf, &enable_vma_readahead); 840 if (ret) 841 return ret; 842 843 return count; 844 } 845 static struct kobj_attribute vma_ra_enabled_attr = __ATTR_RW(vma_ra_enabled); 846 847 static struct attribute *swap_attrs[] = { 848 &vma_ra_enabled_attr.attr, 849 NULL, 850 }; 851 852 static const struct attribute_group swap_attr_group = { 853 .attrs = swap_attrs, 854 }; 855 856 static int __init swap_init(void) 857 { 858 int err; 859 struct kobject *swap_kobj; 860 861 swap_kobj = kobject_create_and_add("swap", mm_kobj); 862 if (!swap_kobj) { 863 pr_err("failed to create swap kobject\n"); 864 return -ENOMEM; 865 } 866 err = sysfs_create_group(swap_kobj, &swap_attr_group); 867 if (err) { 868 pr_err("failed to register swap group\n"); 869 goto delete_obj; 870 } 871 /* Swap cache writeback is LRU based, no tags for it */ 872 mapping_set_no_writeback_tags(&swap_space); 873 return 0; 874 875 delete_obj: 876 kobject_put(swap_kobj); 877 return err; 878 } 879 subsys_initcall(swap_init); 880 #endif 881