1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * linux/mm/swap_state.c 4 * 5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 6 * Swap reorganised 29.12.95, Stephen Tweedie 7 * 8 * Rewritten to use page cache, (C) 1998 Stephen Tweedie 9 */ 10 #include <linux/mm.h> 11 #include <linux/gfp.h> 12 #include <linux/kernel_stat.h> 13 #include <linux/mempolicy.h> 14 #include <linux/swap.h> 15 #include <linux/leafops.h> 16 #include <linux/init.h> 17 #include <linux/pagemap.h> 18 #include <linux/folio_batch.h> 19 #include <linux/backing-dev.h> 20 #include <linux/blk_plug.h> 21 #include <linux/migrate.h> 22 #include <linux/vmalloc.h> 23 #include <linux/huge_mm.h> 24 #include <linux/shmem_fs.h> 25 #include <linux/sysctl.h> 26 #include "internal.h" 27 #include "swap_table.h" 28 #include "swap.h" 29 30 /* Swap readahead cluster size, as a power of 2 pages. */ 31 static int page_cluster; 32 static const int page_cluster_max = 31; 33 34 /* 35 * swapper_space is a fiction, retained to simplify the path through 36 * vmscan's shrink_folio_list. 37 */ 38 static const struct address_space_operations swap_aops = { 39 .dirty_folio = noop_dirty_folio, 40 #ifdef CONFIG_MIGRATION 41 .migrate_folio = migrate_folio, 42 #endif 43 }; 44 45 struct address_space swap_space __read_mostly = { 46 .a_ops = &swap_aops, 47 }; 48 49 static bool enable_vma_readahead __read_mostly = true; 50 51 #define SWAP_RA_ORDER_CEILING 5 52 53 #define SWAP_RA_WIN_SHIFT (PAGE_SHIFT / 2) 54 #define SWAP_RA_HITS_MASK ((1UL << SWAP_RA_WIN_SHIFT) - 1) 55 #define SWAP_RA_HITS_MAX SWAP_RA_HITS_MASK 56 #define SWAP_RA_WIN_MASK (~PAGE_MASK & ~SWAP_RA_HITS_MASK) 57 58 #define SWAP_RA_HITS(v) ((v) & SWAP_RA_HITS_MASK) 59 #define SWAP_RA_WIN(v) (((v) & SWAP_RA_WIN_MASK) >> SWAP_RA_WIN_SHIFT) 60 #define SWAP_RA_ADDR(v) ((v) & PAGE_MASK) 61 62 #define SWAP_RA_VAL(addr, win, hits) \ 63 (((addr) & PAGE_MASK) | \ 64 (((win) << SWAP_RA_WIN_SHIFT) & SWAP_RA_WIN_MASK) | \ 65 ((hits) & SWAP_RA_HITS_MASK)) 66 67 /* Initial readahead hits is 4 to start up with a small window */ 68 #define GET_SWAP_RA_VAL(vma) \ 69 (atomic_long_read(&(vma)->swap_readahead_info) ? : 4) 70 71 static atomic_t swapin_readahead_hits = ATOMIC_INIT(4); 72 73 void show_swap_cache_info(void) 74 { 75 printk("%lu pages in swap cache\n", total_swapcache_pages()); 76 printk("Free swap = %ldkB\n", K(get_nr_swap_pages())); 77 printk("Total swap = %lukB\n", K(total_swap_pages)); 78 } 79 80 /** 81 * swap_cache_get_folio - Looks up a folio in the swap cache. 82 * @entry: swap entry used for the lookup. 83 * 84 * A found folio will be returned unlocked and with its refcount increased. 85 * 86 * Context: Caller must ensure @entry is valid and protect the swap device 87 * with reference count or locks. 88 * Return: Returns the found folio on success, NULL otherwise. The caller 89 * must lock and check if the folio still matches the swap entry before 90 * use (e.g., folio_matches_swap_entry). 91 */ 92 struct folio *swap_cache_get_folio(swp_entry_t entry) 93 { 94 unsigned long swp_tb; 95 struct folio *folio; 96 97 for (;;) { 98 swp_tb = swap_table_get(__swap_entry_to_cluster(entry), 99 swp_cluster_offset(entry)); 100 if (!swp_tb_is_folio(swp_tb)) 101 return NULL; 102 folio = swp_tb_to_folio(swp_tb); 103 if (likely(folio_try_get(folio))) 104 return folio; 105 } 106 107 return NULL; 108 } 109 110 /** 111 * swap_cache_has_folio - Check if a swap slot has cache. 112 * @entry: swap entry indicating the slot. 113 * 114 * Context: Caller must ensure @entry is valid and protect the swap 115 * device with reference count or locks. 116 */ 117 bool swap_cache_has_folio(swp_entry_t entry) 118 { 119 unsigned long swp_tb; 120 121 swp_tb = swap_table_get(__swap_entry_to_cluster(entry), 122 swp_cluster_offset(entry)); 123 return swp_tb_is_folio(swp_tb); 124 } 125 126 /** 127 * swap_cache_get_shadow - Looks up a shadow in the swap cache. 128 * @entry: swap entry used for the lookup. 129 * 130 * Context: Caller must ensure @entry is valid and protect the swap device 131 * with reference count or locks. 132 * Return: Returns either NULL or an XA_VALUE (shadow). 133 */ 134 void *swap_cache_get_shadow(swp_entry_t entry) 135 { 136 unsigned long swp_tb; 137 138 swp_tb = swap_table_get(__swap_entry_to_cluster(entry), 139 swp_cluster_offset(entry)); 140 if (swp_tb_is_shadow(swp_tb)) 141 return swp_tb_to_shadow(swp_tb); 142 return NULL; 143 } 144 145 /** 146 * __swap_cache_add_check - Check if a range is suitable for adding a folio. 147 * @ci: The locked swap cluster 148 * @targ_entry: The target swap entry to check, will be rounded down by @nr 149 * @nr: Number of slots to check, must be a power of 2 150 * @shadowp: Returns the shadow value if one exists in the range 151 * @memcg_id: Returns the memory cgroup id, NULL to ignore cgroup check 152 * 153 * Check if all slots covered by given range have a swap count >= 1. 154 * Retrieves the shadow if there is one. If @memcg_id is not NULL, also 155 * checks if all slots belong to the same cgroup and return the cgroup 156 * private id. 157 * 158 * Context: Caller must lock the cluster. 159 * Return: 0 if success, error code if failed. 160 */ 161 static int __swap_cache_add_check(struct swap_cluster_info *ci, 162 swp_entry_t targ_entry, 163 unsigned long nr, void **shadowp, 164 unsigned short *memcg_id) 165 { 166 unsigned int ci_off, ci_end; 167 unsigned long old_tb; 168 bool is_zero; 169 170 lockdep_assert_held(&ci->lock); 171 172 /* 173 * If the target slot is not swapped out or already cached, return 174 * -ENOENT or -EEXIST. If the batch is not suitable, could be a 175 * race with concurrent free or cache add, return -EBUSY. 176 */ 177 if (unlikely(!ci->table)) 178 return -ENOENT; 179 ci_off = swp_cluster_offset(targ_entry); 180 old_tb = __swap_table_get(ci, ci_off); 181 if (swp_tb_is_folio(old_tb)) 182 return -EEXIST; 183 if (!__swp_tb_get_count(old_tb)) 184 return -ENOENT; 185 if (shadowp && swp_tb_is_shadow(old_tb)) 186 *shadowp = swp_tb_to_shadow(old_tb); 187 if (memcg_id) 188 *memcg_id = __swap_cgroup_get(ci, ci_off); 189 190 if (nr == 1) 191 return 0; 192 193 is_zero = __swap_table_test_zero(ci, ci_off); 194 ci_off = round_down(ci_off, nr); 195 ci_end = ci_off + nr; 196 do { 197 old_tb = __swap_table_get(ci, ci_off); 198 if (unlikely(swp_tb_is_folio(old_tb) || 199 !__swp_tb_get_count(old_tb) || 200 is_zero != __swap_table_test_zero(ci, ci_off) || 201 (memcg_id && *memcg_id != __swap_cgroup_get(ci, ci_off)))) 202 return -EBUSY; 203 } while (++ci_off < ci_end); 204 205 return 0; 206 } 207 208 static void __swap_cache_do_add_folio(struct swap_cluster_info *ci, 209 struct folio *folio, swp_entry_t entry) 210 { 211 unsigned int ci_off = swp_cluster_offset(entry), ci_end; 212 unsigned long nr_pages = folio_nr_pages(folio); 213 unsigned long pfn = folio_pfn(folio); 214 unsigned long old_tb; 215 216 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); 217 VM_WARN_ON_ONCE_FOLIO(folio_test_swapcache(folio), folio); 218 VM_WARN_ON_ONCE_FOLIO(!folio_test_swapbacked(folio), folio); 219 220 ci_end = ci_off + nr_pages; 221 do { 222 old_tb = __swap_table_get(ci, ci_off); 223 VM_WARN_ON_ONCE(swp_tb_is_folio(old_tb)); 224 __swap_table_set(ci, ci_off, pfn_to_swp_tb(pfn, __swp_tb_get_flags(old_tb))); 225 } while (++ci_off < ci_end); 226 227 folio_ref_add(folio, nr_pages); 228 folio_set_swapcache(folio); 229 folio->swap = entry; 230 } 231 232 /** 233 * __swap_cache_add_folio - Add a folio to the swap cache and update stats. 234 * @ci: The locked swap cluster. 235 * @folio: The folio to be added. 236 * @entry: The swap entry corresponding to the folio. 237 * 238 * Unconditionally add a folio to the swap cache. The caller must ensure 239 * all slots are usable and have no conflicts. This assigns entry to 240 * @folio->swap, increases folio refcount by the number of pages, and 241 * updates swap cache stats. 242 * 243 * Context: Caller must ensure the folio is locked and lock the cluster 244 * that holds the entries. 245 */ 246 void __swap_cache_add_folio(struct swap_cluster_info *ci, 247 struct folio *folio, swp_entry_t entry) 248 { 249 unsigned long nr_pages = folio_nr_pages(folio); 250 251 __swap_cache_do_add_folio(ci, folio, entry); 252 node_stat_mod_folio(folio, NR_FILE_PAGES, nr_pages); 253 lruvec_stat_mod_folio(folio, NR_SWAPCACHE, nr_pages); 254 } 255 256 static void __swap_cache_do_del_folio(struct swap_cluster_info *ci, 257 struct folio *folio, 258 swp_entry_t entry, void *shadow) 259 { 260 unsigned long old_tb; 261 struct swap_info_struct *si; 262 unsigned int ci_start, ci_off, ci_end; 263 bool folio_swapped = false, need_free = false; 264 unsigned long nr_pages = folio_nr_pages(folio); 265 266 VM_WARN_ON_ONCE(__swap_entry_to_cluster(entry) != ci); 267 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); 268 VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio); 269 VM_WARN_ON_ONCE_FOLIO(folio_test_writeback(folio), folio); 270 271 si = __swap_entry_to_info(entry); 272 ci_start = swp_cluster_offset(entry); 273 ci_end = ci_start + nr_pages; 274 ci_off = ci_start; 275 do { 276 old_tb = __swap_table_get(ci, ci_off); 277 WARN_ON_ONCE(!swp_tb_is_folio(old_tb) || 278 swp_tb_to_folio(old_tb) != folio); 279 if (__swp_tb_get_count(old_tb)) 280 folio_swapped = true; 281 else 282 need_free = true; 283 /* If shadow is NULL, we set an empty shadow. */ 284 __swap_table_set(ci, ci_off, shadow_to_swp_tb(shadow, 285 __swp_tb_get_flags(old_tb))); 286 } while (++ci_off < ci_end); 287 288 folio->swap.val = 0; 289 folio_clear_swapcache(folio); 290 291 if (!folio_swapped) { 292 __swap_cluster_free_entries(si, ci, ci_start, nr_pages); 293 } else if (need_free) { 294 ci_off = ci_start; 295 do { 296 if (!__swp_tb_get_count(__swap_table_get(ci, ci_off))) 297 __swap_cluster_free_entries(si, ci, ci_off, 1); 298 } while (++ci_off < ci_end); 299 } 300 } 301 302 /** 303 * __swap_cache_del_folio - Removes a folio from the swap cache. 304 * @ci: The locked swap cluster. 305 * @folio: The folio. 306 * @entry: The first swap entry that the folio corresponds to. 307 * @shadow: shadow value to be filled in the swap cache. 308 * 309 * Removes a folio from the swap cache and fills a shadow in place. 310 * This won't put the folio's refcount. The caller has to do that. 311 * 312 * Context: Caller must ensure the folio is locked and in the swap cache 313 * using the index of @entry, and lock the cluster that holds the entries. 314 */ 315 void __swap_cache_del_folio(struct swap_cluster_info *ci, struct folio *folio, 316 swp_entry_t entry, void *shadow) 317 { 318 unsigned long nr_pages = folio_nr_pages(folio); 319 320 __swap_cache_do_del_folio(ci, folio, entry, shadow); 321 node_stat_mod_folio(folio, NR_FILE_PAGES, -nr_pages); 322 lruvec_stat_mod_folio(folio, NR_SWAPCACHE, -nr_pages); 323 } 324 325 /** 326 * swap_cache_del_folio - Removes a folio from the swap cache. 327 * @folio: The folio. 328 * 329 * Same as __swap_cache_del_folio, but handles lock and refcount. The 330 * caller must ensure the folio is either clean or has a swap count 331 * equal to zero, or it may cause data loss. 332 * 333 * Context: Caller must ensure the folio is locked and in the swap cache. 334 */ 335 void swap_cache_del_folio(struct folio *folio) 336 { 337 struct swap_cluster_info *ci; 338 swp_entry_t entry = folio->swap; 339 340 ci = swap_cluster_lock(__swap_entry_to_info(entry), swp_offset(entry)); 341 __swap_cache_del_folio(ci, folio, entry, NULL); 342 swap_cluster_unlock(ci); 343 344 folio_ref_sub(folio, folio_nr_pages(folio)); 345 } 346 347 /** 348 * __swap_cache_replace_folio - Replace a folio in the swap cache. 349 * @ci: The locked swap cluster. 350 * @old: The old folio to be replaced. 351 * @new: The new folio. 352 * 353 * Replace an existing folio in the swap cache with a new folio. The 354 * caller is responsible for setting up the new folio's flag and swap 355 * entries. Replacement will take the new folio's swap entry value as 356 * the starting offset to override all slots covered by the new folio. 357 * 358 * Context: Caller must ensure both folios are locked, and lock the 359 * cluster that holds the old folio to be replaced. 360 */ 361 void __swap_cache_replace_folio(struct swap_cluster_info *ci, 362 struct folio *old, struct folio *new) 363 { 364 swp_entry_t entry = new->swap; 365 unsigned long nr_pages = folio_nr_pages(new); 366 unsigned int ci_off = swp_cluster_offset(entry); 367 unsigned int ci_end = ci_off + nr_pages; 368 unsigned long pfn = folio_pfn(new); 369 unsigned long old_tb; 370 371 VM_WARN_ON_ONCE(!folio_test_swapcache(old) || !folio_test_swapcache(new)); 372 VM_WARN_ON_ONCE(!folio_test_locked(old) || !folio_test_locked(new)); 373 VM_WARN_ON_ONCE(!entry.val); 374 375 /* Swap cache still stores N entries instead of a high-order entry */ 376 do { 377 old_tb = __swap_table_get(ci, ci_off); 378 WARN_ON_ONCE(!swp_tb_is_folio(old_tb) || swp_tb_to_folio(old_tb) != old); 379 __swap_table_set(ci, ci_off, pfn_to_swp_tb(pfn, __swp_tb_get_flags(old_tb))); 380 } while (++ci_off < ci_end); 381 382 /* 383 * If the old folio is partially replaced (e.g., splitting a large 384 * folio, the old folio is shrunk, and new split sub folios replace 385 * the shrunk part), ensure the new folio doesn't overlap it. 386 */ 387 if (IS_ENABLED(CONFIG_DEBUG_VM) && 388 folio_order(old) != folio_order(new)) { 389 ci_off = swp_cluster_offset(old->swap); 390 ci_end = ci_off + folio_nr_pages(old); 391 while (ci_off++ < ci_end) 392 WARN_ON_ONCE(swp_tb_to_folio(__swap_table_get(ci, ci_off)) != old); 393 } 394 } 395 396 /* 397 * Try to allocate a folio of given order in the swap cache. 398 * 399 * This helper resolves the potential races of swap allocation 400 * and prepares a folio to be used for swap IO. May return following 401 * value: 402 * 403 * -ENOMEM / -EBUSY: Order is too large or in conflict with sub slot, 404 * caller should shrink the order and retry 405 * -ENOENT / -EEXIST: Target swap entry is unavailable or cached, the caller 406 * should abort or try to use the cached folio instead 407 */ 408 static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci, 409 swp_entry_t targ_entry, gfp_t gfp, 410 unsigned int order, struct vm_fault *vmf, 411 struct mempolicy *mpol, pgoff_t ilx) 412 { 413 int err; 414 swp_entry_t entry; 415 struct folio *folio; 416 void *shadow = NULL; 417 unsigned short memcg_id; 418 unsigned long address, nr_pages = 1UL << order; 419 struct vm_area_struct *vma = vmf ? vmf->vma : NULL; 420 421 VM_WARN_ON_ONCE(nr_pages > SWAPFILE_CLUSTER); 422 entry.val = round_down(targ_entry.val, nr_pages); 423 424 /* Check if the slot and range are available, skip allocation if not */ 425 spin_lock(&ci->lock); 426 err = __swap_cache_add_check(ci, targ_entry, nr_pages, NULL, NULL); 427 spin_unlock(&ci->lock); 428 if (unlikely(err)) 429 return ERR_PTR(err); 430 431 /* 432 * Limit THP gfp. The limitation is a no-op for typical 433 * GFP_HIGHUSER_MOVABLE but matters for shmem. 434 */ 435 if (order) 436 gfp = thp_shmem_limit_gfp_mask(vma_thp_gfp_mask(vma), gfp); 437 438 if (mpol || !vmf) { 439 folio = folio_alloc_mpol(gfp, order, mpol, ilx, numa_node_id()); 440 } else { 441 address = round_down(vmf->address, PAGE_SIZE << order); 442 folio = vma_alloc_folio(gfp, order, vmf->vma, address); 443 } 444 if (unlikely(!folio)) 445 return ERR_PTR(-ENOMEM); 446 447 /* Double check the range is still not in conflict */ 448 spin_lock(&ci->lock); 449 err = __swap_cache_add_check(ci, targ_entry, nr_pages, &shadow, &memcg_id); 450 if (unlikely(err)) { 451 spin_unlock(&ci->lock); 452 folio_put(folio); 453 return ERR_PTR(err); 454 } 455 456 __folio_set_locked(folio); 457 __folio_set_swapbacked(folio); 458 __swap_cache_do_add_folio(ci, folio, entry); 459 spin_unlock(&ci->lock); 460 461 if (mem_cgroup_swapin_charge_folio(folio, memcg_id, 462 vmf ? vmf->vma->vm_mm : NULL, gfp)) { 463 spin_lock(&ci->lock); 464 __swap_cache_do_del_folio(ci, folio, entry, shadow); 465 spin_unlock(&ci->lock); 466 folio_unlock(folio); 467 /* nr_pages refs from swap cache, 1 from allocation */ 468 folio_put_refs(folio, nr_pages + 1); 469 count_mthp_stat(order, MTHP_STAT_SWPIN_FALLBACK_CHARGE); 470 return ERR_PTR(-ENOMEM); 471 } 472 473 if (order > 1 && folio_memcg_alloc_deferred(folio)) { 474 spin_lock(&ci->lock); 475 __swap_cache_do_del_folio(ci, folio, entry, shadow); 476 spin_unlock(&ci->lock); 477 folio_unlock(folio); 478 /* nr_pages refs from swap cache, 1 from allocation */ 479 folio_put_refs(folio, nr_pages + 1); 480 return ERR_PTR(-ENOMEM); 481 } 482 483 /* memsw uncharges swap when folio is added to swap cache */ 484 memcg1_swapin(folio); 485 if (shadow) 486 workingset_refault(folio, shadow); 487 488 node_stat_mod_folio(folio, NR_FILE_PAGES, nr_pages); 489 lruvec_stat_mod_folio(folio, NR_SWAPCACHE, nr_pages); 490 491 /* Caller will initiate read into locked new_folio */ 492 folio_add_lru(folio); 493 return folio; 494 } 495 496 /** 497 * swap_cache_alloc_folio - Allocate folio for swapped out slot in swap cache. 498 * @targ_entry: swap entry indicating the target slot 499 * @gfp: memory allocation flags 500 * @orders: allocation orders, must be non zero 501 * @vmf: fault information 502 * @mpol: NUMA memory allocation policy to be applied 503 * @ilx: NUMA interleave index, for use only when MPOL_INTERLEAVE 504 * 505 * Allocate a folio in the swap cache for one swap slot, typically before 506 * doing IO (e.g. swap in or zswap writeback). The swap slot indicated by 507 * @targ_entry must have a non-zero swap count (swapped out). 508 * 509 * Context: Caller must protect the swap device with reference count or locks. 510 * Return: Returns the folio if allocation succeeded and folio is in the swap 511 * cache. Returns error code if failed due to race, OOM or invalid arguments. 512 */ 513 struct folio *swap_cache_alloc_folio(swp_entry_t targ_entry, gfp_t gfp, 514 unsigned long orders, struct vm_fault *vmf, 515 struct mempolicy *mpol, pgoff_t ilx) 516 { 517 int order, err; 518 struct folio *ret; 519 struct swap_cluster_info *ci; 520 521 ci = __swap_entry_to_cluster(targ_entry); 522 order = highest_order(orders); 523 524 /* orders must be non-zero, and must not exceed cluster size. */ 525 if (WARN_ON_ONCE(!orders || (1UL << order) > SWAPFILE_CLUSTER)) 526 return ERR_PTR(-EINVAL); 527 528 do { 529 ret = __swap_cache_alloc(ci, targ_entry, gfp, order, 530 vmf, mpol, ilx); 531 if (!IS_ERR(ret)) 532 break; 533 err = PTR_ERR(ret); 534 if (!order || (err && err != -EBUSY && err != -ENOMEM)) 535 break; 536 count_mthp_stat(order, MTHP_STAT_SWPIN_FALLBACK); 537 order = next_order(&orders, order); 538 } while (orders); 539 540 return ret; 541 } 542 543 /* 544 * If we are the only user, then try to free up the swap cache. 545 * 546 * Its ok to check the swapcache flag without the folio lock 547 * here because we are going to recheck again inside 548 * folio_free_swap() _with_ the lock. 549 * - Marcelo 550 */ 551 void free_swap_cache(struct folio *folio) 552 { 553 if (folio_test_swapcache(folio) && !folio_mapped(folio) && 554 folio_trylock(folio)) { 555 folio_free_swap(folio); 556 folio_unlock(folio); 557 } 558 } 559 560 /* 561 * Freeing a folio and also freeing any swap cache associated with 562 * this folio if it is the last user. 563 */ 564 void free_folio_and_swap_cache(struct folio *folio) 565 { 566 free_swap_cache(folio); 567 if (!is_huge_zero_folio(folio)) 568 folio_put(folio); 569 } 570 571 /* 572 * Passed an array of pages, drop them all from swapcache and then release 573 * them. They are removed from the LRU and freed if this is their last use. 574 */ 575 void free_pages_and_swap_cache(struct encoded_page **pages, int nr) 576 { 577 struct folio_batch folios; 578 unsigned int refs[FOLIO_BATCH_SIZE]; 579 580 folio_batch_init(&folios); 581 for (int i = 0; i < nr; i++) { 582 struct folio *folio = page_folio(encoded_page_ptr(pages[i])); 583 584 free_swap_cache(folio); 585 refs[folios.nr] = 1; 586 if (unlikely(encoded_page_flags(pages[i]) & 587 ENCODED_PAGE_BIT_NR_PAGES_NEXT)) 588 refs[folios.nr] = encoded_nr_pages(pages[++i]); 589 590 if (folio_batch_add(&folios, folio) == 0) 591 folios_put_refs(&folios, refs); 592 } 593 if (folios.nr) 594 folios_put_refs(&folios, refs); 595 } 596 597 static inline bool swap_use_vma_readahead(void) 598 { 599 return READ_ONCE(enable_vma_readahead) && !atomic_read(&nr_rotate_swap); 600 } 601 602 /** 603 * swap_update_readahead - Update the readahead statistics of VMA or globally. 604 * @folio: the swap cache folio that just got hit. 605 * @vma: the VMA that should be updated, could be NULL for global update. 606 * @addr: the addr that triggered the swapin, ignored if @vma is NULL. 607 */ 608 void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma, 609 unsigned long addr) 610 { 611 bool readahead, vma_ra = swap_use_vma_readahead(); 612 613 /* 614 * At the moment, we don't support PG_readahead for anon THP 615 * so let's bail out rather than confusing the readahead stat. 616 */ 617 if (unlikely(folio_test_large(folio))) 618 return; 619 620 readahead = folio_test_clear_readahead(folio); 621 if (vma && vma_ra) { 622 unsigned long ra_val; 623 int win, hits; 624 625 ra_val = GET_SWAP_RA_VAL(vma); 626 win = SWAP_RA_WIN(ra_val); 627 hits = SWAP_RA_HITS(ra_val); 628 if (readahead) 629 hits = min_t(int, hits + 1, SWAP_RA_HITS_MAX); 630 atomic_long_set(&vma->swap_readahead_info, 631 SWAP_RA_VAL(addr, win, hits)); 632 } 633 634 if (readahead) { 635 count_vm_event(SWAP_RA_HIT); 636 if (!vma || !vma_ra) 637 atomic_inc(&swapin_readahead_hits); 638 } 639 } 640 641 static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp, 642 struct mempolicy *mpol, pgoff_t ilx, 643 struct swap_iocb **plug, bool readahead) 644 { 645 struct folio *folio; 646 647 do { 648 folio = swap_cache_get_folio(entry); 649 if (folio) 650 return folio; 651 folio = swap_cache_alloc_folio(entry, gfp, BIT(0), NULL, mpol, ilx); 652 } while (PTR_ERR(folio) == -EEXIST); 653 654 if (IS_ERR_OR_NULL(folio)) 655 return NULL; 656 657 swap_read_folio(folio, plug); 658 if (readahead) { 659 folio_set_readahead(folio); 660 count_vm_event(SWAP_RA); 661 } 662 663 return folio; 664 } 665 666 /** 667 * swapin_sync - swap-in one or multiple entries skipping readahead. 668 * @entry: swap entry indicating the target slot 669 * @gfp: memory allocation flags 670 * @orders: allocation orders 671 * @vmf: fault information 672 * @mpol: NUMA memory allocation policy to be applied 673 * @ilx: NUMA interleave index, for use only when MPOL_INTERLEAVE 674 * 675 * This allocates a folio suitable for given @orders, or returns the 676 * existing folio in the swap cache for @entry. This initiates the IO, too, 677 * if needed. @entry is rounded down if @orders allow large allocation. 678 * 679 * Context: Caller must ensure @entry is valid and pin the swap device with refcount. 680 * Return: Returns the folio on success, error code if failed. 681 */ 682 struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders, 683 struct vm_fault *vmf, struct mempolicy *mpol, pgoff_t ilx) 684 { 685 struct folio *folio; 686 687 do { 688 folio = swap_cache_get_folio(entry); 689 if (folio) 690 return folio; 691 folio = swap_cache_alloc_folio(entry, gfp, orders, vmf, mpol, ilx); 692 } while (PTR_ERR(folio) == -EEXIST); 693 694 if (IS_ERR(folio)) 695 return folio; 696 697 swap_read_folio(folio, NULL); 698 return folio; 699 } 700 701 /* 702 * Locate a page of swap in physical memory, reserving swap cache space 703 * and reading the disk if it is not already cached. 704 * A failure return means that either the page allocation failed or that 705 * the swap entry is no longer in use. 706 */ 707 struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, 708 struct vm_area_struct *vma, unsigned long addr, 709 struct swap_iocb **plug) 710 { 711 struct swap_info_struct *si; 712 struct mempolicy *mpol; 713 pgoff_t ilx; 714 struct folio *folio; 715 716 si = get_swap_device(entry); 717 if (!si) 718 return NULL; 719 720 mpol = get_vma_policy(vma, addr, 0, &ilx); 721 folio = swap_cache_read_folio(entry, gfp_mask, mpol, ilx, plug, false); 722 mpol_cond_put(mpol); 723 724 put_swap_device(si); 725 return folio; 726 } 727 728 static unsigned int __swapin_nr_pages(unsigned long prev_offset, 729 unsigned long offset, 730 int hits, 731 int max_pages, 732 int prev_win) 733 { 734 unsigned int pages, last_ra; 735 736 /* 737 * This heuristic has been found to work well on both sequential and 738 * random loads, swapping to hard disk or to SSD: please don't ask 739 * what the "+ 2" means, it just happens to work well, that's all. 740 */ 741 pages = hits + 2; 742 if (pages == 2) { 743 /* 744 * We can have no readahead hits to judge by: but must not get 745 * stuck here forever, so check for an adjacent offset instead 746 * (and don't even bother to check whether swap type is same). 747 */ 748 if (offset != prev_offset + 1 && offset != prev_offset - 1) 749 pages = 1; 750 } else { 751 unsigned int roundup = 4; 752 while (roundup < pages) 753 roundup <<= 1; 754 pages = roundup; 755 } 756 757 if (pages > max_pages) 758 pages = max_pages; 759 760 /* Don't shrink readahead too fast */ 761 last_ra = prev_win / 2; 762 if (pages < last_ra) 763 pages = last_ra; 764 765 return pages; 766 } 767 768 static unsigned long swapin_nr_pages(unsigned long offset) 769 { 770 static unsigned long prev_offset; 771 unsigned int hits, pages, max_pages; 772 static atomic_t last_readahead_pages; 773 774 max_pages = 1 << READ_ONCE(page_cluster); 775 if (max_pages <= 1) 776 return 1; 777 778 hits = atomic_xchg(&swapin_readahead_hits, 0); 779 pages = __swapin_nr_pages(READ_ONCE(prev_offset), offset, hits, 780 max_pages, 781 atomic_read(&last_readahead_pages)); 782 if (!hits) 783 WRITE_ONCE(prev_offset, offset); 784 atomic_set(&last_readahead_pages, pages); 785 786 return pages; 787 } 788 789 /** 790 * swap_cluster_readahead - swap in pages in hope we need them soon 791 * @entry: swap entry of this memory 792 * @gfp_mask: memory allocation flags 793 * @mpol: NUMA memory allocation policy to be applied 794 * @ilx: NUMA interleave index, for use only when MPOL_INTERLEAVE 795 * 796 * Returns the struct folio for entry and addr, after queueing swapin. 797 * 798 * Primitive swap readahead code. We simply read an aligned block of 799 * (1 << page_cluster) entries in the swap area. This method is chosen 800 * because it doesn't cost us any seek time. We also make sure to queue 801 * the 'original' request together with the readahead ones... 802 * 803 * Note: it is intentional that the same NUMA policy and interleave index 804 * are used for every page of the readahead: neighbouring pages on swap 805 * are fairly likely to have been swapped out from the same node. 806 */ 807 struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask, 808 struct mempolicy *mpol, pgoff_t ilx) 809 { 810 struct folio *folio; 811 unsigned long entry_offset = swp_offset(entry); 812 unsigned long offset = entry_offset; 813 unsigned long start_offset, end_offset; 814 unsigned long mask; 815 struct swap_info_struct *si = __swap_entry_to_info(entry); 816 struct blk_plug plug; 817 struct swap_iocb *splug = NULL; 818 swp_entry_t ra_entry; 819 820 mask = swapin_nr_pages(offset) - 1; 821 if (!mask) 822 goto skip; 823 824 /* Read a page_cluster sized and aligned cluster around offset. */ 825 start_offset = offset & ~mask; 826 end_offset = offset | mask; 827 if (!start_offset) /* First page is swap header. */ 828 start_offset++; 829 if (end_offset >= si->max) 830 end_offset = si->max - 1; 831 832 blk_start_plug(&plug); 833 for (offset = start_offset; offset <= end_offset ; offset++) { 834 /* Ok, do the async read-ahead now */ 835 ra_entry = swp_entry(swp_type(entry), offset); 836 folio = swap_cache_read_folio(ra_entry, gfp_mask, mpol, ilx, 837 &splug, offset != entry_offset); 838 if (!folio) 839 continue; 840 folio_put(folio); 841 } 842 blk_finish_plug(&plug); 843 swap_read_unplug(splug); 844 skip: 845 /* The page was likely read above, so no need for plugging here */ 846 return swap_cache_read_folio(entry, gfp_mask, mpol, ilx, NULL, false); 847 } 848 849 static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start, 850 unsigned long *end) 851 { 852 struct vm_area_struct *vma = vmf->vma; 853 unsigned long ra_val; 854 unsigned long faddr, prev_faddr, left, right; 855 unsigned int max_win, hits, prev_win, win; 856 857 max_win = 1 << min(READ_ONCE(page_cluster), SWAP_RA_ORDER_CEILING); 858 if (max_win == 1) 859 return 1; 860 861 faddr = vmf->address; 862 ra_val = GET_SWAP_RA_VAL(vma); 863 prev_faddr = SWAP_RA_ADDR(ra_val); 864 prev_win = SWAP_RA_WIN(ra_val); 865 hits = SWAP_RA_HITS(ra_val); 866 win = __swapin_nr_pages(PFN_DOWN(prev_faddr), PFN_DOWN(faddr), hits, 867 max_win, prev_win); 868 atomic_long_set(&vma->swap_readahead_info, SWAP_RA_VAL(faddr, win, 0)); 869 if (win == 1) 870 return 1; 871 872 if (faddr == prev_faddr + PAGE_SIZE) 873 left = faddr; 874 else if (prev_faddr == faddr + PAGE_SIZE) 875 left = faddr - (win << PAGE_SHIFT) + PAGE_SIZE; 876 else 877 left = faddr - (((win - 1) / 2) << PAGE_SHIFT); 878 right = left + (win << PAGE_SHIFT); 879 if ((long)left < 0) 880 left = 0; 881 *start = max3(left, vma->vm_start, faddr & PMD_MASK); 882 *end = min3(right, vma->vm_end, (faddr & PMD_MASK) + PMD_SIZE); 883 884 return win; 885 } 886 887 /** 888 * swap_vma_readahead - swap in pages in hope we need them soon 889 * @targ_entry: swap entry of the targeted memory 890 * @gfp_mask: memory allocation flags 891 * @mpol: NUMA memory allocation policy to be applied 892 * @targ_ilx: NUMA interleave index, for use only when MPOL_INTERLEAVE 893 * @vmf: fault information 894 * 895 * Returns the struct folio for entry and addr, after queueing swapin. 896 * 897 * Primitive swap readahead code. We simply read in a few pages whose 898 * virtual addresses are around the fault address in the same vma. 899 * 900 * Caller must hold read mmap_lock if vmf->vma is not NULL. 901 * 902 */ 903 static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask, 904 struct mempolicy *mpol, pgoff_t targ_ilx, struct vm_fault *vmf) 905 { 906 struct blk_plug plug; 907 struct swap_iocb *splug = NULL; 908 struct folio *folio; 909 pte_t *pte = NULL, pentry; 910 int win; 911 unsigned long start, end, addr; 912 pgoff_t ilx = targ_ilx; 913 914 win = swap_vma_ra_win(vmf, &start, &end); 915 if (win == 1) 916 goto skip; 917 918 ilx = targ_ilx - PFN_DOWN(vmf->address - start); 919 920 blk_start_plug(&plug); 921 for (addr = start; addr < end; ilx++, addr += PAGE_SIZE) { 922 struct swap_info_struct *si = NULL; 923 softleaf_t entry; 924 925 if (!pte++) { 926 pte = pte_offset_map(vmf->pmd, addr); 927 if (!pte) 928 break; 929 } 930 pentry = ptep_get_lockless(pte); 931 entry = softleaf_from_pte(pentry); 932 933 if (!softleaf_is_swap(entry)) 934 continue; 935 pte_unmap(pte); 936 pte = NULL; 937 /* 938 * Readahead entry may come from a device that we are not 939 * holding a reference to, try to grab a reference, or skip. 940 */ 941 if (swp_type(entry) != swp_type(targ_entry)) { 942 si = get_swap_device(entry); 943 if (!si) 944 continue; 945 } 946 folio = swap_cache_read_folio(entry, gfp_mask, mpol, ilx, 947 &splug, addr != vmf->address); 948 if (si) 949 put_swap_device(si); 950 if (!folio) 951 continue; 952 folio_put(folio); 953 } 954 if (pte) 955 pte_unmap(pte); 956 blk_finish_plug(&plug); 957 swap_read_unplug(splug); 958 skip: 959 /* The folio was likely read above, so no need for plugging here */ 960 folio = swap_cache_read_folio(targ_entry, gfp_mask, mpol, targ_ilx, 961 NULL, false); 962 return folio; 963 } 964 965 /** 966 * swapin_readahead - swap in pages in hope we need them soon 967 * @entry: swap entry of this memory 968 * @gfp_mask: memory allocation flags 969 * @vmf: fault information 970 * 971 * Returns the struct folio for entry and addr, after queueing swapin. 972 * 973 * It's a main entry function for swap readahead. By the configuration, 974 * it will read ahead blocks by cluster-based(ie, physical disk based) 975 * or vma-based(ie, virtual address based on faulty address) readahead. 976 */ 977 struct folio *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask, 978 struct vm_fault *vmf) 979 { 980 struct mempolicy *mpol; 981 pgoff_t ilx; 982 struct folio *folio; 983 984 mpol = get_vma_policy(vmf->vma, vmf->address, 0, &ilx); 985 folio = swap_use_vma_readahead() ? 986 swap_vma_readahead(entry, gfp_mask, mpol, ilx, vmf) : 987 swap_cluster_readahead(entry, gfp_mask, mpol, ilx); 988 mpol_cond_put(mpol); 989 990 return folio; 991 } 992 993 static const struct ctl_table swap_readahead_sysctl_table[] = { 994 { 995 .procname = "page-cluster", 996 .data = &page_cluster, 997 .maxlen = sizeof(int), 998 .mode = 0644, 999 .proc_handler = proc_dointvec_minmax, 1000 .extra1 = SYSCTL_ZERO, 1001 .extra2 = (void *)&page_cluster_max, 1002 } 1003 }; 1004 1005 static void __init swap_readahead_setup(void) 1006 { 1007 unsigned long megs = PAGES_TO_MB(totalram_pages()); 1008 1009 /* Use a smaller cluster for small-memory machines */ 1010 if (megs < 16) 1011 page_cluster = 2; 1012 else 1013 page_cluster = 3; 1014 /* 1015 * Right now other parts of the system means that we 1016 * _really_ don't want to cluster much more 1017 */ 1018 1019 register_sysctl_init("vm", swap_readahead_sysctl_table); 1020 } 1021 1022 #ifdef CONFIG_SYSFS 1023 static ssize_t vma_ra_enabled_show(struct kobject *kobj, 1024 struct kobj_attribute *attr, char *buf) 1025 { 1026 return sysfs_emit(buf, "%s\n", str_true_false(enable_vma_readahead)); 1027 } 1028 static ssize_t vma_ra_enabled_store(struct kobject *kobj, 1029 struct kobj_attribute *attr, 1030 const char *buf, size_t count) 1031 { 1032 ssize_t ret; 1033 1034 ret = kstrtobool(buf, &enable_vma_readahead); 1035 if (ret) 1036 return ret; 1037 1038 return count; 1039 } 1040 static struct kobj_attribute vma_ra_enabled_attr = __ATTR_RW(vma_ra_enabled); 1041 1042 static struct attribute *swap_attrs[] = { 1043 &vma_ra_enabled_attr.attr, 1044 NULL, 1045 }; 1046 1047 static const struct attribute_group swap_attr_group = { 1048 .attrs = swap_attrs, 1049 }; 1050 1051 static int __init swap_sysfs_init(void) 1052 { 1053 int err; 1054 struct kobject *swap_kobj; 1055 1056 swap_kobj = kobject_create_and_add("swap", mm_kobj); 1057 if (!swap_kobj) { 1058 pr_err("failed to create swap kobject\n"); 1059 return -ENOMEM; 1060 } 1061 err = sysfs_create_group(swap_kobj, &swap_attr_group); 1062 if (err) { 1063 pr_err("failed to register swap group\n"); 1064 goto delete_obj; 1065 } 1066 /* Swap cache writeback is LRU based, no tags for it */ 1067 mapping_set_no_writeback_tags(&swap_space); 1068 return 0; 1069 1070 delete_obj: 1071 kobject_put(swap_kobj); 1072 return err; 1073 } 1074 #else 1075 static int __init swap_sysfs_init(void) 1076 { 1077 return 0; 1078 } 1079 #endif 1080 1081 static int __init swap_init(void) 1082 { 1083 swap_readahead_setup(); 1084 1085 return swap_sysfs_init(); 1086 } 1087 subsys_initcall(swap_init); 1088