1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _MM_SWAP_H 3 #define _MM_SWAP_H 4 5 #include <linux/atomic.h> /* for atomic_long_t */ 6 #include <linux/mm.h> /* for PAGE_SHIFT */ 7 #include <linux/memcontrol.h> /* for mem_cgroup_swappiness() */ 8 #include <linux/swap.h> /* for MAX_SWAPFILES_SHIFT, struct swap_info_struct */ 9 10 struct mempolicy; 11 struct swap_iocb; 12 struct swap_memcg_table; 13 struct swap_io_ctx; 14 15 #if defined(MAX_POSSIBLE_PHYSMEM_BITS) 16 #define SWAP_CACHE_PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT) 17 #elif defined(MAX_PHYSMEM_BITS) 18 #define SWAP_CACHE_PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT) 19 #else 20 #define SWAP_CACHE_PFN_BITS (BITS_PER_LONG - PAGE_SHIFT) 21 #endif 22 23 /* Swap table marker, 0x1 means shadow, 0x2 means PFN (SWP_TB_PFN_MARK) */ 24 #define SWAP_CACHE_PFN_MARK_BITS 2 25 /* At least 2 bits are needed to distinguish SWP_TB_COUNT_MAX, 1 and 0 */ 26 #define SWAP_COUNT_MIN_BITS 2 27 /* If there are enough bits besides PFN and marker, store zero flag inline */ 28 #define SWAP_TABLE_HAS_ZEROFLAG ((BITS_PER_LONG - SWAP_CACHE_PFN_MARK_BITS - \ 29 SWAP_CACHE_PFN_BITS) > SWAP_COUNT_MIN_BITS) 30 31 #ifdef CONFIG_THP_SWAP 32 #define SWAPFILE_CLUSTER HPAGE_PMD_NR 33 #define swap_entry_order(order) (order) 34 #else 35 #define SWAPFILE_CLUSTER 256 36 #define swap_entry_order(order) 0 37 #endif 38 39 extern struct swap_info_struct *swap_info[]; 40 41 /* 42 * We use this to track usage of a cluster. A cluster is a block of swap disk 43 * space with SWAPFILE_CLUSTER pages long and naturally aligns in disk. All 44 * free clusters are organized into a list. We fetch an entry from the list to 45 * get a free cluster. 46 * 47 * The flags field determines if a cluster is free. This is 48 * protected by cluster lock. 49 */ 50 struct swap_cluster_info { 51 spinlock_t lock; /* 52 * Protect swap_cluster_info fields 53 * other than list, and swap_info_struct->swap_map 54 * elements corresponding to the swap cluster. 55 */ 56 u16 count; 57 u8 flags; 58 u8 order; 59 atomic_long_t __rcu *table; /* Swap table entries, see mm/swap_table.h */ 60 unsigned int *extend_table; /* For large swap count, protected by ci->lock */ 61 #ifdef CONFIG_MEMCG 62 struct swap_memcg_table *memcg_table; /* Swap table entries' cgroup record */ 63 #endif 64 #if !SWAP_TABLE_HAS_ZEROFLAG 65 unsigned long *zero_bitmap; 66 #endif 67 struct list_head list; 68 }; 69 70 /* All on-list cluster must have a non-zero flag. */ 71 enum swap_cluster_flags { 72 CLUSTER_FLAG_NONE = 0, /* For temporary off-list cluster */ 73 CLUSTER_FLAG_FREE, 74 CLUSTER_FLAG_NONFULL, 75 CLUSTER_FLAG_FRAG, 76 /* Clusters with flags above are allocatable */ 77 CLUSTER_FLAG_USABLE = CLUSTER_FLAG_FRAG, 78 CLUSTER_FLAG_FULL, 79 CLUSTER_FLAG_DISCARD, 80 CLUSTER_FLAG_MAX, 81 }; 82 83 extern int vm_swappiness; 84 85 static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg) 86 { 87 #ifdef CONFIG_MEMCG_V1 88 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && 89 !mem_cgroup_disabled() && !mem_cgroup_is_root(memcg)) 90 return READ_ONCE(memcg->swappiness); 91 #endif 92 return READ_ONCE(vm_swappiness); 93 } 94 95 #ifdef CONFIG_SWAP 96 #include <linux/swapops.h> /* for swp_offset */ 97 #include <linux/blk_types.h> /* for bio_end_io_t */ 98 99 static inline unsigned int swp_cluster_offset(swp_entry_t entry) 100 { 101 return swp_offset(entry) % SWAPFILE_CLUSTER; 102 } 103 104 /* 105 * Callers of all helpers below must ensure the entry, type, or offset is 106 * valid, and protect the swap device with reference count or locks. 107 */ 108 static inline struct swap_info_struct *__swap_type_to_info(int type) 109 { 110 struct swap_info_struct *si; 111 112 si = READ_ONCE(swap_info[type]); /* rcu_dereference() */ 113 VM_WARN_ON_ONCE(percpu_ref_is_zero(&si->users)); /* race with swapoff */ 114 return si; 115 } 116 117 static inline struct swap_info_struct *__swap_entry_to_info(swp_entry_t entry) 118 { 119 return __swap_type_to_info(swp_type(entry)); 120 } 121 122 static inline struct swap_cluster_info *__swap_offset_to_cluster( 123 struct swap_info_struct *si, pgoff_t offset) 124 { 125 VM_WARN_ON_ONCE(percpu_ref_is_zero(&si->users)); /* race with swapoff */ 126 VM_WARN_ON_ONCE(offset >= roundup(si->max, SWAPFILE_CLUSTER)); 127 return &si->cluster_info[offset / SWAPFILE_CLUSTER]; 128 } 129 130 static inline struct swap_cluster_info *__swap_entry_to_cluster(swp_entry_t entry) 131 { 132 return __swap_offset_to_cluster(__swap_entry_to_info(entry), 133 swp_offset(entry)); 134 } 135 136 static __always_inline struct swap_cluster_info *__swap_cluster_lock( 137 struct swap_info_struct *si, unsigned long offset, bool irq) 138 { 139 struct swap_cluster_info *ci = __swap_offset_to_cluster(si, offset); 140 141 /* 142 * Nothing modifies swap cache in an IRQ context. All access to 143 * swap cache is wrapped by swap_cache_* helpers, and swap cache 144 * writeback is handled outside of IRQs. Swapin or swapout never 145 * occurs in IRQ, and neither does in-place split or replace. 146 * 147 * Besides, modifying swap cache requires synchronization with 148 * swap_map, which was never IRQ safe. 149 */ 150 VM_WARN_ON_ONCE(!in_task()); 151 VM_WARN_ON_ONCE(percpu_ref_is_zero(&si->users)); /* race with swapoff */ 152 if (irq) 153 spin_lock_irq(&ci->lock); 154 else 155 spin_lock(&ci->lock); 156 return ci; 157 } 158 159 /** 160 * swap_cluster_lock - Lock and return the swap cluster of given offset. 161 * @si: swap device the cluster belongs to. 162 * @offset: the swap entry offset, pointing to a valid slot. 163 * 164 * Context: The caller must ensure the offset is in the valid range and 165 * protect the swap device with reference count or locks. 166 */ 167 static inline struct swap_cluster_info *swap_cluster_lock( 168 struct swap_info_struct *si, unsigned long offset) 169 { 170 return __swap_cluster_lock(si, offset, false); 171 } 172 173 static inline struct swap_cluster_info *__swap_cluster_get_and_lock( 174 const struct folio *folio, bool irq) 175 { 176 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); 177 VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio); 178 return __swap_cluster_lock(__swap_entry_to_info(folio->swap), 179 swp_offset(folio->swap), irq); 180 } 181 182 /* 183 * swap_cluster_get_and_lock - Locks the cluster that holds a folio's entries. 184 * @folio: The folio. 185 * 186 * This locks and returns the swap cluster that contains a folio's swap 187 * entries. The swap entries of a folio are always in one single cluster. 188 * The folio has to be locked so its swap entries won't change and the 189 * cluster won't be freed. 190 * 191 * Context: Caller must ensure the folio is locked and in the swap cache. 192 * Return: Pointer to the swap cluster. 193 */ 194 static inline struct swap_cluster_info *swap_cluster_get_and_lock( 195 const struct folio *folio) 196 { 197 return __swap_cluster_get_and_lock(folio, false); 198 } 199 200 /* 201 * swap_cluster_get_and_lock_irq - Locks the cluster that holds a folio's entries. 202 * @folio: The folio. 203 * 204 * Same as swap_cluster_get_and_lock but also disable IRQ. 205 * 206 * Context: Caller must ensure the folio is locked and in the swap cache. 207 * Return: Pointer to the swap cluster. 208 */ 209 static inline struct swap_cluster_info *swap_cluster_get_and_lock_irq( 210 const struct folio *folio) 211 { 212 return __swap_cluster_get_and_lock(folio, true); 213 } 214 215 static inline void swap_cluster_unlock(struct swap_cluster_info *ci) 216 { 217 spin_unlock(&ci->lock); 218 } 219 220 static inline void swap_cluster_unlock_irq(struct swap_cluster_info *ci) 221 { 222 spin_unlock_irq(&ci->lock); 223 } 224 225 extern int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp); 226 227 /* 228 * Below are the core routines for doing swap for a folio. 229 * All helpers requires the folio to be locked, and a locked folio 230 * in the swap cache pins the swap entries / slots allocated to the 231 * folio, swap relies heavily on the swap cache and folio lock for 232 * synchronization. 233 * 234 * folio_alloc_swap(): the entry point for a folio to be swapped 235 * out. It allocates swap slots and pins the slots with swap cache. 236 * The slots start with a swap count of zero. The slots are pinned 237 * by swap cache reference which doesn't contribute to swap count. 238 * 239 * folio_dup_swap(): increases the swap count of a folio, usually 240 * during it gets unmapped and a swap entry is installed to replace 241 * it (e.g., swap entry in page table). A swap slot with swap 242 * count == 0 can only be increased by this helper. 243 * 244 * folio_put_swap(): does the opposite thing of folio_dup_swap(). 245 */ 246 int folio_alloc_swap(struct folio *folio); 247 int folio_dup_swap(struct folio *folio, struct page *page); 248 void folio_put_swap(struct folio *folio, struct page *page); 249 250 /* For internal use */ 251 extern void __swap_cluster_free_entries(struct swap_info_struct *si, 252 struct swap_cluster_info *ci, 253 unsigned int ci_off, unsigned int nr_pages); 254 255 /* linux/mm/page_io.c */ 256 int sio_pool_init(void); 257 void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio); 258 void swap_read_submit(struct swap_io_ctx *ctx); 259 void swap_write_submit(struct swap_io_ctx *ctx); 260 int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio); 261 void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio); 262 263 /* linux/mm/swap_state.c */ 264 extern struct address_space swap_space __read_mostly; 265 static inline struct address_space *swap_address_space(swp_entry_t entry) 266 { 267 return &swap_space; 268 } 269 270 /* 271 * Return the swap device position of the swap entry. 272 */ 273 static inline loff_t swap_dev_pos(swp_entry_t entry) 274 { 275 return ((loff_t)swp_offset(entry)) << PAGE_SHIFT; 276 } 277 278 /** 279 * folio_matches_swap_entry - Check if a folio matches a given swap entry. 280 * @folio: The folio. 281 * @entry: The swap entry to check against. 282 * 283 * Context: The caller should have the folio locked to ensure it's stable 284 * and nothing will move it in or out of the swap cache. 285 * Return: true or false. 286 */ 287 static inline bool folio_matches_swap_entry(const struct folio *folio, 288 swp_entry_t entry) 289 { 290 swp_entry_t folio_entry = folio->swap; 291 long nr_pages = folio_nr_pages(folio); 292 293 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); 294 if (!folio_test_swapcache(folio)) 295 return false; 296 VM_WARN_ON_ONCE_FOLIO(!IS_ALIGNED(folio_entry.val, nr_pages), folio); 297 return folio_entry.val == round_down(entry.val, nr_pages); 298 } 299 300 /* 301 * All swap cache helpers below require the caller to ensure the swap entries 302 * used are valid and stabilize the device by any of the following ways: 303 * - Hold a reference by get_swap_device(): this ensures a single entry is 304 * valid and increases the swap device's refcount. 305 * - Locking a folio in the swap cache: this ensures the folio's swap entries 306 * are valid and pinned, also implies reference to the device. 307 * - Locking anything referencing the swap entry: e.g. PTL that protects 308 * swap entries in the page table, similar to locking swap cache folio. 309 * - See the comment of get_swap_device() for more complex usage. 310 */ 311 bool swap_cache_has_folio(swp_entry_t entry); 312 struct folio *swap_cache_get_folio(swp_entry_t entry); 313 void *swap_cache_get_shadow(swp_entry_t entry); 314 void swap_cache_del_folio(struct folio *folio); 315 struct folio *swap_cache_alloc_folio(swp_entry_t target_entry, gfp_t gfp_mask, 316 unsigned long orders, struct vm_fault *vmf, 317 struct mempolicy *mpol, pgoff_t ilx); 318 /* Below helpers require the caller to lock and pass in the swap cluster. */ 319 void __swap_cache_add_folio(struct swap_cluster_info *ci, 320 struct folio *folio, swp_entry_t entry); 321 void __swap_cache_del_folio(struct swap_cluster_info *ci, 322 struct folio *folio, swp_entry_t entry, void *shadow); 323 void __swap_cache_replace_folio(struct swap_cluster_info *ci, 324 struct folio *old, struct folio *new); 325 326 void show_swap_cache_info(void); 327 void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr); 328 struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry, 329 gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr); 330 struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t flag, 331 struct mempolicy *mpol, pgoff_t ilx); 332 struct folio *swapin_readahead(swp_entry_t entry, gfp_t flag, 333 struct vm_fault *vmf); 334 struct folio *swapin_sync(swp_entry_t entry, gfp_t flag, unsigned long orders, 335 struct vm_fault *vmf, struct mempolicy *mpol, pgoff_t ilx); 336 void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma, 337 unsigned long addr); 338 339 #else /* CONFIG_SWAP */ 340 341 static inline struct swap_cluster_info *swap_cluster_get_and_lock( 342 struct folio *folio) 343 { 344 return NULL; 345 } 346 347 static inline struct swap_cluster_info *swap_cluster_get_and_lock_irq( 348 struct folio *folio) 349 { 350 return NULL; 351 } 352 353 static inline void swap_cluster_unlock(struct swap_cluster_info *ci) 354 { 355 } 356 357 static inline void swap_cluster_unlock_irq(struct swap_cluster_info *ci) 358 { 359 } 360 361 static inline struct swap_info_struct *__swap_entry_to_info(swp_entry_t entry) 362 { 363 return NULL; 364 } 365 366 static inline int folio_alloc_swap(struct folio *folio) 367 { 368 return -EINVAL; 369 } 370 371 static inline int folio_dup_swap(struct folio *folio, struct page *page) 372 { 373 return -EINVAL; 374 } 375 376 static inline void folio_put_swap(struct folio *folio, struct page *page) 377 { 378 } 379 380 static inline void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio) 381 { 382 } 383 384 static inline void swap_write_submit(struct swap_io_ctx *ctx) 385 { 386 } 387 388 static inline struct address_space *swap_address_space(swp_entry_t entry) 389 { 390 return NULL; 391 } 392 393 static inline bool folio_matches_swap_entry(const struct folio *folio, swp_entry_t entry) 394 { 395 return false; 396 } 397 398 static inline void show_swap_cache_info(void) 399 { 400 } 401 402 static inline struct folio *swap_cluster_readahead(swp_entry_t entry, 403 gfp_t gfp_mask, struct mempolicy *mpol, pgoff_t ilx) 404 { 405 return NULL; 406 } 407 408 static inline struct folio *swapin_readahead(swp_entry_t swp, gfp_t gfp_mask, 409 struct vm_fault *vmf) 410 { 411 return NULL; 412 } 413 414 static inline struct folio *swapin_sync( 415 swp_entry_t entry, gfp_t flag, unsigned long orders, 416 struct vm_fault *vmf, struct mempolicy *mpol, pgoff_t ilx) 417 { 418 return NULL; 419 } 420 421 static inline void swap_update_readahead(struct folio *folio, 422 struct vm_area_struct *vma, unsigned long addr) 423 { 424 } 425 426 static inline int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio) 427 { 428 return 0; 429 } 430 431 static inline int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp) 432 { 433 return -EINVAL; 434 } 435 436 static inline bool swap_cache_has_folio(swp_entry_t entry) 437 { 438 return false; 439 } 440 441 static inline struct folio *swap_cache_get_folio(swp_entry_t entry) 442 { 443 return NULL; 444 } 445 446 static inline void *swap_cache_get_shadow(swp_entry_t entry) 447 { 448 return NULL; 449 } 450 451 static inline void swap_cache_del_folio(struct folio *folio) 452 { 453 } 454 455 static inline void __swap_cache_del_folio(struct swap_cluster_info *ci, 456 struct folio *folio, swp_entry_t entry, void *shadow) 457 { 458 } 459 460 static inline void __swap_cache_replace_folio(struct swap_cluster_info *ci, 461 struct folio *old, struct folio *new) 462 { 463 } 464 #endif /* CONFIG_SWAP */ 465 466 extern const struct swap_ops swap_bdev_ops; 467 468 int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio, 469 struct list_head *folio_list); 470 471 #endif /* _MM_SWAP_H */ 472