1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _MM_SWAP_H 3 #define _MM_SWAP_H 4 5 struct mempolicy; 6 extern int page_cluster; 7 8 #ifdef CONFIG_SWAP 9 #include <linux/swapops.h> /* for swp_offset */ 10 #include <linux/blk_types.h> /* for bio_end_io_t */ 11 12 /* linux/mm/page_io.c */ 13 int sio_pool_init(void); 14 struct swap_iocb; 15 void swap_read_folio(struct folio *folio, struct swap_iocb **plug); 16 void __swap_read_unplug(struct swap_iocb *plug); 17 static inline void swap_read_unplug(struct swap_iocb *plug) 18 { 19 if (unlikely(plug)) 20 __swap_read_unplug(plug); 21 } 22 void swap_write_unplug(struct swap_iocb *sio); 23 int swap_writeout(struct folio *folio, struct writeback_control *wbc); 24 void __swap_writepage(struct folio *folio, struct writeback_control *wbc); 25 26 /* linux/mm/swap_state.c */ 27 /* One swap address space for each 64M swap space */ 28 #define SWAP_ADDRESS_SPACE_SHIFT 14 29 #define SWAP_ADDRESS_SPACE_PAGES (1 << SWAP_ADDRESS_SPACE_SHIFT) 30 #define SWAP_ADDRESS_SPACE_MASK (SWAP_ADDRESS_SPACE_PAGES - 1) 31 extern struct address_space *swapper_spaces[]; 32 #define swap_address_space(entry) \ 33 (&swapper_spaces[swp_type(entry)][swp_offset(entry) \ 34 >> SWAP_ADDRESS_SPACE_SHIFT]) 35 36 /* 37 * Return the swap device position of the swap entry. 38 */ 39 static inline loff_t swap_dev_pos(swp_entry_t entry) 40 { 41 return ((loff_t)swp_offset(entry)) << PAGE_SHIFT; 42 } 43 44 /* 45 * Return the swap cache index of the swap entry. 46 */ 47 static inline pgoff_t swap_cache_index(swp_entry_t entry) 48 { 49 BUILD_BUG_ON((SWP_OFFSET_MASK | SWAP_ADDRESS_SPACE_MASK) != SWP_OFFSET_MASK); 50 return swp_offset(entry) & SWAP_ADDRESS_SPACE_MASK; 51 } 52 53 void show_swap_cache_info(void); 54 void *get_shadow_from_swap_cache(swp_entry_t entry); 55 int add_to_swap_cache(struct folio *folio, swp_entry_t entry, 56 gfp_t gfp, void **shadowp); 57 void __delete_from_swap_cache(struct folio *folio, 58 swp_entry_t entry, void *shadow); 59 void delete_from_swap_cache(struct folio *folio); 60 void clear_shadow_from_swap_cache(int type, unsigned long begin, 61 unsigned long end); 62 void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr); 63 struct folio *swap_cache_get_folio(swp_entry_t entry, 64 struct vm_area_struct *vma, unsigned long addr); 65 struct folio *filemap_get_incore_folio(struct address_space *mapping, 66 pgoff_t index); 67 68 struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, 69 struct vm_area_struct *vma, unsigned long addr, 70 struct swap_iocb **plug); 71 struct folio *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_flags, 72 struct mempolicy *mpol, pgoff_t ilx, bool *new_page_allocated, 73 bool skip_if_exists); 74 struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t flag, 75 struct mempolicy *mpol, pgoff_t ilx); 76 struct folio *swapin_readahead(swp_entry_t entry, gfp_t flag, 77 struct vm_fault *vmf); 78 79 static inline unsigned int folio_swap_flags(struct folio *folio) 80 { 81 return swp_swap_info(folio->swap)->flags; 82 } 83 84 /* 85 * Return the count of contiguous swap entries that share the same 86 * zeromap status as the starting entry. If is_zeromap is not NULL, 87 * it will return the zeromap status of the starting entry. 88 */ 89 static inline int swap_zeromap_batch(swp_entry_t entry, int max_nr, 90 bool *is_zeromap) 91 { 92 struct swap_info_struct *sis = swp_swap_info(entry); 93 unsigned long start = swp_offset(entry); 94 unsigned long end = start + max_nr; 95 bool first_bit; 96 97 first_bit = test_bit(start, sis->zeromap); 98 if (is_zeromap) 99 *is_zeromap = first_bit; 100 101 if (max_nr <= 1) 102 return max_nr; 103 if (first_bit) 104 return find_next_zero_bit(sis->zeromap, end, start) - start; 105 else 106 return find_next_bit(sis->zeromap, end, start) - start; 107 } 108 109 static inline int non_swapcache_batch(swp_entry_t entry, int max_nr) 110 { 111 struct swap_info_struct *si = swp_swap_info(entry); 112 pgoff_t offset = swp_offset(entry); 113 int i; 114 115 /* 116 * While allocating a large folio and doing mTHP swapin, we need to 117 * ensure all entries are not cached, otherwise, the mTHP folio will 118 * be in conflict with the folio in swap cache. 119 */ 120 for (i = 0; i < max_nr; i++) { 121 if ((si->swap_map[offset + i] & SWAP_HAS_CACHE)) 122 return i; 123 } 124 125 return i; 126 } 127 128 #else /* CONFIG_SWAP */ 129 struct swap_iocb; 130 static inline void swap_read_folio(struct folio *folio, struct swap_iocb **plug) 131 { 132 } 133 static inline void swap_write_unplug(struct swap_iocb *sio) 134 { 135 } 136 137 static inline struct address_space *swap_address_space(swp_entry_t entry) 138 { 139 return NULL; 140 } 141 142 static inline pgoff_t swap_cache_index(swp_entry_t entry) 143 { 144 return 0; 145 } 146 147 static inline void show_swap_cache_info(void) 148 { 149 } 150 151 static inline struct folio *swap_cluster_readahead(swp_entry_t entry, 152 gfp_t gfp_mask, struct mempolicy *mpol, pgoff_t ilx) 153 { 154 return NULL; 155 } 156 157 static inline struct folio *swapin_readahead(swp_entry_t swp, gfp_t gfp_mask, 158 struct vm_fault *vmf) 159 { 160 return NULL; 161 } 162 163 static inline int swap_writeout(struct folio *f, struct writeback_control *wbc) 164 { 165 return 0; 166 } 167 168 static inline void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr) 169 { 170 } 171 172 static inline struct folio *swap_cache_get_folio(swp_entry_t entry, 173 struct vm_area_struct *vma, unsigned long addr) 174 { 175 return NULL; 176 } 177 178 static inline 179 struct folio *filemap_get_incore_folio(struct address_space *mapping, 180 pgoff_t index) 181 { 182 return filemap_get_folio(mapping, index); 183 } 184 185 static inline void *get_shadow_from_swap_cache(swp_entry_t entry) 186 { 187 return NULL; 188 } 189 190 static inline int add_to_swap_cache(struct folio *folio, swp_entry_t entry, 191 gfp_t gfp_mask, void **shadowp) 192 { 193 return -1; 194 } 195 196 static inline void __delete_from_swap_cache(struct folio *folio, 197 swp_entry_t entry, void *shadow) 198 { 199 } 200 201 static inline void delete_from_swap_cache(struct folio *folio) 202 { 203 } 204 205 static inline void clear_shadow_from_swap_cache(int type, unsigned long begin, 206 unsigned long end) 207 { 208 } 209 210 static inline unsigned int folio_swap_flags(struct folio *folio) 211 { 212 return 0; 213 } 214 215 static inline int swap_zeromap_batch(swp_entry_t entry, int max_nr, 216 bool *has_zeromap) 217 { 218 return 0; 219 } 220 221 static inline int non_swapcache_batch(swp_entry_t entry, int max_nr) 222 { 223 return 0; 224 } 225 #endif /* CONFIG_SWAP */ 226 227 /** 228 * folio_index - File index of a folio. 229 * @folio: The folio. 230 * 231 * For a folio which is either in the page cache or the swap cache, 232 * return its index within the address_space it belongs to. If you know 233 * the folio is definitely in the page cache, you can look at the folio's 234 * index directly. 235 * 236 * Return: The index (offset in units of pages) of a folio in its file. 237 */ 238 static inline pgoff_t folio_index(struct folio *folio) 239 { 240 if (unlikely(folio_test_swapcache(folio))) 241 return swap_cache_index(folio->swap); 242 return folio->index; 243 } 244 245 #endif /* _MM_SWAP_H */ 246