1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _MM_SWAP_H 3 #define _MM_SWAP_H 4 5 struct mempolicy; 6 struct swap_iocb; 7 8 extern int page_cluster; 9 10 #ifdef CONFIG_SWAP 11 #include <linux/swapops.h> /* for swp_offset */ 12 #include <linux/blk_types.h> /* for bio_end_io_t */ 13 14 /* linux/mm/page_io.c */ 15 int sio_pool_init(void); 16 struct swap_iocb; 17 void swap_read_folio(struct folio *folio, struct swap_iocb **plug); 18 void __swap_read_unplug(struct swap_iocb *plug); 19 static inline void swap_read_unplug(struct swap_iocb *plug) 20 { 21 if (unlikely(plug)) 22 __swap_read_unplug(plug); 23 } 24 void swap_write_unplug(struct swap_iocb *sio); 25 int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug); 26 void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug); 27 28 /* linux/mm/swap_state.c */ 29 /* One swap address space for each 64M swap space */ 30 #define SWAP_ADDRESS_SPACE_SHIFT 14 31 #define SWAP_ADDRESS_SPACE_PAGES (1 << SWAP_ADDRESS_SPACE_SHIFT) 32 #define SWAP_ADDRESS_SPACE_MASK (SWAP_ADDRESS_SPACE_PAGES - 1) 33 extern struct address_space *swapper_spaces[]; 34 #define swap_address_space(entry) \ 35 (&swapper_spaces[swp_type(entry)][swp_offset(entry) \ 36 >> SWAP_ADDRESS_SPACE_SHIFT]) 37 38 /* 39 * Return the swap device position of the swap entry. 40 */ 41 static inline loff_t swap_dev_pos(swp_entry_t entry) 42 { 43 return ((loff_t)swp_offset(entry)) << PAGE_SHIFT; 44 } 45 46 /* 47 * Return the swap cache index of the swap entry. 48 */ 49 static inline pgoff_t swap_cache_index(swp_entry_t entry) 50 { 51 BUILD_BUG_ON((SWP_OFFSET_MASK | SWAP_ADDRESS_SPACE_MASK) != SWP_OFFSET_MASK); 52 return swp_offset(entry) & SWAP_ADDRESS_SPACE_MASK; 53 } 54 55 void show_swap_cache_info(void); 56 void *get_shadow_from_swap_cache(swp_entry_t entry); 57 int add_to_swap_cache(struct folio *folio, swp_entry_t entry, 58 gfp_t gfp, void **shadowp); 59 void __delete_from_swap_cache(struct folio *folio, 60 swp_entry_t entry, void *shadow); 61 void delete_from_swap_cache(struct folio *folio); 62 void clear_shadow_from_swap_cache(int type, unsigned long begin, 63 unsigned long end); 64 void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr); 65 struct folio *swap_cache_get_folio(swp_entry_t entry, 66 struct vm_area_struct *vma, unsigned long addr); 67 struct folio *filemap_get_incore_folio(struct address_space *mapping, 68 pgoff_t index); 69 70 struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, 71 struct vm_area_struct *vma, unsigned long addr, 72 struct swap_iocb **plug); 73 struct folio *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_flags, 74 struct mempolicy *mpol, pgoff_t ilx, bool *new_page_allocated, 75 bool skip_if_exists); 76 struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t flag, 77 struct mempolicy *mpol, pgoff_t ilx); 78 struct folio *swapin_readahead(swp_entry_t entry, gfp_t flag, 79 struct vm_fault *vmf); 80 81 static inline unsigned int folio_swap_flags(struct folio *folio) 82 { 83 return swp_swap_info(folio->swap)->flags; 84 } 85 86 /* 87 * Return the count of contiguous swap entries that share the same 88 * zeromap status as the starting entry. If is_zeromap is not NULL, 89 * it will return the zeromap status of the starting entry. 90 */ 91 static inline int swap_zeromap_batch(swp_entry_t entry, int max_nr, 92 bool *is_zeromap) 93 { 94 struct swap_info_struct *sis = swp_swap_info(entry); 95 unsigned long start = swp_offset(entry); 96 unsigned long end = start + max_nr; 97 bool first_bit; 98 99 first_bit = test_bit(start, sis->zeromap); 100 if (is_zeromap) 101 *is_zeromap = first_bit; 102 103 if (max_nr <= 1) 104 return max_nr; 105 if (first_bit) 106 return find_next_zero_bit(sis->zeromap, end, start) - start; 107 else 108 return find_next_bit(sis->zeromap, end, start) - start; 109 } 110 111 static inline int non_swapcache_batch(swp_entry_t entry, int max_nr) 112 { 113 struct swap_info_struct *si = swp_swap_info(entry); 114 pgoff_t offset = swp_offset(entry); 115 int i; 116 117 /* 118 * While allocating a large folio and doing mTHP swapin, we need to 119 * ensure all entries are not cached, otherwise, the mTHP folio will 120 * be in conflict with the folio in swap cache. 121 */ 122 for (i = 0; i < max_nr; i++) { 123 if ((si->swap_map[offset + i] & SWAP_HAS_CACHE)) 124 return i; 125 } 126 127 return i; 128 } 129 130 #else /* CONFIG_SWAP */ 131 struct swap_iocb; 132 static inline void swap_read_folio(struct folio *folio, struct swap_iocb **plug) 133 { 134 } 135 static inline void swap_write_unplug(struct swap_iocb *sio) 136 { 137 } 138 139 static inline struct address_space *swap_address_space(swp_entry_t entry) 140 { 141 return NULL; 142 } 143 144 static inline pgoff_t swap_cache_index(swp_entry_t entry) 145 { 146 return 0; 147 } 148 149 static inline void show_swap_cache_info(void) 150 { 151 } 152 153 static inline struct folio *swap_cluster_readahead(swp_entry_t entry, 154 gfp_t gfp_mask, struct mempolicy *mpol, pgoff_t ilx) 155 { 156 return NULL; 157 } 158 159 static inline struct folio *swapin_readahead(swp_entry_t swp, gfp_t gfp_mask, 160 struct vm_fault *vmf) 161 { 162 return NULL; 163 } 164 165 static inline int swap_writeout(struct folio *folio, 166 struct swap_iocb **swap_plug) 167 { 168 return 0; 169 } 170 171 static inline void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr) 172 { 173 } 174 175 static inline struct folio *swap_cache_get_folio(swp_entry_t entry, 176 struct vm_area_struct *vma, unsigned long addr) 177 { 178 return NULL; 179 } 180 181 static inline 182 struct folio *filemap_get_incore_folio(struct address_space *mapping, 183 pgoff_t index) 184 { 185 return filemap_get_folio(mapping, index); 186 } 187 188 static inline void *get_shadow_from_swap_cache(swp_entry_t entry) 189 { 190 return NULL; 191 } 192 193 static inline int add_to_swap_cache(struct folio *folio, swp_entry_t entry, 194 gfp_t gfp_mask, void **shadowp) 195 { 196 return -1; 197 } 198 199 static inline void __delete_from_swap_cache(struct folio *folio, 200 swp_entry_t entry, void *shadow) 201 { 202 } 203 204 static inline void delete_from_swap_cache(struct folio *folio) 205 { 206 } 207 208 static inline void clear_shadow_from_swap_cache(int type, unsigned long begin, 209 unsigned long end) 210 { 211 } 212 213 static inline unsigned int folio_swap_flags(struct folio *folio) 214 { 215 return 0; 216 } 217 218 static inline int swap_zeromap_batch(swp_entry_t entry, int max_nr, 219 bool *has_zeromap) 220 { 221 return 0; 222 } 223 224 static inline int non_swapcache_batch(swp_entry_t entry, int max_nr) 225 { 226 return 0; 227 } 228 #endif /* CONFIG_SWAP */ 229 230 /** 231 * folio_index - File index of a folio. 232 * @folio: The folio. 233 * 234 * For a folio which is either in the page cache or the swap cache, 235 * return its index within the address_space it belongs to. If you know 236 * the folio is definitely in the page cache, you can look at the folio's 237 * index directly. 238 * 239 * Return: The index (offset in units of pages) of a folio in its file. 240 */ 241 static inline pgoff_t folio_index(struct folio *folio) 242 { 243 if (unlikely(folio_test_swapcache(folio))) 244 return swap_cache_index(folio->swap); 245 return folio->index; 246 } 247 248 #endif /* _MM_SWAP_H */ 249