1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_SWAPOPS_H 3 #define _LINUX_SWAPOPS_H 4 5 #include <linux/radix-tree.h> 6 #include <linux/bug.h> 7 #include <linux/mm_types.h> 8 #include <linux/swap.h> 9 10 #ifdef CONFIG_MMU 11 12 #ifdef CONFIG_SWAP 13 #include <linux/swapfile.h> 14 #endif /* CONFIG_SWAP */ 15 16 /* 17 * swapcache pages are stored in the swapper_space radix tree. We want to 18 * get good packing density in that tree, so the index should be dense in 19 * the low-order bits. 20 * 21 * We arrange the `type' and `offset' fields so that `type' is at the six 22 * high-order bits of the swp_entry_t and `offset' is right-aligned in the 23 * remaining bits. Although `type' itself needs only five bits, we allow for 24 * shmem/tmpfs to shift it all up a further one bit: see swp_to_radix_entry(). 25 * 26 * swp_entry_t's are *never* stored anywhere in their arch-dependent format. 27 */ 28 #define SWP_TYPE_SHIFT (BITS_PER_XA_VALUE - MAX_SWAPFILES_SHIFT) 29 #define SWP_OFFSET_MASK ((1UL << SWP_TYPE_SHIFT) - 1) 30 31 /* 32 * Definitions only for PFN swap entries (see leafeant_has_pfn()). To 33 * store PFN, we only need SWP_PFN_BITS bits. Each of the pfn swap entries 34 * can use the extra bits to store other information besides PFN. 35 */ 36 #ifdef MAX_PHYSMEM_BITS 37 #define SWP_PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT) 38 #else /* MAX_PHYSMEM_BITS */ 39 #define SWP_PFN_BITS min_t(int, \ 40 sizeof(phys_addr_t) * 8 - PAGE_SHIFT, \ 41 SWP_TYPE_SHIFT) 42 #endif /* MAX_PHYSMEM_BITS */ 43 #define SWP_PFN_MASK (BIT(SWP_PFN_BITS) - 1) 44 45 /** 46 * Migration swap entry specific bitfield definitions. Layout: 47 * 48 * |----------+--------------------| 49 * | swp_type | swp_offset | 50 * |----------+--------+-+-+-------| 51 * | | resv |D|A| PFN | 52 * |----------+--------+-+-+-------| 53 * 54 * @SWP_MIG_YOUNG_BIT: Whether the page used to have young bit set (bit A) 55 * @SWP_MIG_DIRTY_BIT: Whether the page used to have dirty bit set (bit D) 56 * 57 * Note: A/D bits will be stored in migration entries iff there're enough 58 * free bits in arch specific swp offset. By default we'll ignore A/D bits 59 * when migrating a page. Please refer to migration_entry_supports_ad() 60 * for more information. If there're more bits besides PFN and A/D bits, 61 * they should be reserved and always be zeros. 62 */ 63 #define SWP_MIG_YOUNG_BIT (SWP_PFN_BITS) 64 #define SWP_MIG_DIRTY_BIT (SWP_PFN_BITS + 1) 65 #define SWP_MIG_TOTAL_BITS (SWP_PFN_BITS + 2) 66 67 #define SWP_MIG_YOUNG BIT(SWP_MIG_YOUNG_BIT) 68 #define SWP_MIG_DIRTY BIT(SWP_MIG_DIRTY_BIT) 69 70 /* Clear all flags but only keep swp_entry_t related information */ 71 static inline pte_t pte_swp_clear_flags(pte_t pte) 72 { 73 if (pte_swp_exclusive(pte)) 74 pte = pte_swp_clear_exclusive(pte); 75 if (pte_swp_soft_dirty(pte)) 76 pte = pte_swp_clear_soft_dirty(pte); 77 if (pte_swp_uffd(pte)) 78 pte = pte_swp_clear_uffd(pte); 79 return pte; 80 } 81 82 /* 83 * Store a type+offset into a swp_entry_t in an arch-independent format 84 */ 85 static inline swp_entry_t swp_entry(unsigned long type, pgoff_t offset) 86 { 87 swp_entry_t ret; 88 89 ret.val = (type << SWP_TYPE_SHIFT) | (offset & SWP_OFFSET_MASK); 90 return ret; 91 } 92 93 /* 94 * Extract the `type' field from a swp_entry_t. The swp_entry_t is in 95 * arch-independent format 96 */ 97 static inline unsigned swp_type(swp_entry_t entry) 98 { 99 return (entry.val >> SWP_TYPE_SHIFT); 100 } 101 102 /* 103 * Extract the `offset' field from a swp_entry_t. The swp_entry_t is in 104 * arch-independent format 105 */ 106 static inline pgoff_t swp_offset(swp_entry_t entry) 107 { 108 return entry.val & SWP_OFFSET_MASK; 109 } 110 111 /* 112 * Convert the arch-independent representation of a swp_entry_t into the 113 * arch-dependent pte representation. 114 */ 115 static inline pte_t swp_entry_to_pte(swp_entry_t entry) 116 { 117 swp_entry_t arch_entry; 118 119 arch_entry = __swp_entry(swp_type(entry), swp_offset(entry)); 120 return __swp_entry_to_pte(arch_entry); 121 } 122 123 static inline swp_entry_t radix_to_swp_entry(void *arg) 124 { 125 swp_entry_t entry; 126 127 entry.val = xa_to_value(arg); 128 return entry; 129 } 130 131 static inline void *swp_to_radix_entry(swp_entry_t entry) 132 { 133 return xa_mk_value(entry.val); 134 } 135 136 #if IS_ENABLED(CONFIG_DEVICE_PRIVATE) 137 static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset) 138 { 139 return swp_entry(SWP_DEVICE_READ, offset); 140 } 141 142 static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset) 143 { 144 return swp_entry(SWP_DEVICE_WRITE, offset); 145 } 146 147 static inline swp_entry_t make_device_exclusive_entry(pgoff_t offset) 148 { 149 return swp_entry(SWP_DEVICE_EXCLUSIVE, offset); 150 } 151 152 #else /* CONFIG_DEVICE_PRIVATE */ 153 static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset) 154 { 155 return swp_entry(0, 0); 156 } 157 158 static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset) 159 { 160 return swp_entry(0, 0); 161 } 162 163 static inline swp_entry_t make_device_exclusive_entry(pgoff_t offset) 164 { 165 return swp_entry(0, 0); 166 } 167 168 #endif /* CONFIG_DEVICE_PRIVATE */ 169 170 #ifdef CONFIG_MIGRATION 171 172 static inline swp_entry_t make_readable_migration_entry(pgoff_t offset) 173 { 174 return swp_entry(SWP_MIGRATION_READ, offset); 175 } 176 177 static inline swp_entry_t make_readable_exclusive_migration_entry(pgoff_t offset) 178 { 179 return swp_entry(SWP_MIGRATION_READ_EXCLUSIVE, offset); 180 } 181 182 static inline swp_entry_t make_writable_migration_entry(pgoff_t offset) 183 { 184 return swp_entry(SWP_MIGRATION_WRITE, offset); 185 } 186 187 /* 188 * Returns whether the host has large enough swap offset field to support 189 * carrying over pgtable A/D bits for page migrations. The result is 190 * pretty much arch specific. 191 */ 192 static inline bool migration_entry_supports_ad(void) 193 { 194 #ifdef CONFIG_SWAP 195 return swap_migration_ad_supported; 196 #else /* CONFIG_SWAP */ 197 return false; 198 #endif /* CONFIG_SWAP */ 199 } 200 201 static inline swp_entry_t make_migration_entry_young(swp_entry_t entry) 202 { 203 if (migration_entry_supports_ad()) 204 return swp_entry(swp_type(entry), 205 swp_offset(entry) | SWP_MIG_YOUNG); 206 return entry; 207 } 208 209 static inline swp_entry_t make_migration_entry_dirty(swp_entry_t entry) 210 { 211 if (migration_entry_supports_ad()) 212 return swp_entry(swp_type(entry), 213 swp_offset(entry) | SWP_MIG_DIRTY); 214 return entry; 215 } 216 217 extern void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, 218 unsigned long address); 219 extern void migration_entry_wait_huge(struct vm_area_struct *vma, unsigned long addr, pte_t *pte); 220 #else /* CONFIG_MIGRATION */ 221 static inline swp_entry_t make_readable_migration_entry(pgoff_t offset) 222 { 223 return swp_entry(0, 0); 224 } 225 226 static inline swp_entry_t make_readable_exclusive_migration_entry(pgoff_t offset) 227 { 228 return swp_entry(0, 0); 229 } 230 231 static inline swp_entry_t make_writable_migration_entry(pgoff_t offset) 232 { 233 return swp_entry(0, 0); 234 } 235 236 static inline void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, 237 unsigned long address) { } 238 static inline void migration_entry_wait_huge(struct vm_area_struct *vma, 239 unsigned long addr, pte_t *pte) { } 240 241 static inline swp_entry_t make_migration_entry_young(swp_entry_t entry) 242 { 243 return entry; 244 } 245 246 static inline swp_entry_t make_migration_entry_dirty(swp_entry_t entry) 247 { 248 return entry; 249 } 250 251 #endif /* CONFIG_MIGRATION */ 252 253 #ifdef CONFIG_MEMORY_FAILURE 254 255 /* 256 * Support for hardware poisoned pages 257 */ 258 static inline swp_entry_t make_hwpoison_entry(struct page *page) 259 { 260 BUG_ON(!PageLocked(page)); 261 return swp_entry(SWP_HWPOISON, page_to_pfn(page)); 262 } 263 264 #else 265 266 static inline swp_entry_t make_hwpoison_entry(struct page *page) 267 { 268 return swp_entry(0, 0); 269 } 270 271 #endif 272 273 typedef unsigned long pte_marker; 274 275 #define PTE_MARKER_UFFD_WP BIT(0) 276 /* 277 * "Poisoned" here is meant in the very general sense of "future accesses are 278 * invalid", instead of referring very specifically to hardware memory errors. 279 * This marker is meant to represent any of various different causes of this. 280 * 281 * Note that, when encountered by the faulting logic, PTEs with this marker will 282 * result in VM_FAULT_HWPOISON and thus regardless trigger hardware memory error 283 * logic. 284 */ 285 #define PTE_MARKER_POISONED BIT(1) 286 /* 287 * Indicates that, on fault, this PTE will case a SIGSEGV signal to be 288 * sent. This means guard markers behave in effect as if the region were mapped 289 * PROT_NONE, rather than if they were a memory hole or equivalent. 290 */ 291 #define PTE_MARKER_GUARD BIT(2) 292 #define PTE_MARKER_MASK (BIT(3) - 1) 293 294 static inline swp_entry_t make_pte_marker_entry(pte_marker marker) 295 { 296 return swp_entry(SWP_PTE_MARKER, marker); 297 } 298 299 static inline pte_t make_pte_marker(pte_marker marker) 300 { 301 return swp_entry_to_pte(make_pte_marker_entry(marker)); 302 } 303 304 static inline swp_entry_t make_poisoned_swp_entry(void) 305 { 306 return make_pte_marker_entry(PTE_MARKER_POISONED); 307 } 308 309 static inline swp_entry_t make_guard_swp_entry(void) 310 { 311 return make_pte_marker_entry(PTE_MARKER_GUARD); 312 } 313 314 struct page_vma_mapped_walk; 315 316 #ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES 317 extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, 318 struct page *page); 319 320 void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, 321 struct folio *folio); 322 323 extern void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd); 324 325 static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) 326 { 327 swp_entry_t arch_entry; 328 329 arch_entry = __swp_entry(swp_type(entry), swp_offset(entry)); 330 return __swp_entry_to_pmd(arch_entry); 331 } 332 333 #else /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ 334 static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, 335 struct page *page) 336 { 337 BUILD_BUG(); 338 } 339 340 static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, 341 struct folio *folio) 342 { 343 BUILD_BUG(); 344 } 345 346 static inline void pmd_migration_entry_wait(struct mm_struct *m, pmd_t *p) { } 347 348 static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) 349 { 350 return __pmd(0); 351 } 352 353 #endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ 354 355 #endif /* CONFIG_MMU */ 356 #endif /* _LINUX_SWAPOPS_H */ 357