1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_MM_H 3 #define _LINUX_MM_H 4 5 #include <linux/args.h> 6 #include <linux/errno.h> 7 #include <linux/mmdebug.h> 8 #include <linux/gfp.h> 9 #include <linux/pgalloc_tag.h> 10 #include <linux/bug.h> 11 #include <linux/list.h> 12 #include <linux/mmzone.h> 13 #include <linux/rbtree.h> 14 #include <linux/atomic.h> 15 #include <linux/debug_locks.h> 16 #include <linux/compiler.h> 17 #include <linux/mm_types.h> 18 #include <linux/mmap_lock.h> 19 #include <linux/range.h> 20 #include <linux/pfn.h> 21 #include <linux/percpu-refcount.h> 22 #include <linux/bit_spinlock.h> 23 #include <linux/shrinker.h> 24 #include <linux/resource.h> 25 #include <linux/page_ext.h> 26 #include <linux/err.h> 27 #include <linux/page-flags.h> 28 #include <linux/page_ref.h> 29 #include <linux/overflow.h> 30 #include <linux/sched.h> 31 #include <linux/pgtable.h> 32 #include <linux/kasan.h> 33 #include <linux/memremap.h> 34 #include <linux/slab.h> 35 #include <linux/cacheinfo.h> 36 #include <linux/rcuwait.h> 37 #include <linux/bitmap.h> 38 #include <linux/bitops.h> 39 #include <linux/iommu-debug-pagealloc.h> 40 #include <linux/kcsan-checks.h> 41 #include <linux/vmemmap-optimization.h> 42 43 struct mempolicy; 44 struct anon_vma; 45 struct anon_vma_chain; 46 struct user_struct; 47 struct pt_regs; 48 struct folio_batch; 49 50 void arch_mm_preinit(void); 51 void mm_core_init_early(void); 52 void mm_core_init(void); 53 void init_mm_internals(void); 54 55 extern atomic_long_t _totalram_pages; 56 static inline unsigned long totalram_pages(void) 57 { 58 return (unsigned long)atomic_long_read(&_totalram_pages); 59 } 60 61 static inline void totalram_pages_add(long count) 62 { 63 atomic_long_add(count, &_totalram_pages); 64 } 65 66 extern void * high_memory; 67 68 /* 69 * Convert between pages and MB 70 * 20 is the shift for 1MB (2^20 = 1MB) 71 * PAGE_SHIFT is the shift for page size (e.g., 12 for 4KB pages) 72 * So (20 - PAGE_SHIFT) converts between pages and MB 73 */ 74 #define PAGES_TO_MB(pages) ((pages) >> (20 - PAGE_SHIFT)) 75 #define MB_TO_PAGES(mb) ((mb) << (20 - PAGE_SHIFT)) 76 77 #ifdef CONFIG_SYSCTL 78 extern int sysctl_legacy_va_layout; 79 #else 80 #define sysctl_legacy_va_layout 0 81 #endif 82 83 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS 84 extern const int mmap_rnd_bits_min; 85 extern int mmap_rnd_bits_max __ro_after_init; 86 extern int mmap_rnd_bits __read_mostly; 87 #endif 88 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS 89 extern const int mmap_rnd_compat_bits_min; 90 extern const int mmap_rnd_compat_bits_max; 91 extern int mmap_rnd_compat_bits __read_mostly; 92 #endif 93 94 #ifndef DIRECT_MAP_PHYSMEM_END 95 # ifdef MAX_PHYSMEM_BITS 96 # define DIRECT_MAP_PHYSMEM_END ((1ULL << MAX_PHYSMEM_BITS) - 1) 97 # else 98 # define DIRECT_MAP_PHYSMEM_END (((phys_addr_t)-1)&~(1ULL<<63)) 99 # endif 100 #endif 101 102 #define INVALID_PHYS_ADDR (~(phys_addr_t)0) 103 104 #include <asm/page.h> 105 #include <asm/processor.h> 106 107 #ifndef __pa_symbol 108 #define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0)) 109 #endif 110 111 #ifndef page_to_virt 112 #define page_to_virt(x) __va(PFN_PHYS(page_to_pfn(x))) 113 #endif 114 115 #ifndef lm_alias 116 #define lm_alias(x) __va(__pa_symbol(x)) 117 #endif 118 119 /* 120 * To prevent common memory management code establishing 121 * a zero page mapping on a read fault. 122 * This macro should be defined within <asm/pgtable.h>. 123 * s390 does this to prevent multiplexing of hardware bits 124 * related to the physical page in case of virtualization. 125 */ 126 #ifndef mm_forbids_zeropage 127 #define mm_forbids_zeropage(X) (0) 128 #endif 129 130 /* 131 * On some architectures it is expensive to call memset() for small sizes. 132 * If an architecture decides to implement their own version of 133 * mm_zero_struct_page they should wrap the defines below in a #ifndef and 134 * define their own version of this macro in <asm/pgtable.h> 135 */ 136 #if BITS_PER_LONG == 64 137 /* This function must be updated when the size of struct page grows above 96 138 * or reduces below 56. The idea that compiler optimizes out switch() 139 * statement, and only leaves move/store instructions. Also the compiler can 140 * combine write statements if they are both assignments and can be reordered, 141 * this can result in several of the writes here being dropped. 142 */ 143 #define mm_zero_struct_page(pp) __mm_zero_struct_page(pp) 144 static inline void __mm_zero_struct_page(struct page *page) 145 { 146 unsigned long *_pp = (void *)page; 147 148 /* Check that struct page is either 56, 64, 72, 80, 88 or 96 bytes */ 149 BUILD_BUG_ON(sizeof(struct page) & 7); 150 BUILD_BUG_ON(sizeof(struct page) < 56); 151 BUILD_BUG_ON(sizeof(struct page) > 96); 152 153 switch (sizeof(struct page)) { 154 case 96: 155 _pp[11] = 0; 156 fallthrough; 157 case 88: 158 _pp[10] = 0; 159 fallthrough; 160 case 80: 161 _pp[9] = 0; 162 fallthrough; 163 case 72: 164 _pp[8] = 0; 165 fallthrough; 166 case 64: 167 _pp[7] = 0; 168 fallthrough; 169 case 56: 170 _pp[6] = 0; 171 _pp[5] = 0; 172 _pp[4] = 0; 173 _pp[3] = 0; 174 _pp[2] = 0; 175 _pp[1] = 0; 176 _pp[0] = 0; 177 } 178 } 179 #else 180 #define mm_zero_struct_page(pp) ((void)memset((pp), 0, sizeof(struct page))) 181 #endif 182 183 /* 184 * Default maximum number of active map areas, this limits the number of vmas 185 * per mm struct. Users can overwrite this number by sysctl but there is a 186 * problem. 187 * 188 * When a program's coredump is generated as ELF format, a section is created 189 * per a vma. In ELF, the number of sections is represented in unsigned short. 190 * This means the number of sections should be smaller than 65535 at coredump. 191 * Because the kernel adds some informative sections to a image of program at 192 * generating coredump, we need some margin. The number of extra sections is 193 * 1-3 now and depends on arch. We use "5" as safe margin, here. 194 * 195 * ELF extended numbering allows more than 65535 sections, so 16-bit bound is 196 * not a hard limit any more. Although some userspace tools can be surprised by 197 * that. 198 */ 199 #define MAPCOUNT_ELF_CORE_MARGIN (5) 200 #define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN) 201 202 extern unsigned long sysctl_user_reserve_kbytes; 203 extern unsigned long sysctl_admin_reserve_kbytes; 204 205 #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) 206 bool page_range_contiguous(const struct page *page, unsigned long nr_pages); 207 #else 208 static inline bool page_range_contiguous(const struct page *page, 209 unsigned long nr_pages) 210 { 211 return true; 212 } 213 #endif 214 215 /* to align the pointer to the (next) page boundary */ 216 #define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE) 217 218 /* to align the pointer to the (prev) page boundary */ 219 #define PAGE_ALIGN_DOWN(addr) ALIGN_DOWN(addr, PAGE_SIZE) 220 221 /* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */ 222 #define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), PAGE_SIZE) 223 224 /** 225 * folio_page_idx - Return the number of a page in a folio. 226 * @folio: The folio. 227 * @page: The folio page. 228 * 229 * This function expects that the page is actually part of the folio. 230 * The returned number is relative to the start of the folio. 231 */ 232 static inline unsigned long folio_page_idx(const struct folio *folio, 233 const struct page *page) 234 { 235 return page - &folio->page; 236 } 237 238 static inline struct folio *lru_to_folio(struct list_head *head) 239 { 240 return list_entry((head)->prev, struct folio, lru); 241 } 242 243 void setup_initial_init_mm(void *start_code, void *end_code, 244 void *end_data, void *brk); 245 246 /* 247 * Linux kernel virtual memory manager primitives. 248 * The idea being to have a "virtual" mm in the same way 249 * we have a virtual fs - giving a cleaner interface to the 250 * mm details, and allowing different kinds of memory mappings 251 * (from shared memory to executable loading to arbitrary 252 * mmap() functions). 253 */ 254 255 struct vm_area_struct *vm_area_alloc(struct mm_struct *); 256 struct vm_area_struct *vm_area_dup(struct vm_area_struct *); 257 void vm_area_free(struct vm_area_struct *); 258 259 #ifndef CONFIG_MMU 260 extern struct rb_root nommu_region_tree; 261 extern struct rw_semaphore nommu_region_sem; 262 263 extern unsigned int kobjsize(const void *objp); 264 #endif 265 266 /* 267 * vm_flags in vm_area_struct, see mm_types.h. 268 * When changing, update also include/trace/events/mmflags.h 269 */ 270 271 #define VM_NONE 0x00000000 272 273 /** 274 * typedef vma_flag_t - specifies an individual VMA flag by bit number. 275 * 276 * This value is made type safe by sparse to avoid passing invalid flag values 277 * around. 278 */ 279 typedef int __bitwise vma_flag_t; 280 281 #define DECLARE_VMA_BIT(name, bitnum) \ 282 VMA_ ## name ## _BIT = ((__force vma_flag_t)bitnum) 283 #define DECLARE_VMA_BIT_ALIAS(name, aliased) \ 284 VMA_ ## name ## _BIT = (VMA_ ## aliased ## _BIT) 285 enum { 286 DECLARE_VMA_BIT(READ, 0), 287 DECLARE_VMA_BIT(WRITE, 1), 288 DECLARE_VMA_BIT(EXEC, 2), 289 DECLARE_VMA_BIT(SHARED, 3), 290 /* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */ 291 DECLARE_VMA_BIT(MAYREAD, 4), /* limits for mprotect() etc. */ 292 DECLARE_VMA_BIT(MAYWRITE, 5), 293 DECLARE_VMA_BIT(MAYEXEC, 6), 294 DECLARE_VMA_BIT(MAYSHARE, 7), 295 DECLARE_VMA_BIT(GROWSDOWN, 8), /* general info on the segment */ 296 #ifdef CONFIG_MMU 297 DECLARE_VMA_BIT(UFFD_MISSING, 9),/* missing pages tracking */ 298 #else 299 /* nommu: R/O MAP_PRIVATE mapping that might overlay a file mapping */ 300 DECLARE_VMA_BIT(MAYOVERLAY, 9), 301 #endif /* CONFIG_MMU */ 302 /* Page-ranges managed without "struct page", just pure PFN */ 303 DECLARE_VMA_BIT(PFNMAP, 10), 304 DECLARE_VMA_BIT(MAYBE_GUARD, 11), 305 DECLARE_VMA_BIT(UFFD_WP, 12), /* wrprotect pages tracking */ 306 DECLARE_VMA_BIT(LOCKED, 13), 307 DECLARE_VMA_BIT(IO, 14), /* Memory mapped I/O or similar */ 308 DECLARE_VMA_BIT(SEQ_READ, 15), /* App will access data sequentially */ 309 DECLARE_VMA_BIT(RAND_READ, 16), /* App will not benefit from clustered reads */ 310 DECLARE_VMA_BIT(DONTCOPY, 17), /* Do not copy this vma on fork */ 311 DECLARE_VMA_BIT(DONTEXPAND, 18),/* Cannot expand with mremap() */ 312 DECLARE_VMA_BIT(LOCKONFAULT, 19),/* Lock pages covered when faulted in */ 313 DECLARE_VMA_BIT(ACCOUNT, 20), /* Is a VM accounted object */ 314 DECLARE_VMA_BIT(NORESERVE, 21), /* should the VM suppress accounting */ 315 DECLARE_VMA_BIT(HUGETLB, 22), /* Huge TLB Page VM */ 316 DECLARE_VMA_BIT(SYNC, 23), /* Synchronous page faults */ 317 DECLARE_VMA_BIT(ARCH_1, 24), /* Architecture-specific flag */ 318 DECLARE_VMA_BIT(WIPEONFORK, 25),/* Wipe VMA contents in child. */ 319 DECLARE_VMA_BIT(DONTDUMP, 26), /* Do not include in the core dump */ 320 DECLARE_VMA_BIT(SOFTDIRTY, 27), /* NOT soft dirty clean area */ 321 DECLARE_VMA_BIT(MIXEDMAP, 28), /* Can contain struct page and pure PFN pages */ 322 DECLARE_VMA_BIT(HUGEPAGE, 29), /* MADV_HUGEPAGE marked this vma */ 323 DECLARE_VMA_BIT(NOHUGEPAGE, 30),/* MADV_NOHUGEPAGE marked this vma */ 324 DECLARE_VMA_BIT(MERGEABLE, 31), /* KSM may merge identical pages */ 325 /* These bits are reused, we define specific uses below. */ 326 DECLARE_VMA_BIT(HIGH_ARCH_0, 32), 327 DECLARE_VMA_BIT(HIGH_ARCH_1, 33), 328 DECLARE_VMA_BIT(HIGH_ARCH_2, 34), 329 DECLARE_VMA_BIT(HIGH_ARCH_3, 35), 330 DECLARE_VMA_BIT(HIGH_ARCH_4, 36), 331 DECLARE_VMA_BIT(HIGH_ARCH_5, 37), 332 DECLARE_VMA_BIT(HIGH_ARCH_6, 38), 333 /* 334 * This flag is used to connect VFIO to arch specific KVM code. It 335 * indicates that the memory under this VMA is safe for use with any 336 * non-cachable memory type inside KVM. Some VFIO devices, on some 337 * platforms, are thought to be unsafe and can cause machine crashes 338 * if KVM does not lock down the memory type. 339 */ 340 DECLARE_VMA_BIT(ALLOW_ANY_UNCACHED, 39), 341 #if defined(CONFIG_PPC32) 342 DECLARE_VMA_BIT_ALIAS(DROPPABLE, ARCH_1), 343 #elif defined(CONFIG_64BIT) 344 DECLARE_VMA_BIT(DROPPABLE, 40), 345 #endif 346 DECLARE_VMA_BIT(UFFD_MINOR, 41), 347 DECLARE_VMA_BIT(SEALED, 42), 348 DECLARE_VMA_BIT(UFFD_RWP, 43), 349 /* Flags that reuse flags above. */ 350 DECLARE_VMA_BIT_ALIAS(PKEY_BIT0, HIGH_ARCH_0), 351 DECLARE_VMA_BIT_ALIAS(PKEY_BIT1, HIGH_ARCH_1), 352 DECLARE_VMA_BIT_ALIAS(PKEY_BIT2, HIGH_ARCH_2), 353 DECLARE_VMA_BIT_ALIAS(PKEY_BIT3, HIGH_ARCH_3), 354 DECLARE_VMA_BIT_ALIAS(PKEY_BIT4, HIGH_ARCH_4), 355 #if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_RISCV_USER_CFI) 356 /* 357 * VM_SHADOW_STACK should not be set with VM_SHARED because of lack of 358 * support core mm. 359 * 360 * These VMAs will get a single end guard page. This helps userspace 361 * protect itself from attacks. A single page is enough for current 362 * shadow stack archs (x86). See the comments near alloc_shstk() in 363 * arch/x86/kernel/shstk.c for more details on the guard size. 364 */ 365 DECLARE_VMA_BIT_ALIAS(SHADOW_STACK, HIGH_ARCH_5), 366 #elif defined(CONFIG_ARM64_GCS) 367 /* 368 * arm64's Guarded Control Stack implements similar functionality and 369 * has similar constraints to shadow stacks. 370 */ 371 DECLARE_VMA_BIT_ALIAS(SHADOW_STACK, HIGH_ARCH_6), 372 #endif 373 DECLARE_VMA_BIT_ALIAS(SAO, ARCH_1), /* Strong Access Ordering (powerpc) */ 374 DECLARE_VMA_BIT_ALIAS(GROWSUP, ARCH_1), /* parisc */ 375 DECLARE_VMA_BIT_ALIAS(SPARC_ADI, ARCH_1), /* sparc64 */ 376 DECLARE_VMA_BIT_ALIAS(ARM64_BTI, ARCH_1), /* arm64 */ 377 DECLARE_VMA_BIT_ALIAS(ARCH_CLEAR, ARCH_1), /* sparc64, arm64 */ 378 DECLARE_VMA_BIT_ALIAS(MAPPED_COPY, ARCH_1), /* !CONFIG_MMU */ 379 DECLARE_VMA_BIT_ALIAS(MTE, HIGH_ARCH_4), /* arm64 */ 380 DECLARE_VMA_BIT_ALIAS(MTE_ALLOWED, HIGH_ARCH_5),/* arm64 */ 381 #ifdef CONFIG_STACK_GROWSUP 382 DECLARE_VMA_BIT_ALIAS(STACK, GROWSUP), 383 DECLARE_VMA_BIT_ALIAS(STACK_EARLY, GROWSDOWN), 384 #else 385 DECLARE_VMA_BIT_ALIAS(STACK, GROWSDOWN), 386 #endif 387 }; 388 #undef DECLARE_VMA_BIT 389 #undef DECLARE_VMA_BIT_ALIAS 390 391 #define INIT_VM_FLAG(name) BIT((__force int) VMA_ ## name ## _BIT) 392 #define VM_READ INIT_VM_FLAG(READ) 393 #define VM_WRITE INIT_VM_FLAG(WRITE) 394 #define VM_EXEC INIT_VM_FLAG(EXEC) 395 #define VM_SHARED INIT_VM_FLAG(SHARED) 396 #define VM_MAYREAD INIT_VM_FLAG(MAYREAD) 397 #define VM_MAYWRITE INIT_VM_FLAG(MAYWRITE) 398 #define VM_MAYEXEC INIT_VM_FLAG(MAYEXEC) 399 #define VM_MAYSHARE INIT_VM_FLAG(MAYSHARE) 400 #define VM_GROWSDOWN INIT_VM_FLAG(GROWSDOWN) 401 #ifdef CONFIG_MMU 402 #define VM_UFFD_MISSING INIT_VM_FLAG(UFFD_MISSING) 403 #else 404 #define VM_UFFD_MISSING VM_NONE 405 #define VM_MAYOVERLAY INIT_VM_FLAG(MAYOVERLAY) 406 #endif 407 #define VM_PFNMAP INIT_VM_FLAG(PFNMAP) 408 #define VM_MAYBE_GUARD INIT_VM_FLAG(MAYBE_GUARD) 409 #define VM_UFFD_WP INIT_VM_FLAG(UFFD_WP) 410 #define VM_LOCKED INIT_VM_FLAG(LOCKED) 411 #define VM_IO INIT_VM_FLAG(IO) 412 #define VM_SEQ_READ INIT_VM_FLAG(SEQ_READ) 413 #define VM_RAND_READ INIT_VM_FLAG(RAND_READ) 414 #define VM_DONTCOPY INIT_VM_FLAG(DONTCOPY) 415 #define VM_DONTEXPAND INIT_VM_FLAG(DONTEXPAND) 416 #define VM_LOCKONFAULT INIT_VM_FLAG(LOCKONFAULT) 417 #define VM_ACCOUNT INIT_VM_FLAG(ACCOUNT) 418 #define VM_NORESERVE INIT_VM_FLAG(NORESERVE) 419 #define VM_HUGETLB INIT_VM_FLAG(HUGETLB) 420 #define VM_SYNC INIT_VM_FLAG(SYNC) 421 #define VM_ARCH_1 INIT_VM_FLAG(ARCH_1) 422 #define VM_WIPEONFORK INIT_VM_FLAG(WIPEONFORK) 423 #define VM_DONTDUMP INIT_VM_FLAG(DONTDUMP) 424 #ifdef CONFIG_MEM_SOFT_DIRTY 425 #define VM_SOFTDIRTY INIT_VM_FLAG(SOFTDIRTY) 426 #else 427 #define VM_SOFTDIRTY VM_NONE 428 #endif 429 #define VM_MIXEDMAP INIT_VM_FLAG(MIXEDMAP) 430 #define VM_HUGEPAGE INIT_VM_FLAG(HUGEPAGE) 431 #define VM_NOHUGEPAGE INIT_VM_FLAG(NOHUGEPAGE) 432 #define VM_MERGEABLE INIT_VM_FLAG(MERGEABLE) 433 #define VM_STACK INIT_VM_FLAG(STACK) 434 #ifdef CONFIG_STACK_GROWSUP 435 #define VM_STACK_EARLY INIT_VM_FLAG(STACK_EARLY) 436 #define VMA_STACK_EARLY mk_vma_flags(VMA_STACK_EARLY_BIT) 437 #else 438 #define VM_STACK_EARLY VM_NONE 439 #define VMA_STACK_EARLY EMPTY_VMA_FLAGS 440 #endif 441 #ifdef CONFIG_ARCH_HAS_PKEYS 442 #define VM_PKEY_SHIFT ((__force int)VMA_HIGH_ARCH_0_BIT) 443 /* Despite the naming, these are FLAGS not bits. */ 444 #define VM_PKEY_BIT0 INIT_VM_FLAG(PKEY_BIT0) 445 #define VM_PKEY_BIT1 INIT_VM_FLAG(PKEY_BIT1) 446 #define VM_PKEY_BIT2 INIT_VM_FLAG(PKEY_BIT2) 447 #if CONFIG_ARCH_PKEY_BITS > 3 448 #define VM_PKEY_BIT3 INIT_VM_FLAG(PKEY_BIT3) 449 #else 450 #define VM_PKEY_BIT3 VM_NONE 451 #endif /* CONFIG_ARCH_PKEY_BITS > 3 */ 452 #if CONFIG_ARCH_PKEY_BITS > 4 453 #define VM_PKEY_BIT4 INIT_VM_FLAG(PKEY_BIT4) 454 #else 455 #define VM_PKEY_BIT4 VM_NONE 456 #endif /* CONFIG_ARCH_PKEY_BITS > 4 */ 457 #endif /* CONFIG_ARCH_HAS_PKEYS */ 458 #if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_ARM64_GCS) || \ 459 defined(CONFIG_RISCV_USER_CFI) 460 #define VM_SHADOW_STACK INIT_VM_FLAG(SHADOW_STACK) 461 #define VMA_SHADOW_STACK mk_vma_flags(VMA_SHADOW_STACK_BIT) 462 #define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT, VMA_SHADOW_STACK_BIT) 463 #else 464 #define VM_SHADOW_STACK VM_NONE 465 #define VMA_SHADOW_STACK EMPTY_VMA_FLAGS 466 #define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT) 467 #endif 468 #if defined(CONFIG_PPC64) 469 #define VM_SAO INIT_VM_FLAG(SAO) 470 #elif defined(CONFIG_PARISC) 471 #define VM_GROWSUP INIT_VM_FLAG(GROWSUP) 472 #define VMA_GROWSUP mk_vma_flags(VMA_GROWSUP_BIT) 473 #elif defined(CONFIG_SPARC64) 474 #define VM_SPARC_ADI INIT_VM_FLAG(SPARC_ADI) 475 #define VM_ARCH_CLEAR INIT_VM_FLAG(ARCH_CLEAR) 476 #elif defined(CONFIG_ARM64) 477 #define VM_ARM64_BTI INIT_VM_FLAG(ARM64_BTI) 478 #define VM_ARCH_CLEAR INIT_VM_FLAG(ARCH_CLEAR) 479 #elif !defined(CONFIG_MMU) 480 #define VM_MAPPED_COPY INIT_VM_FLAG(MAPPED_COPY) 481 #endif 482 #ifndef VM_GROWSUP 483 #define VM_GROWSUP VM_NONE 484 #define VMA_GROWSUP EMPTY_VMA_FLAGS 485 #endif 486 #ifdef CONFIG_ARM64_MTE 487 #define VM_MTE INIT_VM_FLAG(MTE) 488 #define VM_MTE_ALLOWED INIT_VM_FLAG(MTE_ALLOWED) 489 #else 490 #define VM_MTE VM_NONE 491 #define VM_MTE_ALLOWED VM_NONE 492 #endif 493 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR 494 #define VM_UFFD_MINOR INIT_VM_FLAG(UFFD_MINOR) 495 #else 496 #define VM_UFFD_MINOR VM_NONE 497 #endif 498 #ifdef CONFIG_USERFAULTFD_RWP 499 #define VM_UFFD_RWP INIT_VM_FLAG(UFFD_RWP) 500 #else 501 #define VM_UFFD_RWP VM_NONE 502 #endif 503 504 /* 505 * vma_flags_t masks for the userfaultfd VMA flags. The two high-bit modes are 506 * gated on the same configs as their VM_* flags above -- both of which imply 507 * 64BIT -- so an out-of-range bit is never fed to mk_vma_flags() on a build 508 * whose bitmap cannot hold it. 509 */ 510 #define VMA_UFFD_MISSING mk_vma_flags(VMA_UFFD_MISSING_BIT) 511 #define VMA_UFFD_WP mk_vma_flags(VMA_UFFD_WP_BIT) 512 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR 513 #define VMA_UFFD_MINOR mk_vma_flags(VMA_UFFD_MINOR_BIT) 514 #else 515 #define VMA_UFFD_MINOR EMPTY_VMA_FLAGS 516 #endif 517 #ifdef CONFIG_USERFAULTFD_RWP 518 #define VMA_UFFD_RWP mk_vma_flags(VMA_UFFD_RWP_BIT) 519 #else 520 #define VMA_UFFD_RWP EMPTY_VMA_FLAGS 521 #endif 522 523 #ifdef CONFIG_64BIT 524 #define VM_ALLOW_ANY_UNCACHED INIT_VM_FLAG(ALLOW_ANY_UNCACHED) 525 #define VM_SEALED INIT_VM_FLAG(SEALED) 526 #else 527 #define VM_ALLOW_ANY_UNCACHED VM_NONE 528 #define VM_SEALED VM_NONE 529 #endif 530 #if defined(CONFIG_64BIT) || defined(CONFIG_PPC32) 531 #define VM_DROPPABLE INIT_VM_FLAG(DROPPABLE) 532 #define VMA_DROPPABLE mk_vma_flags(VMA_DROPPABLE_BIT) 533 #else 534 #define VM_DROPPABLE VM_NONE 535 #define VMA_DROPPABLE EMPTY_VMA_FLAGS 536 #endif 537 538 /* Bits set in the VMA until the stack is in its final location */ 539 #define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY) 540 #define VMA_STACK_INCOMPLETE_SETUP append_vma_flags( \ 541 VMA_STACK_EARLY, VMA_RAND_READ_BIT, VMA_SEQ_READ_BIT) 542 543 #define TASK_EXEC_BIT ((current->personality & READ_IMPLIES_EXEC) ? \ 544 VMA_EXEC_BIT : VMA_READ_BIT) 545 546 /* Common data flag combinations */ 547 #define VMA_DATA_FLAGS_TSK_EXEC mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, \ 548 TASK_EXEC_BIT, VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, \ 549 VMA_MAYEXEC_BIT) 550 #define VMA_DATA_FLAGS_NON_EXEC mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, \ 551 VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, VMA_MAYEXEC_BIT) 552 #define VMA_DATA_FLAGS_EXEC mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, \ 553 VMA_EXEC_BIT, VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, \ 554 VMA_MAYEXEC_BIT) 555 556 #ifndef VMA_DATA_DEFAULT_FLAGS /* arch can override this */ 557 #define VMA_DATA_DEFAULT_FLAGS VMA_DATA_FLAGS_EXEC 558 #endif 559 560 #ifndef VMA_STACK_DEFAULT_FLAGS /* arch can override this */ 561 #define VMA_STACK_DEFAULT_FLAGS VMA_DATA_DEFAULT_FLAGS 562 #endif 563 564 #define VMA_STACK_FLAGS append_vma_flags(VMA_STACK_DEFAULT_FLAGS, \ 565 VMA_STACK_BIT, VMA_ACCOUNT_BIT) 566 567 /* Temporary until VMA flags conversion complete. */ 568 #define VM_STACK_FLAGS vma_flags_to_legacy(VMA_STACK_FLAGS) 569 570 #ifdef CONFIG_MSEAL_SYSTEM_MAPPINGS 571 #define VM_SEALED_SYSMAP VM_SEALED 572 #else 573 #define VM_SEALED_SYSMAP VM_NONE 574 #endif 575 576 /* VMA basic access permission flags */ 577 #define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC) 578 #define VMA_ACCESS_FLAGS mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT) 579 580 /* 581 * Special vmas that are non-mergable, non-mlock()able. 582 */ 583 584 #define VMA_SPECIAL_FLAGS mk_vma_flags(VMA_IO_BIT, VMA_DONTEXPAND_BIT, \ 585 VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT) 586 #define VM_SPECIAL vma_flags_to_legacy(VMA_SPECIAL_FLAGS) 587 588 /* 589 * Physically remapped pages are special. Tell the 590 * rest of the world about it: 591 * IO tells people not to look at these pages 592 * (accesses can have side effects). 593 * PFNMAP tells the core MM that the base pages are just 594 * raw PFN mappings, and do not have a "struct page" associated 595 * with them. 596 * DONTEXPAND 597 * Disable vma merging and expanding with mremap(). 598 * DONTDUMP 599 * Omit vma from core dump, even when VM_IO turned off. 600 */ 601 #define VMA_REMAP_FLAGS mk_vma_flags(VMA_IO_BIT, VMA_PFNMAP_BIT, \ 602 VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT) 603 604 /* This mask prevents VMA from being scanned with khugepaged */ 605 #define VM_NO_KHUGEPAGED (VM_SPECIAL | VM_HUGETLB) 606 607 /* This mask defines which mm->def_flags a process can inherit its parent */ 608 #define VM_INIT_DEF_MASK VM_NOHUGEPAGE 609 610 /* This mask represents all the VMA flag bits used by mlock */ 611 #define VM_LOCKED_MASK (VM_LOCKED | VM_LOCKONFAULT) 612 613 #define VMA_LOCKED_MASK mk_vma_flags(VMA_LOCKED_BIT, VMA_LOCKONFAULT_BIT) 614 615 /* These flags can be updated atomically via VMA/mmap read lock. */ 616 #define VM_ATOMIC_SET_ALLOWED VM_MAYBE_GUARD 617 618 /* Arch-specific flags to clear when updating VM flags on protection change */ 619 #ifndef VM_ARCH_CLEAR 620 #define VM_ARCH_CLEAR VM_NONE 621 #endif 622 #define VM_FLAGS_CLEAR (ARCH_VM_PKEY_FLAGS | VM_ARCH_CLEAR) 623 624 /* 625 * Flags which should be 'sticky' on merge - that is, flags which, when one VMA 626 * possesses it but the other does not, the merged VMA should nonetheless have 627 * applied to it: 628 * 629 * VMA_SOFTDIRTY_BIT - if a VMA is marked soft-dirty, that is has not had its 630 * references cleared via /proc/$pid/clear_refs, any 631 * merged VMA should be considered soft-dirty also as it 632 * operates at a VMA granularity. 633 * 634 * VMA_MAYBE_GUARD_BIT - If a VMA may have guard regions in place it implies 635 * that mapped page tables may contain metadata not 636 * described by the VMA and thus any merged VMA may also 637 * contain this metadata, and thus we must make this flag 638 * sticky. 639 */ 640 #ifdef CONFIG_MEM_SOFT_DIRTY 641 #define VMA_STICKY_FLAGS mk_vma_flags(VMA_SOFTDIRTY_BIT, VMA_MAYBE_GUARD_BIT) 642 #else 643 #define VMA_STICKY_FLAGS mk_vma_flags(VMA_MAYBE_GUARD_BIT) 644 #endif 645 646 /* 647 * VMA flags we ignore for the purposes of merge, i.e. one VMA possessing one 648 * of these flags and the other not does not preclude a merge. 649 * 650 * VMA_STICKY_FLAGS - When merging VMAs, VMA flags must match, unless they 651 * are 'sticky'. If any sticky flags exist in either VMA, 652 * we simply set all of them on the merged VMA. 653 */ 654 #define VMA_IGNORE_MERGE_FLAGS VMA_STICKY_FLAGS 655 656 /* 657 * Flags which should result in page tables being copied on fork. These are 658 * flags which indicate that the VMA maps page tables which cannot be 659 * reconsistuted upon page fault, so necessitate page table copying upon fork. 660 * 661 * Note that these flags should be compared with the DESTINATION VMA not the 662 * source: VM_UFFD_WP and VM_UFFD_RWP may be cleared on the destination 663 * (dup_userfaultfd() -> userfaultfd_reset_ctx() when the parent context did 664 * not negotiate UFFD_FEATURE_EVENT_FORK), while all other flags propagate. 665 * 666 * VM_PFNMAP / VM_MIXEDMAP - These contain kernel-mapped data which cannot be 667 * reasonably reconstructed on page fault. 668 * 669 * VM_UFFD_WP - Encodes metadata about an installed uffd 670 * VM_UFFD_RWP write- or read-write-protect handler, which 671 * cannot be reconstructed on page fault. 672 * 673 * We always copy pgtables when dst_vma has the 674 * uffd PTE bit in use even if it's file-backed 675 * (e.g. shmem). Because when the uffd bit is 676 * in use, the pgtable contains the protection 677 * information, that's something we can't 678 * retrieve from page cache, and skip copying 679 * will lose those info. 680 * 681 * VM_MAYBE_GUARD - Could contain page guard region markers which 682 * by design are a property of the page tables 683 * only and thus cannot be reconstructed on page 684 * fault. 685 */ 686 #define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_UFFD_RWP | \ 687 VM_MAYBE_GUARD) 688 689 /* 690 * mapping from the currently active vm_flags protection bits (the 691 * low four bits) to a page protection mask.. 692 */ 693 694 /* 695 * The default fault flags that should be used by most of the 696 * arch-specific page fault handlers. 697 */ 698 #define FAULT_FLAG_DEFAULT (FAULT_FLAG_ALLOW_RETRY | \ 699 FAULT_FLAG_KILLABLE | \ 700 FAULT_FLAG_INTERRUPTIBLE) 701 702 /** 703 * fault_flag_allow_retry_first - check ALLOW_RETRY the first time 704 * @flags: Fault flags. 705 * 706 * This is mostly used for places where we want to try to avoid taking 707 * the mmap_lock for too long a time when waiting for another condition 708 * to change, in which case we can try to be polite to release the 709 * mmap_lock in the first round to avoid potential starvation of other 710 * processes that would also want the mmap_lock. 711 * 712 * Return: true if the page fault allows retry and this is the first 713 * attempt of the fault handling; false otherwise. 714 */ 715 static inline bool fault_flag_allow_retry_first(enum fault_flag flags) 716 { 717 return (flags & FAULT_FLAG_ALLOW_RETRY) && 718 (!(flags & FAULT_FLAG_TRIED)); 719 } 720 721 #define FAULT_FLAG_TRACE \ 722 { FAULT_FLAG_WRITE, "WRITE" }, \ 723 { FAULT_FLAG_MKWRITE, "MKWRITE" }, \ 724 { FAULT_FLAG_ALLOW_RETRY, "ALLOW_RETRY" }, \ 725 { FAULT_FLAG_RETRY_NOWAIT, "RETRY_NOWAIT" }, \ 726 { FAULT_FLAG_KILLABLE, "KILLABLE" }, \ 727 { FAULT_FLAG_TRIED, "TRIED" }, \ 728 { FAULT_FLAG_USER, "USER" }, \ 729 { FAULT_FLAG_REMOTE, "REMOTE" }, \ 730 { FAULT_FLAG_INSTRUCTION, "INSTRUCTION" }, \ 731 { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }, \ 732 { FAULT_FLAG_VMA_LOCK, "VMA_LOCK" } 733 734 /* 735 * vm_fault is filled by the pagefault handler and passed to the vma's 736 * ->fault function. The vma's ->fault is responsible for returning a bitmask 737 * of VM_FAULT_xxx flags that give details about how the fault was handled. 738 * 739 * MM layer fills up gfp_mask for page allocations but fault handler might 740 * alter it if its implementation requires a different allocation context. 741 * 742 * pgoff should be used in favour of virtual_address, if possible. 743 */ 744 struct vm_fault { 745 const struct { 746 struct vm_area_struct *vma; /* Target VMA */ 747 gfp_t gfp_mask; /* gfp mask to be used for allocations */ 748 pgoff_t pgoff; /* Logical page offset based on vma */ 749 unsigned long address; /* Faulting virtual address - masked */ 750 unsigned long real_address; /* Faulting virtual address - unmasked */ 751 }; 752 enum fault_flag flags; /* FAULT_FLAG_xxx flags 753 * XXX: should really be 'const' */ 754 pmd_t *pmd; /* Pointer to pmd entry matching 755 * the 'address' */ 756 pud_t *pud; /* Pointer to pud entry matching 757 * the 'address' 758 */ 759 union { 760 pte_t orig_pte; /* Value of PTE at the time of fault */ 761 pmd_t orig_pmd; /* Value of PMD at the time of fault, 762 * used by PMD fault only. 763 */ 764 }; 765 766 struct page *cow_page; /* Page handler may use for COW fault */ 767 struct page *page; /* ->fault handlers should return a 768 * page here, unless VM_FAULT_NOPAGE 769 * is set (which is also implied by 770 * VM_FAULT_ERROR). 771 */ 772 /* These three entries are valid only while holding ptl lock */ 773 pte_t *pte; /* Pointer to pte entry matching 774 * the 'address'. NULL if the page 775 * table hasn't been allocated. 776 */ 777 spinlock_t *ptl; /* Page table lock. 778 * Protects pte page table if 'pte' 779 * is not NULL, otherwise pmd. 780 */ 781 pgtable_t prealloc_pte; /* Pre-allocated pte page table. 782 * vm_ops->map_pages() sets up a page 783 * table from atomic context. 784 * do_fault_around() pre-allocates 785 * page table to avoid allocation from 786 * atomic context. 787 */ 788 }; 789 790 struct vm_uffd_ops; 791 792 /* 793 * These are the virtual MM functions - opening of an area, closing and 794 * unmapping it (needed to keep files on disk up-to-date etc), pointer 795 * to the functions called when a no-page or a wp-page exception occurs. 796 */ 797 struct vm_operations_struct { 798 /** 799 * @open: Called when a VMA is remapped, split or forked. Not called 800 * upon first mapping a VMA. 801 * Context: User context. May sleep. Caller holds mmap_lock. 802 */ 803 void (*open)(struct vm_area_struct *vma); 804 /** 805 * @close: Called when the VMA is being removed from the MM. 806 * Context: User context. May sleep. Caller holds mmap_lock. 807 */ 808 void (*close)(struct vm_area_struct *vma); 809 /** 810 * @mapped: Called when the VMA is first mapped in the MM. Not called if 811 * the new VMA is merged with an adjacent VMA. 812 * 813 * The @vm_private_data field is an output field allowing the user to 814 * modify vma->vm_private_data as necessary. 815 * 816 * ONLY valid if set from f_op->mmap_prepare. Will result in an error if 817 * set from f_op->mmap. 818 * 819 * Returns %0 on success, or an error otherwise. On error, the VMA will 820 * be unmapped. 821 * 822 * Context: User context. May sleep. Caller holds mmap_lock. 823 */ 824 int (*mapped)(unsigned long start, unsigned long end, pgoff_t pgoff, 825 const struct file *file, void **vm_private_data); 826 /* Called any time before splitting to check if it's allowed */ 827 int (*may_split)(struct vm_area_struct *vma, unsigned long addr); 828 int (*mremap)(struct vm_area_struct *vma); 829 /* 830 * Called by mprotect() to make driver-specific permission 831 * checks before mprotect() is finalised. The VMA must not 832 * be modified. Returns 0 if mprotect() can proceed. 833 */ 834 int (*mprotect)(struct vm_area_struct *vma, unsigned long start, 835 unsigned long end, unsigned long newflags); 836 vm_fault_t (*fault)(struct vm_fault *vmf); 837 vm_fault_t (*huge_fault)(struct vm_fault *vmf, unsigned int order); 838 vm_fault_t (*map_pages)(struct vm_fault *vmf, 839 pgoff_t start_pgoff, pgoff_t end_pgoff); 840 unsigned long (*pagesize)(struct vm_area_struct *vma); 841 842 /* notification that a previously read-only page is about to become 843 * writable, if an error is returned it will cause a SIGBUS */ 844 vm_fault_t (*page_mkwrite)(struct vm_fault *vmf); 845 846 /* same as page_mkwrite when using VM_PFNMAP|VM_MIXEDMAP */ 847 vm_fault_t (*pfn_mkwrite)(struct vm_fault *vmf); 848 849 /* called by access_process_vm when get_user_pages() fails, typically 850 * for use by special VMAs. See also generic_access_phys() for a generic 851 * implementation useful for any iomem mapping. 852 */ 853 int (*access)(struct vm_area_struct *vma, unsigned long addr, 854 void *buf, int len, int write); 855 856 /* Called by the /proc/PID/maps code to ask the vma whether it 857 * has a special name. Returning non-NULL will also cause this 858 * vma to be dumped unconditionally. */ 859 const char *(*name)(struct vm_area_struct *vma); 860 861 #ifdef CONFIG_NUMA 862 /* 863 * set_policy() op must add a reference to any non-NULL @new mempolicy 864 * to hold the policy upon return. Caller should pass NULL @new to 865 * remove a policy and fall back to surrounding context--i.e. do not 866 * install a MPOL_DEFAULT policy, nor the task or system default 867 * mempolicy. 868 */ 869 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new); 870 871 /* 872 * get_policy() op must add reference [mpol_get()] to any policy at 873 * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure 874 * in mm/mempolicy.c will do this automatically. 875 * get_policy() must NOT add a ref if the policy at (vma,addr) is not 876 * marked as MPOL_SHARED. vma policies are protected by the mmap_lock. 877 * If no [shared/vma] mempolicy exists at the addr, get_policy() op 878 * must return NULL--i.e., do not "fallback" to task or system default 879 * policy. 880 */ 881 struct mempolicy *(*get_policy)(struct vm_area_struct *vma, 882 unsigned long addr, pgoff_t *ilx); 883 #endif 884 #ifdef CONFIG_FIND_NORMAL_PAGE 885 /* 886 * Called by vm_normal_page() for special PTEs in @vma at @addr. This 887 * allows for returning a "normal" page from vm_normal_page() even 888 * though the PTE indicates that the "struct page" either does not exist 889 * or should not be touched: "special". 890 * 891 * Do not add new users: this really only works when a "normal" page 892 * was mapped, but then the PTE got changed to something weird (+ 893 * marked special) that would not make pte_pfn() identify the originally 894 * inserted page. 895 */ 896 struct page *(*find_normal_page)(struct vm_area_struct *vma, 897 unsigned long addr); 898 #endif /* CONFIG_FIND_NORMAL_PAGE */ 899 #ifdef CONFIG_USERFAULTFD 900 const struct vm_uffd_ops *uffd_ops; 901 #endif 902 }; 903 904 #ifdef CONFIG_NUMA_BALANCING 905 static inline void vma_numab_state_init(struct vm_area_struct *vma) 906 { 907 vma->numab_state = NULL; 908 } 909 static inline void vma_numab_state_free(struct vm_area_struct *vma) 910 { 911 kfree(vma->numab_state); 912 } 913 #else 914 static inline void vma_numab_state_init(struct vm_area_struct *vma) {} 915 static inline void vma_numab_state_free(struct vm_area_struct *vma) {} 916 #endif /* CONFIG_NUMA_BALANCING */ 917 918 /* 919 * These must be here rather than mmap_lock.h as dependent on vm_fault type, 920 * declared in this header. 921 */ 922 static inline void release_fault_lock(struct vm_fault *vmf) 923 { 924 if (vmf->flags & FAULT_FLAG_VMA_LOCK) 925 vma_end_read(vmf->vma); 926 else 927 mmap_read_unlock(vmf->vma->vm_mm); 928 } 929 930 static inline void assert_fault_locked(const struct vm_fault *vmf) 931 { 932 if (vmf->flags & FAULT_FLAG_VMA_LOCK) 933 vma_assert_locked(vmf->vma); 934 else 935 mmap_assert_locked(vmf->vma->vm_mm); 936 } 937 938 static inline bool mm_flags_test(int flag, const struct mm_struct *mm) 939 { 940 return test_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags)); 941 } 942 943 static inline bool mm_flags_test_and_set(int flag, struct mm_struct *mm) 944 { 945 return test_and_set_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags)); 946 } 947 948 static inline bool mm_flags_test_and_clear(int flag, struct mm_struct *mm) 949 { 950 return test_and_clear_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags)); 951 } 952 953 static inline void mm_flags_set(int flag, struct mm_struct *mm) 954 { 955 set_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags)); 956 } 957 958 static inline void mm_flags_clear(int flag, struct mm_struct *mm) 959 { 960 clear_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags)); 961 } 962 963 static inline void mm_flags_clear_all(struct mm_struct *mm) 964 { 965 bitmap_zero(ACCESS_PRIVATE(&mm->flags, __mm_flags), NUM_MM_FLAG_BITS); 966 } 967 968 extern const struct vm_operations_struct vma_dummy_vm_ops; 969 970 static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm) 971 { 972 memset(vma, 0, sizeof(*vma)); 973 vma->vm_mm = mm; 974 vma->vm_ops = &vma_dummy_vm_ops; 975 INIT_LIST_HEAD(&vma->anon_vma_chain); 976 vma_lock_init(vma, false); 977 } 978 979 /* Use when VMA is not part of the VMA tree and needs no locking */ 980 static inline void vm_flags_init(struct vm_area_struct *vma, 981 vm_flags_t flags) 982 { 983 VM_WARN_ON_ONCE(!pgtable_supports_soft_dirty() && (flags & VM_SOFTDIRTY)); 984 vma_flags_clear_all(&vma->flags); 985 vma_flags_overwrite_word(&vma->flags, flags); 986 } 987 988 /* 989 * Use when VMA is part of the VMA tree and modifications need coordination 990 * Note: vm_flags_reset and vm_flags_reset_once do not lock the vma and 991 * it should be locked explicitly beforehand. 992 */ 993 static inline void vm_flags_reset(struct vm_area_struct *vma, 994 vm_flags_t flags) 995 { 996 VM_WARN_ON_ONCE(!pgtable_supports_soft_dirty() && (flags & VM_SOFTDIRTY)); 997 vma_assert_write_locked(vma); 998 vm_flags_init(vma, flags); 999 } 1000 1001 static inline void vma_flags_reset_once(struct vm_area_struct *vma, 1002 vma_flags_t *flags) 1003 { 1004 const unsigned long word = flags->__vma_flags[0]; 1005 1006 /* It is assumed only the first system word must be written once. */ 1007 vma_flags_overwrite_word_once(&vma->flags, word); 1008 /* The remainder can be copied normally. */ 1009 if (NUM_VMA_FLAG_BITS > BITS_PER_LONG) { 1010 unsigned long *dst = &vma->flags.__vma_flags[1]; 1011 const unsigned long *src = &flags->__vma_flags[1]; 1012 1013 bitmap_copy(dst, src, NUM_VMA_FLAG_BITS - BITS_PER_LONG); 1014 } 1015 } 1016 1017 static inline void vm_flags_set(struct vm_area_struct *vma, 1018 vm_flags_t flags) 1019 { 1020 vma_start_write(vma); 1021 vma_flags_set_word(&vma->flags, flags); 1022 } 1023 1024 static inline void vm_flags_clear(struct vm_area_struct *vma, 1025 vm_flags_t flags) 1026 { 1027 VM_WARN_ON_ONCE(!pgtable_supports_soft_dirty() && (flags & VM_SOFTDIRTY)); 1028 vma_start_write(vma); 1029 vma_flags_clear_word(&vma->flags, flags); 1030 } 1031 1032 /* 1033 * Use only if VMA is not part of the VMA tree or has no other users and 1034 * therefore needs no locking. 1035 */ 1036 static inline void __vm_flags_mod(struct vm_area_struct *vma, 1037 vm_flags_t set, vm_flags_t clear) 1038 { 1039 vm_flags_init(vma, (vma->vm_flags | set) & ~clear); 1040 } 1041 1042 /* 1043 * Use only when the order of set/clear operations is unimportant, otherwise 1044 * use vm_flags_{set|clear} explicitly. 1045 */ 1046 static inline void vm_flags_mod(struct vm_area_struct *vma, 1047 vm_flags_t set, vm_flags_t clear) 1048 { 1049 vma_start_write(vma); 1050 __vm_flags_mod(vma, set, clear); 1051 } 1052 1053 static __always_inline bool __vma_atomic_valid_flag(struct vm_area_struct *vma, 1054 vma_flag_t bit) 1055 { 1056 const vm_flags_t mask = BIT((__force int)bit); 1057 1058 /* Only specific flags are permitted */ 1059 if (WARN_ON_ONCE(!(mask & VM_ATOMIC_SET_ALLOWED))) 1060 return false; 1061 1062 return true; 1063 } 1064 1065 /* 1066 * Set VMA flag atomically. Requires only VMA/mmap read lock. Only specific 1067 * valid flags are allowed to do this. 1068 */ 1069 static __always_inline void vma_set_atomic_flag(struct vm_area_struct *vma, 1070 vma_flag_t bit) 1071 { 1072 unsigned long *bitmap = vma->flags.__vma_flags; 1073 1074 vma_assert_stabilised(vma); 1075 if (__vma_atomic_valid_flag(vma, bit)) 1076 set_bit((__force int)bit, bitmap); 1077 } 1078 1079 /* 1080 * Test for VMA flag atomically. Requires no locks. Only specific valid flags 1081 * are allowed to do this. 1082 * 1083 * This is necessarily racey, so callers must ensure that serialisation is 1084 * achieved through some other means, or that races are permissible. 1085 */ 1086 static __always_inline bool vma_test_atomic_flag(struct vm_area_struct *vma, 1087 vma_flag_t bit) 1088 { 1089 if (__vma_atomic_valid_flag(vma, bit)) 1090 return test_bit((__force int)bit, &vma->vm_flags); 1091 1092 return false; 1093 } 1094 1095 /* Set an individual VMA flag in flags, non-atomically. */ 1096 static __always_inline void vma_flags_set_flag(vma_flags_t *flags, 1097 vma_flag_t bit) 1098 { 1099 unsigned long *bitmap = flags->__vma_flags; 1100 1101 __set_bit((__force int)bit, bitmap); 1102 } 1103 1104 static __always_inline vma_flags_t __mk_vma_flags(vma_flags_t flags, 1105 size_t count, const vma_flag_t *bits) 1106 { 1107 int i; 1108 1109 for (i = 0; i < count; i++) 1110 vma_flags_set_flag(&flags, bits[i]); 1111 return flags; 1112 } 1113 1114 /* 1115 * Helper macro which bitwise-or combines the specified input flags into a 1116 * vma_flags_t bitmap value. E.g.: 1117 * 1118 * vma_flags_t flags = mk_vma_flags(VMA_IO_BIT, VMA_PFNMAP_BIT, 1119 * VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT); 1120 * 1121 * The compiler cleverly optimises away all of the work and this ends up being 1122 * equivalent to aggregating the values manually. 1123 */ 1124 #define mk_vma_flags(...) __mk_vma_flags(EMPTY_VMA_FLAGS, \ 1125 COUNT_ARGS(__VA_ARGS__), (const vma_flag_t []){__VA_ARGS__}) 1126 1127 /* 1128 * Helper macro which acts like mk_vma_flags, only appending to a copy of the 1129 * specified flags rather than establishing new flags. E.g.: 1130 * 1131 * vma_flags_t flags = append_vma_flags(VMA_STACK_DEFAULT_FLAGS, VMA_STACK_BIT, 1132 * VMA_ACCOUNT_BIT); 1133 */ 1134 #define append_vma_flags(flags, ...) __mk_vma_flags(flags, \ 1135 COUNT_ARGS(__VA_ARGS__), (const vma_flag_t []){__VA_ARGS__}) 1136 1137 /* Calculates the number of set bits in the specified VMA flags. */ 1138 static __always_inline int vma_flags_count(const vma_flags_t *flags) 1139 { 1140 const unsigned long *bitmap = flags->__vma_flags; 1141 1142 return bitmap_weight(bitmap, NUM_VMA_FLAG_BITS); 1143 } 1144 1145 /* 1146 * Test whether a specific VMA flag is set, e.g.: 1147 * 1148 * if (vma_flags_test(flags, VMA_READ_BIT)) { ... } 1149 */ 1150 static __always_inline bool vma_flags_test(const vma_flags_t *flags, 1151 vma_flag_t bit) 1152 { 1153 const unsigned long *bitmap = flags->__vma_flags; 1154 1155 return test_bit((__force int)bit, bitmap); 1156 } 1157 1158 /* 1159 * Obtain a set of VMA flags which contain the overlapping flags contained 1160 * within flags and to_and. 1161 */ 1162 static __always_inline vma_flags_t vma_flags_and_mask(const vma_flags_t *flags, 1163 vma_flags_t to_and) 1164 { 1165 vma_flags_t dst; 1166 unsigned long *bitmap_dst = dst.__vma_flags; 1167 const unsigned long *bitmap = flags->__vma_flags; 1168 const unsigned long *bitmap_to_and = to_and.__vma_flags; 1169 1170 bitmap_and(bitmap_dst, bitmap, bitmap_to_and, NUM_VMA_FLAG_BITS); 1171 return dst; 1172 } 1173 1174 /* 1175 * Obtain a set of VMA flags which contains the specified overlapping flags, 1176 * e.g.: 1177 * 1178 * vma_flags_t read_flags = vma_flags_and(&flags, VMA_READ_BIT, 1179 * VMA_MAY_READ_BIT); 1180 */ 1181 #define vma_flags_and(flags, ...) \ 1182 vma_flags_and_mask(flags, mk_vma_flags(__VA_ARGS__)) 1183 1184 /* Test each of to_test flags in flags, non-atomically. */ 1185 static __always_inline bool vma_flags_test_any_mask(const vma_flags_t *flags, 1186 vma_flags_t to_test) 1187 { 1188 const unsigned long *bitmap = flags->__vma_flags; 1189 const unsigned long *bitmap_to_test = to_test.__vma_flags; 1190 1191 return bitmap_intersects(bitmap_to_test, bitmap, NUM_VMA_FLAG_BITS); 1192 } 1193 1194 /* 1195 * Test whether any specified VMA flag is set, e.g.: 1196 * 1197 * if (vma_flags_test_any(flags, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... } 1198 */ 1199 #define vma_flags_test_any(flags, ...) \ 1200 vma_flags_test_any_mask(flags, mk_vma_flags(__VA_ARGS__)) 1201 1202 /* Test that ALL of the to_test flags are set, non-atomically. */ 1203 static __always_inline bool vma_flags_test_all_mask(const vma_flags_t *flags, 1204 vma_flags_t to_test) 1205 { 1206 const unsigned long *bitmap = flags->__vma_flags; 1207 const unsigned long *bitmap_to_test = to_test.__vma_flags; 1208 1209 return bitmap_subset(bitmap_to_test, bitmap, NUM_VMA_FLAG_BITS); 1210 } 1211 1212 /* 1213 * Test whether ALL specified VMA flags are set, e.g.: 1214 * 1215 * if (vma_flags_test_all(flags, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... } 1216 */ 1217 #define vma_flags_test_all(flags, ...) \ 1218 vma_flags_test_all_mask(flags, mk_vma_flags(__VA_ARGS__)) 1219 1220 /* 1221 * Helper to test that a flag mask of type vma_flags_t has a SINGLE flag set 1222 * (returning false if flagmask has no flags set). 1223 * 1224 * This is defined to make the semantics clearer when testing an optionally 1225 * defined VMA flags mask, e.g.: 1226 * 1227 * if (vma_flags_test_single_mask(&flags, VMA_DROPPABLE)) { ... } 1228 * 1229 * When VMA_DROPPABLE is defined if available, or set to EMPTY_VMA_FLAGS 1230 * otherwise. 1231 */ 1232 static __always_inline bool vma_flags_test_single_mask(const vma_flags_t *flags, 1233 vma_flags_t flagmask) 1234 { 1235 VM_WARN_ON_ONCE(vma_flags_count(&flagmask) > 1); 1236 1237 return vma_flags_test_any_mask(flags, flagmask); 1238 } 1239 1240 /* Set each of the to_set flags in flags, non-atomically. */ 1241 static __always_inline void vma_flags_set_mask(vma_flags_t *flags, 1242 vma_flags_t to_set) 1243 { 1244 unsigned long *bitmap = flags->__vma_flags; 1245 const unsigned long *bitmap_to_set = to_set.__vma_flags; 1246 1247 bitmap_or(bitmap, bitmap, bitmap_to_set, NUM_VMA_FLAG_BITS); 1248 } 1249 1250 /* 1251 * Set all specified VMA flags, e.g.: 1252 * 1253 * vma_flags_set(&flags, VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT); 1254 */ 1255 #define vma_flags_set(flags, ...) \ 1256 vma_flags_set_mask(flags, mk_vma_flags(__VA_ARGS__)) 1257 1258 static __always_inline vma_flags_t __mk_vma_flags_from_masks(size_t count, 1259 const vma_flags_t *masks) 1260 { 1261 vma_flags_t flags = EMPTY_VMA_FLAGS; 1262 size_t i; 1263 1264 for (i = 0; i < count; i++) 1265 vma_flags_set_mask(&flags, masks[i]); 1266 return flags; 1267 } 1268 1269 /* 1270 * Combine pre-computed vma_flags_t masks into one value, e.g.: 1271 * 1272 * vma_flags_t flags = mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_MINOR); 1273 * 1274 * Unlike mk_vma_flags(), which takes bit numbers, this takes whole masks -- 1275 * each of which may be EMPTY_VMA_FLAGS when its feature is unavailable -- so a 1276 * bit that does not exist on the current build is never materialised. 1277 */ 1278 #define mk_vma_flags_from_masks(...) \ 1279 __mk_vma_flags_from_masks(COUNT_ARGS(__VA_ARGS__), \ 1280 (const vma_flags_t []){__VA_ARGS__}) 1281 1282 /* Clear all of the to-clear flags in flags, non-atomically. */ 1283 static __always_inline void vma_flags_clear_mask(vma_flags_t *flags, 1284 vma_flags_t to_clear) 1285 { 1286 unsigned long *bitmap = flags->__vma_flags; 1287 const unsigned long *bitmap_to_clear = to_clear.__vma_flags; 1288 1289 bitmap_andnot(bitmap, bitmap, bitmap_to_clear, NUM_VMA_FLAG_BITS); 1290 } 1291 1292 /* 1293 * Clear all specified individual flags, e.g.: 1294 * 1295 * vma_flags_clear(&flags, VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT); 1296 */ 1297 #define vma_flags_clear(flags, ...) \ 1298 vma_flags_clear_mask(flags, mk_vma_flags(__VA_ARGS__)) 1299 1300 /* 1301 * Obtain a VMA flags value containing those flags that are present in flags or 1302 * flags_other but not in both. 1303 */ 1304 static __always_inline vma_flags_t vma_flags_diff_pair(const vma_flags_t *flags, 1305 const vma_flags_t *flags_other) 1306 { 1307 vma_flags_t dst; 1308 const unsigned long *bitmap_other = flags_other->__vma_flags; 1309 const unsigned long *bitmap = flags->__vma_flags; 1310 unsigned long *bitmap_dst = dst.__vma_flags; 1311 1312 bitmap_xor(bitmap_dst, bitmap, bitmap_other, NUM_VMA_FLAG_BITS); 1313 return dst; 1314 } 1315 1316 /* Determine if flags and flags_other have precisely the same flags set. */ 1317 static __always_inline bool vma_flags_same_pair(const vma_flags_t *flags, 1318 const vma_flags_t *flags_other) 1319 { 1320 const unsigned long *bitmap = flags->__vma_flags; 1321 const unsigned long *bitmap_other = flags_other->__vma_flags; 1322 1323 return bitmap_equal(bitmap, bitmap_other, NUM_VMA_FLAG_BITS); 1324 } 1325 1326 /* Determine if flags and flags_other have precisely the same flags set. */ 1327 static __always_inline bool vma_flags_same_mask(const vma_flags_t *flags, 1328 vma_flags_t flags_other) 1329 { 1330 const unsigned long *bitmap = flags->__vma_flags; 1331 const unsigned long *bitmap_other = flags_other.__vma_flags; 1332 1333 return bitmap_equal(bitmap, bitmap_other, NUM_VMA_FLAG_BITS); 1334 } 1335 1336 /* 1337 * Helper macro to determine if only the specific flags are set, e.g.: 1338 * 1339 * if (vma_flags_same(&flags, VMA_WRITE_BIT) { ... } 1340 */ 1341 #define vma_flags_same(flags, ...) \ 1342 vma_flags_same_mask(flags, mk_vma_flags(__VA_ARGS__)) 1343 1344 /* 1345 * Test whether a specific flag in the VMA is set, e.g.: 1346 * 1347 * if (vma_test(vma, VMA_READ_BIT)) { ... } 1348 */ 1349 static __always_inline bool vma_test(const struct vm_area_struct *vma, 1350 vma_flag_t bit) 1351 { 1352 return vma_flags_test(&vma->flags, bit); 1353 } 1354 1355 /* Helper to test any VMA flags in a VMA . */ 1356 static __always_inline bool vma_test_any_mask(const struct vm_area_struct *vma, 1357 vma_flags_t flags) 1358 { 1359 return vma_flags_test_any_mask(&vma->flags, flags); 1360 } 1361 1362 /* 1363 * Helper macro for testing whether any VMA flags are set in a VMA, 1364 * e.g.: 1365 * 1366 * if (vma_test_any(vma, VMA_IO_BIT, VMA_PFNMAP_BIT, 1367 * VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT)) { ... } 1368 */ 1369 #define vma_test_any(vma, ...) \ 1370 vma_test_any_mask(vma, mk_vma_flags(__VA_ARGS__)) 1371 1372 /* 1373 * Helper to test that ALL specified flags are set in a VMA. 1374 * 1375 * Note: appropriate locks must be held, this function does not acquire them for 1376 * you. 1377 */ 1378 static __always_inline bool vma_test_all_mask(const struct vm_area_struct *vma, 1379 vma_flags_t flags) 1380 { 1381 return vma_flags_test_all_mask(&vma->flags, flags); 1382 } 1383 1384 /* 1385 * Helper macro for checking that ALL specified flags are set in a VMA, e.g.: 1386 * 1387 * if (vma_test_all(vma, VMA_READ_BIT, VMA_MAYREAD_BIT) { ... } 1388 */ 1389 #define vma_test_all(vma, ...) \ 1390 vma_test_all_mask(vma, mk_vma_flags(__VA_ARGS__)) 1391 1392 /* 1393 * Helper to test that a flag mask of type vma_flags_t has a SINGLE flag set 1394 * (returning false if flagmask has no flags set). 1395 * 1396 * This is useful when a flag needs to be either defined or not depending upon 1397 * kernel configuration, e.g.: 1398 * 1399 * if (vma_test_single_mask(vma, VMA_DROPPABLE)) { ... } 1400 * 1401 * When VMA_DROPPABLE is defined if available, or set to EMPTY_VMA_FLAGS 1402 * otherwise. 1403 */ 1404 static __always_inline bool 1405 vma_test_single_mask(const struct vm_area_struct *vma, vma_flags_t flagmask) 1406 { 1407 return vma_flags_test_single_mask(&vma->flags, flagmask); 1408 } 1409 1410 /* 1411 * Helper to set all VMA flags in a VMA. 1412 * 1413 * Note: appropriate locks must be held, this function does not acquire them for 1414 * you. 1415 */ 1416 static __always_inline void vma_set_flags_mask(struct vm_area_struct *vma, 1417 vma_flags_t flags) 1418 { 1419 vma_flags_set_mask(&vma->flags, flags); 1420 } 1421 1422 /* 1423 * Helper macro for specifying VMA flags in a VMA, e.g.: 1424 * 1425 * vma_set_flags(vma, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT, 1426 * VMA_DONTDUMP_BIT); 1427 * 1428 * Note: appropriate locks must be held, this function does not acquire them for 1429 * you. 1430 */ 1431 #define vma_set_flags(vma, ...) \ 1432 vma_set_flags_mask(vma, mk_vma_flags(__VA_ARGS__)) 1433 1434 /* Helper to clear all VMA flags in a VMA. */ 1435 static __always_inline void vma_clear_flags_mask(struct vm_area_struct *vma, 1436 vma_flags_t flags) 1437 { 1438 vma_flags_clear_mask(&vma->flags, flags); 1439 } 1440 1441 /* 1442 * Helper macro for clearing VMA flags, e.g.: 1443 * 1444 * vma_clear_flags(vma, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT, 1445 * VMA_DONTDUMP_BIT); 1446 */ 1447 #define vma_clear_flags(vma, ...) \ 1448 vma_clear_flags_mask(vma, mk_vma_flags(__VA_ARGS__)) 1449 1450 /* 1451 * Test whether a specific VMA flag is set in a VMA descriptor, e.g.: 1452 * 1453 * if (vma_desc_test(desc, VMA_READ_BIT)) { ... } 1454 */ 1455 static __always_inline bool vma_desc_test(const struct vm_area_desc *desc, 1456 vma_flag_t bit) 1457 { 1458 return vma_flags_test(&desc->vma_flags, bit); 1459 } 1460 1461 /* Helper to test any VMA flags in a VMA descriptor. */ 1462 static __always_inline bool vma_desc_test_any_mask(const struct vm_area_desc *desc, 1463 vma_flags_t flags) 1464 { 1465 return vma_flags_test_any_mask(&desc->vma_flags, flags); 1466 } 1467 1468 /* 1469 * Helper macro for testing whether any VMA flags are set in a VMA descriptor, 1470 * e.g.: 1471 * 1472 * if (vma_desc_test_any(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, 1473 * VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT)) { ... } 1474 */ 1475 #define vma_desc_test_any(desc, ...) \ 1476 vma_desc_test_any_mask(desc, mk_vma_flags(__VA_ARGS__)) 1477 1478 /* Helper to test all VMA flags in a VMA descriptor. */ 1479 static __always_inline bool vma_desc_test_all_mask(const struct vm_area_desc *desc, 1480 vma_flags_t flags) 1481 { 1482 return vma_flags_test_all_mask(&desc->vma_flags, flags); 1483 } 1484 1485 /* 1486 * Helper macro for testing whether ALL VMA flags are set in a VMA descriptor, 1487 * e.g.: 1488 * 1489 * if (vma_desc_test_all(desc, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... } 1490 */ 1491 #define vma_desc_test_all(desc, ...) \ 1492 vma_desc_test_all_mask(desc, mk_vma_flags(__VA_ARGS__)) 1493 1494 /* Helper to set all VMA flags in a VMA descriptor. */ 1495 static __always_inline void vma_desc_set_flags_mask(struct vm_area_desc *desc, 1496 vma_flags_t flags) 1497 { 1498 vma_flags_set_mask(&desc->vma_flags, flags); 1499 } 1500 1501 /* 1502 * Helper macro for specifying VMA flags for an input pointer to a struct 1503 * vm_area_desc object describing a proposed VMA, e.g.: 1504 * 1505 * vma_desc_set_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT, 1506 * VMA_DONTDUMP_BIT); 1507 */ 1508 #define vma_desc_set_flags(desc, ...) \ 1509 vma_desc_set_flags_mask(desc, mk_vma_flags(__VA_ARGS__)) 1510 1511 /* Helper to clear all VMA flags in a VMA descriptor. */ 1512 static __always_inline void vma_desc_clear_flags_mask(struct vm_area_desc *desc, 1513 vma_flags_t flags) 1514 { 1515 vma_flags_clear_mask(&desc->vma_flags, flags); 1516 } 1517 1518 /* 1519 * Helper macro for clearing VMA flags for an input pointer to a struct 1520 * vm_area_desc object describing a proposed VMA, e.g.: 1521 * 1522 * vma_desc_clear_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT, 1523 * VMA_DONTDUMP_BIT); 1524 */ 1525 #define vma_desc_clear_flags(desc, ...) \ 1526 vma_desc_clear_flags_mask(desc, mk_vma_flags(__VA_ARGS__)) 1527 1528 static inline void vma_set_anonymous(struct vm_area_struct *vma) 1529 { 1530 vma->vm_ops = NULL; 1531 } 1532 1533 static inline bool vma_is_anonymous(const struct vm_area_struct *vma) 1534 { 1535 return !vma->vm_ops; 1536 } 1537 1538 /* 1539 * Indicate if the VMA is a heap for the given task; for 1540 * /proc/PID/maps that is the heap of the main task. 1541 */ 1542 static inline bool vma_is_initial_heap(const struct vm_area_struct *vma) 1543 { 1544 return vma->vm_start < vma->vm_mm->brk && 1545 vma->vm_end > vma->vm_mm->start_brk; 1546 } 1547 1548 /* 1549 * Indicate if the VMA is a stack for the given task; for 1550 * /proc/PID/maps that is the stack of the main task. 1551 */ 1552 static inline bool vma_is_initial_stack(const struct vm_area_struct *vma) 1553 { 1554 /* 1555 * We make no effort to guess what a given thread considers to be 1556 * its "stack". It's not even well-defined for programs written 1557 * languages like Go. 1558 */ 1559 return vma->vm_start <= vma->vm_mm->start_stack && 1560 vma->vm_end >= vma->vm_mm->start_stack; 1561 } 1562 1563 static inline bool vma_flags_can_grow(const vma_flags_t *flags) 1564 { 1565 if (vma_flags_test_single_mask(flags, VMA_GROWSUP)) 1566 return true; 1567 if (vma_flags_test(flags, VMA_GROWSDOWN_BIT)) 1568 return true; 1569 1570 return false; 1571 } 1572 1573 static inline bool vma_can_grow(const struct vm_area_struct *vma) 1574 { 1575 return vma_flags_can_grow(&vma->flags); 1576 } 1577 1578 static inline bool vma_is_temporary_stack(const struct vm_area_struct *vma) 1579 { 1580 if (!vma_can_grow(vma)) 1581 return false; 1582 1583 if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == 1584 VM_STACK_INCOMPLETE_SETUP) 1585 return true; 1586 1587 return false; 1588 } 1589 1590 static inline bool vma_is_foreign(const struct vm_area_struct *vma) 1591 { 1592 if (!current->mm) 1593 return true; 1594 1595 if (current->mm != vma->vm_mm) 1596 return true; 1597 1598 return false; 1599 } 1600 1601 static inline bool vma_is_accessible(const struct vm_area_struct *vma) 1602 { 1603 return vma->vm_flags & VM_ACCESS_FLAGS; 1604 } 1605 1606 static inline bool is_shared_maywrite(const vma_flags_t *flags) 1607 { 1608 return vma_flags_test_all(flags, VMA_SHARED_BIT, VMA_MAYWRITE_BIT); 1609 } 1610 1611 static inline bool vma_is_shared_maywrite(const struct vm_area_struct *vma) 1612 { 1613 return is_shared_maywrite(&vma->flags); 1614 } 1615 1616 /** 1617 * vma_kernel_pagesize - Default page size granularity for this VMA. 1618 * @vma: The user mapping. 1619 * 1620 * The kernel page size specifies in which granularity VMA modifications 1621 * can be performed. Folios in this VMA will be aligned to, and at least 1622 * the size of the number of bytes returned by this function. 1623 * 1624 * The default kernel page size is not affected by Transparent Huge Pages 1625 * being in effect. 1626 * 1627 * Return: The default page size granularity for this VMA. 1628 */ 1629 static inline unsigned long vma_kernel_pagesize(struct vm_area_struct *vma) 1630 { 1631 if (unlikely(vma->vm_ops && vma->vm_ops->pagesize)) 1632 return vma->vm_ops->pagesize(vma); 1633 return PAGE_SIZE; 1634 } 1635 1636 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma); 1637 1638 static inline 1639 struct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max) 1640 { 1641 return mas_find(&vmi->mas, max - 1); 1642 } 1643 1644 static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi) 1645 { 1646 /* 1647 * Uses mas_find() to get the first VMA when the iterator starts. 1648 * Calling mas_next() could skip the first entry. 1649 */ 1650 return mas_find(&vmi->mas, ULONG_MAX); 1651 } 1652 1653 static inline 1654 struct vm_area_struct *vma_iter_next_range(struct vma_iterator *vmi) 1655 { 1656 return mas_next_range(&vmi->mas, ULONG_MAX); 1657 } 1658 1659 1660 static inline struct vm_area_struct *vma_prev(struct vma_iterator *vmi) 1661 { 1662 return mas_prev(&vmi->mas, 0); 1663 } 1664 1665 static inline int vma_iter_clear_gfp(struct vma_iterator *vmi, 1666 unsigned long start, unsigned long end, gfp_t gfp) 1667 { 1668 __mas_set_range(&vmi->mas, start, end - 1); 1669 mas_store_gfp(&vmi->mas, NULL, gfp); 1670 if (unlikely(mas_is_err(&vmi->mas))) 1671 return -ENOMEM; 1672 1673 return 0; 1674 } 1675 1676 /* Free any unused preallocations */ 1677 static inline void vma_iter_free(struct vma_iterator *vmi) 1678 { 1679 mas_destroy(&vmi->mas); 1680 } 1681 1682 static inline int vma_iter_bulk_store(struct vma_iterator *vmi, 1683 struct vm_area_struct *vma) 1684 { 1685 vmi->mas.index = vma->vm_start; 1686 vmi->mas.last = vma->vm_end - 1; 1687 mas_store(&vmi->mas, vma); 1688 if (unlikely(mas_is_err(&vmi->mas))) 1689 return -ENOMEM; 1690 1691 vma_mark_attached(vma); 1692 return 0; 1693 } 1694 1695 static inline void vma_iter_invalidate(struct vma_iterator *vmi) 1696 { 1697 mas_pause(&vmi->mas); 1698 } 1699 1700 static inline void vma_iter_set(struct vma_iterator *vmi, unsigned long addr) 1701 { 1702 mas_set(&vmi->mas, addr); 1703 } 1704 1705 #define for_each_vma(__vmi, __vma) \ 1706 while (((__vma) = vma_next(&(__vmi))) != NULL) 1707 1708 /* The MM code likes to work with exclusive end addresses */ 1709 #define for_each_vma_range(__vmi, __vma, __end) \ 1710 while (((__vma) = vma_find(&(__vmi), (__end))) != NULL) 1711 1712 #ifdef CONFIG_SHMEM 1713 /* 1714 * The vma_is_shmem is not inline because it is used only by slow 1715 * paths in userfault. 1716 */ 1717 bool vma_is_shmem(const struct vm_area_struct *vma); 1718 bool vma_is_anon_shmem(const struct vm_area_struct *vma); 1719 #else 1720 static inline bool vma_is_shmem(const struct vm_area_struct *vma) { return false; } 1721 static inline bool vma_is_anon_shmem(const struct vm_area_struct *vma) { return false; } 1722 #endif 1723 1724 int vma_is_stack_for_current(const struct vm_area_struct *vma); 1725 1726 /* flush_tlb_range() takes a vma, not a mm, and can care about flags */ 1727 #define TLB_FLUSH_VMA(mm,flags) { .vm_mm = (mm), .vm_flags = (flags) } 1728 1729 struct mmu_gather; 1730 struct inode; 1731 1732 extern void prep_compound_page(struct page *page, unsigned int order); 1733 1734 static inline unsigned int folio_large_order(const struct folio *folio) 1735 { 1736 return folio->_flags_1 & 0xff; 1737 } 1738 1739 #ifdef NR_PAGES_IN_LARGE_FOLIO 1740 static inline unsigned long folio_large_nr_pages(const struct folio *folio) 1741 { 1742 return folio->_nr_pages; 1743 } 1744 #else 1745 static inline unsigned long folio_large_nr_pages(const struct folio *folio) 1746 { 1747 return 1L << folio_large_order(folio); 1748 } 1749 #endif 1750 1751 /* 1752 * compound_order() can be called without holding a reference, which means 1753 * that niceties like page_folio() don't work. These callers should be 1754 * prepared to handle wild return values. For example, PG_head may be 1755 * set before the order is initialised, or this may be a tail page. 1756 * See compaction.c for some good examples. 1757 */ 1758 static inline unsigned int compound_order(const struct page *page) 1759 { 1760 const struct folio *folio = (struct folio *)page; 1761 1762 if (!test_bit(PG_head, &folio->flags.f)) 1763 return 0; 1764 return folio_large_order(folio); 1765 } 1766 1767 /** 1768 * folio_order - The allocation order of a folio. 1769 * @folio: The folio. 1770 * 1771 * A folio is composed of 2^order pages. See get_order() for the definition 1772 * of order. 1773 * 1774 * Return: The order of the folio. 1775 */ 1776 static inline unsigned int folio_order(const struct folio *folio) 1777 { 1778 if (!folio_test_large(folio)) 1779 return 0; 1780 return folio_large_order(folio); 1781 } 1782 1783 /** 1784 * folio_reset_order - Reset the folio order and derived _nr_pages 1785 * @folio: The folio. 1786 * 1787 * Reset the order and derived _nr_pages to 0. Must only be used in the 1788 * process of splitting large folios. 1789 */ 1790 static inline void folio_reset_order(struct folio *folio) 1791 { 1792 if (WARN_ON_ONCE(!folio_test_large(folio))) 1793 return; 1794 folio->_flags_1 &= ~0xffUL; 1795 #ifdef NR_PAGES_IN_LARGE_FOLIO 1796 folio->_nr_pages = 0; 1797 #endif 1798 } 1799 1800 #include <linux/huge_mm.h> 1801 1802 /* 1803 * Methods to modify the page usage count. 1804 * 1805 * What counts for a page usage: 1806 * - cache mapping (page->mapping) 1807 * - private data (page->private) 1808 * - page mapped in a task's page tables, each mapping 1809 * is counted separately 1810 * 1811 * Also, many kernel routines increase the page count before a critical 1812 * routine so they can be sure the page doesn't go away from under them. 1813 */ 1814 1815 /* 1816 * Drop a ref, return true if the refcount fell to zero (the page has no users) 1817 */ 1818 static inline int put_page_testzero(struct page *page) 1819 { 1820 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 1821 return page_ref_dec_and_test(page); 1822 } 1823 1824 static inline int folio_put_testzero(struct folio *folio) 1825 { 1826 return put_page_testzero(&folio->page); 1827 } 1828 1829 /* 1830 * Try to grab a ref unless the page has a refcount of zero, return false if 1831 * that is the case. 1832 * This can be called when MMU is off so it must not access 1833 * any of the virtual mappings. 1834 */ 1835 static inline bool get_page_unless_zero(struct page *page) 1836 { 1837 return page_ref_add_unless_zero(page, 1); 1838 } 1839 1840 static inline struct folio *folio_get_nontail_page(struct page *page) 1841 { 1842 if (unlikely(!get_page_unless_zero(page))) 1843 return NULL; 1844 return (struct folio *)page; 1845 } 1846 1847 extern int page_is_ram(unsigned long pfn); 1848 1849 enum { 1850 REGION_INTERSECTS, 1851 REGION_DISJOINT, 1852 REGION_MIXED, 1853 }; 1854 1855 int region_intersects(resource_size_t offset, size_t size, unsigned long flags, 1856 unsigned long desc); 1857 1858 /* Support for virtually mapped pages */ 1859 struct page *vmalloc_to_page(const void *addr); 1860 unsigned long vmalloc_to_pfn(const void *addr); 1861 1862 /* 1863 * Determine if an address is within the vmalloc range 1864 * 1865 * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there 1866 * is no special casing required. 1867 */ 1868 #ifdef CONFIG_MMU 1869 extern bool is_vmalloc_addr(const void *x); 1870 extern int is_vmalloc_or_module_addr(const void *x); 1871 #else 1872 static inline bool is_vmalloc_addr(const void *x) 1873 { 1874 return false; 1875 } 1876 static inline int is_vmalloc_or_module_addr(const void *x) 1877 { 1878 return 0; 1879 } 1880 #endif 1881 1882 /* 1883 * How many times the entire folio is mapped as a single unit (eg by a 1884 * PMD or PUD entry). This is probably not what you want, except for 1885 * debugging purposes or implementation of other core folio_*() primitives. 1886 */ 1887 static inline int folio_entire_mapcount(const struct folio *folio) 1888 { 1889 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 1890 if (!IS_ENABLED(CONFIG_64BIT) && unlikely(folio_large_order(folio) == 1)) 1891 return 0; 1892 return atomic_read(&folio->_entire_mapcount) + 1; 1893 } 1894 1895 static inline int folio_large_mapcount(const struct folio *folio) 1896 { 1897 VM_WARN_ON_FOLIO(!folio_test_large(folio), folio); 1898 return atomic_read(&folio->_large_mapcount) + 1; 1899 } 1900 1901 /** 1902 * folio_mapcount() - Number of mappings of this folio. 1903 * @folio: The folio. 1904 * 1905 * The folio mapcount corresponds to the number of present user page table 1906 * entries that reference any part of a folio. Each such present user page 1907 * table entry must be paired with exactly on folio reference. 1908 * 1909 * For ordindary folios, each user page table entry (PTE/PMD/PUD/...) counts 1910 * exactly once. 1911 * 1912 * For hugetlb folios, each abstracted "hugetlb" user page table entry that 1913 * references the entire folio counts exactly once, even when such special 1914 * page table entries are comprised of multiple ordinary page table entries. 1915 * 1916 * Will report 0 for pages which cannot be mapped into userspace, such as 1917 * slab, page tables and similar. 1918 * 1919 * Return: The number of times this folio is mapped. 1920 */ 1921 static inline int folio_mapcount(const struct folio *folio) 1922 { 1923 int mapcount; 1924 1925 if (likely(!folio_test_large(folio))) { 1926 mapcount = atomic_read(&folio->_mapcount) + 1; 1927 if (page_mapcount_is_type(mapcount)) 1928 mapcount = 0; 1929 return mapcount; 1930 } 1931 return folio_large_mapcount(folio); 1932 } 1933 1934 /** 1935 * folio_mapped - Is this folio mapped into userspace? 1936 * @folio: The folio. 1937 * 1938 * Return: True if any page in this folio is referenced by user page tables. 1939 */ 1940 static inline bool folio_mapped(const struct folio *folio) 1941 { 1942 return folio_mapcount(folio) >= 1; 1943 } 1944 1945 static inline struct page *virt_to_head_page(const void *x) 1946 { 1947 struct page *page = virt_to_page(x); 1948 1949 return compound_head(page); 1950 } 1951 1952 static inline struct folio *virt_to_folio(const void *x) 1953 { 1954 struct page *page = virt_to_page(x); 1955 1956 return page_folio(page); 1957 } 1958 1959 void __folio_put(struct folio *folio); 1960 1961 void split_page(struct page *page, unsigned int order); 1962 void folio_copy(struct folio *dst, struct folio *src); 1963 int folio_mc_copy(struct folio *dst, struct folio *src); 1964 1965 unsigned long nr_free_buffer_pages(void); 1966 1967 /* Returns the number of bytes in this potentially compound page. */ 1968 static inline unsigned long page_size(const struct page *page) 1969 { 1970 return PAGE_SIZE << compound_order(page); 1971 } 1972 1973 /* Returns the number of bits needed for the number of bytes in a page */ 1974 static inline unsigned int page_shift(struct page *page) 1975 { 1976 return PAGE_SHIFT + compound_order(page); 1977 } 1978 1979 /** 1980 * thp_order - Order of a transparent huge page. 1981 * @page: Head page of a transparent huge page. 1982 */ 1983 static inline unsigned int thp_order(struct page *page) 1984 { 1985 VM_BUG_ON_PGFLAGS(PageTail(page), page); 1986 return compound_order(page); 1987 } 1988 1989 /** 1990 * thp_size - Size of a transparent huge page. 1991 * @page: Head page of a transparent huge page. 1992 * 1993 * Return: Number of bytes in this page. 1994 */ 1995 static inline unsigned long thp_size(struct page *page) 1996 { 1997 return PAGE_SIZE << thp_order(page); 1998 } 1999 2000 #ifdef CONFIG_MMU 2001 /* 2002 * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when 2003 * servicing faults for write access. In the normal case, do always want 2004 * pte_mkwrite. But get_user_pages can cause write faults for mappings 2005 * that do not have writing enabled, when used by access_process_vm. 2006 */ 2007 static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) 2008 { 2009 if (likely(vma->vm_flags & VM_WRITE)) 2010 pte = pte_mkwrite(pte, vma); 2011 return pte; 2012 } 2013 2014 vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *page); 2015 void set_pte_range(struct vm_fault *vmf, struct folio *folio, 2016 struct page *page, unsigned int nr, unsigned long addr); 2017 2018 vm_fault_t finish_fault(struct vm_fault *vmf); 2019 #endif 2020 2021 /* 2022 * Multiple processes may "see" the same page. E.g. for untouched 2023 * mappings of /dev/null, all processes see the same page full of 2024 * zeroes, and text pages of executables and shared libraries have 2025 * only one copy in memory, at most, normally. 2026 * 2027 * For the non-reserved pages, page_count(page) denotes a reference count. 2028 * page_count() == 0 means the page is free. page->lru is then used for 2029 * freelist management in the buddy allocator. 2030 * page_count() > 0 means the page has been allocated. 2031 * 2032 * Pages are allocated by the slab allocator in order to provide memory 2033 * to kmalloc and kmem_cache_alloc. In this case, the management of the 2034 * page, and the fields in 'struct page' are the responsibility of mm/slab.c 2035 * unless a particular usage is carefully commented. (the responsibility of 2036 * freeing the kmalloc memory is the caller's, of course). 2037 * 2038 * A page may be used by anyone else who does a __get_free_page(). 2039 * In this case, page_count still tracks the references, and should only 2040 * be used through the normal accessor functions. The top bits of page->flags 2041 * and page->virtual store page management information, but all other fields 2042 * are unused and could be used privately, carefully. The management of this 2043 * page is the responsibility of the one who allocated it, and those who have 2044 * subsequently been given references to it. 2045 * 2046 * The other pages (we may call them "pagecache pages") are completely 2047 * managed by the Linux memory manager: I/O, buffers, swapping etc. 2048 * The following discussion applies only to them. 2049 * 2050 * A pagecache page contains an opaque `private' member, which belongs to the 2051 * page's address_space. Usually, this is the address of a circular list of 2052 * the page's disk buffers. It tells the VM to call into the filesystem to 2053 * release these pages. 2054 * 2055 * A folio may belong to an inode's memory mapping. In this case, 2056 * folio->mapping points to the inode, and folio->index is the file 2057 * offset of the folio, in units of PAGE_SIZE. 2058 * 2059 * If pagecache folios are not associated with an inode, they are said to be 2060 * anonymous folios. These may become associated with the swapcache, and in that 2061 * case PG_swapcache is set, and folio->private is an offset into the swapcache. 2062 * 2063 * In either case (swapcache or inode backed), the pagecache itself holds one 2064 * reference to the folio. Attaching filesystem private data via 2065 * folio_attach_private() also increments the refcount. Each user mapping also 2066 * has a reference to the folio. 2067 * 2068 * The pagecache pages are stored in a per-mapping radix tree, which is 2069 * rooted at mapping->i_pages, and indexed by offset. 2070 * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space 2071 * lists, we instead now tag pages as dirty/writeback in the radix tree. 2072 * 2073 * All pagecache pages may be subject to I/O: 2074 * - inode pages may need to be read from disk, 2075 * - inode pages which have been modified and are MAP_SHARED may need 2076 * to be written back to the inode on disk, 2077 * - anonymous pages (including MAP_PRIVATE file mappings) which have been 2078 * modified may need to be swapped out to swap space and (later) to be read 2079 * back into memory. 2080 */ 2081 2082 /* 127: arbitrary random number, small enough to assemble well */ 2083 #define folio_ref_zero_or_close_to_overflow(folio) \ 2084 ((unsigned int) folio_ref_count(folio) + 127u <= 127u) 2085 2086 /** 2087 * folio_get - Increment the reference count on a folio. 2088 * @folio: The folio. 2089 * 2090 * Context: May be called in any context, as long as you know that 2091 * you have a refcount on the folio. If you do not already have one, 2092 * folio_try_get() may be the right interface for you to use. 2093 */ 2094 static inline void folio_get(struct folio *folio) 2095 { 2096 VM_BUG_ON_FOLIO(folio_ref_zero_or_close_to_overflow(folio), folio); 2097 folio_ref_inc(folio); 2098 } 2099 2100 static inline void get_page(struct page *page) 2101 { 2102 struct folio *folio = page_folio(page); 2103 if (WARN_ON_ONCE(folio_test_slab(folio))) 2104 return; 2105 if (WARN_ON_ONCE(folio_test_large_kmalloc(folio))) 2106 return; 2107 folio_get(folio); 2108 } 2109 2110 static inline __must_check bool try_get_page(struct page *page) 2111 { 2112 page = compound_head(page); 2113 if (WARN_ON_ONCE(page_ref_count(page) <= 0)) 2114 return false; 2115 page_ref_inc(page); 2116 return true; 2117 } 2118 2119 /** 2120 * folio_put - Decrement the reference count on a folio. 2121 * @folio: The folio. 2122 * 2123 * If the folio's reference count reaches zero, the memory will be 2124 * released back to the page allocator and may be used by another 2125 * allocation immediately. Do not access the memory or the struct folio 2126 * after calling folio_put() unless you can be sure that it wasn't the 2127 * last reference. 2128 * 2129 * Context: May be called in process or interrupt context, but not in NMI 2130 * context. May be called while holding a spinlock. 2131 */ 2132 static inline void folio_put(struct folio *folio) 2133 { 2134 if (folio_put_testzero(folio)) 2135 __folio_put(folio); 2136 } 2137 2138 /** 2139 * folio_put_refs - Reduce the reference count on a folio. 2140 * @folio: The folio. 2141 * @refs: The amount to subtract from the folio's reference count. 2142 * 2143 * If the folio's reference count reaches zero, the memory will be 2144 * released back to the page allocator and may be used by another 2145 * allocation immediately. Do not access the memory or the struct folio 2146 * after calling folio_put_refs() unless you can be sure that these weren't 2147 * the last references. 2148 * 2149 * Context: May be called in process or interrupt context, but not in NMI 2150 * context. May be called while holding a spinlock. 2151 */ 2152 static inline void folio_put_refs(struct folio *folio, int refs) 2153 { 2154 if (folio_ref_sub_and_test(folio, refs)) 2155 __folio_put(folio); 2156 } 2157 2158 void folios_put_refs(struct folio_batch *folios, unsigned int *refs); 2159 2160 /* 2161 * union release_pages_arg - an array of pages or folios 2162 * 2163 * release_pages() releases a simple array of multiple pages, and 2164 * accepts various different forms of said page array: either 2165 * a regular old boring array of pages, an array of folios, or 2166 * an array of encoded page pointers. 2167 * 2168 * The transparent union syntax for this kind of "any of these 2169 * argument types" is all kinds of ugly, so look away. 2170 */ 2171 typedef union { 2172 struct page **pages; 2173 struct folio **folios; 2174 struct encoded_page **encoded_pages; 2175 } release_pages_arg __attribute__ ((__transparent_union__)); 2176 2177 void release_pages(release_pages_arg, int nr); 2178 2179 /** 2180 * folios_put - Decrement the reference count on an array of folios. 2181 * @folios: The folios. 2182 * 2183 * Like folio_put(), but for a batch of folios. This is more efficient 2184 * than writing the loop yourself as it will optimise the locks which need 2185 * to be taken if the folios are freed. The folios batch is returned 2186 * empty and ready to be reused for another batch; there is no need to 2187 * reinitialise it. 2188 * 2189 * Context: May be called in process or interrupt context, but not in NMI 2190 * context. May be called while holding a spinlock. 2191 */ 2192 static inline void folios_put(struct folio_batch *folios) 2193 { 2194 folios_put_refs(folios, NULL); 2195 } 2196 2197 static inline void put_page(struct page *page) 2198 { 2199 struct folio *folio = page_folio(page); 2200 2201 if (folio_test_slab(folio) || folio_test_large_kmalloc(folio)) 2202 return; 2203 2204 folio_put(folio); 2205 } 2206 2207 /* 2208 * GUP_PIN_COUNTING_BIAS, and the associated functions that use it, overload 2209 * the page's refcount so that two separate items are tracked: the original page 2210 * reference count, and also a new count of how many pin_user_pages() calls were 2211 * made against the page. ("gup-pinned" is another term for the latter). 2212 * 2213 * With this scheme, pin_user_pages() becomes special: such pages are marked as 2214 * distinct from normal pages. As such, the unpin_user_page() call (and its 2215 * variants) must be used in order to release gup-pinned pages. 2216 * 2217 * Choice of value: 2218 * 2219 * By making GUP_PIN_COUNTING_BIAS a power of two, debugging of page reference 2220 * counts with respect to pin_user_pages() and unpin_user_page() becomes 2221 * simpler, due to the fact that adding an even power of two to the page 2222 * refcount has the effect of using only the upper N bits, for the code that 2223 * counts up using the bias value. This means that the lower bits are left for 2224 * the exclusive use of the original code that increments and decrements by one 2225 * (or at least, by much smaller values than the bias value). 2226 * 2227 * Of course, once the lower bits overflow into the upper bits (and this is 2228 * OK, because subtraction recovers the original values), then visual inspection 2229 * no longer suffices to directly view the separate counts. However, for normal 2230 * applications that don't have huge page reference counts, this won't be an 2231 * issue. 2232 * 2233 * Locking: the lockless algorithm described in folio_try_get_rcu() 2234 * provides safe operation for get_user_pages(), folio_mkclean() and 2235 * other calls that race to set up page table entries. 2236 */ 2237 #define GUP_PIN_COUNTING_BIAS (1U << 10) 2238 2239 void unpin_user_page(struct page *page); 2240 void unpin_folio(struct folio *folio); 2241 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages, 2242 bool make_dirty); 2243 void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages, 2244 bool make_dirty); 2245 void unpin_user_pages(struct page **pages, unsigned long npages); 2246 void unpin_user_folio(struct folio *folio, unsigned long npages); 2247 void unpin_folios(struct folio **folios, unsigned long nfolios); 2248 2249 /** 2250 * vma_flags_is_cow_mapping() - Do these VMA flags imply a CoW mapping? 2251 * @flags: The VMA flags to check. 2252 * 2253 * Mappings which could be CoW'd (subject to Copy-On-Write faults) are 2254 * described as CoW mappings. 2255 * 2256 * All mappings backed by anonymous folios (all anonymous mappings and most 2257 * MAP_PRIVATE-file backed ranges) are CoW mappings. 2258 * 2259 * All other mappings (including all MAP_SHARED mappings) are non-CoW. 2260 * 2261 * The criteria are !VMA_SHARED_BIT, VMA_MAYWRITE_BIT. 2262 * 2263 * VMA_MAYWRITE_BIT is checked instead of VMA_WRITE_BIT to account for both 2264 * future mprotect() calls which can render a read-only mapping writable, and 2265 * GUP with FOLL_FORCE (e.g. ptrace) which can CoW a read-only mapping. 2266 * 2267 * - No anonymous mapping can ever clear VMA_MAYWRITE_BIT. 2268 * 2269 * - Writes to anonymous mappings do not immediately result in CoW faults but 2270 * may do so after the process is forked or if a read is followed by a 2271 * write. 2272 * 2273 * - Writes to MAP_PRIVATE file-backed mappings result in CoW faults and may 2274 * do so again after fork. 2275 * 2276 * - MAP_SHARED mappings of a file opened read-only are transformed into 2277 * VMA_MAYSHARE_BIT, !VMA_SHARED_BIT, !VMA_MAYWRITE_BIT mappings, so remain 2278 * non-CoW. 2279 * 2280 * - Drivers may clear VMA_MAYWRITE_BIT but do so at mmap() time and cannot 2281 * mark themselves anonymous. Having cleared this flag it is not valid for 2282 * them to leave the VMA_WRITE_BIT flag set. 2283 * 2284 * As a consequence, the anonymous reverse mapping only tracks CoW mappings. 2285 * 2286 * Returns: true if the flags indicate a CoW mapping, otherwise false. 2287 */ 2288 static inline bool vma_flags_is_cow_mapping(const vma_flags_t *flags) 2289 { 2290 return vma_flags_test(flags, VMA_MAYWRITE_BIT) && 2291 !vma_flags_test(flags, VMA_SHARED_BIT); 2292 } 2293 2294 /** 2295 * vma_is_cow_mapping() - Is this VMA a CoW mapping? 2296 * @vma: The VMA to check. 2297 * 2298 * See vma_flags_is_cow_mapping() for details. 2299 * 2300 * Returns: true if the VMA is a CoW mapping, otherwise false. 2301 */ 2302 static inline bool vma_is_cow_mapping(const struct vm_area_struct *vma) 2303 { 2304 return vma_flags_is_cow_mapping(&vma->flags); 2305 } 2306 2307 /** 2308 * vma_desc_is_cow_mapping() - Is this VMA descriptor a CoW mapping? 2309 * @desc: The VMA descriptor to check. 2310 * 2311 * See vma_flags_is_cow_mapping() for details. 2312 * 2313 * Returns: true if the VMA descriptor describes a CoW mapping, otherwise 2314 * false. 2315 */ 2316 static inline bool vma_desc_is_cow_mapping(struct vm_area_desc *desc) 2317 { 2318 return vma_flags_is_cow_mapping(&desc->vma_flags); 2319 } 2320 2321 #ifndef CONFIG_MMU 2322 static inline bool is_nommu_shared_mapping(vm_flags_t flags) 2323 { 2324 /* 2325 * NOMMU shared mappings are ordinary MAP_SHARED mappings and selected 2326 * R/O MAP_PRIVATE file mappings that are an effective R/O overlay of 2327 * a file mapping. R/O MAP_PRIVATE mappings might still modify 2328 * underlying memory if ptrace is active, so this is only possible if 2329 * ptrace does not apply. Note that there is no mprotect() to upgrade 2330 * write permissions later. 2331 */ 2332 return flags & (VM_MAYSHARE | VM_MAYOVERLAY); 2333 } 2334 2335 static inline bool is_nommu_shared_vma_flags(const vma_flags_t *flags) 2336 { 2337 return vma_flags_test_any(flags, VMA_MAYSHARE_BIT, VMA_MAYOVERLAY_BIT); 2338 } 2339 #endif 2340 2341 #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) 2342 #define SECTION_IN_PAGE_FLAGS 2343 #endif 2344 2345 /* 2346 * The identification function is mainly used by the buddy allocator for 2347 * determining if two pages could be buddies. We are not really identifying 2348 * the zone since we could be using the section number id if we do not have 2349 * node id available in page flags. 2350 * We only guarantee that it will return the same value for two combinable 2351 * pages in a zone. 2352 */ 2353 static inline int page_zone_id(struct page *page) 2354 { 2355 return (page->flags.f >> ZONEID_PGSHIFT) & ZONEID_MASK; 2356 } 2357 2358 #ifdef NODE_NOT_IN_PAGE_FLAGS 2359 int memdesc_nid(const memdesc_flags_t *mdf); 2360 #else 2361 #ifdef CONFIG_NUMA 2362 static inline int memdesc_nid(const memdesc_flags_t *mdf) 2363 { 2364 ASSERT_EXCLUSIVE_BITS(mdf->f, NODES_MASK << NODES_PGSHIFT); 2365 return (mdf->f >> NODES_PGSHIFT) & NODES_MASK; 2366 } 2367 #else 2368 static inline int memdesc_nid(const memdesc_flags_t *mdf) 2369 { 2370 return 0; 2371 } 2372 #endif 2373 #endif 2374 2375 static inline int page_to_nid(const struct page *page) 2376 { 2377 return memdesc_nid(&(PF_POISONED_CHECK(page)->flags)); 2378 } 2379 2380 static inline int folio_nid(const struct folio *folio) 2381 { 2382 return memdesc_nid(&folio->flags); 2383 } 2384 2385 #ifdef CONFIG_NUMA_BALANCING 2386 /* page access time bits needs to hold at least 4 seconds */ 2387 #define PAGE_ACCESS_TIME_MIN_BITS 12 2388 #if LAST_CPUPID_SHIFT < PAGE_ACCESS_TIME_MIN_BITS 2389 #define PAGE_ACCESS_TIME_BUCKETS \ 2390 (PAGE_ACCESS_TIME_MIN_BITS - LAST_CPUPID_SHIFT) 2391 #else 2392 #define PAGE_ACCESS_TIME_BUCKETS 0 2393 #endif 2394 2395 #define PAGE_ACCESS_TIME_MASK \ 2396 (LAST_CPUPID_MASK << PAGE_ACCESS_TIME_BUCKETS) 2397 2398 static inline int cpu_pid_to_cpupid(int cpu, int pid) 2399 { 2400 return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK); 2401 } 2402 2403 static inline int cpupid_to_pid(int cpupid) 2404 { 2405 return cpupid & LAST__PID_MASK; 2406 } 2407 2408 static inline int cpupid_to_cpu(int cpupid) 2409 { 2410 return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK; 2411 } 2412 2413 static inline int cpupid_to_nid(int cpupid) 2414 { 2415 return cpu_to_node(cpupid_to_cpu(cpupid)); 2416 } 2417 2418 static inline bool cpupid_pid_unset(int cpupid) 2419 { 2420 return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK); 2421 } 2422 2423 static inline bool cpupid_cpu_unset(int cpupid) 2424 { 2425 return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK); 2426 } 2427 2428 static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid) 2429 { 2430 return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid); 2431 } 2432 2433 #define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid) 2434 #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS 2435 static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid) 2436 { 2437 return xchg(&folio->_last_cpupid, cpupid & LAST_CPUPID_MASK); 2438 } 2439 2440 static inline int folio_last_cpupid(struct folio *folio) 2441 { 2442 return folio->_last_cpupid; 2443 } 2444 static inline void page_cpupid_reset_last(struct page *page) 2445 { 2446 page->_last_cpupid = -1 & LAST_CPUPID_MASK; 2447 } 2448 #else 2449 static inline int folio_last_cpupid(struct folio *folio) 2450 { 2451 return (folio->flags.f >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK; 2452 } 2453 2454 int folio_xchg_last_cpupid(struct folio *folio, int cpupid); 2455 2456 static inline void page_cpupid_reset_last(struct page *page) 2457 { 2458 page->flags.f |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT; 2459 } 2460 #endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */ 2461 2462 static inline int folio_xchg_access_time(struct folio *folio, int time) 2463 { 2464 int last_time; 2465 2466 last_time = folio_xchg_last_cpupid(folio, 2467 time >> PAGE_ACCESS_TIME_BUCKETS); 2468 return last_time << PAGE_ACCESS_TIME_BUCKETS; 2469 } 2470 2471 static inline void vma_set_access_pid_bit(struct vm_area_struct *vma) 2472 { 2473 unsigned int pid_bit; 2474 2475 pid_bit = hash_32(current->pid, ilog2(BITS_PER_LONG)); 2476 if (vma->numab_state && !test_bit(pid_bit, &vma->numab_state->pids_active[1])) { 2477 __set_bit(pid_bit, &vma->numab_state->pids_active[1]); 2478 } 2479 } 2480 2481 bool folio_use_access_time(struct folio *folio); 2482 #else /* !CONFIG_NUMA_BALANCING */ 2483 static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid) 2484 { 2485 return folio_nid(folio); /* XXX */ 2486 } 2487 2488 static inline int folio_xchg_access_time(struct folio *folio, int time) 2489 { 2490 return 0; 2491 } 2492 2493 static inline int folio_last_cpupid(struct folio *folio) 2494 { 2495 return folio_nid(folio); /* XXX */ 2496 } 2497 2498 static inline int cpupid_to_nid(int cpupid) 2499 { 2500 return -1; 2501 } 2502 2503 static inline int cpupid_to_pid(int cpupid) 2504 { 2505 return -1; 2506 } 2507 2508 static inline int cpupid_to_cpu(int cpupid) 2509 { 2510 return -1; 2511 } 2512 2513 static inline int cpu_pid_to_cpupid(int nid, int pid) 2514 { 2515 return -1; 2516 } 2517 2518 static inline bool cpupid_pid_unset(int cpupid) 2519 { 2520 return true; 2521 } 2522 2523 static inline void page_cpupid_reset_last(struct page *page) 2524 { 2525 } 2526 2527 static inline bool cpupid_match_pid(struct task_struct *task, int cpupid) 2528 { 2529 return false; 2530 } 2531 2532 static inline void vma_set_access_pid_bit(struct vm_area_struct *vma) 2533 { 2534 } 2535 static inline bool folio_use_access_time(struct folio *folio) 2536 { 2537 return false; 2538 } 2539 #endif /* CONFIG_NUMA_BALANCING */ 2540 2541 #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) 2542 2543 /* 2544 * KASAN per-page tags are stored xor'ed with 0xff. This allows to avoid 2545 * setting tags for all pages to native kernel tag value 0xff, as the default 2546 * value 0x00 maps to 0xff. 2547 */ 2548 2549 static inline u8 page_kasan_tag(const struct page *page) 2550 { 2551 u8 tag = KASAN_TAG_KERNEL; 2552 2553 if (kasan_enabled()) { 2554 tag = (page->flags.f >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK; 2555 tag ^= 0xff; 2556 } 2557 2558 return tag; 2559 } 2560 2561 static inline void page_kasan_tag_set(struct page *page, u8 tag) 2562 { 2563 unsigned long old_flags, flags; 2564 2565 if (!kasan_enabled()) 2566 return; 2567 2568 tag ^= 0xff; 2569 old_flags = READ_ONCE(page->flags.f); 2570 do { 2571 flags = old_flags; 2572 flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT); 2573 flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT; 2574 } while (unlikely(!try_cmpxchg(&page->flags.f, &old_flags, flags))); 2575 } 2576 2577 static inline void page_kasan_tag_reset(struct page *page) 2578 { 2579 if (kasan_enabled()) 2580 page_kasan_tag_set(page, KASAN_TAG_KERNEL); 2581 } 2582 2583 #else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */ 2584 2585 static inline u8 page_kasan_tag(const struct page *page) 2586 { 2587 return 0xff; 2588 } 2589 2590 static inline void page_kasan_tag_set(struct page *page, u8 tag) { } 2591 static inline void page_kasan_tag_reset(struct page *page) { } 2592 2593 #endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */ 2594 2595 static inline struct zone *page_zone(const struct page *page) 2596 { 2597 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)]; 2598 } 2599 2600 static inline pg_data_t *page_pgdat(const struct page *page) 2601 { 2602 return NODE_DATA(page_to_nid(page)); 2603 } 2604 2605 static inline pg_data_t *folio_pgdat(const struct folio *folio) 2606 { 2607 return NODE_DATA(folio_nid(folio)); 2608 } 2609 2610 static inline struct zone *folio_zone(const struct folio *folio) 2611 { 2612 return &folio_pgdat(folio)->node_zones[folio_zonenum(folio)]; 2613 } 2614 2615 #ifdef SECTION_IN_PAGE_FLAGS 2616 static inline void set_page_section(struct page *page, unsigned long section) 2617 { 2618 page->flags.f &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT); 2619 page->flags.f |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; 2620 } 2621 2622 static inline void set_page_section_from_pfn(struct page *page, 2623 unsigned long pfn) 2624 { 2625 set_page_section(page, pfn_to_section_nr(pfn)); 2626 } 2627 2628 static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) 2629 { 2630 ASSERT_EXCLUSIVE_BITS(mdf->f, SECTIONS_MASK << SECTIONS_PGSHIFT); 2631 return (mdf->f >> SECTIONS_PGSHIFT) & SECTIONS_MASK; 2632 } 2633 #else /* !SECTION_IN_PAGE_FLAGS */ 2634 static inline void set_page_section_from_pfn(struct page *page, 2635 unsigned long pfn) 2636 { 2637 } 2638 2639 static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) 2640 { 2641 return 0; 2642 } 2643 #endif /* SECTION_IN_PAGE_FLAGS */ 2644 2645 /** 2646 * folio_pfn - Return the Page Frame Number of a folio. 2647 * @folio: The folio. 2648 * 2649 * A folio may contain multiple pages. The pages have consecutive 2650 * Page Frame Numbers. 2651 * 2652 * Return: The Page Frame Number of the first page in the folio. 2653 */ 2654 static inline unsigned long folio_pfn(const struct folio *folio) 2655 { 2656 return page_to_pfn(&folio->page); 2657 } 2658 2659 static inline struct folio *pfn_folio(unsigned long pfn) 2660 { 2661 return page_folio(pfn_to_page(pfn)); 2662 } 2663 2664 #ifdef CONFIG_MMU 2665 static inline pte_t mk_pte(const struct page *page, pgprot_t pgprot) 2666 { 2667 return pfn_pte(page_to_pfn(page), pgprot); 2668 } 2669 2670 /** 2671 * folio_mk_pte - Create a PTE for this folio 2672 * @folio: The folio to create a PTE for 2673 * @pgprot: The page protection bits to use 2674 * 2675 * Create a page table entry for the first page of this folio. 2676 * This is suitable for passing to set_ptes(). 2677 * 2678 * Return: A page table entry suitable for mapping this folio. 2679 */ 2680 static inline pte_t folio_mk_pte(const struct folio *folio, pgprot_t pgprot) 2681 { 2682 return pfn_pte(folio_pfn(folio), pgprot); 2683 } 2684 2685 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2686 /** 2687 * folio_mk_pmd - Create a PMD for this folio 2688 * @folio: The folio to create a PMD for 2689 * @pgprot: The page protection bits to use 2690 * 2691 * Create a page table entry for the first page of this folio. 2692 * This is suitable for passing to set_pmd_at(). 2693 * 2694 * Return: A page table entry suitable for mapping this folio. 2695 */ 2696 static inline pmd_t folio_mk_pmd(const struct folio *folio, pgprot_t pgprot) 2697 { 2698 return pmd_mkhuge(pfn_pmd(folio_pfn(folio), pgprot)); 2699 } 2700 2701 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 2702 /** 2703 * folio_mk_pud - Create a PUD for this folio 2704 * @folio: The folio to create a PUD for 2705 * @pgprot: The page protection bits to use 2706 * 2707 * Create a page table entry for the first page of this folio. 2708 * This is suitable for passing to set_pud_at(). 2709 * 2710 * Return: A page table entry suitable for mapping this folio. 2711 */ 2712 static inline pud_t folio_mk_pud(const struct folio *folio, pgprot_t pgprot) 2713 { 2714 return pud_mkhuge(pfn_pud(folio_pfn(folio), pgprot)); 2715 } 2716 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 2717 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 2718 #endif /* CONFIG_MMU */ 2719 2720 static inline bool folio_has_pincount(const struct folio *folio) 2721 { 2722 if (IS_ENABLED(CONFIG_64BIT)) 2723 return folio_test_large(folio); 2724 return folio_order(folio) > 1; 2725 } 2726 2727 /** 2728 * folio_maybe_dma_pinned - Report if a folio may be pinned for DMA. 2729 * @folio: The folio. 2730 * 2731 * This function checks if a folio has been pinned via a call to 2732 * a function in the pin_user_pages() family. 2733 * 2734 * For small folios, the return value is partially fuzzy: false is not fuzzy, 2735 * because it means "definitely not pinned for DMA", but true means "probably 2736 * pinned for DMA, but possibly a false positive due to having at least 2737 * GUP_PIN_COUNTING_BIAS worth of normal folio references". 2738 * 2739 * False positives are OK, because: a) it's unlikely for a folio to 2740 * get that many refcounts, and b) all the callers of this routine are 2741 * expected to be able to deal gracefully with a false positive. 2742 * 2743 * For most large folios, the result will be exactly correct. That's because 2744 * we have more tracking data available: the _pincount field is used 2745 * instead of the GUP_PIN_COUNTING_BIAS scheme. 2746 * 2747 * For more information, please see Documentation/core-api/pin_user_pages.rst. 2748 * 2749 * Return: True, if it is likely that the folio has been "dma-pinned". 2750 * False, if the folio is definitely not dma-pinned. 2751 */ 2752 static inline bool folio_maybe_dma_pinned(struct folio *folio) 2753 { 2754 if (folio_has_pincount(folio)) 2755 return atomic_read(&folio->_pincount) > 0; 2756 2757 /* 2758 * folio_ref_count() is signed. If that refcount overflows, then 2759 * folio_ref_count() returns a negative value, and callers will avoid 2760 * further incrementing the refcount. 2761 * 2762 * Here, for that overflow case, use the sign bit to count a little 2763 * bit higher via unsigned math, and thus still get an accurate result. 2764 */ 2765 return ((unsigned int)folio_ref_count(folio)) >= 2766 GUP_PIN_COUNTING_BIAS; 2767 } 2768 2769 /* 2770 * This should most likely only be called during fork() to see whether we 2771 * should break the cow immediately for an anon page on the src mm. 2772 * 2773 * The caller has to hold the PT lock and the vma->vm_mm->->write_protect_seq. 2774 */ 2775 static inline bool folio_needs_cow_for_dma(struct vm_area_struct *vma, 2776 struct folio *folio) 2777 { 2778 VM_BUG_ON(!(raw_read_seqcount(&vma->vm_mm->write_protect_seq) & 1)); 2779 2780 if (!mm_flags_test(MMF_HAS_PINNED, vma->vm_mm)) 2781 return false; 2782 2783 return folio_maybe_dma_pinned(folio); 2784 } 2785 2786 /** 2787 * is_zero_page - Query if a page is a zero page 2788 * @page: The page to query 2789 * 2790 * This returns true if @page is one of the permanent zero pages. 2791 */ 2792 static inline bool is_zero_page(const struct page *page) 2793 { 2794 return is_zero_pfn(page_to_pfn(page)); 2795 } 2796 2797 /** 2798 * is_zero_folio - Query if a folio is a zero page 2799 * @folio: The folio to query 2800 * 2801 * This returns true if @folio is one of the permanent zero pages. 2802 */ 2803 static inline bool is_zero_folio(const struct folio *folio) 2804 { 2805 return is_zero_page(&folio->page); 2806 } 2807 2808 /* MIGRATE_CMA and ZONE_MOVABLE do not allow pin folios */ 2809 #ifdef CONFIG_MIGRATION 2810 static inline bool folio_is_longterm_pinnable(struct folio *folio) 2811 { 2812 #ifdef CONFIG_CMA 2813 int mt = folio_migratetype(folio); 2814 2815 if (mt == MIGRATE_CMA || mt == MIGRATE_ISOLATE) 2816 return false; 2817 #endif 2818 /* The zero page can be "pinned" but gets special handling. */ 2819 if (is_zero_folio(folio)) 2820 return true; 2821 2822 /* Coherent device memory must always allow eviction. */ 2823 if (folio_is_device_coherent(folio)) 2824 return false; 2825 2826 /* 2827 * Filesystems can only tolerate transient delays to truncate and 2828 * hole-punch operations 2829 */ 2830 if (folio_is_fsdax(folio)) 2831 return false; 2832 2833 /* Otherwise, non-movable zone folios can be pinned. */ 2834 return !folio_is_zone_movable(folio); 2835 2836 } 2837 #else 2838 static inline bool folio_is_longterm_pinnable(struct folio *folio) 2839 { 2840 return true; 2841 } 2842 #endif 2843 2844 static inline void set_page_zone(struct page *page, enum zone_type zone) 2845 { 2846 page->flags.f &= ~(ZONES_MASK << ZONES_PGSHIFT); 2847 page->flags.f |= (zone & ZONES_MASK) << ZONES_PGSHIFT; 2848 } 2849 2850 static inline void set_page_node(struct page *page, unsigned long node) 2851 { 2852 page->flags.f &= ~(NODES_MASK << NODES_PGSHIFT); 2853 page->flags.f |= (node & NODES_MASK) << NODES_PGSHIFT; 2854 } 2855 2856 static inline void set_page_links(struct page *page, enum zone_type zone, 2857 unsigned long node, unsigned long pfn) 2858 { 2859 set_page_zone(page, zone); 2860 set_page_node(page, node); 2861 set_page_section_from_pfn(page, pfn); 2862 } 2863 2864 /** 2865 * folio_nr_pages - The number of pages in the folio. 2866 * @folio: The folio. 2867 * 2868 * Return: A positive power of two. 2869 */ 2870 static inline unsigned long folio_nr_pages(const struct folio *folio) 2871 { 2872 if (!folio_test_large(folio)) 2873 return 1; 2874 return folio_large_nr_pages(folio); 2875 } 2876 2877 /* 2878 * compound_nr() returns the number of pages in this potentially compound 2879 * page. compound_nr() can be called on a tail page, and is defined to 2880 * return 1 in that case. 2881 */ 2882 static inline unsigned long compound_nr(const struct page *page) 2883 { 2884 const struct folio *folio = (struct folio *)page; 2885 2886 if (!test_bit(PG_head, &folio->flags.f)) 2887 return 1; 2888 return folio_large_nr_pages(folio); 2889 } 2890 2891 /** 2892 * folio_next - Move to the next physical folio. 2893 * @folio: The folio we're currently operating on. 2894 * 2895 * If you have physically contiguous memory which may span more than 2896 * one folio (eg a &struct bio_vec), use this function to move from one 2897 * folio to the next. Do not use it if the memory is only virtually 2898 * contiguous as the folios are almost certainly not adjacent to each 2899 * other. This is the folio equivalent to writing ``page++``. 2900 * 2901 * Context: We assume that the folios are refcounted and/or locked at a 2902 * higher level and do not adjust the reference counts. 2903 * Return: The next struct folio. 2904 */ 2905 static inline struct folio *folio_next(struct folio *folio) 2906 { 2907 return (struct folio *)folio_page(folio, folio_nr_pages(folio)); 2908 } 2909 2910 /** 2911 * folio_shift - The size of the memory described by this folio. 2912 * @folio: The folio. 2913 * 2914 * A folio represents a number of bytes which is a power-of-two in size. 2915 * This function tells you which power-of-two the folio is. See also 2916 * folio_size() and folio_order(). 2917 * 2918 * Context: The caller should have a reference on the folio to prevent 2919 * it from being split. It is not necessary for the folio to be locked. 2920 * Return: The base-2 logarithm of the size of this folio. 2921 */ 2922 static inline unsigned int folio_shift(const struct folio *folio) 2923 { 2924 return PAGE_SHIFT + folio_order(folio); 2925 } 2926 2927 /** 2928 * folio_size - The number of bytes in a folio. 2929 * @folio: The folio. 2930 * 2931 * Context: The caller should have a reference on the folio to prevent 2932 * it from being split. It is not necessary for the folio to be locked. 2933 * Return: The number of bytes in this folio. 2934 */ 2935 static inline size_t folio_size(const struct folio *folio) 2936 { 2937 return PAGE_SIZE << folio_order(folio); 2938 } 2939 2940 /** 2941 * folio_maybe_mapped_shared - Whether the folio is mapped into the page 2942 * tables of more than one MM 2943 * @folio: The folio. 2944 * 2945 * This function checks if the folio maybe currently mapped into more than one 2946 * MM ("maybe mapped shared"), or if the folio is certainly mapped into a single 2947 * MM ("mapped exclusively"). 2948 * 2949 * For KSM folios, this function also returns "mapped shared" when a folio is 2950 * mapped multiple times into the same MM, because the individual page mappings 2951 * are independent. 2952 * 2953 * For small anonymous folios and anonymous hugetlb folios, the return 2954 * value will be exactly correct: non-KSM folios can only be mapped at most once 2955 * into an MM, and they cannot be partially mapped. KSM folios are 2956 * considered shared even if mapped multiple times into the same MM. 2957 * 2958 * For other folios, the result can be fuzzy: 2959 * #. For partially-mappable large folios (THP), the return value can wrongly 2960 * indicate "mapped shared" (false positive) if a folio was mapped by 2961 * more than two MMs at one point in time. 2962 * #. For pagecache folios (including hugetlb), the return value can wrongly 2963 * indicate "mapped shared" (false positive) when two VMAs in the same MM 2964 * cover the same file range. 2965 * 2966 * Further, this function only considers current page table mappings that 2967 * are tracked using the folio mapcount(s). 2968 * 2969 * This function does not consider: 2970 * #. If the folio might get mapped in the (near) future (e.g., swapcache, 2971 * pagecache, temporary unmapping for migration). 2972 * #. If the folio is mapped differently (VM_PFNMAP). 2973 * #. If hugetlb page table sharing applies. Callers might want to check 2974 * hugetlb_pmd_shared(). 2975 * 2976 * Return: Whether the folio is estimated to be mapped into more than one MM. 2977 */ 2978 static inline bool folio_maybe_mapped_shared(struct folio *folio) 2979 { 2980 int mapcount = folio_mapcount(folio); 2981 2982 /* Only partially-mappable folios require more care. */ 2983 if (!folio_test_large(folio) || unlikely(folio_test_hugetlb(folio))) 2984 return mapcount > 1; 2985 2986 /* 2987 * vm_insert_page() without CONFIG_TRANSPARENT_HUGEPAGE ... 2988 * simply assume "mapped shared", nobody should really care 2989 * about this for arbitrary kernel allocations. 2990 */ 2991 if (!IS_ENABLED(CONFIG_MM_ID)) 2992 return true; 2993 2994 /* 2995 * A single mapping implies "mapped exclusively", even if the 2996 * folio flag says something different: it's easier to handle this 2997 * case here instead of on the RMAP hot path. 2998 */ 2999 if (mapcount <= 1) 3000 return false; 3001 return test_bit(FOLIO_MM_IDS_SHARED_BITNUM, &folio->_mm_ids); 3002 } 3003 3004 /** 3005 * folio_expected_ref_count - calculate the expected folio refcount 3006 * @folio: the folio 3007 * 3008 * Calculate the expected folio refcount, taking references from the pagecache, 3009 * swapcache, private data (folio->private != NULL) and page table mappings into 3010 * account. Useful in combination with folio_ref_count() to detect unexpected 3011 * references (e.g., GUP or other temporary references). 3012 * 3013 * Does currently not consider references from the LRU cache. If the folio 3014 * was isolated from the LRU (which is the case during migration or split), 3015 * the LRU cache does not apply. 3016 * 3017 * Calling this function on an unmapped folio -- !folio_mapped() -- that is 3018 * locked will return a stable result. 3019 * 3020 * Calling this function on a mapped folio will not result in a stable result, 3021 * because nothing stops additional page table mappings from coming (e.g., 3022 * fork()) or going (e.g., munmap()). 3023 * 3024 * Calling this function without the folio lock will also not result in a 3025 * stable result: for example, the folio might get dropped from the swapcache 3026 * concurrently. 3027 * 3028 * However, even when called without the folio lock or on a mapped folio, 3029 * this function can be used to detect unexpected references early (for example, 3030 * if it makes sense to even lock the folio and unmap it). 3031 * 3032 * The caller must add any reference (e.g., from folio_try_get()) it might be 3033 * holding itself to the result. 3034 * 3035 * Returns: the expected folio refcount. 3036 */ 3037 static inline int folio_expected_ref_count(const struct folio *folio) 3038 { 3039 const int order = folio_order(folio); 3040 int ref_count = 0; 3041 3042 if (WARN_ON_ONCE(page_has_type(&folio->page) && !folio_test_hugetlb(folio))) 3043 return 0; 3044 3045 /* One reference per page from the swapcache. */ 3046 ref_count += folio_test_swapcache(folio) << order; 3047 3048 if (!folio_test_anon(folio)) { 3049 /* 3050 * One reference per page from the pagecache. 3051 * Use data_race() since folio might not be locked. 3052 */ 3053 ref_count += !!data_race(folio->mapping) << order; 3054 /* 3055 * One reference from filesystem private data. 3056 */ 3057 ref_count += folio_has_attached_private(folio); 3058 } 3059 3060 /* One reference per page table mapping. */ 3061 return ref_count + folio_mapcount(folio); 3062 } 3063 3064 #ifndef HAVE_ARCH_MAKE_FOLIO_ACCESSIBLE 3065 static inline int arch_make_folio_accessible(struct folio *folio) 3066 { 3067 return 0; 3068 } 3069 #endif 3070 3071 /* 3072 * Some inline functions in vmstat.h depend on page_zone() 3073 */ 3074 #include <linux/vmstat.h> 3075 3076 #if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL) 3077 #define HASHED_PAGE_VIRTUAL 3078 #endif 3079 3080 #if defined(WANT_PAGE_VIRTUAL) 3081 static inline void *page_address(const struct page *page) 3082 { 3083 return page->virtual; 3084 } 3085 static inline void set_page_address(struct page *page, void *address) 3086 { 3087 page->virtual = address; 3088 } 3089 #define page_address_init() do { } while(0) 3090 #endif 3091 3092 #if defined(HASHED_PAGE_VIRTUAL) 3093 void *page_address(const struct page *page); 3094 void set_page_address(struct page *page, void *virtual); 3095 void page_address_init(void); 3096 #endif 3097 3098 static __always_inline void *lowmem_page_address(const struct page *page) 3099 { 3100 return page_to_virt(page); 3101 } 3102 3103 #if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL) 3104 #define page_address(page) lowmem_page_address(page) 3105 #define set_page_address(page, address) do { } while(0) 3106 #define page_address_init() do { } while(0) 3107 #endif 3108 3109 static inline void *folio_address(const struct folio *folio) 3110 { 3111 return page_address(&folio->page); 3112 } 3113 3114 /* 3115 * Return true only if the page has been allocated with 3116 * ALLOC_NO_WATERMARKS and the low watermark was not 3117 * met implying that the system is under some pressure. 3118 */ 3119 static inline bool page_is_pfmemalloc(const struct page *page) 3120 { 3121 /* 3122 * lru.next has bit 1 set if the page is allocated from the 3123 * pfmemalloc reserves. Callers may simply overwrite it if 3124 * they do not need to preserve that information. 3125 */ 3126 return (uintptr_t)page->lru.next & BIT(1); 3127 } 3128 3129 /* 3130 * Return true only if the folio has been allocated with 3131 * ALLOC_NO_WATERMARKS and the low watermark was not 3132 * met implying that the system is under some pressure. 3133 */ 3134 static inline bool folio_is_pfmemalloc(const struct folio *folio) 3135 { 3136 /* 3137 * lru.next has bit 1 set if the page is allocated from the 3138 * pfmemalloc reserves. Callers may simply overwrite it if 3139 * they do not need to preserve that information. 3140 */ 3141 return (uintptr_t)folio->lru.next & BIT(1); 3142 } 3143 3144 /* 3145 * Only to be called by the page allocator on a freshly allocated 3146 * page. 3147 */ 3148 static inline void set_page_pfmemalloc(struct page *page) 3149 { 3150 page->lru.next = (void *)BIT(1); 3151 } 3152 3153 static inline void clear_page_pfmemalloc(struct page *page) 3154 { 3155 page->lru.next = NULL; 3156 } 3157 3158 /* 3159 * Can be called by the pagefault handler when it gets a VM_FAULT_OOM. 3160 */ 3161 extern void pagefault_out_of_memory(void); 3162 3163 #define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK) 3164 #define offset_in_folio(folio, p) ((unsigned long)(p) & (folio_size(folio) - 1)) 3165 3166 /* 3167 * Parameter block passed down to zap_pte_range in exceptional cases. 3168 */ 3169 struct zap_details { 3170 struct folio *single_folio; /* Locked folio to be unmapped */ 3171 bool skip_cows; /* Do not zap COWed private pages */ 3172 bool reclaim_pt; /* Need reclaim page tables? */ 3173 bool reaping; /* Reaping, do not block. */ 3174 zap_flags_t zap_flags; /* Extra flags for zapping */ 3175 }; 3176 3177 /* 3178 * Whether to drop the pte markers, for example, the uffd-wp information for 3179 * file-backed memory. This should only be specified when we will completely 3180 * drop the page in the mm, either by truncation or unmapping of the vma. By 3181 * default, the flag is not set. 3182 */ 3183 #define ZAP_FLAG_DROP_MARKER ((__force zap_flags_t) BIT(0)) 3184 /* Set in unmap_vmas() to indicate a final unmap call. Only used by hugetlb */ 3185 #define ZAP_FLAG_UNMAP ((__force zap_flags_t) BIT(1)) 3186 3187 #ifdef CONFIG_MMU 3188 extern bool can_do_mlock(void); 3189 #else 3190 static inline bool can_do_mlock(void) { return false; } 3191 #endif 3192 extern int user_shm_lock(size_t, struct ucounts *); 3193 extern void user_shm_unlock(size_t, struct ucounts *); 3194 3195 struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr, 3196 pte_t pte); 3197 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, 3198 pte_t pte); 3199 struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma, 3200 unsigned long addr, pmd_t pmd); 3201 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr, 3202 pmd_t pmd); 3203 struct page *vm_normal_page_pud(struct vm_area_struct *vma, unsigned long addr, 3204 pud_t pud); 3205 3206 void zap_special_vma_range(struct vm_area_struct *vma, unsigned long address, 3207 unsigned long size); 3208 void zap_vma_range(struct vm_area_struct *vma, unsigned long address, 3209 unsigned long size); 3210 /** 3211 * zap_vma - zap all page table entries in a vma 3212 * @vma: The vma to zap. 3213 */ 3214 static inline void zap_vma(struct vm_area_struct *vma) 3215 { 3216 zap_vma_range(vma, vma->vm_start, vma->vm_end - vma->vm_start); 3217 } 3218 struct mmu_notifier_range; 3219 3220 void free_pgd_range(struct mmu_gather *tlb, unsigned long addr, 3221 unsigned long end, unsigned long floor, unsigned long ceiling); 3222 int 3223 copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma); 3224 int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, 3225 void *buf, int len, int write); 3226 3227 struct follow_pfnmap_args { 3228 /** 3229 * Inputs: 3230 * @vma: Pointer to @vm_area_struct struct 3231 * @address: the virtual address to walk 3232 */ 3233 struct vm_area_struct *vma; 3234 unsigned long address; 3235 /** 3236 * Internals: 3237 * 3238 * The caller shouldn't touch any of these. 3239 */ 3240 spinlock_t *lock; 3241 pte_t *ptep; 3242 /** 3243 * Outputs: 3244 * 3245 * @pfn: the PFN of the address 3246 * @addr_mask: address mask covering pfn 3247 * @pgprot: the pgprot_t of the mapping 3248 * @writable: whether the mapping is writable 3249 * @special: whether the mapping is a special mapping (real PFN maps) 3250 */ 3251 unsigned long pfn; 3252 unsigned long addr_mask; 3253 pgprot_t pgprot; 3254 bool writable; 3255 bool special; 3256 }; 3257 int follow_pfnmap_start(struct follow_pfnmap_args *args); 3258 void follow_pfnmap_end(struct follow_pfnmap_args *args); 3259 3260 extern void truncate_pagecache(struct inode *inode, loff_t new); 3261 extern void truncate_setsize(struct inode *inode, loff_t newsize); 3262 void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to); 3263 void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end); 3264 int generic_error_remove_folio(struct address_space *mapping, 3265 struct folio *folio); 3266 3267 struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm, 3268 unsigned long address, struct pt_regs *regs); 3269 3270 #ifdef CONFIG_MMU 3271 extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma, 3272 unsigned long address, unsigned int flags, 3273 struct pt_regs *regs); 3274 extern int fixup_user_fault(struct mm_struct *mm, 3275 unsigned long address, unsigned int fault_flags, 3276 bool *unlocked); 3277 void unmap_mapping_pages(struct address_space *mapping, 3278 pgoff_t start, pgoff_t nr, bool even_cows); 3279 void unmap_mapping_range(struct address_space *mapping, 3280 loff_t const holebegin, loff_t const holelen, int even_cows); 3281 #else 3282 static inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma, 3283 unsigned long address, unsigned int flags, 3284 struct pt_regs *regs) 3285 { 3286 /* should never happen if there's no MMU */ 3287 BUG(); 3288 return VM_FAULT_SIGBUS; 3289 } 3290 static inline int fixup_user_fault(struct mm_struct *mm, unsigned long address, 3291 unsigned int fault_flags, bool *unlocked) 3292 { 3293 /* should never happen if there's no MMU */ 3294 BUG(); 3295 return -EFAULT; 3296 } 3297 static inline void unmap_mapping_pages(struct address_space *mapping, 3298 pgoff_t start, pgoff_t nr, bool even_cows) { } 3299 static inline void unmap_mapping_range(struct address_space *mapping, 3300 loff_t const holebegin, loff_t const holelen, int even_cows) { } 3301 #endif 3302 3303 static inline void unmap_shared_mapping_range(struct address_space *mapping, 3304 loff_t const holebegin, loff_t const holelen) 3305 { 3306 unmap_mapping_range(mapping, holebegin, holelen, 0); 3307 } 3308 3309 static inline struct vm_area_struct *vma_lookup(struct mm_struct *mm, 3310 unsigned long addr); 3311 3312 extern int access_process_vm(struct task_struct *tsk, unsigned long addr, 3313 void *buf, int len, unsigned int gup_flags); 3314 extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, 3315 void *buf, int len, unsigned int gup_flags); 3316 3317 int copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, 3318 void *buf, int len, unsigned int gup_flags); 3319 int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, 3320 void *buf, int len, unsigned int gup_flags); 3321 3322 long get_user_pages_remote(struct mm_struct *mm, 3323 unsigned long start, unsigned long nr_pages, 3324 unsigned int gup_flags, struct page **pages, 3325 int *locked); 3326 long pin_user_pages_remote(struct mm_struct *mm, 3327 unsigned long start, unsigned long nr_pages, 3328 unsigned int gup_flags, struct page **pages, 3329 int *locked); 3330 3331 /* 3332 * Retrieves a single page alongside its VMA. Does not support FOLL_NOWAIT. 3333 */ 3334 static inline struct page *get_user_page_vma_remote(struct mm_struct *mm, 3335 unsigned long addr, 3336 int gup_flags, 3337 struct vm_area_struct **vmap) 3338 { 3339 struct page *page; 3340 struct vm_area_struct *vma; 3341 int got; 3342 3343 if (WARN_ON_ONCE(unlikely(gup_flags & FOLL_NOWAIT))) 3344 return ERR_PTR(-EINVAL); 3345 3346 got = get_user_pages_remote(mm, addr, 1, gup_flags, &page, NULL); 3347 3348 if (got < 0) 3349 return ERR_PTR(got); 3350 3351 vma = vma_lookup(mm, addr); 3352 if (WARN_ON_ONCE(!vma)) { 3353 put_page(page); 3354 return ERR_PTR(-EINVAL); 3355 } 3356 3357 *vmap = vma; 3358 return page; 3359 } 3360 3361 long get_user_pages(unsigned long start, unsigned long nr_pages, 3362 unsigned int gup_flags, struct page **pages); 3363 long pin_user_pages(unsigned long start, unsigned long nr_pages, 3364 unsigned int gup_flags, struct page **pages); 3365 long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, 3366 struct page **pages, unsigned int gup_flags); 3367 long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages, 3368 struct page **pages, unsigned int gup_flags); 3369 long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end, 3370 struct folio **folios, unsigned int max_folios, 3371 pgoff_t *offset); 3372 int folio_add_pins(struct folio *folio, unsigned int pins); 3373 3374 int get_user_pages_fast(unsigned long start, int nr_pages, 3375 unsigned int gup_flags, struct page **pages); 3376 int pin_user_pages_fast(unsigned long start, int nr_pages, 3377 unsigned int gup_flags, struct page **pages); 3378 void folio_add_pin(struct folio *folio); 3379 3380 int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc); 3381 int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, 3382 const struct task_struct *task, bool bypass_rlim); 3383 3384 struct kvec; 3385 struct page *get_dump_page(unsigned long addr, int *locked); 3386 3387 bool folio_mark_dirty(struct folio *folio); 3388 bool folio_mark_dirty_lock(struct folio *folio); 3389 bool set_page_dirty(struct page *page); 3390 int set_page_dirty_lock(struct page *page); 3391 3392 int get_cmdline(struct task_struct *task, char *buffer, int buflen); 3393 3394 /* 3395 * Flags used by change_protection(). For now we make it a bitmap so 3396 * that we can pass in multiple flags just like parameters. However 3397 * for now all the callers are only use one of the flags at the same 3398 * time. 3399 */ 3400 /* 3401 * Whether we should manually check if we can map individual PTEs writable, 3402 * because something (e.g., COW, uffd-wp) blocks that from happening for all 3403 * PTEs automatically in a writable mapping. 3404 */ 3405 #define MM_CP_TRY_CHANGE_WRITABLE (1UL << 0) 3406 /* Whether this protection change is for NUMA hints */ 3407 #define MM_CP_PROT_NUMA (1UL << 1) 3408 /* Whether this change is for write protecting */ 3409 #define MM_CP_UFFD_WP (1UL << 2) /* do wp */ 3410 #define MM_CP_UFFD_WP_RESOLVE (1UL << 3) /* Resolve wp */ 3411 #define MM_CP_UFFD_WP_ALL (MM_CP_UFFD_WP | \ 3412 MM_CP_UFFD_WP_RESOLVE) 3413 /* Whether this change is for uffd RWP */ 3414 #define MM_CP_UFFD_RWP (1UL << 4) /* do rwp */ 3415 #define MM_CP_UFFD_RWP_RESOLVE (1UL << 5) /* resolve rwp */ 3416 #define MM_CP_UFFD_RWP_ALL (MM_CP_UFFD_RWP | \ 3417 MM_CP_UFFD_RWP_RESOLVE) 3418 3419 bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, 3420 pte_t pte); 3421 extern long change_protection(struct mmu_gather *tlb, 3422 struct vm_area_struct *vma, unsigned long start, 3423 unsigned long end, unsigned long cp_flags); 3424 extern int mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, 3425 struct vm_area_struct *vma, struct vm_area_struct **pprev, 3426 unsigned long start, unsigned long end, vm_flags_t newflags); 3427 3428 /* 3429 * doesn't attempt to fault and will return short. 3430 */ 3431 int get_user_pages_fast_only(unsigned long start, int nr_pages, 3432 unsigned int gup_flags, struct page **pages); 3433 3434 static inline bool get_user_page_fast_only(unsigned long addr, 3435 unsigned int gup_flags, struct page **pagep) 3436 { 3437 return get_user_pages_fast_only(addr, 1, gup_flags, pagep) == 1; 3438 } 3439 /* 3440 * per-process(per-mm_struct) statistics. 3441 */ 3442 static inline unsigned long get_mm_counter(struct mm_struct *mm, int member) 3443 { 3444 return percpu_counter_read_positive(&mm->rss_stat[member]); 3445 } 3446 3447 static inline unsigned long get_mm_counter_sum(struct mm_struct *mm, int member) 3448 { 3449 return percpu_counter_sum_positive(&mm->rss_stat[member]); 3450 } 3451 3452 void mm_trace_rss_stat(struct mm_struct *mm, int member); 3453 3454 static inline void add_mm_counter(struct mm_struct *mm, int member, long value) 3455 { 3456 percpu_counter_add(&mm->rss_stat[member], value); 3457 3458 mm_trace_rss_stat(mm, member); 3459 } 3460 3461 static inline void inc_mm_counter(struct mm_struct *mm, int member) 3462 { 3463 percpu_counter_inc(&mm->rss_stat[member]); 3464 3465 mm_trace_rss_stat(mm, member); 3466 } 3467 3468 static inline void dec_mm_counter(struct mm_struct *mm, int member) 3469 { 3470 percpu_counter_dec(&mm->rss_stat[member]); 3471 3472 mm_trace_rss_stat(mm, member); 3473 } 3474 3475 /* Optimized variant when folio is already known not to be anon */ 3476 static inline int mm_counter_file(struct folio *folio) 3477 { 3478 if (folio_test_swapbacked(folio)) 3479 return MM_SHMEMPAGES; 3480 return MM_FILEPAGES; 3481 } 3482 3483 static inline int mm_counter(struct folio *folio) 3484 { 3485 if (folio_test_anon(folio)) 3486 return MM_ANONPAGES; 3487 return mm_counter_file(folio); 3488 } 3489 3490 static inline unsigned long get_mm_rss(struct mm_struct *mm) 3491 { 3492 return get_mm_counter(mm, MM_FILEPAGES) + 3493 get_mm_counter(mm, MM_ANONPAGES) + 3494 get_mm_counter(mm, MM_SHMEMPAGES); 3495 } 3496 3497 static inline unsigned long get_mm_rss_sum(struct mm_struct *mm) 3498 { 3499 return get_mm_counter_sum(mm, MM_FILEPAGES) + 3500 get_mm_counter_sum(mm, MM_ANONPAGES) + 3501 get_mm_counter_sum(mm, MM_SHMEMPAGES); 3502 } 3503 3504 static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm) 3505 { 3506 return max(mm->hiwater_rss, get_mm_rss(mm)); 3507 } 3508 3509 static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm) 3510 { 3511 return max(mm->hiwater_vm, mm->total_vm); 3512 } 3513 3514 static inline void update_hiwater_rss(struct mm_struct *mm) 3515 { 3516 unsigned long _rss = get_mm_rss(mm); 3517 3518 if (data_race(mm->hiwater_rss) < _rss) 3519 data_race(mm->hiwater_rss = _rss); 3520 } 3521 3522 static inline void update_hiwater_vm(struct mm_struct *mm) 3523 { 3524 if (mm->hiwater_vm < mm->total_vm) 3525 mm->hiwater_vm = mm->total_vm; 3526 } 3527 3528 static inline void reset_mm_hiwater_rss(struct mm_struct *mm) 3529 { 3530 mm->hiwater_rss = get_mm_rss(mm); 3531 } 3532 3533 static inline void setmax_mm_hiwater_rss(unsigned long *maxrss, 3534 struct mm_struct *mm) 3535 { 3536 unsigned long hiwater_rss = get_mm_hiwater_rss(mm); 3537 3538 if (*maxrss < hiwater_rss) 3539 *maxrss = hiwater_rss; 3540 } 3541 3542 #ifndef CONFIG_ARCH_HAS_PTE_SPECIAL 3543 static inline int pte_special(pte_t pte) 3544 { 3545 return 0; 3546 } 3547 3548 static inline pte_t pte_mkspecial(pte_t pte) 3549 { 3550 return pte; 3551 } 3552 #endif 3553 3554 #ifndef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP 3555 static inline bool pmd_special(pmd_t pmd) 3556 { 3557 return false; 3558 } 3559 3560 static inline pmd_t pmd_mkspecial(pmd_t pmd) 3561 { 3562 return pmd; 3563 } 3564 #endif /* CONFIG_ARCH_SUPPORTS_PMD_PFNMAP */ 3565 3566 #ifndef CONFIG_ARCH_SUPPORTS_PUD_PFNMAP 3567 static inline bool pud_special(pud_t pud) 3568 { 3569 return false; 3570 } 3571 3572 static inline pud_t pud_mkspecial(pud_t pud) 3573 { 3574 return pud; 3575 } 3576 #endif /* CONFIG_ARCH_SUPPORTS_PUD_PFNMAP */ 3577 3578 extern pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, 3579 spinlock_t **ptl); 3580 3581 #ifdef __PAGETABLE_P4D_FOLDED 3582 static inline int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, 3583 unsigned long address) 3584 { 3585 return 0; 3586 } 3587 #else 3588 int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address); 3589 #endif 3590 3591 #if defined(__PAGETABLE_PUD_FOLDED) || !defined(CONFIG_MMU) 3592 static inline int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, 3593 unsigned long address) 3594 { 3595 return 0; 3596 } 3597 static inline void mm_inc_nr_puds(struct mm_struct *mm) {} 3598 static inline void mm_dec_nr_puds(struct mm_struct *mm) {} 3599 3600 #else 3601 int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address); 3602 3603 static inline void mm_inc_nr_puds(struct mm_struct *mm) 3604 { 3605 if (mm_pud_folded(mm)) 3606 return; 3607 atomic_long_add(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes); 3608 } 3609 3610 static inline void mm_dec_nr_puds(struct mm_struct *mm) 3611 { 3612 if (mm_pud_folded(mm)) 3613 return; 3614 atomic_long_sub(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes); 3615 } 3616 #endif 3617 3618 #if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU) 3619 static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud, 3620 unsigned long address) 3621 { 3622 return 0; 3623 } 3624 3625 static inline void mm_inc_nr_pmds(struct mm_struct *mm) {} 3626 static inline void mm_dec_nr_pmds(struct mm_struct *mm) {} 3627 3628 #else 3629 int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address); 3630 3631 static inline void mm_inc_nr_pmds(struct mm_struct *mm) 3632 { 3633 if (mm_pmd_folded(mm)) 3634 return; 3635 atomic_long_add(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes); 3636 } 3637 3638 static inline void mm_dec_nr_pmds(struct mm_struct *mm) 3639 { 3640 if (mm_pmd_folded(mm)) 3641 return; 3642 atomic_long_sub(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes); 3643 } 3644 #endif 3645 3646 #ifdef CONFIG_MMU 3647 static inline void mm_pgtables_bytes_init(struct mm_struct *mm) 3648 { 3649 atomic_long_set(&mm->pgtables_bytes, 0); 3650 } 3651 3652 static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm) 3653 { 3654 return atomic_long_read(&mm->pgtables_bytes); 3655 } 3656 3657 static inline void mm_inc_nr_ptes(struct mm_struct *mm) 3658 { 3659 atomic_long_add(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes); 3660 } 3661 3662 static inline void mm_dec_nr_ptes(struct mm_struct *mm) 3663 { 3664 atomic_long_sub(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes); 3665 } 3666 #else 3667 3668 static inline void mm_pgtables_bytes_init(struct mm_struct *mm) {} 3669 static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm) 3670 { 3671 return 0; 3672 } 3673 3674 static inline void mm_inc_nr_ptes(struct mm_struct *mm) {} 3675 static inline void mm_dec_nr_ptes(struct mm_struct *mm) {} 3676 #endif 3677 3678 int __pte_alloc(struct mm_struct *mm, pmd_t *pmd); 3679 int __pte_alloc_kernel(pmd_t *pmd); 3680 3681 #if defined(CONFIG_MMU) 3682 3683 static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd, 3684 unsigned long address) 3685 { 3686 return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ? 3687 NULL : p4d_offset(pgd, address); 3688 } 3689 3690 static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d, 3691 unsigned long address) 3692 { 3693 return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ? 3694 NULL : pud_offset(p4d, address); 3695 } 3696 3697 static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) 3698 { 3699 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))? 3700 NULL: pmd_offset(pud, address); 3701 } 3702 #endif /* CONFIG_MMU */ 3703 3704 enum pt_flags { 3705 PT_kernel = PG_referenced, 3706 PT_reserved = PG_reserved, 3707 /* High bits are used for zone/node/section */ 3708 }; 3709 3710 static inline struct ptdesc *virt_to_ptdesc(const void *x) 3711 { 3712 return page_ptdesc(virt_to_page(x)); 3713 } 3714 3715 /** 3716 * ptdesc_address - Virtual address of page table. 3717 * @pt: Page table descriptor. 3718 * 3719 * Return: The first byte of the page table described by @pt. 3720 */ 3721 static inline void *ptdesc_address(const struct ptdesc *pt) 3722 { 3723 return folio_address(ptdesc_folio(pt)); 3724 } 3725 3726 static inline bool pagetable_is_reserved(struct ptdesc *pt) 3727 { 3728 return test_bit(PT_reserved, &pt->pt_flags.f); 3729 } 3730 3731 /** 3732 * ptdesc_set_kernel - Mark a ptdesc used to map the kernel 3733 * @ptdesc: The ptdesc to be marked 3734 * 3735 * Kernel page tables often need special handling. Set a flag so that 3736 * the handling code knows this ptdesc will not be used for userspace. 3737 */ 3738 static inline void ptdesc_set_kernel(struct ptdesc *ptdesc) 3739 { 3740 set_bit(PT_kernel, &ptdesc->pt_flags.f); 3741 } 3742 3743 /** 3744 * ptdesc_clear_kernel - Mark a ptdesc as no longer used to map the kernel 3745 * @ptdesc: The ptdesc to be unmarked 3746 * 3747 * Use when the ptdesc is no longer used to map the kernel and no longer 3748 * needs special handling. 3749 */ 3750 static inline void ptdesc_clear_kernel(struct ptdesc *ptdesc) 3751 { 3752 /* 3753 * Note: the 'PG_referenced' bit does not strictly need to be 3754 * cleared before freeing the page. But this is nice for 3755 * symmetry. 3756 */ 3757 clear_bit(PT_kernel, &ptdesc->pt_flags.f); 3758 } 3759 3760 /** 3761 * ptdesc_test_kernel - Check if a ptdesc is used to map the kernel 3762 * @ptdesc: The ptdesc being tested 3763 * 3764 * Call to tell if the ptdesc used to map the kernel. 3765 */ 3766 static inline bool ptdesc_test_kernel(const struct ptdesc *ptdesc) 3767 { 3768 return test_bit(PT_kernel, &ptdesc->pt_flags.f); 3769 } 3770 3771 /** 3772 * pagetable_alloc - Allocate pagetables 3773 * @gfp: GFP flags 3774 * @order: desired pagetable order 3775 * 3776 * pagetable_alloc allocates memory for page tables as well as a page table 3777 * descriptor to describe that memory. 3778 * 3779 * Return: The ptdesc describing the allocated page tables. 3780 */ 3781 static inline struct ptdesc *pagetable_alloc_noprof(gfp_t gfp, unsigned int order) 3782 { 3783 struct page *page = alloc_pages_noprof(gfp | __GFP_COMP, order); 3784 3785 return page_ptdesc(page); 3786 } 3787 #define pagetable_alloc(...) alloc_hooks(pagetable_alloc_noprof(__VA_ARGS__)) 3788 3789 static inline void __pagetable_free(struct ptdesc *pt) 3790 { 3791 struct page *page = ptdesc_page(pt); 3792 3793 __free_pages(page, compound_order(page)); 3794 } 3795 3796 #ifdef CONFIG_ASYNC_KERNEL_PGTABLE_FREE 3797 void pagetable_free_kernel(struct ptdesc *pt); 3798 #else 3799 static inline void pagetable_free_kernel(struct ptdesc *pt) 3800 { 3801 __pagetable_free(pt); 3802 } 3803 #endif 3804 /** 3805 * pagetable_free - Free pagetables 3806 * @pt: The page table descriptor 3807 * 3808 * pagetable_free frees the memory of all page tables described by a page 3809 * table descriptor and the memory for the descriptor itself. 3810 */ 3811 static inline void pagetable_free(struct ptdesc *pt) 3812 { 3813 if (ptdesc_test_kernel(pt)) { 3814 ptdesc_clear_kernel(pt); 3815 pagetable_free_kernel(pt); 3816 } else { 3817 __pagetable_free(pt); 3818 } 3819 } 3820 3821 #if defined(CONFIG_SPLIT_PTE_PTLOCKS) 3822 #if ALLOC_SPLIT_PTLOCKS 3823 void __init ptlock_cache_init(void); 3824 bool ptlock_alloc(struct ptdesc *ptdesc); 3825 void ptlock_free(struct ptdesc *ptdesc); 3826 3827 static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc) 3828 { 3829 return ptdesc->ptl; 3830 } 3831 #else /* ALLOC_SPLIT_PTLOCKS */ 3832 static inline void ptlock_cache_init(void) 3833 { 3834 } 3835 3836 static inline bool ptlock_alloc(struct ptdesc *ptdesc) 3837 { 3838 return true; 3839 } 3840 3841 static inline void ptlock_free(struct ptdesc *ptdesc) 3842 { 3843 } 3844 3845 static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc) 3846 { 3847 return &ptdesc->ptl; 3848 } 3849 #endif /* ALLOC_SPLIT_PTLOCKS */ 3850 3851 static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd) 3852 { 3853 return ptlock_ptr(page_ptdesc(pmd_page(*pmd))); 3854 } 3855 3856 static inline spinlock_t *ptep_lockptr(struct mm_struct *mm, pte_t *pte) 3857 { 3858 BUILD_BUG_ON(IS_ENABLED(CONFIG_HIGHPTE)); 3859 BUILD_BUG_ON(MAX_PTRS_PER_PTE * sizeof(pte_t) > PAGE_SIZE); 3860 return ptlock_ptr(virt_to_ptdesc(pte)); 3861 } 3862 3863 static inline bool ptlock_init(struct ptdesc *ptdesc) 3864 { 3865 /* 3866 * prep_new_page() initialize page->private (and therefore page->ptl) 3867 * with 0. Make sure nobody took it in use in between. 3868 * 3869 * It can happen if arch try to use slab for page table allocation: 3870 * slab code uses page->slab_cache, which share storage with page->ptl. 3871 */ 3872 VM_BUG_ON_PAGE(*(unsigned long *)&ptdesc->ptl, ptdesc_page(ptdesc)); 3873 if (!ptlock_alloc(ptdesc)) 3874 return false; 3875 spin_lock_init(ptlock_ptr(ptdesc)); 3876 return true; 3877 } 3878 3879 #else /* !defined(CONFIG_SPLIT_PTE_PTLOCKS) */ 3880 /* 3881 * We use mm->page_table_lock to guard all pagetable pages of the mm. 3882 */ 3883 static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd) 3884 { 3885 return &mm->page_table_lock; 3886 } 3887 static inline spinlock_t *ptep_lockptr(struct mm_struct *mm, pte_t *pte) 3888 { 3889 return &mm->page_table_lock; 3890 } 3891 static inline void ptlock_cache_init(void) {} 3892 static inline bool ptlock_init(struct ptdesc *ptdesc) { return true; } 3893 static inline void ptlock_free(struct ptdesc *ptdesc) {} 3894 #endif /* defined(CONFIG_SPLIT_PTE_PTLOCKS) */ 3895 3896 static inline void __pagetable_ctor(struct ptdesc *ptdesc) 3897 { 3898 struct folio *folio = ptdesc_folio(ptdesc); 3899 3900 __folio_set_pgtable(folio); 3901 lruvec_stat_add_folio(folio, NR_PAGETABLE); 3902 } 3903 3904 static inline void pagetable_dtor(struct ptdesc *ptdesc) 3905 { 3906 struct folio *folio = ptdesc_folio(ptdesc); 3907 3908 ptlock_free(ptdesc); 3909 __folio_clear_pgtable(folio); 3910 lruvec_stat_sub_folio(folio, NR_PAGETABLE); 3911 } 3912 3913 static inline void pagetable_dtor_free(struct ptdesc *ptdesc) 3914 { 3915 pagetable_dtor(ptdesc); 3916 pagetable_free(ptdesc); 3917 } 3918 3919 static inline bool pagetable_pte_ctor(struct mm_struct *mm, 3920 struct ptdesc *ptdesc) 3921 { 3922 if (mm != &init_mm && !ptlock_init(ptdesc)) 3923 return false; 3924 __pagetable_ctor(ptdesc); 3925 return true; 3926 } 3927 3928 pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp); 3929 3930 static inline pte_t *pte_offset_map(pmd_t *pmd, unsigned long addr) 3931 { 3932 return __pte_offset_map(pmd, addr, NULL); 3933 } 3934 3935 pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, 3936 unsigned long addr, spinlock_t **ptlp); 3937 3938 pte_t *pte_offset_map_ro_nolock(struct mm_struct *mm, pmd_t *pmd, 3939 unsigned long addr, spinlock_t **ptlp); 3940 pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, 3941 unsigned long addr, pmd_t *pmdvalp, 3942 spinlock_t **ptlp); 3943 3944 #define pte_unmap_unlock(pte, ptl) do { \ 3945 spin_unlock(ptl); \ 3946 pte_unmap(pte); \ 3947 } while (0) 3948 3949 #define pte_alloc(mm, pmd) (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd)) 3950 3951 #define pte_alloc_map(mm, pmd, address) \ 3952 (pte_alloc(mm, pmd) ? NULL : pte_offset_map(pmd, address)) 3953 3954 #define pte_alloc_map_lock(mm, pmd, address, ptlp) \ 3955 (pte_alloc(mm, pmd) ? \ 3956 NULL : pte_offset_map_lock(mm, pmd, address, ptlp)) 3957 3958 #define pte_alloc_kernel(pmd, address) \ 3959 ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd))? \ 3960 NULL: pte_offset_kernel(pmd, address)) 3961 3962 #if defined(CONFIG_SPLIT_PMD_PTLOCKS) 3963 3964 static inline struct page *pmd_pgtable_page(pmd_t *pmd) 3965 { 3966 unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1); 3967 return virt_to_page((void *)((unsigned long) pmd & mask)); 3968 } 3969 3970 static inline struct ptdesc *pmd_ptdesc(pmd_t *pmd) 3971 { 3972 return page_ptdesc(pmd_pgtable_page(pmd)); 3973 } 3974 3975 static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd) 3976 { 3977 return ptlock_ptr(pmd_ptdesc(pmd)); 3978 } 3979 3980 static inline bool pmd_ptlock_init(struct ptdesc *ptdesc) 3981 { 3982 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3983 ptdesc->pmd_huge_pte = NULL; 3984 #endif 3985 return ptlock_init(ptdesc); 3986 } 3987 3988 #define pmd_huge_pte(mm, pmd) (pmd_ptdesc(pmd)->pmd_huge_pte) 3989 3990 #else 3991 3992 static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd) 3993 { 3994 return &mm->page_table_lock; 3995 } 3996 3997 static inline bool pmd_ptlock_init(struct ptdesc *ptdesc) { return true; } 3998 3999 #define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte) 4000 4001 #endif 4002 4003 static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd) 4004 { 4005 spinlock_t *ptl = pmd_lockptr(mm, pmd); 4006 spin_lock(ptl); 4007 return ptl; 4008 } 4009 4010 static inline bool pagetable_pmd_ctor(struct mm_struct *mm, 4011 struct ptdesc *ptdesc) 4012 { 4013 if (mm != &init_mm && !pmd_ptlock_init(ptdesc)) 4014 return false; 4015 ptdesc_pmd_pts_init(ptdesc); 4016 __pagetable_ctor(ptdesc); 4017 return true; 4018 } 4019 4020 /* 4021 * No scalability reason to split PUD locks yet, but follow the same pattern 4022 * as the PMD locks to make it easier if we decide to. The VM should not be 4023 * considered ready to switch to split PUD locks yet; there may be places 4024 * which need to be converted from page_table_lock. 4025 */ 4026 static inline spinlock_t *pud_lockptr(struct mm_struct *mm, pud_t *pud) 4027 { 4028 return &mm->page_table_lock; 4029 } 4030 4031 static inline spinlock_t *pud_lock(struct mm_struct *mm, pud_t *pud) 4032 { 4033 spinlock_t *ptl = pud_lockptr(mm, pud); 4034 4035 spin_lock(ptl); 4036 return ptl; 4037 } 4038 4039 static inline void pagetable_pud_ctor(struct ptdesc *ptdesc) 4040 { 4041 __pagetable_ctor(ptdesc); 4042 } 4043 4044 static inline void pagetable_p4d_ctor(struct ptdesc *ptdesc) 4045 { 4046 __pagetable_ctor(ptdesc); 4047 } 4048 4049 static inline void pagetable_pgd_ctor(struct ptdesc *ptdesc) 4050 { 4051 __pagetable_ctor(ptdesc); 4052 } 4053 4054 extern void __init pagecache_init(void); 4055 extern void free_initmem(void); 4056 4057 /* 4058 * Free reserved pages within range [PAGE_ALIGN(start), end & PAGE_MASK) 4059 * into the buddy system. The freed pages will be poisoned with pattern 4060 * "poison" if it's within range [0, UCHAR_MAX]. 4061 * Return pages freed into the buddy system. 4062 */ 4063 extern unsigned long free_reserved_area(void *start, void *end, 4064 int poison, const char *s); 4065 4066 extern void adjust_managed_page_count(struct page *page, long count); 4067 4068 void free_reserved_pages(struct page *page, unsigned int order); 4069 4070 static inline void free_reserved_page(struct page *page) 4071 { 4072 free_reserved_pages(page, 0); 4073 } 4074 4075 static inline void free_reserved_ptdesc(struct ptdesc *pt) 4076 { 4077 free_reserved_page(ptdesc_page(pt)); 4078 } 4079 4080 /* 4081 * Default method to free all the __init memory into the buddy system. 4082 * The freed pages will be poisoned with pattern "poison" if it's within 4083 * range [0, UCHAR_MAX]. 4084 * Return pages freed into the buddy system. 4085 */ 4086 static inline unsigned long free_initmem_default(int poison) 4087 { 4088 extern char __init_begin[], __init_end[]; 4089 4090 return free_reserved_area(&__init_begin, &__init_end, 4091 poison, "unused kernel image (initmem)"); 4092 } 4093 4094 static inline unsigned long get_num_physpages(void) 4095 { 4096 int nid; 4097 unsigned long phys_pages = 0; 4098 4099 for_each_online_node(nid) 4100 phys_pages += node_present_pages(nid); 4101 4102 return phys_pages; 4103 } 4104 4105 /* 4106 * FIXME: Using memblock node mappings, an architecture may initialise its 4107 * zones, allocate the backing mem_map and account for memory holes in an 4108 * architecture independent manner. 4109 * 4110 * An architecture is expected to register range of page frames backed by 4111 * physical memory with memblock_add[_node]() before calling 4112 * free_area_init() passing in the PFN each zone ends at. At a basic 4113 * usage, an architecture is expected to do something like 4114 * 4115 * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn, 4116 * max_highmem_pfn}; 4117 * for_each_valid_physical_page_range() 4118 * memblock_add_node(base, size, nid, MEMBLOCK_NONE) 4119 * free_area_init(max_zone_pfns); 4120 */ 4121 void arch_zone_limits_init(unsigned long *max_zone_pfn); 4122 unsigned long node_map_pfn_alignment(void); 4123 extern unsigned long absent_pages_in_range(unsigned long start_pfn, 4124 unsigned long end_pfn); 4125 extern void get_pfn_range_for_nid(unsigned int nid, 4126 unsigned long *start_pfn, unsigned long *end_pfn); 4127 4128 #ifndef CONFIG_NUMA 4129 static inline int early_pfn_to_nid(unsigned long pfn) 4130 { 4131 return 0; 4132 } 4133 #else 4134 /* please see mm/page_alloc.c */ 4135 extern int __meminit early_pfn_to_nid(unsigned long pfn); 4136 #endif 4137 4138 extern void mem_init(void); 4139 extern void __init mmap_init(void); 4140 4141 extern void __show_mem(unsigned int flags, const nodemask_t *nodemask, int max_zone_idx); 4142 static inline void show_mem(void) 4143 { 4144 __show_mem(0, NULL, MAX_NR_ZONES - 1); 4145 } 4146 extern long si_mem_available(void); 4147 extern void si_meminfo(struct sysinfo * val); 4148 extern void si_meminfo_node(struct sysinfo *val, int nid); 4149 4150 extern __printf(3, 4) 4151 void warn_alloc(gfp_t gfp_mask, const nodemask_t *nodemask, const char *fmt, ...); 4152 4153 extern void setup_per_cpu_pageset(void); 4154 4155 /* nommu.c */ 4156 extern atomic_long_t mmap_pages_allocated; 4157 extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t); 4158 4159 /* interval_tree.c */ 4160 void mapping_rmap_tree_insert(struct vm_area_struct *vma, 4161 struct address_space *mapping); 4162 void mapping_rmap_tree_insert_after(struct vm_area_struct *vma, 4163 struct vm_area_struct *prev, 4164 struct address_space *mapping); 4165 void mapping_rmap_tree_remove(struct vm_area_struct *vma, 4166 struct address_space *mapping); 4167 struct vm_area_struct * 4168 mapping_rmap_tree_iter_first(struct address_space *mapping, 4169 pgoff_t pgoff_start, pgoff_t pgoff_last); 4170 struct vm_area_struct * 4171 mapping_rmap_tree_iter_next(struct vm_area_struct *vma, 4172 pgoff_t pgoff_start, pgoff_t pgoff_last); 4173 4174 #define mapping_rmap_tree_foreach(vma, mapping, pgoff_start, pgoff_last) \ 4175 for (vma = mapping_rmap_tree_iter_first(mapping, pgoff_start, \ 4176 pgoff_last); \ 4177 vma; vma = mapping_rmap_tree_iter_next(vma, pgoff_start, \ 4178 pgoff_last)) 4179 4180 void anon_rmap_tree_insert(struct anon_vma_chain *avc, 4181 struct anon_vma *anon_vma); 4182 void anon_rmap_tree_remove(struct anon_vma_chain *avc, 4183 struct anon_vma *anon_vma); 4184 struct anon_vma_chain * 4185 anon_rmap_tree_iter_first(struct anon_vma *anon_vma, 4186 pgoff_t pgoff_start, pgoff_t pgoff_last); 4187 struct anon_vma_chain * 4188 anon_rmap_tree_iter_next(struct anon_vma_chain *avc, 4189 pgoff_t pgoff_start, pgoff_t pgoff_last); 4190 #ifdef CONFIG_DEBUG_VM_RB 4191 void anon_rmap_tree_verify(struct anon_vma_chain *avc); 4192 #endif 4193 4194 #define anon_rmap_tree_foreach(avc, anon_vma, pgoff_start, pgoff_last) \ 4195 for (avc = anon_rmap_tree_iter_first(anon_vma, pgoff_start, pgoff_last); \ 4196 avc; avc = anon_rmap_tree_iter_next(avc, pgoff_start, pgoff_last)) 4197 4198 /* mmap.c */ 4199 extern int __vm_enough_memory(const struct mm_struct *mm, long pages, int cap_sys_admin); 4200 extern void exit_mmap(struct mm_struct *); 4201 bool mmap_read_lock_maybe_expand(struct mm_struct *mm, struct vm_area_struct *vma, 4202 unsigned long addr, bool write); 4203 4204 static inline int check_data_rlimit(unsigned long rlim, 4205 unsigned long new, 4206 unsigned long start, 4207 unsigned long end_data, 4208 unsigned long start_data) 4209 { 4210 if (rlim < RLIM_INFINITY) { 4211 if (((new - start) + (end_data - start_data)) > rlim) 4212 return -ENOSPC; 4213 } 4214 4215 return 0; 4216 } 4217 4218 extern int mm_take_all_locks(struct mm_struct *mm); 4219 extern void mm_drop_all_locks(struct mm_struct *mm); 4220 4221 extern int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file); 4222 extern int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file); 4223 extern struct file *get_mm_exe_file(struct mm_struct *mm); 4224 extern struct file *get_task_exe_file(struct task_struct *task); 4225 4226 extern void vm_stat_account(struct mm_struct *, vm_flags_t, long npages); 4227 4228 extern bool vma_is_special_mapping(const struct vm_area_struct *vma, 4229 const struct vm_special_mapping *sm); 4230 struct vm_area_struct *_install_special_mapping(struct mm_struct *mm, 4231 unsigned long addr, unsigned long len, 4232 vm_flags_t vm_flags, 4233 const struct vm_special_mapping *spec); 4234 4235 unsigned long randomize_stack_top(unsigned long stack_top); 4236 unsigned long randomize_page(unsigned long start, unsigned long range); 4237 4238 unsigned long 4239 __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, 4240 unsigned long pgoff, unsigned long flags, 4241 vma_flags_t vma_flags); 4242 4243 static inline unsigned long 4244 get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, 4245 unsigned long pgoff, unsigned long flags) 4246 { 4247 return __get_unmapped_area(file, addr, len, pgoff, flags, 4248 EMPTY_VMA_FLAGS); 4249 } 4250 4251 unsigned long do_mmap(struct file *file, unsigned long addr, 4252 unsigned long len, unsigned long prot, unsigned long flags, 4253 vma_flags_t vma_flags, unsigned long pgoff, unsigned long *populate, 4254 struct list_head *uf); 4255 extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, 4256 unsigned long start, size_t len, struct list_head *uf, 4257 bool unlock); 4258 int do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, 4259 struct mm_struct *mm, unsigned long start, 4260 unsigned long end, struct list_head *uf, bool unlock); 4261 extern int do_munmap(struct mm_struct *, unsigned long, size_t, 4262 struct list_head *uf); 4263 extern int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior); 4264 4265 #ifdef CONFIG_MMU 4266 extern int __mm_populate(unsigned long addr, unsigned long len, 4267 int ignore_errors); 4268 static inline void mm_populate(unsigned long addr, unsigned long len) 4269 { 4270 /* Ignore errors */ 4271 (void) __mm_populate(addr, len, 1); 4272 } 4273 #else 4274 static inline void mm_populate(unsigned long addr, unsigned long len) {} 4275 #endif 4276 4277 /* This takes the mm semaphore itself */ 4278 int __must_check vm_brk_flags(unsigned long addr, unsigned long request, bool is_exec); 4279 int vm_munmap(unsigned long start, size_t len); 4280 unsigned long __must_check vm_mmap(struct file *file, unsigned long addr, 4281 unsigned long len, unsigned long prot, 4282 unsigned long flag, unsigned long offset); 4283 unsigned long __must_check vm_mmap_shadow_stack(unsigned long addr, 4284 unsigned long len, unsigned long flags); 4285 4286 struct vm_unmapped_area_info { 4287 #define VM_UNMAPPED_AREA_TOPDOWN 1 4288 unsigned long flags; 4289 unsigned long length; 4290 unsigned long low_limit; 4291 unsigned long high_limit; 4292 unsigned long align_mask; 4293 unsigned long align_offset; 4294 unsigned long start_gap; 4295 }; 4296 4297 extern unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info); 4298 4299 /* truncate.c */ 4300 void truncate_inode_pages(struct address_space *mapping, loff_t lstart); 4301 void truncate_inode_pages_range(struct address_space *mapping, loff_t lstart, 4302 uoff_t lend); 4303 void truncate_inode_pages_final(struct address_space *mapping); 4304 4305 /* generic vm_area_ops exported for stackable file systems */ 4306 extern vm_fault_t filemap_fault(struct vm_fault *vmf); 4307 extern vm_fault_t filemap_map_pages(struct vm_fault *vmf, 4308 pgoff_t start_pgoff, pgoff_t end_pgoff); 4309 extern vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf); 4310 4311 extern unsigned long stack_guard_gap; 4312 /* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */ 4313 int expand_stack_locked(struct vm_area_struct *vma, unsigned long address); 4314 struct vm_area_struct *expand_stack(struct mm_struct * mm, unsigned long addr); 4315 4316 /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */ 4317 extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr); 4318 extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr, 4319 struct vm_area_struct **pprev); 4320 4321 /* 4322 * Look up the first VMA which intersects the interval [start_addr, end_addr) 4323 * NULL if none. Assume start_addr < end_addr. 4324 */ 4325 struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, 4326 unsigned long start_addr, unsigned long end_addr); 4327 4328 /** 4329 * vma_lookup() - Find a VMA at a specific address 4330 * @mm: The process address space. 4331 * @addr: The user address. 4332 * 4333 * Return: The vm_area_struct at the given address, %NULL otherwise. 4334 */ 4335 static inline 4336 struct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr) 4337 { 4338 return mtree_load(&mm->mm_mt, addr); 4339 } 4340 4341 static inline unsigned long stack_guard_start_gap(const struct vm_area_struct *vma) 4342 { 4343 if (vma->vm_flags & VM_GROWSDOWN) 4344 return stack_guard_gap; 4345 4346 /* See reasoning around the VM_SHADOW_STACK definition */ 4347 if (vma->vm_flags & VM_SHADOW_STACK) 4348 return PAGE_SIZE; 4349 4350 return 0; 4351 } 4352 4353 static inline unsigned long vm_start_gap(const struct vm_area_struct *vma) 4354 { 4355 unsigned long gap = stack_guard_start_gap(vma); 4356 unsigned long vm_start = vma->vm_start; 4357 4358 vm_start -= gap; 4359 if (vm_start > vma->vm_start) 4360 vm_start = 0; 4361 return vm_start; 4362 } 4363 4364 static inline unsigned long vm_end_gap(const struct vm_area_struct *vma) 4365 { 4366 unsigned long vm_end = vma->vm_end; 4367 4368 if (vma->vm_flags & VM_GROWSUP) { 4369 vm_end += stack_guard_gap; 4370 if (vm_end < vma->vm_end) 4371 vm_end = -PAGE_SIZE; 4372 } 4373 return vm_end; 4374 } 4375 4376 static inline unsigned long vma_pages(const struct vm_area_struct *vma) 4377 { 4378 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; 4379 } 4380 4381 /** 4382 * vma_start_pgoff() - Get the page offset of the start of @vma 4383 * @vma: The VMA whose page offset is required. 4384 * 4385 * If the VMA is file-backed, this is the page offset into the file. 4386 * 4387 * If @vma is anonymous, this is the virtual page offset of the start of the 4388 * VMA - if unfaulted, then vma->vm_start >> PAGE_SHIFT, if faulted then the 4389 * virtual page offset at the time of first fault. 4390 * 4391 * If @vma is a MAP_PRIVATE file-backed mapping, then this returns the 4392 * page offset within the file. 4393 * 4394 * Edge cases: nommu does not abide by these and CoW MAP_PRIVATE-pfnmap regions 4395 * have their page offset set to the first PFN in the range. 4396 * 4397 * Returns: The page offset of the start of @vma. 4398 */ 4399 static inline pgoff_t vma_start_pgoff(const struct vm_area_struct *vma) 4400 { 4401 return vma->vm_pgoff; 4402 } 4403 4404 /** 4405 * vma_end_pgoff() - Get the page offset of the exclusive end of @vma 4406 * @vma: The VMA whose end page offset is required. 4407 * 4408 * This returns the exclusive end page offset of @vma, which is useful for 4409 * expressing page offset ranges. 4410 * 4411 * See the description of vma_start_pgoff() for a description of VMA page 4412 * offsets. 4413 * 4414 * Returns: The exclusive end page offset of @vma. 4415 */ 4416 static inline pgoff_t vma_end_pgoff(const struct vm_area_struct *vma) 4417 { 4418 return vma_start_pgoff(vma) + vma_pages(vma); 4419 } 4420 4421 /** 4422 * vma_last_pgoff() - Get the page offset of the last page in @vma 4423 * @vma: The VMA whose last page offset is required. 4424 * 4425 * This returns the last page offset contained within @vma. 4426 * 4427 * See the description of vma_start_pgoff() for a description of VMA page 4428 * offsets. 4429 * 4430 * Returns: The last page offset of @vma. 4431 */ 4432 static inline pgoff_t vma_last_pgoff(const struct vm_area_struct *vma) 4433 { 4434 return vma_end_pgoff(vma) - 1; 4435 } 4436 4437 /** 4438 * vma_start_anon_pgoff() - Get the anonymous page offset of the start of @vma 4439 * @vma: The VMA whose anonymous page offset is required. 4440 * 4441 * If unfaulted, then this is vma->vm_start >> PAGE_SHIFT, if faulted then the 4442 * anonymous page offset at the time of first fault. 4443 * 4444 * If the VMA is anonymous, this returns the same value as vma_start_pgoff(). 4445 * 4446 * This value is used for tracking MAP_PRIVATE file-backed mappings by their 4447 * anonymous page offset. 4448 * 4449 * Returns: The anonymous page offset of the start of @vma. 4450 */ 4451 static inline pgoff_t vma_start_anon_pgoff(const struct vm_area_struct *vma) 4452 { 4453 pgoff_t pgoff = 0; 4454 4455 #ifdef CONFIG_64BIT 4456 pgoff += vma->__vm_anon_pgoff_hi; 4457 pgoff <<= 32; 4458 #endif 4459 pgoff += vma->__vm_anon_pgoff_lo; 4460 return pgoff; 4461 } 4462 4463 /** 4464 * vma_end_anon_pgoff() - Get the anonymous page offset of the exclusive end of 4465 * @vma. 4466 * @vma: The VMA whose end anonymous page offset is required. 4467 * 4468 * This returns the anonymous exclusive end page offset of @vma, which is useful 4469 * for expressing page offset ranges. 4470 * 4471 * See the description of vma_start_anon_pgoff() for a description of VMA 4472 * anonymous page offsets. 4473 * 4474 * Returns: The exclusive end anonymous page offset of @vma. 4475 */ 4476 static inline pgoff_t vma_end_anon_pgoff(const struct vm_area_struct *vma) 4477 { 4478 return vma_start_anon_pgoff(vma) + vma_pages(vma); 4479 } 4480 4481 /** 4482 * vma_last_anon_pgoff() - Get the anonymous page offset of the last page in 4483 * @vma. 4484 * @vma: The VMA whose last anonymous page offset is required. 4485 * 4486 * See the description of vma_start_anon_pgoff() for a description of VMA 4487 * anonymous page offsets. 4488 * 4489 * Returns: The last anonymous page offset of @vma. 4490 */ 4491 static inline pgoff_t vma_last_anon_pgoff(const struct vm_area_struct *vma) 4492 { 4493 return vma_end_anon_pgoff(vma) - 1; 4494 } 4495 4496 static inline unsigned long vma_desc_size(const struct vm_area_desc *desc) 4497 { 4498 return desc->end - desc->start; 4499 } 4500 4501 static inline unsigned long vma_desc_pages(const struct vm_area_desc *desc) 4502 { 4503 return vma_desc_size(desc) >> PAGE_SHIFT; 4504 } 4505 4506 /** 4507 * mmap_action_remap - helper for mmap_prepare hook to specify that a pure PFN 4508 * remap is required. 4509 * @desc: The VMA descriptor for the VMA requiring remap. 4510 * @start: The virtual address to start the remap from, must be within the VMA. 4511 * @start_pfn: The first PFN in the range to remap. 4512 * @size: The size of the range to remap, in bytes, at most spanning to the end 4513 * of the VMA. 4514 */ 4515 static inline void mmap_action_remap(struct vm_area_desc *desc, 4516 unsigned long start, 4517 unsigned long start_pfn, 4518 unsigned long size) 4519 { 4520 struct mmap_action *action = &desc->action; 4521 4522 /* [start, start + size) must be within the VMA. */ 4523 WARN_ON_ONCE(start < desc->start || start >= desc->end); 4524 WARN_ON_ONCE(start + size > desc->end); 4525 4526 action->type = MMAP_REMAP_PFN; 4527 action->remap.start = start; 4528 action->remap.start_pfn = start_pfn; 4529 action->remap.size = size; 4530 action->remap.pgprot = desc->page_prot; 4531 } 4532 4533 /** 4534 * mmap_action_remap_full - helper for mmap_prepare hook to specify that the 4535 * entirety of a VMA should be PFN remapped. 4536 * @desc: The VMA descriptor for the VMA requiring remap. 4537 * @start_pfn: The first PFN in the range to remap. 4538 */ 4539 static inline void mmap_action_remap_full(struct vm_area_desc *desc, 4540 unsigned long start_pfn) 4541 { 4542 mmap_action_remap(desc, desc->start, start_pfn, vma_desc_size(desc)); 4543 } 4544 4545 /** 4546 * mmap_action_ioremap - helper for mmap_prepare hook to specify that a pure PFN 4547 * I/O remap is required. 4548 * @desc: The VMA descriptor for the VMA requiring remap. 4549 * @start: The virtual address to start the remap from, must be within the VMA. 4550 * @start_pfn: The first PFN in the range to remap. 4551 * @size: The size of the range to remap, in bytes, at most spanning to the end 4552 * of the VMA. 4553 */ 4554 static inline void mmap_action_ioremap(struct vm_area_desc *desc, 4555 unsigned long start, 4556 unsigned long start_pfn, 4557 unsigned long size) 4558 { 4559 mmap_action_remap(desc, start, start_pfn, size); 4560 desc->action.type = MMAP_IO_REMAP_PFN; 4561 } 4562 4563 /** 4564 * mmap_action_ioremap_full - helper for mmap_prepare hook to specify that the 4565 * entirety of a VMA should be PFN I/O remapped. 4566 * @desc: The VMA descriptor for the VMA requiring remap. 4567 * @start_pfn: The first PFN in the range to remap. 4568 */ 4569 static inline void mmap_action_ioremap_full(struct vm_area_desc *desc, 4570 unsigned long start_pfn) 4571 { 4572 mmap_action_ioremap(desc, desc->start, start_pfn, vma_desc_size(desc)); 4573 } 4574 4575 /** 4576 * mmap_action_simple_ioremap - helper for mmap_prepare hook to specify that the 4577 * physical range in [start_phys_addr, start_phys_addr + size) should be I/O 4578 * remapped. 4579 * @desc: The VMA descriptor for the VMA requiring remap. 4580 * @start_phys_addr: Start of the physical memory to be mapped. 4581 * @size: Size of the area to map. 4582 * 4583 * NOTE: Some drivers might want to tweak desc->page_prot for purposes of 4584 * write-combine or similar. 4585 */ 4586 static inline void mmap_action_simple_ioremap(struct vm_area_desc *desc, 4587 phys_addr_t start_phys_addr, 4588 unsigned long size) 4589 { 4590 struct mmap_action *action = &desc->action; 4591 4592 action->simple_ioremap.start_phys_addr = start_phys_addr; 4593 action->simple_ioremap.size = size; 4594 action->type = MMAP_SIMPLE_IO_REMAP; 4595 } 4596 4597 /** 4598 * mmap_action_map_kernel_pages - helper for mmap_prepare hook to specify that 4599 * @num kernel pages contained in the @pages array should be mapped to userland 4600 * starting at virtual address @start. 4601 * @desc: The VMA descriptor for the VMA requiring kernel pags to be mapped. 4602 * @start: The virtual address from which to map them. 4603 * @pages: An array of struct page pointers describing the memory to map. 4604 * @nr_pages: The number of entries in the @pages aray. 4605 */ 4606 static inline void mmap_action_map_kernel_pages(struct vm_area_desc *desc, 4607 unsigned long start, struct page **pages, 4608 unsigned long nr_pages) 4609 { 4610 struct mmap_action *action = &desc->action; 4611 4612 action->type = MMAP_MAP_KERNEL_PAGES; 4613 action->map_kernel.start = start; 4614 action->map_kernel.pages = pages; 4615 action->map_kernel.nr_pages = nr_pages; 4616 action->map_kernel.pgoff = desc->pgoff; 4617 } 4618 4619 /** 4620 * mmap_action_map_kernel_pages_full - helper for mmap_prepare hook to specify that 4621 * kernel pages contained in the @pages array should be mapped to userland 4622 * from @desc->start to @desc->end. 4623 * @desc: The VMA descriptor for the VMA requiring kernel pags to be mapped. 4624 * @pages: An array of struct page pointers describing the memory to map. 4625 * 4626 * The caller must ensure that @pages contains sufficient entries to cover the 4627 * entire range described by @desc. 4628 */ 4629 static inline void mmap_action_map_kernel_pages_full(struct vm_area_desc *desc, 4630 struct page **pages) 4631 { 4632 mmap_action_map_kernel_pages(desc, desc->start, pages, 4633 vma_desc_pages(desc)); 4634 } 4635 4636 int mmap_action_prepare(struct vm_area_desc *desc); 4637 int mmap_action_complete(struct vm_area_struct *vma, 4638 struct mmap_action *action, bool is_compat); 4639 4640 /* Look up the first VMA which exactly match the interval vm_start ... vm_end */ 4641 static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm, 4642 unsigned long vm_start, unsigned long vm_end) 4643 { 4644 struct vm_area_struct *vma = vma_lookup(mm, vm_start); 4645 4646 if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end)) 4647 vma = NULL; 4648 4649 return vma; 4650 } 4651 4652 /** 4653 * range_is_subset - Is the specified inner range a subset of the outer range? 4654 * @outer_start: The start of the outer range. 4655 * @outer_end: The exclusive end of the outer range. 4656 * @inner_start: The start of the inner range. 4657 * @inner_end: The exclusive end of the inner range. 4658 * 4659 * Returns: %true if [inner_start, inner_end) is a subset of [outer_start, 4660 * outer_end), otherwise %false. 4661 */ 4662 static inline bool range_is_subset(unsigned long outer_start, 4663 unsigned long outer_end, 4664 unsigned long inner_start, 4665 unsigned long inner_end) 4666 { 4667 return outer_start <= inner_start && inner_end <= outer_end; 4668 } 4669 4670 /** 4671 * range_in_vma - is the specified [@start, @end) range a subset of the VMA? 4672 * @vma: The VMA against which we want to check [@start, @end). 4673 * @start: The start of the range we wish to check. 4674 * @end: The exclusive end of the range we wish to check. 4675 * 4676 * Returns: %true if [@start, @end) is a subset of [@vma->vm_start, 4677 * @vma->vm_end), %false otherwise. 4678 */ 4679 static inline bool range_in_vma(const struct vm_area_struct *vma, 4680 unsigned long start, unsigned long end) 4681 { 4682 if (!vma) 4683 return false; 4684 4685 return range_is_subset(vma->vm_start, vma->vm_end, start, end); 4686 } 4687 4688 /** 4689 * range_in_vma_desc - is the specified [@start, @end) range a subset of the VMA 4690 * described by @desc, a VMA descriptor? 4691 * @desc: The VMA descriptor against which we want to check [@start, @end). 4692 * @start: The start of the range we wish to check. 4693 * @end: The exclusive end of the range we wish to check. 4694 * 4695 * Returns: %true if [@start, @end) is a subset of [@desc->start, @desc->end), 4696 * %false otherwise. 4697 */ 4698 static inline bool range_in_vma_desc(const struct vm_area_desc *desc, 4699 unsigned long start, unsigned long end) 4700 { 4701 if (!desc) 4702 return false; 4703 4704 return range_is_subset(desc->start, desc->end, start, end); 4705 } 4706 4707 #ifdef CONFIG_MMU 4708 pgprot_t vm_get_page_prot(vm_flags_t vm_flags); 4709 4710 static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags) 4711 { 4712 const vm_flags_t vm_flags = vma_flags_to_legacy(vma_flags); 4713 4714 return vm_get_page_prot(vm_flags); 4715 } 4716 4717 static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma) 4718 { 4719 return vma_flags_to_page_prot(vma->flags); 4720 } 4721 4722 void vma_set_page_prot(struct vm_area_struct *vma); 4723 #else 4724 static inline pgprot_t vm_get_page_prot(vm_flags_t vm_flags) 4725 { 4726 return __pgprot(0); 4727 } 4728 static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags) 4729 { 4730 return __pgprot(0); 4731 } 4732 static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma) 4733 { 4734 return __pgprot(0); 4735 } 4736 static inline void vma_set_page_prot(struct vm_area_struct *vma) 4737 { 4738 vma->vm_page_prot = vma_get_page_prot(vma); 4739 } 4740 #endif 4741 4742 void vma_set_file(struct vm_area_struct *vma, struct file *file); 4743 4744 #ifdef CONFIG_NUMA_BALANCING 4745 unsigned long change_prot_numa(struct vm_area_struct *vma, 4746 unsigned long start, unsigned long end); 4747 #endif 4748 4749 struct vm_area_struct *find_extend_vma_locked(struct mm_struct *, 4750 unsigned long addr); 4751 int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, 4752 unsigned long pfn, unsigned long size, pgprot_t pgprot); 4753 4754 int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); 4755 int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, 4756 struct page **pages, unsigned long *num); 4757 int map_kernel_pages_prepare(struct vm_area_desc *desc); 4758 int map_kernel_pages_complete(struct vm_area_struct *vma, 4759 struct mmap_action *action); 4760 int vm_map_pages(struct vm_area_struct *vma, struct page **pages, 4761 unsigned long num); 4762 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, 4763 unsigned long num); 4764 vm_fault_t vmf_insert_page_mkwrite(struct vm_fault *vmf, struct page *page, 4765 bool write); 4766 vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr, 4767 unsigned long pfn); 4768 vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, 4769 unsigned long pfn, pgprot_t pgprot); 4770 vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr, 4771 unsigned long pfn); 4772 vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma, 4773 unsigned long addr, unsigned long pfn); 4774 int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len); 4775 4776 static inline vm_fault_t vmf_insert_page(struct vm_area_struct *vma, 4777 unsigned long addr, struct page *page) 4778 { 4779 int err = vm_insert_page(vma, addr, page); 4780 4781 if (err == -ENOMEM) 4782 return VM_FAULT_OOM; 4783 if (err < 0 && err != -EBUSY) 4784 return VM_FAULT_SIGBUS; 4785 4786 return VM_FAULT_NOPAGE; 4787 } 4788 4789 #ifndef io_remap_pfn_range_pfn 4790 static inline unsigned long io_remap_pfn_range_pfn(unsigned long pfn, 4791 unsigned long size) 4792 { 4793 return pfn; 4794 } 4795 #endif 4796 4797 static inline int io_remap_pfn_range(struct vm_area_struct *vma, 4798 unsigned long addr, unsigned long orig_pfn, 4799 unsigned long size, pgprot_t orig_prot) 4800 { 4801 const unsigned long pfn = io_remap_pfn_range_pfn(orig_pfn, size); 4802 const pgprot_t prot = pgprot_decrypted(orig_prot); 4803 4804 return remap_pfn_range(vma, addr, pfn, size, prot); 4805 } 4806 4807 static inline vm_fault_t vmf_error(int err) 4808 { 4809 if (err == -ENOMEM) 4810 return VM_FAULT_OOM; 4811 else if (err == -EHWPOISON) 4812 return VM_FAULT_HWPOISON; 4813 return VM_FAULT_SIGBUS; 4814 } 4815 4816 /* 4817 * Convert errno to return value for ->page_mkwrite() calls. 4818 * 4819 * This should eventually be merged with vmf_error() above, but will need a 4820 * careful audit of all vmf_error() callers. 4821 */ 4822 static inline vm_fault_t vmf_fs_error(int err) 4823 { 4824 if (err == 0) 4825 return VM_FAULT_LOCKED; 4826 if (err == -EFAULT || err == -EAGAIN) 4827 return VM_FAULT_NOPAGE; 4828 if (err == -ENOMEM) 4829 return VM_FAULT_OOM; 4830 /* -ENOSPC, -EDQUOT, -EIO ... */ 4831 return VM_FAULT_SIGBUS; 4832 } 4833 4834 static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags) 4835 { 4836 if (vm_fault & VM_FAULT_OOM) 4837 return -ENOMEM; 4838 if (vm_fault & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE)) 4839 return (foll_flags & FOLL_HWPOISON) ? -EHWPOISON : -EFAULT; 4840 if (vm_fault & (VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV)) 4841 return -EFAULT; 4842 return 0; 4843 } 4844 4845 /* 4846 * Indicates whether GUP can follow a PROT_NONE mapped page, or whether 4847 * a (NUMA hinting or userfaultfd RWP) fault is required. 4848 */ 4849 static inline bool gup_can_follow_protnone(const struct vm_area_struct *vma, 4850 unsigned int flags) 4851 { 4852 /* 4853 * VM_UFFD_RWP uses protnone as an access-tracking marker, not for 4854 * NUMA hinting. GUP must always take a fault so the access is 4855 * delivered to userfaultfd, regardless of FOLL_HONOR_NUMA_FAULT. 4856 * 4857 * Only do so while the VMA is accessible. If it has been made 4858 * inaccessible (e.g. mprotect(PROT_NONE)), fall through to the guard 4859 * below: forcing a fault there would loop, as handle_mm_fault() makes 4860 * no progress on protnone in an inaccessible VMA, and the access is 4861 * denied regardless of RWP anyway. 4862 */ 4863 if (vma_test_single_mask(vma, VMA_UFFD_RWP) && vma_is_accessible(vma)) 4864 return false; 4865 4866 /* 4867 * If callers don't want to honor NUMA hinting faults, no need to 4868 * determine if we would actually have to trigger a NUMA hinting fault. 4869 */ 4870 if (!(flags & FOLL_HONOR_NUMA_FAULT)) 4871 return true; 4872 4873 /* 4874 * NUMA hinting faults don't apply in inaccessible (PROT_NONE) VMAs. 4875 * 4876 * Requiring a fault here even for inaccessible VMAs would mean that 4877 * FOLL_FORCE cannot make any progress, because handle_mm_fault() 4878 * refuses to process NUMA hinting faults in inaccessible VMAs. 4879 */ 4880 return !vma_is_accessible(vma); 4881 } 4882 4883 typedef int (*pte_fn_t)(pte_t *pte, unsigned long addr, void *data); 4884 extern int apply_to_page_range(struct mm_struct *mm, unsigned long address, 4885 unsigned long size, pte_fn_t fn, void *data); 4886 extern int apply_to_existing_page_range(struct mm_struct *mm, 4887 unsigned long address, unsigned long size, 4888 pte_fn_t fn, void *data); 4889 4890 #ifdef CONFIG_PAGE_POISONING 4891 extern void __kernel_poison_pages(struct page *page, int numpages); 4892 extern void __kernel_unpoison_pages(struct page *page, int numpages); 4893 extern bool _page_poisoning_enabled_early; 4894 DECLARE_STATIC_KEY_FALSE(_page_poisoning_enabled); 4895 static inline bool page_poisoning_enabled(void) 4896 { 4897 return _page_poisoning_enabled_early; 4898 } 4899 /* 4900 * For use in fast paths after init_mem_debugging() has run, or when a 4901 * false negative result is not harmful when called too early. 4902 */ 4903 static inline bool page_poisoning_enabled_static(void) 4904 { 4905 return static_branch_unlikely(&_page_poisoning_enabled); 4906 } 4907 static inline void kernel_poison_pages(struct page *page, int numpages) 4908 { 4909 if (page_poisoning_enabled_static()) 4910 __kernel_poison_pages(page, numpages); 4911 } 4912 static inline void kernel_unpoison_pages(struct page *page, int numpages) 4913 { 4914 if (page_poisoning_enabled_static()) 4915 __kernel_unpoison_pages(page, numpages); 4916 } 4917 #else 4918 static inline bool page_poisoning_enabled(void) { return false; } 4919 static inline bool page_poisoning_enabled_static(void) { return false; } 4920 static inline void __kernel_poison_pages(struct page *page, int nunmpages) { } 4921 static inline void kernel_poison_pages(struct page *page, int numpages) { } 4922 static inline void kernel_unpoison_pages(struct page *page, int numpages) { } 4923 #endif 4924 4925 DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc); 4926 static inline bool want_init_on_alloc(gfp_t flags) 4927 { 4928 if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, 4929 &init_on_alloc)) 4930 return true; 4931 return flags & __GFP_ZERO; 4932 } 4933 4934 DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free); 4935 static inline bool want_init_on_free(void) 4936 { 4937 return static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON, 4938 &init_on_free); 4939 } 4940 4941 extern bool _debug_pagealloc_enabled_early; 4942 DECLARE_STATIC_KEY_FALSE(_debug_pagealloc_enabled); 4943 4944 static inline bool debug_pagealloc_enabled(void) 4945 { 4946 return IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) && 4947 _debug_pagealloc_enabled_early; 4948 } 4949 4950 /* 4951 * For use in fast paths after mem_debugging_and_hardening_init() has run, 4952 * or when a false negative result is not harmful when called too early. 4953 */ 4954 static inline bool debug_pagealloc_enabled_static(void) 4955 { 4956 if (!IS_ENABLED(CONFIG_DEBUG_PAGEALLOC)) 4957 return false; 4958 4959 return static_branch_unlikely(&_debug_pagealloc_enabled); 4960 } 4961 4962 /* 4963 * To support DEBUG_PAGEALLOC architecture must ensure that 4964 * __kernel_map_pages() never fails 4965 */ 4966 extern void __kernel_map_pages(struct page *page, int numpages, int enable); 4967 #ifdef CONFIG_DEBUG_PAGEALLOC 4968 static inline void debug_pagealloc_map_pages(struct page *page, int numpages) 4969 { 4970 iommu_debug_check_unmapped(page, numpages); 4971 4972 if (debug_pagealloc_enabled_static()) 4973 __kernel_map_pages(page, numpages, 1); 4974 } 4975 4976 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) 4977 { 4978 iommu_debug_check_unmapped(page, numpages); 4979 4980 if (debug_pagealloc_enabled_static()) 4981 __kernel_map_pages(page, numpages, 0); 4982 } 4983 4984 extern unsigned int _debug_guardpage_minorder; 4985 DECLARE_STATIC_KEY_FALSE(_debug_guardpage_enabled); 4986 4987 static inline unsigned int debug_guardpage_minorder(void) 4988 { 4989 return _debug_guardpage_minorder; 4990 } 4991 4992 static inline bool debug_guardpage_enabled(void) 4993 { 4994 return static_branch_unlikely(&_debug_guardpage_enabled); 4995 } 4996 4997 static inline bool page_is_guard(const struct page *page) 4998 { 4999 if (!debug_guardpage_enabled()) 5000 return false; 5001 5002 return PageGuard(page); 5003 } 5004 5005 bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order); 5006 static inline bool set_page_guard(struct zone *zone, struct page *page, 5007 unsigned int order) 5008 { 5009 if (!debug_guardpage_enabled()) 5010 return false; 5011 return __set_page_guard(zone, page, order); 5012 } 5013 5014 void __clear_page_guard(struct zone *zone, struct page *page, unsigned int order); 5015 static inline void clear_page_guard(struct zone *zone, struct page *page, 5016 unsigned int order) 5017 { 5018 if (!debug_guardpage_enabled()) 5019 return; 5020 __clear_page_guard(zone, page, order); 5021 } 5022 5023 #else /* CONFIG_DEBUG_PAGEALLOC */ 5024 static inline void debug_pagealloc_map_pages(struct page *page, int numpages) {} 5025 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) {} 5026 static inline unsigned int debug_guardpage_minorder(void) { return 0; } 5027 static inline bool debug_guardpage_enabled(void) { return false; } 5028 static inline bool page_is_guard(const struct page *page) { return false; } 5029 static inline bool set_page_guard(struct zone *zone, struct page *page, 5030 unsigned int order) { return false; } 5031 static inline void clear_page_guard(struct zone *zone, struct page *page, 5032 unsigned int order) {} 5033 #endif /* CONFIG_DEBUG_PAGEALLOC */ 5034 5035 #ifndef clear_pages 5036 /** 5037 * clear_pages() - clear a page range for kernel-internal use. 5038 * @addr: start address 5039 * @npages: number of pages 5040 * 5041 * Use clear_user_pages() instead when clearing a page range to be 5042 * mapped to user space. 5043 * 5044 * Does absolutely no exception handling. 5045 * 5046 * Note that even though the clearing operation is preemptible, clear_pages() 5047 * does not (and on architectures where it reduces to a few long-running 5048 * instructions, might not be able to) call cond_resched() to check if 5049 * rescheduling is required. 5050 * 5051 * When running under preemptible models this is not a problem. Under 5052 * cooperatively scheduled models, however, the caller is expected to 5053 * limit @npages to no more than PROCESS_PAGES_NON_PREEMPT_BATCH. 5054 */ 5055 static inline void clear_pages(void *addr, unsigned int npages) 5056 { 5057 do { 5058 clear_page(addr); 5059 addr += PAGE_SIZE; 5060 } while (--npages); 5061 } 5062 #endif 5063 5064 #ifndef PROCESS_PAGES_NON_PREEMPT_BATCH 5065 #ifdef clear_pages 5066 /* 5067 * The architecture defines clear_pages(), and we assume that it is 5068 * generally "fast". So choose a batch size large enough to allow the processor 5069 * headroom for optimizing the operation and yet small enough that we see 5070 * reasonable preemption latency for when this optimization is not possible 5071 * (ex. slow microarchitectures, memory bandwidth saturation.) 5072 * 5073 * With a value of 32MB and assuming a memory bandwidth of ~10GBps, this should 5074 * result in worst case preemption latency of around 3ms when clearing pages. 5075 * 5076 * (See comment above clear_pages() for why preemption latency is a concern 5077 * here.) 5078 */ 5079 #define PROCESS_PAGES_NON_PREEMPT_BATCH (SZ_32M >> PAGE_SHIFT) 5080 #else /* !clear_pages */ 5081 /* 5082 * The architecture does not provide a clear_pages() implementation. Assume 5083 * that clear_page() -- which clear_pages() will fallback to -- is relatively 5084 * slow and choose a small value for PROCESS_PAGES_NON_PREEMPT_BATCH. 5085 */ 5086 #define PROCESS_PAGES_NON_PREEMPT_BATCH 1 5087 #endif 5088 #endif 5089 5090 #ifdef __HAVE_ARCH_GATE_AREA 5091 extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm); 5092 extern int in_gate_area_no_mm(unsigned long addr); 5093 extern int in_gate_area(struct mm_struct *mm, unsigned long addr); 5094 #else 5095 static inline struct vm_area_struct *get_gate_vma(struct mm_struct *mm) 5096 { 5097 return NULL; 5098 } 5099 static inline int in_gate_area_no_mm(unsigned long addr) { return 0; } 5100 static inline int in_gate_area(struct mm_struct *mm, unsigned long addr) 5101 { 5102 return 0; 5103 } 5104 #endif /* __HAVE_ARCH_GATE_AREA */ 5105 5106 bool process_shares_mm(const struct task_struct *p, const struct mm_struct *mm); 5107 5108 void drop_slab(void); 5109 5110 #ifndef CONFIG_MMU 5111 #define randomize_va_space 0 5112 #else 5113 extern int randomize_va_space; 5114 #endif 5115 5116 const char * arch_vma_name(struct vm_area_struct *vma); 5117 #ifdef CONFIG_MMU 5118 void print_vma_addr(char *prefix, unsigned long rip); 5119 #else 5120 static inline void print_vma_addr(char *prefix, unsigned long rip) 5121 { 5122 } 5123 #endif 5124 5125 struct page * __populate_section_memmap(unsigned long pfn, 5126 unsigned long nr_pages, int nid, struct vmem_altmap *altmap, 5127 struct dev_pagemap *pgmap); 5128 void *vmemmap_alloc_block(unsigned long size, int node); 5129 struct vmem_altmap; 5130 void *vmemmap_alloc_block_buf(unsigned long size, int node, 5131 struct vmem_altmap *altmap); 5132 void vmemmap_verify(pte_t *, int, unsigned long, unsigned long); 5133 void vmemmap_set_pmd(pmd_t *pmd, void *p, int node, 5134 unsigned long addr, unsigned long next); 5135 int vmemmap_check_pmd(pmd_t *pmd, int node, 5136 unsigned long addr, unsigned long next); 5137 int vmemmap_populate_basepages(unsigned long start, unsigned long end, 5138 int node, struct vmem_altmap *altmap); 5139 int vmemmap_populate_hugepages(unsigned long start, unsigned long end, 5140 int node, struct vmem_altmap *altmap); 5141 int vmemmap_populate(unsigned long start, unsigned long end, int node, 5142 struct vmem_altmap *altmap); 5143 void vmemmap_wrprotect_hvo(unsigned long start, unsigned long end, int node, 5144 unsigned long headsize); 5145 void vmemmap_populate_print_last(void); 5146 #ifdef CONFIG_MEMORY_HOTPLUG 5147 void vmemmap_free(unsigned long start, unsigned long end, 5148 struct vmem_altmap *altmap); 5149 #endif 5150 5151 #ifdef CONFIG_SPARSEMEM_VMEMMAP 5152 static inline unsigned long vmem_altmap_offset(const struct vmem_altmap *altmap) 5153 { 5154 /* number of pfns from base where pfn_to_page() is valid */ 5155 if (altmap) 5156 return altmap->reserve + altmap->free; 5157 return 0; 5158 } 5159 5160 static inline void vmem_altmap_free(struct vmem_altmap *altmap, 5161 unsigned long nr_pfns) 5162 { 5163 altmap->alloc -= nr_pfns; 5164 } 5165 #else 5166 static inline unsigned long vmem_altmap_offset(const struct vmem_altmap *altmap) 5167 { 5168 return 0; 5169 } 5170 5171 static inline void vmem_altmap_free(struct vmem_altmap *altmap, 5172 unsigned long nr_pfns) 5173 { 5174 } 5175 #endif 5176 5177 #ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP 5178 static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap, 5179 struct dev_pagemap *pgmap) 5180 { 5181 unsigned long nr_pages; 5182 unsigned long nr_vmemmap_pages; 5183 5184 if (!IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION)) 5185 return false; 5186 5187 if (!pgmap || !is_power_of_2(sizeof(struct page))) 5188 return false; 5189 5190 nr_pages = pgmap_vmemmap_nr(pgmap); 5191 nr_vmemmap_pages = ((nr_pages * sizeof(struct page)) >> PAGE_SHIFT); 5192 /* 5193 * For vmemmap optimization with DAX we need minimum 2 vmemmap 5194 * pages. See layout diagram in Documentation/mm/vmemmap_dedup.rst 5195 */ 5196 return !altmap && (nr_vmemmap_pages > VMEMMAP_OPTIMIZATION_PAGES); 5197 } 5198 /* 5199 * If we don't have an architecture override, use the generic rule 5200 */ 5201 #ifndef vmemmap_can_optimize 5202 #define vmemmap_can_optimize __vmemmap_can_optimize 5203 #endif 5204 5205 #else 5206 static inline bool vmemmap_can_optimize(struct vmem_altmap *altmap, 5207 struct dev_pagemap *pgmap) 5208 { 5209 return false; 5210 } 5211 #endif 5212 5213 enum mf_flags { 5214 MF_COUNT_INCREASED = 1 << 0, 5215 MF_ACTION_REQUIRED = 1 << 1, 5216 MF_MUST_KILL = 1 << 2, 5217 MF_SOFT_OFFLINE = 1 << 3, 5218 MF_UNPOISON = 1 << 4, 5219 MF_SW_SIMULATED = 1 << 5, 5220 MF_NO_RETRY = 1 << 6, 5221 MF_MEM_PRE_REMOVE = 1 << 7, 5222 }; 5223 int mf_dax_kill_procs(struct address_space *mapping, pgoff_t index, 5224 unsigned long count, int mf_flags); 5225 extern int memory_failure(unsigned long pfn, int flags); 5226 extern int unpoison_memory(unsigned long pfn); 5227 extern atomic_long_t num_poisoned_pages __read_mostly; 5228 extern int soft_offline_page(unsigned long pfn, int flags); 5229 #ifdef CONFIG_MEMORY_FAILURE 5230 /* 5231 * Sysfs entries for memory failure handling statistics. 5232 */ 5233 extern const struct attribute_group memory_failure_attr_group; 5234 extern void memory_failure_queue(unsigned long pfn, int flags); 5235 void num_poisoned_pages_inc(unsigned long pfn); 5236 void num_poisoned_pages_sub(unsigned long pfn, long i); 5237 #else 5238 static inline void memory_failure_queue(unsigned long pfn, int flags) 5239 { 5240 } 5241 5242 static inline void num_poisoned_pages_inc(unsigned long pfn) 5243 { 5244 } 5245 5246 static inline void num_poisoned_pages_sub(unsigned long pfn, long i) 5247 { 5248 } 5249 #endif 5250 5251 #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG) 5252 extern void memblk_nr_poison_inc(unsigned long pfn); 5253 extern void memblk_nr_poison_sub(unsigned long pfn, long i); 5254 #else 5255 static inline void memblk_nr_poison_inc(unsigned long pfn) 5256 { 5257 } 5258 5259 static inline void memblk_nr_poison_sub(unsigned long pfn, long i) 5260 { 5261 } 5262 #endif 5263 5264 #ifndef arch_memory_failure 5265 static inline int arch_memory_failure(unsigned long pfn, int flags) 5266 { 5267 return -ENXIO; 5268 } 5269 #endif 5270 5271 #ifndef arch_is_platform_page 5272 static inline bool arch_is_platform_page(u64 paddr) 5273 { 5274 return false; 5275 } 5276 #endif 5277 5278 /* 5279 * Error handlers for various types of pages. 5280 */ 5281 enum mf_result { 5282 MF_IGNORED, /* Error: cannot be handled */ 5283 MF_FAILED, /* Error: handling failed */ 5284 MF_DELAYED, /* Will be handled later */ 5285 MF_RECOVERED, /* Successfully recovered */ 5286 }; 5287 5288 enum mf_action_page_type { 5289 MF_MSG_KERNEL, 5290 MF_MSG_KERNEL_HIGH_ORDER, 5291 MF_MSG_DIFFERENT_COMPOUND, 5292 MF_MSG_HUGE, 5293 MF_MSG_FREE_HUGE, 5294 MF_MSG_GET_HWPOISON, 5295 MF_MSG_UNMAP_FAILED, 5296 MF_MSG_DIRTY_SWAPCACHE, 5297 MF_MSG_CLEAN_SWAPCACHE, 5298 MF_MSG_DIRTY_MLOCKED_LRU, 5299 MF_MSG_CLEAN_MLOCKED_LRU, 5300 MF_MSG_DIRTY_UNEVICTABLE_LRU, 5301 MF_MSG_CLEAN_UNEVICTABLE_LRU, 5302 MF_MSG_DIRTY_LRU, 5303 MF_MSG_CLEAN_LRU, 5304 MF_MSG_TRUNCATED_LRU, 5305 MF_MSG_BUDDY, 5306 MF_MSG_DAX, 5307 MF_MSG_UNSPLIT_THP, 5308 MF_MSG_ALREADY_POISONED, 5309 MF_MSG_PFN_MAP, 5310 MF_MSG_UNKNOWN, 5311 }; 5312 5313 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS) 5314 void folio_zero_user(struct folio *folio, unsigned long addr_hint); 5315 int copy_user_large_folio(struct folio *dst, struct folio *src, 5316 unsigned long addr_hint, 5317 struct vm_area_struct *vma); 5318 long copy_folio_from_user(struct folio *dst_folio, 5319 const void __user *usr_src, 5320 bool allow_pagefault); 5321 5322 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */ 5323 5324 #if MAX_NUMNODES > 1 5325 void __init setup_nr_node_ids(void); 5326 #else 5327 static inline void setup_nr_node_ids(void) {} 5328 #endif 5329 5330 extern int memcmp_pages(struct page *page1, struct page *page2); 5331 5332 static inline int pages_identical(struct page *page1, struct page *page2) 5333 { 5334 return !memcmp_pages(page1, page2); 5335 } 5336 5337 #ifdef CONFIG_MAPPING_DIRTY_HELPERS 5338 unsigned long clean_record_shared_mapping_range(struct address_space *mapping, 5339 pgoff_t first_index, pgoff_t nr, 5340 pgoff_t bitmap_pgoff, 5341 unsigned long *bitmap, 5342 pgoff_t *start, 5343 pgoff_t *end); 5344 5345 unsigned long wp_shared_mapping_range(struct address_space *mapping, 5346 pgoff_t first_index, pgoff_t nr); 5347 #endif 5348 5349 #ifdef CONFIG_ANON_VMA_NAME 5350 int set_anon_vma_name(unsigned long addr, unsigned long size, 5351 const char __user *uname); 5352 #else 5353 static inline 5354 int set_anon_vma_name(unsigned long addr, unsigned long size, 5355 const char __user *uname) 5356 { 5357 return -EINVAL; 5358 } 5359 #endif 5360 5361 #ifdef CONFIG_UNACCEPTED_MEMORY 5362 5363 bool range_contains_unaccepted_memory(phys_addr_t start, unsigned long size); 5364 void accept_memory(phys_addr_t start, unsigned long size); 5365 5366 #else 5367 5368 static inline bool range_contains_unaccepted_memory(phys_addr_t start, 5369 unsigned long size) 5370 { 5371 return false; 5372 } 5373 5374 static inline void accept_memory(phys_addr_t start, unsigned long size) 5375 { 5376 } 5377 5378 #endif 5379 5380 static inline bool pfn_is_unaccepted_memory(unsigned long pfn) 5381 { 5382 return range_contains_unaccepted_memory(pfn << PAGE_SHIFT, PAGE_SIZE); 5383 } 5384 5385 void vma_pgtable_walk_begin(struct vm_area_struct *vma); 5386 void vma_pgtable_walk_end(struct vm_area_struct *vma); 5387 5388 int reserve_mem_find_by_name(const char *name, phys_addr_t *start, phys_addr_t *size); 5389 int reserve_mem_release_by_name(const char *name); 5390 5391 #ifdef CONFIG_64BIT 5392 void mseal_mmap_page_zero(void); 5393 #else 5394 static inline void mseal_mmap_page_zero(void) {} 5395 #endif 5396 5397 /* 5398 * user_alloc_needs_zeroing checks if a user folio from page allocator needs to 5399 * be zeroed or not. 5400 */ 5401 static inline bool user_alloc_needs_zeroing(void) 5402 { 5403 /* 5404 * for user folios, arch with cache aliasing requires cache flush and 5405 * arc changes folio->flags to make icache coherent with dcache, so 5406 * always return false to make caller use 5407 * clear_user_page()/clear_user_highpage(). 5408 */ 5409 return cpu_dcache_is_aliasing() || cpu_icache_is_aliasing() || 5410 !static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, 5411 &init_on_alloc); 5412 } 5413 5414 int arch_get_shadow_stack_status(struct task_struct *t, unsigned long __user *status); 5415 int arch_set_shadow_stack_status(struct task_struct *t, unsigned long status); 5416 int arch_lock_shadow_stack_status(struct task_struct *t, unsigned long status); 5417 5418 /* 5419 * DMA mapping IDs for page_pool 5420 * 5421 * When DMA-mapping a page, page_pool allocates an ID (from an xarray) and 5422 * stashes it in the upper bits of page->pp_magic. We always want to be able to 5423 * unambiguously identify page pool pages (using page_pool_page_is_pp()). Non-PP 5424 * pages can have arbitrary kernel pointers stored in the same field as pp_magic 5425 * (since it overlaps with page->lru.next), so we must ensure that we cannot 5426 * mistake a valid kernel pointer with any of the values we write into this 5427 * field. 5428 * 5429 * On architectures that set POISON_POINTER_DELTA, this is already ensured, 5430 * since this value becomes part of PP_SIGNATURE; meaning we can just use the 5431 * space between the PP_SIGNATURE value (without POISON_POINTER_DELTA), and the 5432 * lowest bits of POISON_POINTER_DELTA. On arches where POISON_POINTER_DELTA is 5433 * 0, we use the lowest bit of PAGE_OFFSET as the boundary if that value is 5434 * known at compile-time. 5435 * 5436 * If the value of PAGE_OFFSET is not known at compile time, or if it is too 5437 * small to leave at least 8 bits available above PP_SIGNATURE, we define the 5438 * number of bits to be 0, which turns off the DMA index tracking altogether 5439 * (see page_pool_register_dma_index()). 5440 */ 5441 #define PP_DMA_INDEX_SHIFT (1 + __fls(PP_SIGNATURE - POISON_POINTER_DELTA)) 5442 #if POISON_POINTER_DELTA > 0 5443 /* PP_SIGNATURE includes POISON_POINTER_DELTA, so limit the size of the DMA 5444 * index to not overlap with that if set 5445 */ 5446 #define PP_DMA_INDEX_BITS MIN(32, __ffs(POISON_POINTER_DELTA) - PP_DMA_INDEX_SHIFT) 5447 #else 5448 /* Use the lowest bit of PAGE_OFFSET if there's at least 8 bits available; see above */ 5449 #define PP_DMA_INDEX_MIN_OFFSET (1 << (PP_DMA_INDEX_SHIFT + 8)) 5450 #define PP_DMA_INDEX_BITS ((__builtin_constant_p(PAGE_OFFSET) && \ 5451 PAGE_OFFSET >= PP_DMA_INDEX_MIN_OFFSET && \ 5452 !(PAGE_OFFSET & (PP_DMA_INDEX_MIN_OFFSET - 1))) ? \ 5453 MIN(32, __ffs(PAGE_OFFSET) - PP_DMA_INDEX_SHIFT) : 0) 5454 5455 #endif 5456 5457 #define PP_DMA_INDEX_MASK GENMASK(PP_DMA_INDEX_BITS + PP_DMA_INDEX_SHIFT - 1, \ 5458 PP_DMA_INDEX_SHIFT) 5459 5460 /* Mask used for checking in page_pool_page_is_pp() below. page->pp_magic is 5461 * OR'ed with PP_SIGNATURE after the allocation in order to preserve bit 0 for 5462 * the head page of compound page and bit 1 for pfmemalloc page, as well as the 5463 * bits used for the DMA index. page_is_pfmemalloc() is checked in 5464 * __page_pool_put_page() to avoid recycling the pfmemalloc page. 5465 */ 5466 #define PP_MAGIC_MASK ~(PP_DMA_INDEX_MASK | 0x3UL) 5467 5468 #ifdef CONFIG_PAGE_POOL 5469 static inline bool page_pool_page_is_pp(const struct page *page) 5470 { 5471 return (page->pp_magic & PP_MAGIC_MASK) == PP_SIGNATURE; 5472 } 5473 #else 5474 static inline bool page_pool_page_is_pp(const struct page *page) 5475 { 5476 return false; 5477 } 5478 #endif 5479 5480 #define PAGE_SNAPSHOT_FAITHFUL (1 << 0) 5481 #define PAGE_SNAPSHOT_PG_BUDDY (1 << 1) 5482 #define PAGE_SNAPSHOT_PG_IDLE (1 << 2) 5483 5484 struct page_snapshot { 5485 struct folio folio_snapshot; 5486 struct page page_snapshot; 5487 unsigned long pfn; 5488 unsigned long idx; 5489 unsigned long flags; 5490 }; 5491 5492 static inline bool snapshot_page_is_faithful(const struct page_snapshot *ps) 5493 { 5494 return ps->flags & PAGE_SNAPSHOT_FAITHFUL; 5495 } 5496 5497 void snapshot_page(struct page_snapshot *ps, const struct page *page); 5498 5499 void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, 5500 struct vm_area_struct *vma, unsigned long addr, 5501 bool uffd_wp); 5502 5503 #endif /* _LINUX_MM_H */ 5504