1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_MM_H 3 #define _LINUX_MM_H 4 5 #include <linux/errno.h> 6 #include <linux/mmdebug.h> 7 #include <linux/gfp.h> 8 #include <linux/pgalloc_tag.h> 9 #include <linux/bug.h> 10 #include <linux/list.h> 11 #include <linux/mmzone.h> 12 #include <linux/rbtree.h> 13 #include <linux/atomic.h> 14 #include <linux/debug_locks.h> 15 #include <linux/mm_types.h> 16 #include <linux/mmap_lock.h> 17 #include <linux/range.h> 18 #include <linux/pfn.h> 19 #include <linux/percpu-refcount.h> 20 #include <linux/bit_spinlock.h> 21 #include <linux/shrinker.h> 22 #include <linux/resource.h> 23 #include <linux/page_ext.h> 24 #include <linux/err.h> 25 #include <linux/page-flags.h> 26 #include <linux/page_ref.h> 27 #include <linux/overflow.h> 28 #include <linux/sizes.h> 29 #include <linux/sched.h> 30 #include <linux/pgtable.h> 31 #include <linux/kasan.h> 32 #include <linux/memremap.h> 33 #include <linux/slab.h> 34 35 struct mempolicy; 36 struct anon_vma; 37 struct anon_vma_chain; 38 struct user_struct; 39 struct pt_regs; 40 struct folio_batch; 41 42 extern int sysctl_page_lock_unfairness; 43 44 void mm_core_init(void); 45 void init_mm_internals(void); 46 47 #ifndef CONFIG_NUMA /* Don't use mapnrs, do it properly */ 48 extern unsigned long max_mapnr; 49 50 static inline void set_max_mapnr(unsigned long limit) 51 { 52 max_mapnr = limit; 53 } 54 #else 55 static inline void set_max_mapnr(unsigned long limit) { } 56 #endif 57 58 extern atomic_long_t _totalram_pages; 59 static inline unsigned long totalram_pages(void) 60 { 61 return (unsigned long)atomic_long_read(&_totalram_pages); 62 } 63 64 static inline void totalram_pages_inc(void) 65 { 66 atomic_long_inc(&_totalram_pages); 67 } 68 69 static inline void totalram_pages_dec(void) 70 { 71 atomic_long_dec(&_totalram_pages); 72 } 73 74 static inline void totalram_pages_add(long count) 75 { 76 atomic_long_add(count, &_totalram_pages); 77 } 78 79 extern void * high_memory; 80 extern int page_cluster; 81 extern const int page_cluster_max; 82 83 #ifdef CONFIG_SYSCTL 84 extern int sysctl_legacy_va_layout; 85 #else 86 #define sysctl_legacy_va_layout 0 87 #endif 88 89 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS 90 extern const int mmap_rnd_bits_min; 91 extern int mmap_rnd_bits_max __ro_after_init; 92 extern int mmap_rnd_bits __read_mostly; 93 #endif 94 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS 95 extern const int mmap_rnd_compat_bits_min; 96 extern const int mmap_rnd_compat_bits_max; 97 extern int mmap_rnd_compat_bits __read_mostly; 98 #endif 99 100 #include <asm/page.h> 101 #include <asm/processor.h> 102 103 #ifndef __pa_symbol 104 #define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0)) 105 #endif 106 107 #ifndef page_to_virt 108 #define page_to_virt(x) __va(PFN_PHYS(page_to_pfn(x))) 109 #endif 110 111 #ifndef lm_alias 112 #define lm_alias(x) __va(__pa_symbol(x)) 113 #endif 114 115 /* 116 * To prevent common memory management code establishing 117 * a zero page mapping on a read fault. 118 * This macro should be defined within <asm/pgtable.h>. 119 * s390 does this to prevent multiplexing of hardware bits 120 * related to the physical page in case of virtualization. 121 */ 122 #ifndef mm_forbids_zeropage 123 #define mm_forbids_zeropage(X) (0) 124 #endif 125 126 /* 127 * On some architectures it is expensive to call memset() for small sizes. 128 * If an architecture decides to implement their own version of 129 * mm_zero_struct_page they should wrap the defines below in a #ifndef and 130 * define their own version of this macro in <asm/pgtable.h> 131 */ 132 #if BITS_PER_LONG == 64 133 /* This function must be updated when the size of struct page grows above 96 134 * or reduces below 56. The idea that compiler optimizes out switch() 135 * statement, and only leaves move/store instructions. Also the compiler can 136 * combine write statements if they are both assignments and can be reordered, 137 * this can result in several of the writes here being dropped. 138 */ 139 #define mm_zero_struct_page(pp) __mm_zero_struct_page(pp) 140 static inline void __mm_zero_struct_page(struct page *page) 141 { 142 unsigned long *_pp = (void *)page; 143 144 /* Check that struct page is either 56, 64, 72, 80, 88 or 96 bytes */ 145 BUILD_BUG_ON(sizeof(struct page) & 7); 146 BUILD_BUG_ON(sizeof(struct page) < 56); 147 BUILD_BUG_ON(sizeof(struct page) > 96); 148 149 switch (sizeof(struct page)) { 150 case 96: 151 _pp[11] = 0; 152 fallthrough; 153 case 88: 154 _pp[10] = 0; 155 fallthrough; 156 case 80: 157 _pp[9] = 0; 158 fallthrough; 159 case 72: 160 _pp[8] = 0; 161 fallthrough; 162 case 64: 163 _pp[7] = 0; 164 fallthrough; 165 case 56: 166 _pp[6] = 0; 167 _pp[5] = 0; 168 _pp[4] = 0; 169 _pp[3] = 0; 170 _pp[2] = 0; 171 _pp[1] = 0; 172 _pp[0] = 0; 173 } 174 } 175 #else 176 #define mm_zero_struct_page(pp) ((void)memset((pp), 0, sizeof(struct page))) 177 #endif 178 179 /* 180 * Default maximum number of active map areas, this limits the number of vmas 181 * per mm struct. Users can overwrite this number by sysctl but there is a 182 * problem. 183 * 184 * When a program's coredump is generated as ELF format, a section is created 185 * per a vma. In ELF, the number of sections is represented in unsigned short. 186 * This means the number of sections should be smaller than 65535 at coredump. 187 * Because the kernel adds some informative sections to a image of program at 188 * generating coredump, we need some margin. The number of extra sections is 189 * 1-3 now and depends on arch. We use "5" as safe margin, here. 190 * 191 * ELF extended numbering allows more than 65535 sections, so 16-bit bound is 192 * not a hard limit any more. Although some userspace tools can be surprised by 193 * that. 194 */ 195 #define MAPCOUNT_ELF_CORE_MARGIN (5) 196 #define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN) 197 198 extern int sysctl_max_map_count; 199 200 extern unsigned long sysctl_user_reserve_kbytes; 201 extern unsigned long sysctl_admin_reserve_kbytes; 202 203 extern int sysctl_overcommit_memory; 204 extern int sysctl_overcommit_ratio; 205 extern unsigned long sysctl_overcommit_kbytes; 206 207 int overcommit_ratio_handler(struct ctl_table *, int, void *, size_t *, 208 loff_t *); 209 int overcommit_kbytes_handler(struct ctl_table *, int, void *, size_t *, 210 loff_t *); 211 int overcommit_policy_handler(struct ctl_table *, int, void *, size_t *, 212 loff_t *); 213 214 #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) 215 #define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n)) 216 #define folio_page_idx(folio, p) (page_to_pfn(p) - folio_pfn(folio)) 217 #else 218 #define nth_page(page,n) ((page) + (n)) 219 #define folio_page_idx(folio, p) ((p) - &(folio)->page) 220 #endif 221 222 /* to align the pointer to the (next) page boundary */ 223 #define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE) 224 225 /* to align the pointer to the (prev) page boundary */ 226 #define PAGE_ALIGN_DOWN(addr) ALIGN_DOWN(addr, PAGE_SIZE) 227 228 /* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */ 229 #define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), PAGE_SIZE) 230 231 static inline struct folio *lru_to_folio(struct list_head *head) 232 { 233 return list_entry((head)->prev, struct folio, lru); 234 } 235 236 void setup_initial_init_mm(void *start_code, void *end_code, 237 void *end_data, void *brk); 238 239 /* 240 * Linux kernel virtual memory manager primitives. 241 * The idea being to have a "virtual" mm in the same way 242 * we have a virtual fs - giving a cleaner interface to the 243 * mm details, and allowing different kinds of memory mappings 244 * (from shared memory to executable loading to arbitrary 245 * mmap() functions). 246 */ 247 248 struct vm_area_struct *vm_area_alloc(struct mm_struct *); 249 struct vm_area_struct *vm_area_dup(struct vm_area_struct *); 250 void vm_area_free(struct vm_area_struct *); 251 /* Use only if VMA has no other users */ 252 void __vm_area_free(struct vm_area_struct *vma); 253 254 #ifndef CONFIG_MMU 255 extern struct rb_root nommu_region_tree; 256 extern struct rw_semaphore nommu_region_sem; 257 258 extern unsigned int kobjsize(const void *objp); 259 #endif 260 261 /* 262 * vm_flags in vm_area_struct, see mm_types.h. 263 * When changing, update also include/trace/events/mmflags.h 264 */ 265 #define VM_NONE 0x00000000 266 267 #define VM_READ 0x00000001 /* currently active flags */ 268 #define VM_WRITE 0x00000002 269 #define VM_EXEC 0x00000004 270 #define VM_SHARED 0x00000008 271 272 /* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */ 273 #define VM_MAYREAD 0x00000010 /* limits for mprotect() etc */ 274 #define VM_MAYWRITE 0x00000020 275 #define VM_MAYEXEC 0x00000040 276 #define VM_MAYSHARE 0x00000080 277 278 #define VM_GROWSDOWN 0x00000100 /* general info on the segment */ 279 #ifdef CONFIG_MMU 280 #define VM_UFFD_MISSING 0x00000200 /* missing pages tracking */ 281 #else /* CONFIG_MMU */ 282 #define VM_MAYOVERLAY 0x00000200 /* nommu: R/O MAP_PRIVATE mapping that might overlay a file mapping */ 283 #define VM_UFFD_MISSING 0 284 #endif /* CONFIG_MMU */ 285 #define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */ 286 #define VM_UFFD_WP 0x00001000 /* wrprotect pages tracking */ 287 288 #define VM_LOCKED 0x00002000 289 #define VM_IO 0x00004000 /* Memory mapped I/O or similar */ 290 291 /* Used by sys_madvise() */ 292 #define VM_SEQ_READ 0x00008000 /* App will access data sequentially */ 293 #define VM_RAND_READ 0x00010000 /* App will not benefit from clustered reads */ 294 295 #define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */ 296 #define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */ 297 #define VM_LOCKONFAULT 0x00080000 /* Lock the pages covered when they are faulted in */ 298 #define VM_ACCOUNT 0x00100000 /* Is a VM accounted object */ 299 #define VM_NORESERVE 0x00200000 /* should the VM suppress accounting */ 300 #define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */ 301 #define VM_SYNC 0x00800000 /* Synchronous page faults */ 302 #define VM_ARCH_1 0x01000000 /* Architecture-specific flag */ 303 #define VM_WIPEONFORK 0x02000000 /* Wipe VMA contents in child. */ 304 #define VM_DONTDUMP 0x04000000 /* Do not include in the core dump */ 305 306 #ifdef CONFIG_MEM_SOFT_DIRTY 307 # define VM_SOFTDIRTY 0x08000000 /* Not soft dirty clean area */ 308 #else 309 # define VM_SOFTDIRTY 0 310 #endif 311 312 #define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */ 313 #define VM_HUGEPAGE 0x20000000 /* MADV_HUGEPAGE marked this vma */ 314 #define VM_NOHUGEPAGE 0x40000000 /* MADV_NOHUGEPAGE marked this vma */ 315 #define VM_MERGEABLE 0x80000000 /* KSM may merge identical pages */ 316 317 #ifdef CONFIG_ARCH_USES_HIGH_VMA_FLAGS 318 #define VM_HIGH_ARCH_BIT_0 32 /* bit only usable on 64-bit architectures */ 319 #define VM_HIGH_ARCH_BIT_1 33 /* bit only usable on 64-bit architectures */ 320 #define VM_HIGH_ARCH_BIT_2 34 /* bit only usable on 64-bit architectures */ 321 #define VM_HIGH_ARCH_BIT_3 35 /* bit only usable on 64-bit architectures */ 322 #define VM_HIGH_ARCH_BIT_4 36 /* bit only usable on 64-bit architectures */ 323 #define VM_HIGH_ARCH_BIT_5 37 /* bit only usable on 64-bit architectures */ 324 #define VM_HIGH_ARCH_0 BIT(VM_HIGH_ARCH_BIT_0) 325 #define VM_HIGH_ARCH_1 BIT(VM_HIGH_ARCH_BIT_1) 326 #define VM_HIGH_ARCH_2 BIT(VM_HIGH_ARCH_BIT_2) 327 #define VM_HIGH_ARCH_3 BIT(VM_HIGH_ARCH_BIT_3) 328 #define VM_HIGH_ARCH_4 BIT(VM_HIGH_ARCH_BIT_4) 329 #define VM_HIGH_ARCH_5 BIT(VM_HIGH_ARCH_BIT_5) 330 #endif /* CONFIG_ARCH_USES_HIGH_VMA_FLAGS */ 331 332 #ifdef CONFIG_ARCH_HAS_PKEYS 333 # define VM_PKEY_SHIFT VM_HIGH_ARCH_BIT_0 334 # define VM_PKEY_BIT0 VM_HIGH_ARCH_0 /* A protection key is a 4-bit value */ 335 # define VM_PKEY_BIT1 VM_HIGH_ARCH_1 /* on x86 and 5-bit value on ppc64 */ 336 # define VM_PKEY_BIT2 VM_HIGH_ARCH_2 337 # define VM_PKEY_BIT3 VM_HIGH_ARCH_3 338 #ifdef CONFIG_PPC 339 # define VM_PKEY_BIT4 VM_HIGH_ARCH_4 340 #else 341 # define VM_PKEY_BIT4 0 342 #endif 343 #endif /* CONFIG_ARCH_HAS_PKEYS */ 344 345 #ifdef CONFIG_X86_USER_SHADOW_STACK 346 /* 347 * VM_SHADOW_STACK should not be set with VM_SHARED because of lack of 348 * support core mm. 349 * 350 * These VMAs will get a single end guard page. This helps userspace protect 351 * itself from attacks. A single page is enough for current shadow stack archs 352 * (x86). See the comments near alloc_shstk() in arch/x86/kernel/shstk.c 353 * for more details on the guard size. 354 */ 355 # define VM_SHADOW_STACK VM_HIGH_ARCH_5 356 #else 357 # define VM_SHADOW_STACK VM_NONE 358 #endif 359 360 #if defined(CONFIG_X86) 361 # define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */ 362 #elif defined(CONFIG_PPC) 363 # define VM_SAO VM_ARCH_1 /* Strong Access Ordering (powerpc) */ 364 #elif defined(CONFIG_PARISC) 365 # define VM_GROWSUP VM_ARCH_1 366 #elif defined(CONFIG_SPARC64) 367 # define VM_SPARC_ADI VM_ARCH_1 /* Uses ADI tag for access control */ 368 # define VM_ARCH_CLEAR VM_SPARC_ADI 369 #elif defined(CONFIG_ARM64) 370 # define VM_ARM64_BTI VM_ARCH_1 /* BTI guarded page, a.k.a. GP bit */ 371 # define VM_ARCH_CLEAR VM_ARM64_BTI 372 #elif !defined(CONFIG_MMU) 373 # define VM_MAPPED_COPY VM_ARCH_1 /* T if mapped copy of data (nommu mmap) */ 374 #endif 375 376 #if defined(CONFIG_ARM64_MTE) 377 # define VM_MTE VM_HIGH_ARCH_0 /* Use Tagged memory for access control */ 378 # define VM_MTE_ALLOWED VM_HIGH_ARCH_1 /* Tagged memory permitted */ 379 #else 380 # define VM_MTE VM_NONE 381 # define VM_MTE_ALLOWED VM_NONE 382 #endif 383 384 #ifndef VM_GROWSUP 385 # define VM_GROWSUP VM_NONE 386 #endif 387 388 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR 389 # define VM_UFFD_MINOR_BIT 38 390 # define VM_UFFD_MINOR BIT(VM_UFFD_MINOR_BIT) /* UFFD minor faults */ 391 #else /* !CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */ 392 # define VM_UFFD_MINOR VM_NONE 393 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */ 394 395 /* 396 * This flag is used to connect VFIO to arch specific KVM code. It 397 * indicates that the memory under this VMA is safe for use with any 398 * non-cachable memory type inside KVM. Some VFIO devices, on some 399 * platforms, are thought to be unsafe and can cause machine crashes 400 * if KVM does not lock down the memory type. 401 */ 402 #ifdef CONFIG_64BIT 403 #define VM_ALLOW_ANY_UNCACHED_BIT 39 404 #define VM_ALLOW_ANY_UNCACHED BIT(VM_ALLOW_ANY_UNCACHED_BIT) 405 #else 406 #define VM_ALLOW_ANY_UNCACHED VM_NONE 407 #endif 408 409 /* Bits set in the VMA until the stack is in its final location */ 410 #define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY) 411 412 #define TASK_EXEC ((current->personality & READ_IMPLIES_EXEC) ? VM_EXEC : 0) 413 414 /* Common data flag combinations */ 415 #define VM_DATA_FLAGS_TSK_EXEC (VM_READ | VM_WRITE | TASK_EXEC | \ 416 VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) 417 #define VM_DATA_FLAGS_NON_EXEC (VM_READ | VM_WRITE | VM_MAYREAD | \ 418 VM_MAYWRITE | VM_MAYEXEC) 419 #define VM_DATA_FLAGS_EXEC (VM_READ | VM_WRITE | VM_EXEC | \ 420 VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) 421 422 #ifndef VM_DATA_DEFAULT_FLAGS /* arch can override this */ 423 #define VM_DATA_DEFAULT_FLAGS VM_DATA_FLAGS_EXEC 424 #endif 425 426 #ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */ 427 #define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS 428 #endif 429 430 #define VM_STARTGAP_FLAGS (VM_GROWSDOWN | VM_SHADOW_STACK) 431 432 #ifdef CONFIG_STACK_GROWSUP 433 #define VM_STACK VM_GROWSUP 434 #define VM_STACK_EARLY VM_GROWSDOWN 435 #else 436 #define VM_STACK VM_GROWSDOWN 437 #define VM_STACK_EARLY 0 438 #endif 439 440 #define VM_STACK_FLAGS (VM_STACK | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT) 441 442 /* VMA basic access permission flags */ 443 #define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC) 444 445 446 /* 447 * Special vmas that are non-mergable, non-mlock()able. 448 */ 449 #define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_MIXEDMAP) 450 451 /* This mask prevents VMA from being scanned with khugepaged */ 452 #define VM_NO_KHUGEPAGED (VM_SPECIAL | VM_HUGETLB) 453 454 /* This mask defines which mm->def_flags a process can inherit its parent */ 455 #define VM_INIT_DEF_MASK VM_NOHUGEPAGE 456 457 /* This mask represents all the VMA flag bits used by mlock */ 458 #define VM_LOCKED_MASK (VM_LOCKED | VM_LOCKONFAULT) 459 460 /* Arch-specific flags to clear when updating VM flags on protection change */ 461 #ifndef VM_ARCH_CLEAR 462 # define VM_ARCH_CLEAR VM_NONE 463 #endif 464 #define VM_FLAGS_CLEAR (ARCH_VM_PKEY_FLAGS | VM_ARCH_CLEAR) 465 466 /* 467 * mapping from the currently active vm_flags protection bits (the 468 * low four bits) to a page protection mask.. 469 */ 470 471 /* 472 * The default fault flags that should be used by most of the 473 * arch-specific page fault handlers. 474 */ 475 #define FAULT_FLAG_DEFAULT (FAULT_FLAG_ALLOW_RETRY | \ 476 FAULT_FLAG_KILLABLE | \ 477 FAULT_FLAG_INTERRUPTIBLE) 478 479 /** 480 * fault_flag_allow_retry_first - check ALLOW_RETRY the first time 481 * @flags: Fault flags. 482 * 483 * This is mostly used for places where we want to try to avoid taking 484 * the mmap_lock for too long a time when waiting for another condition 485 * to change, in which case we can try to be polite to release the 486 * mmap_lock in the first round to avoid potential starvation of other 487 * processes that would also want the mmap_lock. 488 * 489 * Return: true if the page fault allows retry and this is the first 490 * attempt of the fault handling; false otherwise. 491 */ 492 static inline bool fault_flag_allow_retry_first(enum fault_flag flags) 493 { 494 return (flags & FAULT_FLAG_ALLOW_RETRY) && 495 (!(flags & FAULT_FLAG_TRIED)); 496 } 497 498 #define FAULT_FLAG_TRACE \ 499 { FAULT_FLAG_WRITE, "WRITE" }, \ 500 { FAULT_FLAG_MKWRITE, "MKWRITE" }, \ 501 { FAULT_FLAG_ALLOW_RETRY, "ALLOW_RETRY" }, \ 502 { FAULT_FLAG_RETRY_NOWAIT, "RETRY_NOWAIT" }, \ 503 { FAULT_FLAG_KILLABLE, "KILLABLE" }, \ 504 { FAULT_FLAG_TRIED, "TRIED" }, \ 505 { FAULT_FLAG_USER, "USER" }, \ 506 { FAULT_FLAG_REMOTE, "REMOTE" }, \ 507 { FAULT_FLAG_INSTRUCTION, "INSTRUCTION" }, \ 508 { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }, \ 509 { FAULT_FLAG_VMA_LOCK, "VMA_LOCK" } 510 511 /* 512 * vm_fault is filled by the pagefault handler and passed to the vma's 513 * ->fault function. The vma's ->fault is responsible for returning a bitmask 514 * of VM_FAULT_xxx flags that give details about how the fault was handled. 515 * 516 * MM layer fills up gfp_mask for page allocations but fault handler might 517 * alter it if its implementation requires a different allocation context. 518 * 519 * pgoff should be used in favour of virtual_address, if possible. 520 */ 521 struct vm_fault { 522 const struct { 523 struct vm_area_struct *vma; /* Target VMA */ 524 gfp_t gfp_mask; /* gfp mask to be used for allocations */ 525 pgoff_t pgoff; /* Logical page offset based on vma */ 526 unsigned long address; /* Faulting virtual address - masked */ 527 unsigned long real_address; /* Faulting virtual address - unmasked */ 528 }; 529 enum fault_flag flags; /* FAULT_FLAG_xxx flags 530 * XXX: should really be 'const' */ 531 pmd_t *pmd; /* Pointer to pmd entry matching 532 * the 'address' */ 533 pud_t *pud; /* Pointer to pud entry matching 534 * the 'address' 535 */ 536 union { 537 pte_t orig_pte; /* Value of PTE at the time of fault */ 538 pmd_t orig_pmd; /* Value of PMD at the time of fault, 539 * used by PMD fault only. 540 */ 541 }; 542 543 struct page *cow_page; /* Page handler may use for COW fault */ 544 struct page *page; /* ->fault handlers should return a 545 * page here, unless VM_FAULT_NOPAGE 546 * is set (which is also implied by 547 * VM_FAULT_ERROR). 548 */ 549 /* These three entries are valid only while holding ptl lock */ 550 pte_t *pte; /* Pointer to pte entry matching 551 * the 'address'. NULL if the page 552 * table hasn't been allocated. 553 */ 554 spinlock_t *ptl; /* Page table lock. 555 * Protects pte page table if 'pte' 556 * is not NULL, otherwise pmd. 557 */ 558 pgtable_t prealloc_pte; /* Pre-allocated pte page table. 559 * vm_ops->map_pages() sets up a page 560 * table from atomic context. 561 * do_fault_around() pre-allocates 562 * page table to avoid allocation from 563 * atomic context. 564 */ 565 }; 566 567 /* 568 * These are the virtual MM functions - opening of an area, closing and 569 * unmapping it (needed to keep files on disk up-to-date etc), pointer 570 * to the functions called when a no-page or a wp-page exception occurs. 571 */ 572 struct vm_operations_struct { 573 void (*open)(struct vm_area_struct * area); 574 /** 575 * @close: Called when the VMA is being removed from the MM. 576 * Context: User context. May sleep. Caller holds mmap_lock. 577 */ 578 void (*close)(struct vm_area_struct * area); 579 /* Called any time before splitting to check if it's allowed */ 580 int (*may_split)(struct vm_area_struct *area, unsigned long addr); 581 int (*mremap)(struct vm_area_struct *area); 582 /* 583 * Called by mprotect() to make driver-specific permission 584 * checks before mprotect() is finalised. The VMA must not 585 * be modified. Returns 0 if mprotect() can proceed. 586 */ 587 int (*mprotect)(struct vm_area_struct *vma, unsigned long start, 588 unsigned long end, unsigned long newflags); 589 vm_fault_t (*fault)(struct vm_fault *vmf); 590 vm_fault_t (*huge_fault)(struct vm_fault *vmf, unsigned int order); 591 vm_fault_t (*map_pages)(struct vm_fault *vmf, 592 pgoff_t start_pgoff, pgoff_t end_pgoff); 593 unsigned long (*pagesize)(struct vm_area_struct * area); 594 595 /* notification that a previously read-only page is about to become 596 * writable, if an error is returned it will cause a SIGBUS */ 597 vm_fault_t (*page_mkwrite)(struct vm_fault *vmf); 598 599 /* same as page_mkwrite when using VM_PFNMAP|VM_MIXEDMAP */ 600 vm_fault_t (*pfn_mkwrite)(struct vm_fault *vmf); 601 602 /* called by access_process_vm when get_user_pages() fails, typically 603 * for use by special VMAs. See also generic_access_phys() for a generic 604 * implementation useful for any iomem mapping. 605 */ 606 int (*access)(struct vm_area_struct *vma, unsigned long addr, 607 void *buf, int len, int write); 608 609 /* Called by the /proc/PID/maps code to ask the vma whether it 610 * has a special name. Returning non-NULL will also cause this 611 * vma to be dumped unconditionally. */ 612 const char *(*name)(struct vm_area_struct *vma); 613 614 #ifdef CONFIG_NUMA 615 /* 616 * set_policy() op must add a reference to any non-NULL @new mempolicy 617 * to hold the policy upon return. Caller should pass NULL @new to 618 * remove a policy and fall back to surrounding context--i.e. do not 619 * install a MPOL_DEFAULT policy, nor the task or system default 620 * mempolicy. 621 */ 622 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new); 623 624 /* 625 * get_policy() op must add reference [mpol_get()] to any policy at 626 * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure 627 * in mm/mempolicy.c will do this automatically. 628 * get_policy() must NOT add a ref if the policy at (vma,addr) is not 629 * marked as MPOL_SHARED. vma policies are protected by the mmap_lock. 630 * If no [shared/vma] mempolicy exists at the addr, get_policy() op 631 * must return NULL--i.e., do not "fallback" to task or system default 632 * policy. 633 */ 634 struct mempolicy *(*get_policy)(struct vm_area_struct *vma, 635 unsigned long addr, pgoff_t *ilx); 636 #endif 637 /* 638 * Called by vm_normal_page() for special PTEs to find the 639 * page for @addr. This is useful if the default behavior 640 * (using pte_page()) would not find the correct page. 641 */ 642 struct page *(*find_special_page)(struct vm_area_struct *vma, 643 unsigned long addr); 644 }; 645 646 #ifdef CONFIG_NUMA_BALANCING 647 static inline void vma_numab_state_init(struct vm_area_struct *vma) 648 { 649 vma->numab_state = NULL; 650 } 651 static inline void vma_numab_state_free(struct vm_area_struct *vma) 652 { 653 kfree(vma->numab_state); 654 } 655 #else 656 static inline void vma_numab_state_init(struct vm_area_struct *vma) {} 657 static inline void vma_numab_state_free(struct vm_area_struct *vma) {} 658 #endif /* CONFIG_NUMA_BALANCING */ 659 660 #ifdef CONFIG_PER_VMA_LOCK 661 /* 662 * Try to read-lock a vma. The function is allowed to occasionally yield false 663 * locked result to avoid performance overhead, in which case we fall back to 664 * using mmap_lock. The function should never yield false unlocked result. 665 */ 666 static inline bool vma_start_read(struct vm_area_struct *vma) 667 { 668 /* 669 * Check before locking. A race might cause false locked result. 670 * We can use READ_ONCE() for the mm_lock_seq here, and don't need 671 * ACQUIRE semantics, because this is just a lockless check whose result 672 * we don't rely on for anything - the mm_lock_seq read against which we 673 * need ordering is below. 674 */ 675 if (READ_ONCE(vma->vm_lock_seq) == READ_ONCE(vma->vm_mm->mm_lock_seq)) 676 return false; 677 678 if (unlikely(down_read_trylock(&vma->vm_lock->lock) == 0)) 679 return false; 680 681 /* 682 * Overflow might produce false locked result. 683 * False unlocked result is impossible because we modify and check 684 * vma->vm_lock_seq under vma->vm_lock protection and mm->mm_lock_seq 685 * modification invalidates all existing locks. 686 * 687 * We must use ACQUIRE semantics for the mm_lock_seq so that if we are 688 * racing with vma_end_write_all(), we only start reading from the VMA 689 * after it has been unlocked. 690 * This pairs with RELEASE semantics in vma_end_write_all(). 691 */ 692 if (unlikely(vma->vm_lock_seq == smp_load_acquire(&vma->vm_mm->mm_lock_seq))) { 693 up_read(&vma->vm_lock->lock); 694 return false; 695 } 696 return true; 697 } 698 699 static inline void vma_end_read(struct vm_area_struct *vma) 700 { 701 rcu_read_lock(); /* keeps vma alive till the end of up_read */ 702 up_read(&vma->vm_lock->lock); 703 rcu_read_unlock(); 704 } 705 706 /* WARNING! Can only be used if mmap_lock is expected to be write-locked */ 707 static bool __is_vma_write_locked(struct vm_area_struct *vma, int *mm_lock_seq) 708 { 709 mmap_assert_write_locked(vma->vm_mm); 710 711 /* 712 * current task is holding mmap_write_lock, both vma->vm_lock_seq and 713 * mm->mm_lock_seq can't be concurrently modified. 714 */ 715 *mm_lock_seq = vma->vm_mm->mm_lock_seq; 716 return (vma->vm_lock_seq == *mm_lock_seq); 717 } 718 719 /* 720 * Begin writing to a VMA. 721 * Exclude concurrent readers under the per-VMA lock until the currently 722 * write-locked mmap_lock is dropped or downgraded. 723 */ 724 static inline void vma_start_write(struct vm_area_struct *vma) 725 { 726 int mm_lock_seq; 727 728 if (__is_vma_write_locked(vma, &mm_lock_seq)) 729 return; 730 731 down_write(&vma->vm_lock->lock); 732 /* 733 * We should use WRITE_ONCE() here because we can have concurrent reads 734 * from the early lockless pessimistic check in vma_start_read(). 735 * We don't really care about the correctness of that early check, but 736 * we should use WRITE_ONCE() for cleanliness and to keep KCSAN happy. 737 */ 738 WRITE_ONCE(vma->vm_lock_seq, mm_lock_seq); 739 up_write(&vma->vm_lock->lock); 740 } 741 742 static inline void vma_assert_write_locked(struct vm_area_struct *vma) 743 { 744 int mm_lock_seq; 745 746 VM_BUG_ON_VMA(!__is_vma_write_locked(vma, &mm_lock_seq), vma); 747 } 748 749 static inline void vma_assert_locked(struct vm_area_struct *vma) 750 { 751 if (!rwsem_is_locked(&vma->vm_lock->lock)) 752 vma_assert_write_locked(vma); 753 } 754 755 static inline void vma_mark_detached(struct vm_area_struct *vma, bool detached) 756 { 757 /* When detaching vma should be write-locked */ 758 if (detached) 759 vma_assert_write_locked(vma); 760 vma->detached = detached; 761 } 762 763 static inline void release_fault_lock(struct vm_fault *vmf) 764 { 765 if (vmf->flags & FAULT_FLAG_VMA_LOCK) 766 vma_end_read(vmf->vma); 767 else 768 mmap_read_unlock(vmf->vma->vm_mm); 769 } 770 771 static inline void assert_fault_locked(struct vm_fault *vmf) 772 { 773 if (vmf->flags & FAULT_FLAG_VMA_LOCK) 774 vma_assert_locked(vmf->vma); 775 else 776 mmap_assert_locked(vmf->vma->vm_mm); 777 } 778 779 struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, 780 unsigned long address); 781 782 #else /* CONFIG_PER_VMA_LOCK */ 783 784 static inline bool vma_start_read(struct vm_area_struct *vma) 785 { return false; } 786 static inline void vma_end_read(struct vm_area_struct *vma) {} 787 static inline void vma_start_write(struct vm_area_struct *vma) {} 788 static inline void vma_assert_write_locked(struct vm_area_struct *vma) 789 { mmap_assert_write_locked(vma->vm_mm); } 790 static inline void vma_mark_detached(struct vm_area_struct *vma, 791 bool detached) {} 792 793 static inline struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, 794 unsigned long address) 795 { 796 return NULL; 797 } 798 799 static inline void vma_assert_locked(struct vm_area_struct *vma) 800 { 801 mmap_assert_locked(vma->vm_mm); 802 } 803 804 static inline void release_fault_lock(struct vm_fault *vmf) 805 { 806 mmap_read_unlock(vmf->vma->vm_mm); 807 } 808 809 static inline void assert_fault_locked(struct vm_fault *vmf) 810 { 811 mmap_assert_locked(vmf->vma->vm_mm); 812 } 813 814 #endif /* CONFIG_PER_VMA_LOCK */ 815 816 extern const struct vm_operations_struct vma_dummy_vm_ops; 817 818 /* 819 * WARNING: vma_init does not initialize vma->vm_lock. 820 * Use vm_area_alloc()/vm_area_free() if vma needs locking. 821 */ 822 static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm) 823 { 824 memset(vma, 0, sizeof(*vma)); 825 vma->vm_mm = mm; 826 vma->vm_ops = &vma_dummy_vm_ops; 827 INIT_LIST_HEAD(&vma->anon_vma_chain); 828 vma_mark_detached(vma, false); 829 vma_numab_state_init(vma); 830 } 831 832 /* Use when VMA is not part of the VMA tree and needs no locking */ 833 static inline void vm_flags_init(struct vm_area_struct *vma, 834 vm_flags_t flags) 835 { 836 ACCESS_PRIVATE(vma, __vm_flags) = flags; 837 } 838 839 /* 840 * Use when VMA is part of the VMA tree and modifications need coordination 841 * Note: vm_flags_reset and vm_flags_reset_once do not lock the vma and 842 * it should be locked explicitly beforehand. 843 */ 844 static inline void vm_flags_reset(struct vm_area_struct *vma, 845 vm_flags_t flags) 846 { 847 vma_assert_write_locked(vma); 848 vm_flags_init(vma, flags); 849 } 850 851 static inline void vm_flags_reset_once(struct vm_area_struct *vma, 852 vm_flags_t flags) 853 { 854 vma_assert_write_locked(vma); 855 WRITE_ONCE(ACCESS_PRIVATE(vma, __vm_flags), flags); 856 } 857 858 static inline void vm_flags_set(struct vm_area_struct *vma, 859 vm_flags_t flags) 860 { 861 vma_start_write(vma); 862 ACCESS_PRIVATE(vma, __vm_flags) |= flags; 863 } 864 865 static inline void vm_flags_clear(struct vm_area_struct *vma, 866 vm_flags_t flags) 867 { 868 vma_start_write(vma); 869 ACCESS_PRIVATE(vma, __vm_flags) &= ~flags; 870 } 871 872 /* 873 * Use only if VMA is not part of the VMA tree or has no other users and 874 * therefore needs no locking. 875 */ 876 static inline void __vm_flags_mod(struct vm_area_struct *vma, 877 vm_flags_t set, vm_flags_t clear) 878 { 879 vm_flags_init(vma, (vma->vm_flags | set) & ~clear); 880 } 881 882 /* 883 * Use only when the order of set/clear operations is unimportant, otherwise 884 * use vm_flags_{set|clear} explicitly. 885 */ 886 static inline void vm_flags_mod(struct vm_area_struct *vma, 887 vm_flags_t set, vm_flags_t clear) 888 { 889 vma_start_write(vma); 890 __vm_flags_mod(vma, set, clear); 891 } 892 893 static inline void vma_set_anonymous(struct vm_area_struct *vma) 894 { 895 vma->vm_ops = NULL; 896 } 897 898 static inline bool vma_is_anonymous(struct vm_area_struct *vma) 899 { 900 return !vma->vm_ops; 901 } 902 903 /* 904 * Indicate if the VMA is a heap for the given task; for 905 * /proc/PID/maps that is the heap of the main task. 906 */ 907 static inline bool vma_is_initial_heap(const struct vm_area_struct *vma) 908 { 909 return vma->vm_start < vma->vm_mm->brk && 910 vma->vm_end > vma->vm_mm->start_brk; 911 } 912 913 /* 914 * Indicate if the VMA is a stack for the given task; for 915 * /proc/PID/maps that is the stack of the main task. 916 */ 917 static inline bool vma_is_initial_stack(const struct vm_area_struct *vma) 918 { 919 /* 920 * We make no effort to guess what a given thread considers to be 921 * its "stack". It's not even well-defined for programs written 922 * languages like Go. 923 */ 924 return vma->vm_start <= vma->vm_mm->start_stack && 925 vma->vm_end >= vma->vm_mm->start_stack; 926 } 927 928 static inline bool vma_is_temporary_stack(struct vm_area_struct *vma) 929 { 930 int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP); 931 932 if (!maybe_stack) 933 return false; 934 935 if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == 936 VM_STACK_INCOMPLETE_SETUP) 937 return true; 938 939 return false; 940 } 941 942 static inline bool vma_is_foreign(struct vm_area_struct *vma) 943 { 944 if (!current->mm) 945 return true; 946 947 if (current->mm != vma->vm_mm) 948 return true; 949 950 return false; 951 } 952 953 static inline bool vma_is_accessible(struct vm_area_struct *vma) 954 { 955 return vma->vm_flags & VM_ACCESS_FLAGS; 956 } 957 958 static inline bool is_shared_maywrite(vm_flags_t vm_flags) 959 { 960 return (vm_flags & (VM_SHARED | VM_MAYWRITE)) == 961 (VM_SHARED | VM_MAYWRITE); 962 } 963 964 static inline bool vma_is_shared_maywrite(struct vm_area_struct *vma) 965 { 966 return is_shared_maywrite(vma->vm_flags); 967 } 968 969 static inline 970 struct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max) 971 { 972 return mas_find(&vmi->mas, max - 1); 973 } 974 975 static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi) 976 { 977 /* 978 * Uses mas_find() to get the first VMA when the iterator starts. 979 * Calling mas_next() could skip the first entry. 980 */ 981 return mas_find(&vmi->mas, ULONG_MAX); 982 } 983 984 static inline 985 struct vm_area_struct *vma_iter_next_range(struct vma_iterator *vmi) 986 { 987 return mas_next_range(&vmi->mas, ULONG_MAX); 988 } 989 990 991 static inline struct vm_area_struct *vma_prev(struct vma_iterator *vmi) 992 { 993 return mas_prev(&vmi->mas, 0); 994 } 995 996 static inline 997 struct vm_area_struct *vma_iter_prev_range(struct vma_iterator *vmi) 998 { 999 return mas_prev_range(&vmi->mas, 0); 1000 } 1001 1002 static inline unsigned long vma_iter_addr(struct vma_iterator *vmi) 1003 { 1004 return vmi->mas.index; 1005 } 1006 1007 static inline unsigned long vma_iter_end(struct vma_iterator *vmi) 1008 { 1009 return vmi->mas.last + 1; 1010 } 1011 static inline int vma_iter_bulk_alloc(struct vma_iterator *vmi, 1012 unsigned long count) 1013 { 1014 return mas_expected_entries(&vmi->mas, count); 1015 } 1016 1017 static inline int vma_iter_clear_gfp(struct vma_iterator *vmi, 1018 unsigned long start, unsigned long end, gfp_t gfp) 1019 { 1020 __mas_set_range(&vmi->mas, start, end - 1); 1021 mas_store_gfp(&vmi->mas, NULL, gfp); 1022 if (unlikely(mas_is_err(&vmi->mas))) 1023 return -ENOMEM; 1024 1025 return 0; 1026 } 1027 1028 /* Free any unused preallocations */ 1029 static inline void vma_iter_free(struct vma_iterator *vmi) 1030 { 1031 mas_destroy(&vmi->mas); 1032 } 1033 1034 static inline int vma_iter_bulk_store(struct vma_iterator *vmi, 1035 struct vm_area_struct *vma) 1036 { 1037 vmi->mas.index = vma->vm_start; 1038 vmi->mas.last = vma->vm_end - 1; 1039 mas_store(&vmi->mas, vma); 1040 if (unlikely(mas_is_err(&vmi->mas))) 1041 return -ENOMEM; 1042 1043 return 0; 1044 } 1045 1046 static inline void vma_iter_invalidate(struct vma_iterator *vmi) 1047 { 1048 mas_pause(&vmi->mas); 1049 } 1050 1051 static inline void vma_iter_set(struct vma_iterator *vmi, unsigned long addr) 1052 { 1053 mas_set(&vmi->mas, addr); 1054 } 1055 1056 #define for_each_vma(__vmi, __vma) \ 1057 while (((__vma) = vma_next(&(__vmi))) != NULL) 1058 1059 /* The MM code likes to work with exclusive end addresses */ 1060 #define for_each_vma_range(__vmi, __vma, __end) \ 1061 while (((__vma) = vma_find(&(__vmi), (__end))) != NULL) 1062 1063 #ifdef CONFIG_SHMEM 1064 /* 1065 * The vma_is_shmem is not inline because it is used only by slow 1066 * paths in userfault. 1067 */ 1068 bool vma_is_shmem(struct vm_area_struct *vma); 1069 bool vma_is_anon_shmem(struct vm_area_struct *vma); 1070 #else 1071 static inline bool vma_is_shmem(struct vm_area_struct *vma) { return false; } 1072 static inline bool vma_is_anon_shmem(struct vm_area_struct *vma) { return false; } 1073 #endif 1074 1075 int vma_is_stack_for_current(struct vm_area_struct *vma); 1076 1077 /* flush_tlb_range() takes a vma, not a mm, and can care about flags */ 1078 #define TLB_FLUSH_VMA(mm,flags) { .vm_mm = (mm), .vm_flags = (flags) } 1079 1080 struct mmu_gather; 1081 struct inode; 1082 1083 /* 1084 * compound_order() can be called without holding a reference, which means 1085 * that niceties like page_folio() don't work. These callers should be 1086 * prepared to handle wild return values. For example, PG_head may be 1087 * set before the order is initialised, or this may be a tail page. 1088 * See compaction.c for some good examples. 1089 */ 1090 static inline unsigned int compound_order(struct page *page) 1091 { 1092 struct folio *folio = (struct folio *)page; 1093 1094 if (!test_bit(PG_head, &folio->flags)) 1095 return 0; 1096 return folio->_flags_1 & 0xff; 1097 } 1098 1099 /** 1100 * folio_order - The allocation order of a folio. 1101 * @folio: The folio. 1102 * 1103 * A folio is composed of 2^order pages. See get_order() for the definition 1104 * of order. 1105 * 1106 * Return: The order of the folio. 1107 */ 1108 static inline unsigned int folio_order(struct folio *folio) 1109 { 1110 if (!folio_test_large(folio)) 1111 return 0; 1112 return folio->_flags_1 & 0xff; 1113 } 1114 1115 #include <linux/huge_mm.h> 1116 1117 /* 1118 * Methods to modify the page usage count. 1119 * 1120 * What counts for a page usage: 1121 * - cache mapping (page->mapping) 1122 * - private data (page->private) 1123 * - page mapped in a task's page tables, each mapping 1124 * is counted separately 1125 * 1126 * Also, many kernel routines increase the page count before a critical 1127 * routine so they can be sure the page doesn't go away from under them. 1128 */ 1129 1130 /* 1131 * Drop a ref, return true if the refcount fell to zero (the page has no users) 1132 */ 1133 static inline int put_page_testzero(struct page *page) 1134 { 1135 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 1136 return page_ref_dec_and_test(page); 1137 } 1138 1139 static inline int folio_put_testzero(struct folio *folio) 1140 { 1141 return put_page_testzero(&folio->page); 1142 } 1143 1144 /* 1145 * Try to grab a ref unless the page has a refcount of zero, return false if 1146 * that is the case. 1147 * This can be called when MMU is off so it must not access 1148 * any of the virtual mappings. 1149 */ 1150 static inline bool get_page_unless_zero(struct page *page) 1151 { 1152 return page_ref_add_unless(page, 1, 0); 1153 } 1154 1155 static inline struct folio *folio_get_nontail_page(struct page *page) 1156 { 1157 if (unlikely(!get_page_unless_zero(page))) 1158 return NULL; 1159 return (struct folio *)page; 1160 } 1161 1162 extern int page_is_ram(unsigned long pfn); 1163 1164 enum { 1165 REGION_INTERSECTS, 1166 REGION_DISJOINT, 1167 REGION_MIXED, 1168 }; 1169 1170 int region_intersects(resource_size_t offset, size_t size, unsigned long flags, 1171 unsigned long desc); 1172 1173 /* Support for virtually mapped pages */ 1174 struct page *vmalloc_to_page(const void *addr); 1175 unsigned long vmalloc_to_pfn(const void *addr); 1176 1177 /* 1178 * Determine if an address is within the vmalloc range 1179 * 1180 * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there 1181 * is no special casing required. 1182 */ 1183 #ifdef CONFIG_MMU 1184 extern bool is_vmalloc_addr(const void *x); 1185 extern int is_vmalloc_or_module_addr(const void *x); 1186 #else 1187 static inline bool is_vmalloc_addr(const void *x) 1188 { 1189 return false; 1190 } 1191 static inline int is_vmalloc_or_module_addr(const void *x) 1192 { 1193 return 0; 1194 } 1195 #endif 1196 1197 /* 1198 * How many times the entire folio is mapped as a single unit (eg by a 1199 * PMD or PUD entry). This is probably not what you want, except for 1200 * debugging purposes - it does not include PTE-mapped sub-pages; look 1201 * at folio_mapcount() or page_mapcount() instead. 1202 */ 1203 static inline int folio_entire_mapcount(const struct folio *folio) 1204 { 1205 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 1206 return atomic_read(&folio->_entire_mapcount) + 1; 1207 } 1208 1209 /* 1210 * The atomic page->_mapcount, starts from -1: so that transitions 1211 * both from it and to it can be tracked, using atomic_inc_and_test 1212 * and atomic_add_negative(-1). 1213 */ 1214 static inline void page_mapcount_reset(struct page *page) 1215 { 1216 atomic_set(&(page)->_mapcount, -1); 1217 } 1218 1219 /** 1220 * page_mapcount() - Number of times this precise page is mapped. 1221 * @page: The page. 1222 * 1223 * The number of times this page is mapped. If this page is part of 1224 * a large folio, it includes the number of times this page is mapped 1225 * as part of that folio. 1226 * 1227 * Will report 0 for pages which cannot be mapped into userspace, eg 1228 * slab, page tables and similar. 1229 */ 1230 static inline int page_mapcount(struct page *page) 1231 { 1232 int mapcount = atomic_read(&page->_mapcount) + 1; 1233 1234 /* Handle page_has_type() pages */ 1235 if (mapcount < PAGE_MAPCOUNT_RESERVE + 1) 1236 mapcount = 0; 1237 if (unlikely(PageCompound(page))) 1238 mapcount += folio_entire_mapcount(page_folio(page)); 1239 1240 return mapcount; 1241 } 1242 1243 static inline int folio_large_mapcount(const struct folio *folio) 1244 { 1245 VM_WARN_ON_FOLIO(!folio_test_large(folio), folio); 1246 return atomic_read(&folio->_large_mapcount) + 1; 1247 } 1248 1249 /** 1250 * folio_mapcount() - Number of mappings of this folio. 1251 * @folio: The folio. 1252 * 1253 * The folio mapcount corresponds to the number of present user page table 1254 * entries that reference any part of a folio. Each such present user page 1255 * table entry must be paired with exactly on folio reference. 1256 * 1257 * For ordindary folios, each user page table entry (PTE/PMD/PUD/...) counts 1258 * exactly once. 1259 * 1260 * For hugetlb folios, each abstracted "hugetlb" user page table entry that 1261 * references the entire folio counts exactly once, even when such special 1262 * page table entries are comprised of multiple ordinary page table entries. 1263 * 1264 * Will report 0 for pages which cannot be mapped into userspace, such as 1265 * slab, page tables and similar. 1266 * 1267 * Return: The number of times this folio is mapped. 1268 */ 1269 static inline int folio_mapcount(const struct folio *folio) 1270 { 1271 int mapcount; 1272 1273 if (likely(!folio_test_large(folio))) { 1274 mapcount = atomic_read(&folio->_mapcount) + 1; 1275 /* Handle page_has_type() pages */ 1276 if (mapcount < PAGE_MAPCOUNT_RESERVE + 1) 1277 mapcount = 0; 1278 return mapcount; 1279 } 1280 return folio_large_mapcount(folio); 1281 } 1282 1283 /** 1284 * folio_mapped - Is this folio mapped into userspace? 1285 * @folio: The folio. 1286 * 1287 * Return: True if any page in this folio is referenced by user page tables. 1288 */ 1289 static inline bool folio_mapped(const struct folio *folio) 1290 { 1291 return folio_mapcount(folio) >= 1; 1292 } 1293 1294 /* 1295 * Return true if this page is mapped into pagetables. 1296 * For compound page it returns true if any sub-page of compound page is mapped, 1297 * even if this particular sub-page is not itself mapped by any PTE or PMD. 1298 */ 1299 static inline bool page_mapped(const struct page *page) 1300 { 1301 return folio_mapped(page_folio(page)); 1302 } 1303 1304 static inline struct page *virt_to_head_page(const void *x) 1305 { 1306 struct page *page = virt_to_page(x); 1307 1308 return compound_head(page); 1309 } 1310 1311 static inline struct folio *virt_to_folio(const void *x) 1312 { 1313 struct page *page = virt_to_page(x); 1314 1315 return page_folio(page); 1316 } 1317 1318 void __folio_put(struct folio *folio); 1319 1320 void put_pages_list(struct list_head *pages); 1321 1322 void split_page(struct page *page, unsigned int order); 1323 void folio_copy(struct folio *dst, struct folio *src); 1324 1325 unsigned long nr_free_buffer_pages(void); 1326 1327 /* Returns the number of bytes in this potentially compound page. */ 1328 static inline unsigned long page_size(struct page *page) 1329 { 1330 return PAGE_SIZE << compound_order(page); 1331 } 1332 1333 /* Returns the number of bits needed for the number of bytes in a page */ 1334 static inline unsigned int page_shift(struct page *page) 1335 { 1336 return PAGE_SHIFT + compound_order(page); 1337 } 1338 1339 /** 1340 * thp_order - Order of a transparent huge page. 1341 * @page: Head page of a transparent huge page. 1342 */ 1343 static inline unsigned int thp_order(struct page *page) 1344 { 1345 VM_BUG_ON_PGFLAGS(PageTail(page), page); 1346 return compound_order(page); 1347 } 1348 1349 /** 1350 * thp_size - Size of a transparent huge page. 1351 * @page: Head page of a transparent huge page. 1352 * 1353 * Return: Number of bytes in this page. 1354 */ 1355 static inline unsigned long thp_size(struct page *page) 1356 { 1357 return PAGE_SIZE << thp_order(page); 1358 } 1359 1360 #ifdef CONFIG_MMU 1361 /* 1362 * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when 1363 * servicing faults for write access. In the normal case, do always want 1364 * pte_mkwrite. But get_user_pages can cause write faults for mappings 1365 * that do not have writing enabled, when used by access_process_vm. 1366 */ 1367 static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) 1368 { 1369 if (likely(vma->vm_flags & VM_WRITE)) 1370 pte = pte_mkwrite(pte, vma); 1371 return pte; 1372 } 1373 1374 vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page); 1375 void set_pte_range(struct vm_fault *vmf, struct folio *folio, 1376 struct page *page, unsigned int nr, unsigned long addr); 1377 1378 vm_fault_t finish_fault(struct vm_fault *vmf); 1379 #endif 1380 1381 /* 1382 * Multiple processes may "see" the same page. E.g. for untouched 1383 * mappings of /dev/null, all processes see the same page full of 1384 * zeroes, and text pages of executables and shared libraries have 1385 * only one copy in memory, at most, normally. 1386 * 1387 * For the non-reserved pages, page_count(page) denotes a reference count. 1388 * page_count() == 0 means the page is free. page->lru is then used for 1389 * freelist management in the buddy allocator. 1390 * page_count() > 0 means the page has been allocated. 1391 * 1392 * Pages are allocated by the slab allocator in order to provide memory 1393 * to kmalloc and kmem_cache_alloc. In this case, the management of the 1394 * page, and the fields in 'struct page' are the responsibility of mm/slab.c 1395 * unless a particular usage is carefully commented. (the responsibility of 1396 * freeing the kmalloc memory is the caller's, of course). 1397 * 1398 * A page may be used by anyone else who does a __get_free_page(). 1399 * In this case, page_count still tracks the references, and should only 1400 * be used through the normal accessor functions. The top bits of page->flags 1401 * and page->virtual store page management information, but all other fields 1402 * are unused and could be used privately, carefully. The management of this 1403 * page is the responsibility of the one who allocated it, and those who have 1404 * subsequently been given references to it. 1405 * 1406 * The other pages (we may call them "pagecache pages") are completely 1407 * managed by the Linux memory manager: I/O, buffers, swapping etc. 1408 * The following discussion applies only to them. 1409 * 1410 * A pagecache page contains an opaque `private' member, which belongs to the 1411 * page's address_space. Usually, this is the address of a circular list of 1412 * the page's disk buffers. PG_private must be set to tell the VM to call 1413 * into the filesystem to release these pages. 1414 * 1415 * A page may belong to an inode's memory mapping. In this case, page->mapping 1416 * is the pointer to the inode, and page->index is the file offset of the page, 1417 * in units of PAGE_SIZE. 1418 * 1419 * If pagecache pages are not associated with an inode, they are said to be 1420 * anonymous pages. These may become associated with the swapcache, and in that 1421 * case PG_swapcache is set, and page->private is an offset into the swapcache. 1422 * 1423 * In either case (swapcache or inode backed), the pagecache itself holds one 1424 * reference to the page. Setting PG_private should also increment the 1425 * refcount. The each user mapping also has a reference to the page. 1426 * 1427 * The pagecache pages are stored in a per-mapping radix tree, which is 1428 * rooted at mapping->i_pages, and indexed by offset. 1429 * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space 1430 * lists, we instead now tag pages as dirty/writeback in the radix tree. 1431 * 1432 * All pagecache pages may be subject to I/O: 1433 * - inode pages may need to be read from disk, 1434 * - inode pages which have been modified and are MAP_SHARED may need 1435 * to be written back to the inode on disk, 1436 * - anonymous pages (including MAP_PRIVATE file mappings) which have been 1437 * modified may need to be swapped out to swap space and (later) to be read 1438 * back into memory. 1439 */ 1440 1441 #if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_FS_DAX) 1442 DECLARE_STATIC_KEY_FALSE(devmap_managed_key); 1443 1444 bool __put_devmap_managed_folio_refs(struct folio *folio, int refs); 1445 static inline bool put_devmap_managed_folio_refs(struct folio *folio, int refs) 1446 { 1447 if (!static_branch_unlikely(&devmap_managed_key)) 1448 return false; 1449 if (!folio_is_zone_device(folio)) 1450 return false; 1451 return __put_devmap_managed_folio_refs(folio, refs); 1452 } 1453 #else /* CONFIG_ZONE_DEVICE && CONFIG_FS_DAX */ 1454 static inline bool put_devmap_managed_folio_refs(struct folio *folio, int refs) 1455 { 1456 return false; 1457 } 1458 #endif /* CONFIG_ZONE_DEVICE && CONFIG_FS_DAX */ 1459 1460 /* 127: arbitrary random number, small enough to assemble well */ 1461 #define folio_ref_zero_or_close_to_overflow(folio) \ 1462 ((unsigned int) folio_ref_count(folio) + 127u <= 127u) 1463 1464 /** 1465 * folio_get - Increment the reference count on a folio. 1466 * @folio: The folio. 1467 * 1468 * Context: May be called in any context, as long as you know that 1469 * you have a refcount on the folio. If you do not already have one, 1470 * folio_try_get() may be the right interface for you to use. 1471 */ 1472 static inline void folio_get(struct folio *folio) 1473 { 1474 VM_BUG_ON_FOLIO(folio_ref_zero_or_close_to_overflow(folio), folio); 1475 folio_ref_inc(folio); 1476 } 1477 1478 static inline void get_page(struct page *page) 1479 { 1480 folio_get(page_folio(page)); 1481 } 1482 1483 static inline __must_check bool try_get_page(struct page *page) 1484 { 1485 page = compound_head(page); 1486 if (WARN_ON_ONCE(page_ref_count(page) <= 0)) 1487 return false; 1488 page_ref_inc(page); 1489 return true; 1490 } 1491 1492 /** 1493 * folio_put - Decrement the reference count on a folio. 1494 * @folio: The folio. 1495 * 1496 * If the folio's reference count reaches zero, the memory will be 1497 * released back to the page allocator and may be used by another 1498 * allocation immediately. Do not access the memory or the struct folio 1499 * after calling folio_put() unless you can be sure that it wasn't the 1500 * last reference. 1501 * 1502 * Context: May be called in process or interrupt context, but not in NMI 1503 * context. May be called while holding a spinlock. 1504 */ 1505 static inline void folio_put(struct folio *folio) 1506 { 1507 if (folio_put_testzero(folio)) 1508 __folio_put(folio); 1509 } 1510 1511 /** 1512 * folio_put_refs - Reduce the reference count on a folio. 1513 * @folio: The folio. 1514 * @refs: The amount to subtract from the folio's reference count. 1515 * 1516 * If the folio's reference count reaches zero, the memory will be 1517 * released back to the page allocator and may be used by another 1518 * allocation immediately. Do not access the memory or the struct folio 1519 * after calling folio_put_refs() unless you can be sure that these weren't 1520 * the last references. 1521 * 1522 * Context: May be called in process or interrupt context, but not in NMI 1523 * context. May be called while holding a spinlock. 1524 */ 1525 static inline void folio_put_refs(struct folio *folio, int refs) 1526 { 1527 if (folio_ref_sub_and_test(folio, refs)) 1528 __folio_put(folio); 1529 } 1530 1531 void folios_put_refs(struct folio_batch *folios, unsigned int *refs); 1532 1533 /* 1534 * union release_pages_arg - an array of pages or folios 1535 * 1536 * release_pages() releases a simple array of multiple pages, and 1537 * accepts various different forms of said page array: either 1538 * a regular old boring array of pages, an array of folios, or 1539 * an array of encoded page pointers. 1540 * 1541 * The transparent union syntax for this kind of "any of these 1542 * argument types" is all kinds of ugly, so look away. 1543 */ 1544 typedef union { 1545 struct page **pages; 1546 struct folio **folios; 1547 struct encoded_page **encoded_pages; 1548 } release_pages_arg __attribute__ ((__transparent_union__)); 1549 1550 void release_pages(release_pages_arg, int nr); 1551 1552 /** 1553 * folios_put - Decrement the reference count on an array of folios. 1554 * @folios: The folios. 1555 * 1556 * Like folio_put(), but for a batch of folios. This is more efficient 1557 * than writing the loop yourself as it will optimise the locks which need 1558 * to be taken if the folios are freed. The folios batch is returned 1559 * empty and ready to be reused for another batch; there is no need to 1560 * reinitialise it. 1561 * 1562 * Context: May be called in process or interrupt context, but not in NMI 1563 * context. May be called while holding a spinlock. 1564 */ 1565 static inline void folios_put(struct folio_batch *folios) 1566 { 1567 folios_put_refs(folios, NULL); 1568 } 1569 1570 static inline void put_page(struct page *page) 1571 { 1572 struct folio *folio = page_folio(page); 1573 1574 /* 1575 * For some devmap managed pages we need to catch refcount transition 1576 * from 2 to 1: 1577 */ 1578 if (put_devmap_managed_folio_refs(folio, 1)) 1579 return; 1580 folio_put(folio); 1581 } 1582 1583 /* 1584 * GUP_PIN_COUNTING_BIAS, and the associated functions that use it, overload 1585 * the page's refcount so that two separate items are tracked: the original page 1586 * reference count, and also a new count of how many pin_user_pages() calls were 1587 * made against the page. ("gup-pinned" is another term for the latter). 1588 * 1589 * With this scheme, pin_user_pages() becomes special: such pages are marked as 1590 * distinct from normal pages. As such, the unpin_user_page() call (and its 1591 * variants) must be used in order to release gup-pinned pages. 1592 * 1593 * Choice of value: 1594 * 1595 * By making GUP_PIN_COUNTING_BIAS a power of two, debugging of page reference 1596 * counts with respect to pin_user_pages() and unpin_user_page() becomes 1597 * simpler, due to the fact that adding an even power of two to the page 1598 * refcount has the effect of using only the upper N bits, for the code that 1599 * counts up using the bias value. This means that the lower bits are left for 1600 * the exclusive use of the original code that increments and decrements by one 1601 * (or at least, by much smaller values than the bias value). 1602 * 1603 * Of course, once the lower bits overflow into the upper bits (and this is 1604 * OK, because subtraction recovers the original values), then visual inspection 1605 * no longer suffices to directly view the separate counts. However, for normal 1606 * applications that don't have huge page reference counts, this won't be an 1607 * issue. 1608 * 1609 * Locking: the lockless algorithm described in folio_try_get_rcu() 1610 * provides safe operation for get_user_pages(), page_mkclean() and 1611 * other calls that race to set up page table entries. 1612 */ 1613 #define GUP_PIN_COUNTING_BIAS (1U << 10) 1614 1615 void unpin_user_page(struct page *page); 1616 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages, 1617 bool make_dirty); 1618 void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages, 1619 bool make_dirty); 1620 void unpin_user_pages(struct page **pages, unsigned long npages); 1621 1622 static inline bool is_cow_mapping(vm_flags_t flags) 1623 { 1624 return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; 1625 } 1626 1627 #ifndef CONFIG_MMU 1628 static inline bool is_nommu_shared_mapping(vm_flags_t flags) 1629 { 1630 /* 1631 * NOMMU shared mappings are ordinary MAP_SHARED mappings and selected 1632 * R/O MAP_PRIVATE file mappings that are an effective R/O overlay of 1633 * a file mapping. R/O MAP_PRIVATE mappings might still modify 1634 * underlying memory if ptrace is active, so this is only possible if 1635 * ptrace does not apply. Note that there is no mprotect() to upgrade 1636 * write permissions later. 1637 */ 1638 return flags & (VM_MAYSHARE | VM_MAYOVERLAY); 1639 } 1640 #endif 1641 1642 #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) 1643 #define SECTION_IN_PAGE_FLAGS 1644 #endif 1645 1646 /* 1647 * The identification function is mainly used by the buddy allocator for 1648 * determining if two pages could be buddies. We are not really identifying 1649 * the zone since we could be using the section number id if we do not have 1650 * node id available in page flags. 1651 * We only guarantee that it will return the same value for two combinable 1652 * pages in a zone. 1653 */ 1654 static inline int page_zone_id(struct page *page) 1655 { 1656 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK; 1657 } 1658 1659 #ifdef NODE_NOT_IN_PAGE_FLAGS 1660 int page_to_nid(const struct page *page); 1661 #else 1662 static inline int page_to_nid(const struct page *page) 1663 { 1664 return (PF_POISONED_CHECK(page)->flags >> NODES_PGSHIFT) & NODES_MASK; 1665 } 1666 #endif 1667 1668 static inline int folio_nid(const struct folio *folio) 1669 { 1670 return page_to_nid(&folio->page); 1671 } 1672 1673 #ifdef CONFIG_NUMA_BALANCING 1674 /* page access time bits needs to hold at least 4 seconds */ 1675 #define PAGE_ACCESS_TIME_MIN_BITS 12 1676 #if LAST_CPUPID_SHIFT < PAGE_ACCESS_TIME_MIN_BITS 1677 #define PAGE_ACCESS_TIME_BUCKETS \ 1678 (PAGE_ACCESS_TIME_MIN_BITS - LAST_CPUPID_SHIFT) 1679 #else 1680 #define PAGE_ACCESS_TIME_BUCKETS 0 1681 #endif 1682 1683 #define PAGE_ACCESS_TIME_MASK \ 1684 (LAST_CPUPID_MASK << PAGE_ACCESS_TIME_BUCKETS) 1685 1686 static inline int cpu_pid_to_cpupid(int cpu, int pid) 1687 { 1688 return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK); 1689 } 1690 1691 static inline int cpupid_to_pid(int cpupid) 1692 { 1693 return cpupid & LAST__PID_MASK; 1694 } 1695 1696 static inline int cpupid_to_cpu(int cpupid) 1697 { 1698 return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK; 1699 } 1700 1701 static inline int cpupid_to_nid(int cpupid) 1702 { 1703 return cpu_to_node(cpupid_to_cpu(cpupid)); 1704 } 1705 1706 static inline bool cpupid_pid_unset(int cpupid) 1707 { 1708 return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK); 1709 } 1710 1711 static inline bool cpupid_cpu_unset(int cpupid) 1712 { 1713 return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK); 1714 } 1715 1716 static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid) 1717 { 1718 return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid); 1719 } 1720 1721 #define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid) 1722 #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS 1723 static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid) 1724 { 1725 return xchg(&folio->_last_cpupid, cpupid & LAST_CPUPID_MASK); 1726 } 1727 1728 static inline int folio_last_cpupid(struct folio *folio) 1729 { 1730 return folio->_last_cpupid; 1731 } 1732 static inline void page_cpupid_reset_last(struct page *page) 1733 { 1734 page->_last_cpupid = -1 & LAST_CPUPID_MASK; 1735 } 1736 #else 1737 static inline int folio_last_cpupid(struct folio *folio) 1738 { 1739 return (folio->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK; 1740 } 1741 1742 int folio_xchg_last_cpupid(struct folio *folio, int cpupid); 1743 1744 static inline void page_cpupid_reset_last(struct page *page) 1745 { 1746 page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT; 1747 } 1748 #endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */ 1749 1750 static inline int folio_xchg_access_time(struct folio *folio, int time) 1751 { 1752 int last_time; 1753 1754 last_time = folio_xchg_last_cpupid(folio, 1755 time >> PAGE_ACCESS_TIME_BUCKETS); 1756 return last_time << PAGE_ACCESS_TIME_BUCKETS; 1757 } 1758 1759 static inline void vma_set_access_pid_bit(struct vm_area_struct *vma) 1760 { 1761 unsigned int pid_bit; 1762 1763 pid_bit = hash_32(current->pid, ilog2(BITS_PER_LONG)); 1764 if (vma->numab_state && !test_bit(pid_bit, &vma->numab_state->pids_active[1])) { 1765 __set_bit(pid_bit, &vma->numab_state->pids_active[1]); 1766 } 1767 } 1768 #else /* !CONFIG_NUMA_BALANCING */ 1769 static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid) 1770 { 1771 return folio_nid(folio); /* XXX */ 1772 } 1773 1774 static inline int folio_xchg_access_time(struct folio *folio, int time) 1775 { 1776 return 0; 1777 } 1778 1779 static inline int folio_last_cpupid(struct folio *folio) 1780 { 1781 return folio_nid(folio); /* XXX */ 1782 } 1783 1784 static inline int cpupid_to_nid(int cpupid) 1785 { 1786 return -1; 1787 } 1788 1789 static inline int cpupid_to_pid(int cpupid) 1790 { 1791 return -1; 1792 } 1793 1794 static inline int cpupid_to_cpu(int cpupid) 1795 { 1796 return -1; 1797 } 1798 1799 static inline int cpu_pid_to_cpupid(int nid, int pid) 1800 { 1801 return -1; 1802 } 1803 1804 static inline bool cpupid_pid_unset(int cpupid) 1805 { 1806 return true; 1807 } 1808 1809 static inline void page_cpupid_reset_last(struct page *page) 1810 { 1811 } 1812 1813 static inline bool cpupid_match_pid(struct task_struct *task, int cpupid) 1814 { 1815 return false; 1816 } 1817 1818 static inline void vma_set_access_pid_bit(struct vm_area_struct *vma) 1819 { 1820 } 1821 #endif /* CONFIG_NUMA_BALANCING */ 1822 1823 #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) 1824 1825 /* 1826 * KASAN per-page tags are stored xor'ed with 0xff. This allows to avoid 1827 * setting tags for all pages to native kernel tag value 0xff, as the default 1828 * value 0x00 maps to 0xff. 1829 */ 1830 1831 static inline u8 page_kasan_tag(const struct page *page) 1832 { 1833 u8 tag = KASAN_TAG_KERNEL; 1834 1835 if (kasan_enabled()) { 1836 tag = (page->flags >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK; 1837 tag ^= 0xff; 1838 } 1839 1840 return tag; 1841 } 1842 1843 static inline void page_kasan_tag_set(struct page *page, u8 tag) 1844 { 1845 unsigned long old_flags, flags; 1846 1847 if (!kasan_enabled()) 1848 return; 1849 1850 tag ^= 0xff; 1851 old_flags = READ_ONCE(page->flags); 1852 do { 1853 flags = old_flags; 1854 flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT); 1855 flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT; 1856 } while (unlikely(!try_cmpxchg(&page->flags, &old_flags, flags))); 1857 } 1858 1859 static inline void page_kasan_tag_reset(struct page *page) 1860 { 1861 if (kasan_enabled()) 1862 page_kasan_tag_set(page, KASAN_TAG_KERNEL); 1863 } 1864 1865 #else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */ 1866 1867 static inline u8 page_kasan_tag(const struct page *page) 1868 { 1869 return 0xff; 1870 } 1871 1872 static inline void page_kasan_tag_set(struct page *page, u8 tag) { } 1873 static inline void page_kasan_tag_reset(struct page *page) { } 1874 1875 #endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */ 1876 1877 static inline struct zone *page_zone(const struct page *page) 1878 { 1879 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)]; 1880 } 1881 1882 static inline pg_data_t *page_pgdat(const struct page *page) 1883 { 1884 return NODE_DATA(page_to_nid(page)); 1885 } 1886 1887 static inline struct zone *folio_zone(const struct folio *folio) 1888 { 1889 return page_zone(&folio->page); 1890 } 1891 1892 static inline pg_data_t *folio_pgdat(const struct folio *folio) 1893 { 1894 return page_pgdat(&folio->page); 1895 } 1896 1897 #ifdef SECTION_IN_PAGE_FLAGS 1898 static inline void set_page_section(struct page *page, unsigned long section) 1899 { 1900 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT); 1901 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; 1902 } 1903 1904 static inline unsigned long page_to_section(const struct page *page) 1905 { 1906 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK; 1907 } 1908 #endif 1909 1910 /** 1911 * folio_pfn - Return the Page Frame Number of a folio. 1912 * @folio: The folio. 1913 * 1914 * A folio may contain multiple pages. The pages have consecutive 1915 * Page Frame Numbers. 1916 * 1917 * Return: The Page Frame Number of the first page in the folio. 1918 */ 1919 static inline unsigned long folio_pfn(struct folio *folio) 1920 { 1921 return page_to_pfn(&folio->page); 1922 } 1923 1924 static inline struct folio *pfn_folio(unsigned long pfn) 1925 { 1926 return page_folio(pfn_to_page(pfn)); 1927 } 1928 1929 /** 1930 * folio_maybe_dma_pinned - Report if a folio may be pinned for DMA. 1931 * @folio: The folio. 1932 * 1933 * This function checks if a folio has been pinned via a call to 1934 * a function in the pin_user_pages() family. 1935 * 1936 * For small folios, the return value is partially fuzzy: false is not fuzzy, 1937 * because it means "definitely not pinned for DMA", but true means "probably 1938 * pinned for DMA, but possibly a false positive due to having at least 1939 * GUP_PIN_COUNTING_BIAS worth of normal folio references". 1940 * 1941 * False positives are OK, because: a) it's unlikely for a folio to 1942 * get that many refcounts, and b) all the callers of this routine are 1943 * expected to be able to deal gracefully with a false positive. 1944 * 1945 * For large folios, the result will be exactly correct. That's because 1946 * we have more tracking data available: the _pincount field is used 1947 * instead of the GUP_PIN_COUNTING_BIAS scheme. 1948 * 1949 * For more information, please see Documentation/core-api/pin_user_pages.rst. 1950 * 1951 * Return: True, if it is likely that the page has been "dma-pinned". 1952 * False, if the page is definitely not dma-pinned. 1953 */ 1954 static inline bool folio_maybe_dma_pinned(struct folio *folio) 1955 { 1956 if (folio_test_large(folio)) 1957 return atomic_read(&folio->_pincount) > 0; 1958 1959 /* 1960 * folio_ref_count() is signed. If that refcount overflows, then 1961 * folio_ref_count() returns a negative value, and callers will avoid 1962 * further incrementing the refcount. 1963 * 1964 * Here, for that overflow case, use the sign bit to count a little 1965 * bit higher via unsigned math, and thus still get an accurate result. 1966 */ 1967 return ((unsigned int)folio_ref_count(folio)) >= 1968 GUP_PIN_COUNTING_BIAS; 1969 } 1970 1971 static inline bool page_maybe_dma_pinned(struct page *page) 1972 { 1973 return folio_maybe_dma_pinned(page_folio(page)); 1974 } 1975 1976 /* 1977 * This should most likely only be called during fork() to see whether we 1978 * should break the cow immediately for an anon page on the src mm. 1979 * 1980 * The caller has to hold the PT lock and the vma->vm_mm->->write_protect_seq. 1981 */ 1982 static inline bool folio_needs_cow_for_dma(struct vm_area_struct *vma, 1983 struct folio *folio) 1984 { 1985 VM_BUG_ON(!(raw_read_seqcount(&vma->vm_mm->write_protect_seq) & 1)); 1986 1987 if (!test_bit(MMF_HAS_PINNED, &vma->vm_mm->flags)) 1988 return false; 1989 1990 return folio_maybe_dma_pinned(folio); 1991 } 1992 1993 /** 1994 * is_zero_page - Query if a page is a zero page 1995 * @page: The page to query 1996 * 1997 * This returns true if @page is one of the permanent zero pages. 1998 */ 1999 static inline bool is_zero_page(const struct page *page) 2000 { 2001 return is_zero_pfn(page_to_pfn(page)); 2002 } 2003 2004 /** 2005 * is_zero_folio - Query if a folio is a zero page 2006 * @folio: The folio to query 2007 * 2008 * This returns true if @folio is one of the permanent zero pages. 2009 */ 2010 static inline bool is_zero_folio(const struct folio *folio) 2011 { 2012 return is_zero_page(&folio->page); 2013 } 2014 2015 /* MIGRATE_CMA and ZONE_MOVABLE do not allow pin folios */ 2016 #ifdef CONFIG_MIGRATION 2017 static inline bool folio_is_longterm_pinnable(struct folio *folio) 2018 { 2019 #ifdef CONFIG_CMA 2020 int mt = folio_migratetype(folio); 2021 2022 if (mt == MIGRATE_CMA || mt == MIGRATE_ISOLATE) 2023 return false; 2024 #endif 2025 /* The zero page can be "pinned" but gets special handling. */ 2026 if (is_zero_folio(folio)) 2027 return true; 2028 2029 /* Coherent device memory must always allow eviction. */ 2030 if (folio_is_device_coherent(folio)) 2031 return false; 2032 2033 /* Otherwise, non-movable zone folios can be pinned. */ 2034 return !folio_is_zone_movable(folio); 2035 2036 } 2037 #else 2038 static inline bool folio_is_longterm_pinnable(struct folio *folio) 2039 { 2040 return true; 2041 } 2042 #endif 2043 2044 static inline void set_page_zone(struct page *page, enum zone_type zone) 2045 { 2046 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT); 2047 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT; 2048 } 2049 2050 static inline void set_page_node(struct page *page, unsigned long node) 2051 { 2052 page->flags &= ~(NODES_MASK << NODES_PGSHIFT); 2053 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT; 2054 } 2055 2056 static inline void set_page_links(struct page *page, enum zone_type zone, 2057 unsigned long node, unsigned long pfn) 2058 { 2059 set_page_zone(page, zone); 2060 set_page_node(page, node); 2061 #ifdef SECTION_IN_PAGE_FLAGS 2062 set_page_section(page, pfn_to_section_nr(pfn)); 2063 #endif 2064 } 2065 2066 /** 2067 * folio_nr_pages - The number of pages in the folio. 2068 * @folio: The folio. 2069 * 2070 * Return: A positive power of two. 2071 */ 2072 static inline long folio_nr_pages(const struct folio *folio) 2073 { 2074 if (!folio_test_large(folio)) 2075 return 1; 2076 #ifdef CONFIG_64BIT 2077 return folio->_folio_nr_pages; 2078 #else 2079 return 1L << (folio->_flags_1 & 0xff); 2080 #endif 2081 } 2082 2083 /* Only hugetlbfs can allocate folios larger than MAX_ORDER */ 2084 #ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE 2085 #define MAX_FOLIO_NR_PAGES (1UL << PUD_ORDER) 2086 #else 2087 #define MAX_FOLIO_NR_PAGES MAX_ORDER_NR_PAGES 2088 #endif 2089 2090 /* 2091 * compound_nr() returns the number of pages in this potentially compound 2092 * page. compound_nr() can be called on a tail page, and is defined to 2093 * return 1 in that case. 2094 */ 2095 static inline unsigned long compound_nr(struct page *page) 2096 { 2097 struct folio *folio = (struct folio *)page; 2098 2099 if (!test_bit(PG_head, &folio->flags)) 2100 return 1; 2101 #ifdef CONFIG_64BIT 2102 return folio->_folio_nr_pages; 2103 #else 2104 return 1L << (folio->_flags_1 & 0xff); 2105 #endif 2106 } 2107 2108 /** 2109 * thp_nr_pages - The number of regular pages in this huge page. 2110 * @page: The head page of a huge page. 2111 */ 2112 static inline int thp_nr_pages(struct page *page) 2113 { 2114 return folio_nr_pages((struct folio *)page); 2115 } 2116 2117 /** 2118 * folio_next - Move to the next physical folio. 2119 * @folio: The folio we're currently operating on. 2120 * 2121 * If you have physically contiguous memory which may span more than 2122 * one folio (eg a &struct bio_vec), use this function to move from one 2123 * folio to the next. Do not use it if the memory is only virtually 2124 * contiguous as the folios are almost certainly not adjacent to each 2125 * other. This is the folio equivalent to writing ``page++``. 2126 * 2127 * Context: We assume that the folios are refcounted and/or locked at a 2128 * higher level and do not adjust the reference counts. 2129 * Return: The next struct folio. 2130 */ 2131 static inline struct folio *folio_next(struct folio *folio) 2132 { 2133 return (struct folio *)folio_page(folio, folio_nr_pages(folio)); 2134 } 2135 2136 /** 2137 * folio_shift - The size of the memory described by this folio. 2138 * @folio: The folio. 2139 * 2140 * A folio represents a number of bytes which is a power-of-two in size. 2141 * This function tells you which power-of-two the folio is. See also 2142 * folio_size() and folio_order(). 2143 * 2144 * Context: The caller should have a reference on the folio to prevent 2145 * it from being split. It is not necessary for the folio to be locked. 2146 * Return: The base-2 logarithm of the size of this folio. 2147 */ 2148 static inline unsigned int folio_shift(struct folio *folio) 2149 { 2150 return PAGE_SHIFT + folio_order(folio); 2151 } 2152 2153 /** 2154 * folio_size - The number of bytes in a folio. 2155 * @folio: The folio. 2156 * 2157 * Context: The caller should have a reference on the folio to prevent 2158 * it from being split. It is not necessary for the folio to be locked. 2159 * Return: The number of bytes in this folio. 2160 */ 2161 static inline size_t folio_size(struct folio *folio) 2162 { 2163 return PAGE_SIZE << folio_order(folio); 2164 } 2165 2166 /** 2167 * folio_likely_mapped_shared - Estimate if the folio is mapped into the page 2168 * tables of more than one MM 2169 * @folio: The folio. 2170 * 2171 * This function checks if the folio is currently mapped into more than one 2172 * MM ("mapped shared"), or if the folio is only mapped into a single MM 2173 * ("mapped exclusively"). 2174 * 2175 * As precise information is not easily available for all folios, this function 2176 * estimates the number of MMs ("sharers") that are currently mapping a folio 2177 * using the number of times the first page of the folio is currently mapped 2178 * into page tables. 2179 * 2180 * For small anonymous folios (except KSM folios) and anonymous hugetlb folios, 2181 * the return value will be exactly correct, because they can only be mapped 2182 * at most once into an MM, and they cannot be partially mapped. 2183 * 2184 * For other folios, the result can be fuzzy: 2185 * #. For partially-mappable large folios (THP), the return value can wrongly 2186 * indicate "mapped exclusively" (false negative) when the folio is 2187 * only partially mapped into at least one MM. 2188 * #. For pagecache folios (including hugetlb), the return value can wrongly 2189 * indicate "mapped shared" (false positive) when two VMAs in the same MM 2190 * cover the same file range. 2191 * #. For (small) KSM folios, the return value can wrongly indicate "mapped 2192 * shared" (false positive), when the folio is mapped multiple times into 2193 * the same MM. 2194 * 2195 * Further, this function only considers current page table mappings that 2196 * are tracked using the folio mapcount(s). 2197 * 2198 * This function does not consider: 2199 * #. If the folio might get mapped in the (near) future (e.g., swapcache, 2200 * pagecache, temporary unmapping for migration). 2201 * #. If the folio is mapped differently (VM_PFNMAP). 2202 * #. If hugetlb page table sharing applies. Callers might want to check 2203 * hugetlb_pmd_shared(). 2204 * 2205 * Return: Whether the folio is estimated to be mapped into more than one MM. 2206 */ 2207 static inline bool folio_likely_mapped_shared(struct folio *folio) 2208 { 2209 int mapcount = folio_mapcount(folio); 2210 2211 /* Only partially-mappable folios require more care. */ 2212 if (!folio_test_large(folio) || unlikely(folio_test_hugetlb(folio))) 2213 return mapcount > 1; 2214 2215 /* A single mapping implies "mapped exclusively". */ 2216 if (mapcount <= 1) 2217 return false; 2218 2219 /* If any page is mapped more than once we treat it "mapped shared". */ 2220 if (folio_entire_mapcount(folio) || mapcount > folio_nr_pages(folio)) 2221 return true; 2222 2223 /* Let's guess based on the first subpage. */ 2224 return atomic_read(&folio->_mapcount) > 0; 2225 } 2226 2227 #ifndef HAVE_ARCH_MAKE_PAGE_ACCESSIBLE 2228 static inline int arch_make_page_accessible(struct page *page) 2229 { 2230 return 0; 2231 } 2232 #endif 2233 2234 #ifndef HAVE_ARCH_MAKE_FOLIO_ACCESSIBLE 2235 static inline int arch_make_folio_accessible(struct folio *folio) 2236 { 2237 int ret; 2238 long i, nr = folio_nr_pages(folio); 2239 2240 for (i = 0; i < nr; i++) { 2241 ret = arch_make_page_accessible(folio_page(folio, i)); 2242 if (ret) 2243 break; 2244 } 2245 2246 return ret; 2247 } 2248 #endif 2249 2250 /* 2251 * Some inline functions in vmstat.h depend on page_zone() 2252 */ 2253 #include <linux/vmstat.h> 2254 2255 #if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL) 2256 #define HASHED_PAGE_VIRTUAL 2257 #endif 2258 2259 #if defined(WANT_PAGE_VIRTUAL) 2260 static inline void *page_address(const struct page *page) 2261 { 2262 return page->virtual; 2263 } 2264 static inline void set_page_address(struct page *page, void *address) 2265 { 2266 page->virtual = address; 2267 } 2268 #define page_address_init() do { } while(0) 2269 #endif 2270 2271 #if defined(HASHED_PAGE_VIRTUAL) 2272 void *page_address(const struct page *page); 2273 void set_page_address(struct page *page, void *virtual); 2274 void page_address_init(void); 2275 #endif 2276 2277 static __always_inline void *lowmem_page_address(const struct page *page) 2278 { 2279 return page_to_virt(page); 2280 } 2281 2282 #if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL) 2283 #define page_address(page) lowmem_page_address(page) 2284 #define set_page_address(page, address) do { } while(0) 2285 #define page_address_init() do { } while(0) 2286 #endif 2287 2288 static inline void *folio_address(const struct folio *folio) 2289 { 2290 return page_address(&folio->page); 2291 } 2292 2293 extern pgoff_t __page_file_index(struct page *page); 2294 2295 /* 2296 * Return the pagecache index of the passed page. Regular pagecache pages 2297 * use ->index whereas swapcache pages use swp_offset(->private) 2298 */ 2299 static inline pgoff_t page_index(struct page *page) 2300 { 2301 if (unlikely(PageSwapCache(page))) 2302 return __page_file_index(page); 2303 return page->index; 2304 } 2305 2306 /* 2307 * Return true only if the page has been allocated with 2308 * ALLOC_NO_WATERMARKS and the low watermark was not 2309 * met implying that the system is under some pressure. 2310 */ 2311 static inline bool page_is_pfmemalloc(const struct page *page) 2312 { 2313 /* 2314 * lru.next has bit 1 set if the page is allocated from the 2315 * pfmemalloc reserves. Callers may simply overwrite it if 2316 * they do not need to preserve that information. 2317 */ 2318 return (uintptr_t)page->lru.next & BIT(1); 2319 } 2320 2321 /* 2322 * Return true only if the folio has been allocated with 2323 * ALLOC_NO_WATERMARKS and the low watermark was not 2324 * met implying that the system is under some pressure. 2325 */ 2326 static inline bool folio_is_pfmemalloc(const struct folio *folio) 2327 { 2328 /* 2329 * lru.next has bit 1 set if the page is allocated from the 2330 * pfmemalloc reserves. Callers may simply overwrite it if 2331 * they do not need to preserve that information. 2332 */ 2333 return (uintptr_t)folio->lru.next & BIT(1); 2334 } 2335 2336 /* 2337 * Only to be called by the page allocator on a freshly allocated 2338 * page. 2339 */ 2340 static inline void set_page_pfmemalloc(struct page *page) 2341 { 2342 page->lru.next = (void *)BIT(1); 2343 } 2344 2345 static inline void clear_page_pfmemalloc(struct page *page) 2346 { 2347 page->lru.next = NULL; 2348 } 2349 2350 /* 2351 * Can be called by the pagefault handler when it gets a VM_FAULT_OOM. 2352 */ 2353 extern void pagefault_out_of_memory(void); 2354 2355 #define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK) 2356 #define offset_in_thp(page, p) ((unsigned long)(p) & (thp_size(page) - 1)) 2357 #define offset_in_folio(folio, p) ((unsigned long)(p) & (folio_size(folio) - 1)) 2358 2359 /* 2360 * Parameter block passed down to zap_pte_range in exceptional cases. 2361 */ 2362 struct zap_details { 2363 struct folio *single_folio; /* Locked folio to be unmapped */ 2364 bool even_cows; /* Zap COWed private pages too? */ 2365 zap_flags_t zap_flags; /* Extra flags for zapping */ 2366 }; 2367 2368 /* 2369 * Whether to drop the pte markers, for example, the uffd-wp information for 2370 * file-backed memory. This should only be specified when we will completely 2371 * drop the page in the mm, either by truncation or unmapping of the vma. By 2372 * default, the flag is not set. 2373 */ 2374 #define ZAP_FLAG_DROP_MARKER ((__force zap_flags_t) BIT(0)) 2375 /* Set in unmap_vmas() to indicate a final unmap call. Only used by hugetlb */ 2376 #define ZAP_FLAG_UNMAP ((__force zap_flags_t) BIT(1)) 2377 2378 #ifdef CONFIG_SCHED_MM_CID 2379 void sched_mm_cid_before_execve(struct task_struct *t); 2380 void sched_mm_cid_after_execve(struct task_struct *t); 2381 void sched_mm_cid_fork(struct task_struct *t); 2382 void sched_mm_cid_exit_signals(struct task_struct *t); 2383 static inline int task_mm_cid(struct task_struct *t) 2384 { 2385 return t->mm_cid; 2386 } 2387 #else 2388 static inline void sched_mm_cid_before_execve(struct task_struct *t) { } 2389 static inline void sched_mm_cid_after_execve(struct task_struct *t) { } 2390 static inline void sched_mm_cid_fork(struct task_struct *t) { } 2391 static inline void sched_mm_cid_exit_signals(struct task_struct *t) { } 2392 static inline int task_mm_cid(struct task_struct *t) 2393 { 2394 /* 2395 * Use the processor id as a fall-back when the mm cid feature is 2396 * disabled. This provides functional per-cpu data structure accesses 2397 * in user-space, althrough it won't provide the memory usage benefits. 2398 */ 2399 return raw_smp_processor_id(); 2400 } 2401 #endif 2402 2403 #ifdef CONFIG_MMU 2404 extern bool can_do_mlock(void); 2405 #else 2406 static inline bool can_do_mlock(void) { return false; } 2407 #endif 2408 extern int user_shm_lock(size_t, struct ucounts *); 2409 extern void user_shm_unlock(size_t, struct ucounts *); 2410 2411 struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr, 2412 pte_t pte); 2413 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, 2414 pte_t pte); 2415 struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma, 2416 unsigned long addr, pmd_t pmd); 2417 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr, 2418 pmd_t pmd); 2419 2420 void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, 2421 unsigned long size); 2422 void zap_page_range_single(struct vm_area_struct *vma, unsigned long address, 2423 unsigned long size, struct zap_details *details); 2424 static inline void zap_vma_pages(struct vm_area_struct *vma) 2425 { 2426 zap_page_range_single(vma, vma->vm_start, 2427 vma->vm_end - vma->vm_start, NULL); 2428 } 2429 void unmap_vmas(struct mmu_gather *tlb, struct ma_state *mas, 2430 struct vm_area_struct *start_vma, unsigned long start, 2431 unsigned long end, unsigned long tree_end, bool mm_wr_locked); 2432 2433 struct mmu_notifier_range; 2434 2435 void free_pgd_range(struct mmu_gather *tlb, unsigned long addr, 2436 unsigned long end, unsigned long floor, unsigned long ceiling); 2437 int 2438 copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma); 2439 int follow_pte(struct vm_area_struct *vma, unsigned long address, 2440 pte_t **ptepp, spinlock_t **ptlp); 2441 int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, 2442 void *buf, int len, int write); 2443 2444 extern void truncate_pagecache(struct inode *inode, loff_t new); 2445 extern void truncate_setsize(struct inode *inode, loff_t newsize); 2446 void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to); 2447 void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end); 2448 int generic_error_remove_folio(struct address_space *mapping, 2449 struct folio *folio); 2450 2451 struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm, 2452 unsigned long address, struct pt_regs *regs); 2453 2454 #ifdef CONFIG_MMU 2455 extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma, 2456 unsigned long address, unsigned int flags, 2457 struct pt_regs *regs); 2458 extern int fixup_user_fault(struct mm_struct *mm, 2459 unsigned long address, unsigned int fault_flags, 2460 bool *unlocked); 2461 void unmap_mapping_pages(struct address_space *mapping, 2462 pgoff_t start, pgoff_t nr, bool even_cows); 2463 void unmap_mapping_range(struct address_space *mapping, 2464 loff_t const holebegin, loff_t const holelen, int even_cows); 2465 #else 2466 static inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma, 2467 unsigned long address, unsigned int flags, 2468 struct pt_regs *regs) 2469 { 2470 /* should never happen if there's no MMU */ 2471 BUG(); 2472 return VM_FAULT_SIGBUS; 2473 } 2474 static inline int fixup_user_fault(struct mm_struct *mm, unsigned long address, 2475 unsigned int fault_flags, bool *unlocked) 2476 { 2477 /* should never happen if there's no MMU */ 2478 BUG(); 2479 return -EFAULT; 2480 } 2481 static inline void unmap_mapping_pages(struct address_space *mapping, 2482 pgoff_t start, pgoff_t nr, bool even_cows) { } 2483 static inline void unmap_mapping_range(struct address_space *mapping, 2484 loff_t const holebegin, loff_t const holelen, int even_cows) { } 2485 #endif 2486 2487 static inline void unmap_shared_mapping_range(struct address_space *mapping, 2488 loff_t const holebegin, loff_t const holelen) 2489 { 2490 unmap_mapping_range(mapping, holebegin, holelen, 0); 2491 } 2492 2493 static inline struct vm_area_struct *vma_lookup(struct mm_struct *mm, 2494 unsigned long addr); 2495 2496 extern int access_process_vm(struct task_struct *tsk, unsigned long addr, 2497 void *buf, int len, unsigned int gup_flags); 2498 extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, 2499 void *buf, int len, unsigned int gup_flags); 2500 2501 long get_user_pages_remote(struct mm_struct *mm, 2502 unsigned long start, unsigned long nr_pages, 2503 unsigned int gup_flags, struct page **pages, 2504 int *locked); 2505 long pin_user_pages_remote(struct mm_struct *mm, 2506 unsigned long start, unsigned long nr_pages, 2507 unsigned int gup_flags, struct page **pages, 2508 int *locked); 2509 2510 /* 2511 * Retrieves a single page alongside its VMA. Does not support FOLL_NOWAIT. 2512 */ 2513 static inline struct page *get_user_page_vma_remote(struct mm_struct *mm, 2514 unsigned long addr, 2515 int gup_flags, 2516 struct vm_area_struct **vmap) 2517 { 2518 struct page *page; 2519 struct vm_area_struct *vma; 2520 int got; 2521 2522 if (WARN_ON_ONCE(unlikely(gup_flags & FOLL_NOWAIT))) 2523 return ERR_PTR(-EINVAL); 2524 2525 got = get_user_pages_remote(mm, addr, 1, gup_flags, &page, NULL); 2526 2527 if (got < 0) 2528 return ERR_PTR(got); 2529 2530 vma = vma_lookup(mm, addr); 2531 if (WARN_ON_ONCE(!vma)) { 2532 put_page(page); 2533 return ERR_PTR(-EINVAL); 2534 } 2535 2536 *vmap = vma; 2537 return page; 2538 } 2539 2540 long get_user_pages(unsigned long start, unsigned long nr_pages, 2541 unsigned int gup_flags, struct page **pages); 2542 long pin_user_pages(unsigned long start, unsigned long nr_pages, 2543 unsigned int gup_flags, struct page **pages); 2544 long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, 2545 struct page **pages, unsigned int gup_flags); 2546 long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages, 2547 struct page **pages, unsigned int gup_flags); 2548 2549 int get_user_pages_fast(unsigned long start, int nr_pages, 2550 unsigned int gup_flags, struct page **pages); 2551 int pin_user_pages_fast(unsigned long start, int nr_pages, 2552 unsigned int gup_flags, struct page **pages); 2553 void folio_add_pin(struct folio *folio); 2554 2555 int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc); 2556 int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, 2557 struct task_struct *task, bool bypass_rlim); 2558 2559 struct kvec; 2560 struct page *get_dump_page(unsigned long addr); 2561 2562 bool folio_mark_dirty(struct folio *folio); 2563 bool set_page_dirty(struct page *page); 2564 int set_page_dirty_lock(struct page *page); 2565 2566 int get_cmdline(struct task_struct *task, char *buffer, int buflen); 2567 2568 extern unsigned long move_page_tables(struct vm_area_struct *vma, 2569 unsigned long old_addr, struct vm_area_struct *new_vma, 2570 unsigned long new_addr, unsigned long len, 2571 bool need_rmap_locks, bool for_stack); 2572 2573 /* 2574 * Flags used by change_protection(). For now we make it a bitmap so 2575 * that we can pass in multiple flags just like parameters. However 2576 * for now all the callers are only use one of the flags at the same 2577 * time. 2578 */ 2579 /* 2580 * Whether we should manually check if we can map individual PTEs writable, 2581 * because something (e.g., COW, uffd-wp) blocks that from happening for all 2582 * PTEs automatically in a writable mapping. 2583 */ 2584 #define MM_CP_TRY_CHANGE_WRITABLE (1UL << 0) 2585 /* Whether this protection change is for NUMA hints */ 2586 #define MM_CP_PROT_NUMA (1UL << 1) 2587 /* Whether this change is for write protecting */ 2588 #define MM_CP_UFFD_WP (1UL << 2) /* do wp */ 2589 #define MM_CP_UFFD_WP_RESOLVE (1UL << 3) /* Resolve wp */ 2590 #define MM_CP_UFFD_WP_ALL (MM_CP_UFFD_WP | \ 2591 MM_CP_UFFD_WP_RESOLVE) 2592 2593 bool vma_needs_dirty_tracking(struct vm_area_struct *vma); 2594 bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot); 2595 static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma) 2596 { 2597 /* 2598 * We want to check manually if we can change individual PTEs writable 2599 * if we can't do that automatically for all PTEs in a mapping. For 2600 * private mappings, that's always the case when we have write 2601 * permissions as we properly have to handle COW. 2602 */ 2603 if (vma->vm_flags & VM_SHARED) 2604 return vma_wants_writenotify(vma, vma->vm_page_prot); 2605 return !!(vma->vm_flags & VM_WRITE); 2606 2607 } 2608 bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, 2609 pte_t pte); 2610 extern long change_protection(struct mmu_gather *tlb, 2611 struct vm_area_struct *vma, unsigned long start, 2612 unsigned long end, unsigned long cp_flags); 2613 extern int mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, 2614 struct vm_area_struct *vma, struct vm_area_struct **pprev, 2615 unsigned long start, unsigned long end, unsigned long newflags); 2616 2617 /* 2618 * doesn't attempt to fault and will return short. 2619 */ 2620 int get_user_pages_fast_only(unsigned long start, int nr_pages, 2621 unsigned int gup_flags, struct page **pages); 2622 2623 static inline bool get_user_page_fast_only(unsigned long addr, 2624 unsigned int gup_flags, struct page **pagep) 2625 { 2626 return get_user_pages_fast_only(addr, 1, gup_flags, pagep) == 1; 2627 } 2628 /* 2629 * per-process(per-mm_struct) statistics. 2630 */ 2631 static inline unsigned long get_mm_counter(struct mm_struct *mm, int member) 2632 { 2633 return percpu_counter_read_positive(&mm->rss_stat[member]); 2634 } 2635 2636 void mm_trace_rss_stat(struct mm_struct *mm, int member); 2637 2638 static inline void add_mm_counter(struct mm_struct *mm, int member, long value) 2639 { 2640 percpu_counter_add(&mm->rss_stat[member], value); 2641 2642 mm_trace_rss_stat(mm, member); 2643 } 2644 2645 static inline void inc_mm_counter(struct mm_struct *mm, int member) 2646 { 2647 percpu_counter_inc(&mm->rss_stat[member]); 2648 2649 mm_trace_rss_stat(mm, member); 2650 } 2651 2652 static inline void dec_mm_counter(struct mm_struct *mm, int member) 2653 { 2654 percpu_counter_dec(&mm->rss_stat[member]); 2655 2656 mm_trace_rss_stat(mm, member); 2657 } 2658 2659 /* Optimized variant when folio is already known not to be anon */ 2660 static inline int mm_counter_file(struct folio *folio) 2661 { 2662 if (folio_test_swapbacked(folio)) 2663 return MM_SHMEMPAGES; 2664 return MM_FILEPAGES; 2665 } 2666 2667 static inline int mm_counter(struct folio *folio) 2668 { 2669 if (folio_test_anon(folio)) 2670 return MM_ANONPAGES; 2671 return mm_counter_file(folio); 2672 } 2673 2674 static inline unsigned long get_mm_rss(struct mm_struct *mm) 2675 { 2676 return get_mm_counter(mm, MM_FILEPAGES) + 2677 get_mm_counter(mm, MM_ANONPAGES) + 2678 get_mm_counter(mm, MM_SHMEMPAGES); 2679 } 2680 2681 static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm) 2682 { 2683 return max(mm->hiwater_rss, get_mm_rss(mm)); 2684 } 2685 2686 static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm) 2687 { 2688 return max(mm->hiwater_vm, mm->total_vm); 2689 } 2690 2691 static inline void update_hiwater_rss(struct mm_struct *mm) 2692 { 2693 unsigned long _rss = get_mm_rss(mm); 2694 2695 if ((mm)->hiwater_rss < _rss) 2696 (mm)->hiwater_rss = _rss; 2697 } 2698 2699 static inline void update_hiwater_vm(struct mm_struct *mm) 2700 { 2701 if (mm->hiwater_vm < mm->total_vm) 2702 mm->hiwater_vm = mm->total_vm; 2703 } 2704 2705 static inline void reset_mm_hiwater_rss(struct mm_struct *mm) 2706 { 2707 mm->hiwater_rss = get_mm_rss(mm); 2708 } 2709 2710 static inline void setmax_mm_hiwater_rss(unsigned long *maxrss, 2711 struct mm_struct *mm) 2712 { 2713 unsigned long hiwater_rss = get_mm_hiwater_rss(mm); 2714 2715 if (*maxrss < hiwater_rss) 2716 *maxrss = hiwater_rss; 2717 } 2718 2719 #ifndef CONFIG_ARCH_HAS_PTE_SPECIAL 2720 static inline int pte_special(pte_t pte) 2721 { 2722 return 0; 2723 } 2724 2725 static inline pte_t pte_mkspecial(pte_t pte) 2726 { 2727 return pte; 2728 } 2729 #endif 2730 2731 #ifndef CONFIG_ARCH_HAS_PTE_DEVMAP 2732 static inline int pte_devmap(pte_t pte) 2733 { 2734 return 0; 2735 } 2736 #endif 2737 2738 extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr, 2739 spinlock_t **ptl); 2740 static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, 2741 spinlock_t **ptl) 2742 { 2743 pte_t *ptep; 2744 __cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl)); 2745 return ptep; 2746 } 2747 2748 #ifdef __PAGETABLE_P4D_FOLDED 2749 static inline int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, 2750 unsigned long address) 2751 { 2752 return 0; 2753 } 2754 #else 2755 int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address); 2756 #endif 2757 2758 #if defined(__PAGETABLE_PUD_FOLDED) || !defined(CONFIG_MMU) 2759 static inline int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, 2760 unsigned long address) 2761 { 2762 return 0; 2763 } 2764 static inline void mm_inc_nr_puds(struct mm_struct *mm) {} 2765 static inline void mm_dec_nr_puds(struct mm_struct *mm) {} 2766 2767 #else 2768 int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address); 2769 2770 static inline void mm_inc_nr_puds(struct mm_struct *mm) 2771 { 2772 if (mm_pud_folded(mm)) 2773 return; 2774 atomic_long_add(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes); 2775 } 2776 2777 static inline void mm_dec_nr_puds(struct mm_struct *mm) 2778 { 2779 if (mm_pud_folded(mm)) 2780 return; 2781 atomic_long_sub(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes); 2782 } 2783 #endif 2784 2785 #if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU) 2786 static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud, 2787 unsigned long address) 2788 { 2789 return 0; 2790 } 2791 2792 static inline void mm_inc_nr_pmds(struct mm_struct *mm) {} 2793 static inline void mm_dec_nr_pmds(struct mm_struct *mm) {} 2794 2795 #else 2796 int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address); 2797 2798 static inline void mm_inc_nr_pmds(struct mm_struct *mm) 2799 { 2800 if (mm_pmd_folded(mm)) 2801 return; 2802 atomic_long_add(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes); 2803 } 2804 2805 static inline void mm_dec_nr_pmds(struct mm_struct *mm) 2806 { 2807 if (mm_pmd_folded(mm)) 2808 return; 2809 atomic_long_sub(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes); 2810 } 2811 #endif 2812 2813 #ifdef CONFIG_MMU 2814 static inline void mm_pgtables_bytes_init(struct mm_struct *mm) 2815 { 2816 atomic_long_set(&mm->pgtables_bytes, 0); 2817 } 2818 2819 static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm) 2820 { 2821 return atomic_long_read(&mm->pgtables_bytes); 2822 } 2823 2824 static inline void mm_inc_nr_ptes(struct mm_struct *mm) 2825 { 2826 atomic_long_add(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes); 2827 } 2828 2829 static inline void mm_dec_nr_ptes(struct mm_struct *mm) 2830 { 2831 atomic_long_sub(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes); 2832 } 2833 #else 2834 2835 static inline void mm_pgtables_bytes_init(struct mm_struct *mm) {} 2836 static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm) 2837 { 2838 return 0; 2839 } 2840 2841 static inline void mm_inc_nr_ptes(struct mm_struct *mm) {} 2842 static inline void mm_dec_nr_ptes(struct mm_struct *mm) {} 2843 #endif 2844 2845 int __pte_alloc(struct mm_struct *mm, pmd_t *pmd); 2846 int __pte_alloc_kernel(pmd_t *pmd); 2847 2848 #if defined(CONFIG_MMU) 2849 2850 static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd, 2851 unsigned long address) 2852 { 2853 return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ? 2854 NULL : p4d_offset(pgd, address); 2855 } 2856 2857 static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d, 2858 unsigned long address) 2859 { 2860 return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ? 2861 NULL : pud_offset(p4d, address); 2862 } 2863 2864 static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) 2865 { 2866 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))? 2867 NULL: pmd_offset(pud, address); 2868 } 2869 #endif /* CONFIG_MMU */ 2870 2871 static inline struct ptdesc *virt_to_ptdesc(const void *x) 2872 { 2873 return page_ptdesc(virt_to_page(x)); 2874 } 2875 2876 static inline void *ptdesc_to_virt(const struct ptdesc *pt) 2877 { 2878 return page_to_virt(ptdesc_page(pt)); 2879 } 2880 2881 static inline void *ptdesc_address(const struct ptdesc *pt) 2882 { 2883 return folio_address(ptdesc_folio(pt)); 2884 } 2885 2886 static inline bool pagetable_is_reserved(struct ptdesc *pt) 2887 { 2888 return folio_test_reserved(ptdesc_folio(pt)); 2889 } 2890 2891 /** 2892 * pagetable_alloc - Allocate pagetables 2893 * @gfp: GFP flags 2894 * @order: desired pagetable order 2895 * 2896 * pagetable_alloc allocates memory for page tables as well as a page table 2897 * descriptor to describe that memory. 2898 * 2899 * Return: The ptdesc describing the allocated page tables. 2900 */ 2901 static inline struct ptdesc *pagetable_alloc_noprof(gfp_t gfp, unsigned int order) 2902 { 2903 struct page *page = alloc_pages_noprof(gfp | __GFP_COMP, order); 2904 2905 return page_ptdesc(page); 2906 } 2907 #define pagetable_alloc(...) alloc_hooks(pagetable_alloc_noprof(__VA_ARGS__)) 2908 2909 /** 2910 * pagetable_free - Free pagetables 2911 * @pt: The page table descriptor 2912 * 2913 * pagetable_free frees the memory of all page tables described by a page 2914 * table descriptor and the memory for the descriptor itself. 2915 */ 2916 static inline void pagetable_free(struct ptdesc *pt) 2917 { 2918 struct page *page = ptdesc_page(pt); 2919 2920 __free_pages(page, compound_order(page)); 2921 } 2922 2923 #if USE_SPLIT_PTE_PTLOCKS 2924 #if ALLOC_SPLIT_PTLOCKS 2925 void __init ptlock_cache_init(void); 2926 bool ptlock_alloc(struct ptdesc *ptdesc); 2927 void ptlock_free(struct ptdesc *ptdesc); 2928 2929 static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc) 2930 { 2931 return ptdesc->ptl; 2932 } 2933 #else /* ALLOC_SPLIT_PTLOCKS */ 2934 static inline void ptlock_cache_init(void) 2935 { 2936 } 2937 2938 static inline bool ptlock_alloc(struct ptdesc *ptdesc) 2939 { 2940 return true; 2941 } 2942 2943 static inline void ptlock_free(struct ptdesc *ptdesc) 2944 { 2945 } 2946 2947 static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc) 2948 { 2949 return &ptdesc->ptl; 2950 } 2951 #endif /* ALLOC_SPLIT_PTLOCKS */ 2952 2953 static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd) 2954 { 2955 return ptlock_ptr(page_ptdesc(pmd_page(*pmd))); 2956 } 2957 2958 static inline bool ptlock_init(struct ptdesc *ptdesc) 2959 { 2960 /* 2961 * prep_new_page() initialize page->private (and therefore page->ptl) 2962 * with 0. Make sure nobody took it in use in between. 2963 * 2964 * It can happen if arch try to use slab for page table allocation: 2965 * slab code uses page->slab_cache, which share storage with page->ptl. 2966 */ 2967 VM_BUG_ON_PAGE(*(unsigned long *)&ptdesc->ptl, ptdesc_page(ptdesc)); 2968 if (!ptlock_alloc(ptdesc)) 2969 return false; 2970 spin_lock_init(ptlock_ptr(ptdesc)); 2971 return true; 2972 } 2973 2974 #else /* !USE_SPLIT_PTE_PTLOCKS */ 2975 /* 2976 * We use mm->page_table_lock to guard all pagetable pages of the mm. 2977 */ 2978 static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd) 2979 { 2980 return &mm->page_table_lock; 2981 } 2982 static inline void ptlock_cache_init(void) {} 2983 static inline bool ptlock_init(struct ptdesc *ptdesc) { return true; } 2984 static inline void ptlock_free(struct ptdesc *ptdesc) {} 2985 #endif /* USE_SPLIT_PTE_PTLOCKS */ 2986 2987 static inline bool pagetable_pte_ctor(struct ptdesc *ptdesc) 2988 { 2989 struct folio *folio = ptdesc_folio(ptdesc); 2990 2991 if (!ptlock_init(ptdesc)) 2992 return false; 2993 __folio_set_pgtable(folio); 2994 lruvec_stat_add_folio(folio, NR_PAGETABLE); 2995 return true; 2996 } 2997 2998 static inline void pagetable_pte_dtor(struct ptdesc *ptdesc) 2999 { 3000 struct folio *folio = ptdesc_folio(ptdesc); 3001 3002 ptlock_free(ptdesc); 3003 __folio_clear_pgtable(folio); 3004 lruvec_stat_sub_folio(folio, NR_PAGETABLE); 3005 } 3006 3007 pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp); 3008 static inline pte_t *pte_offset_map(pmd_t *pmd, unsigned long addr) 3009 { 3010 return __pte_offset_map(pmd, addr, NULL); 3011 } 3012 3013 pte_t *__pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, 3014 unsigned long addr, spinlock_t **ptlp); 3015 static inline pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, 3016 unsigned long addr, spinlock_t **ptlp) 3017 { 3018 pte_t *pte; 3019 3020 __cond_lock(*ptlp, pte = __pte_offset_map_lock(mm, pmd, addr, ptlp)); 3021 return pte; 3022 } 3023 3024 pte_t *pte_offset_map_nolock(struct mm_struct *mm, pmd_t *pmd, 3025 unsigned long addr, spinlock_t **ptlp); 3026 3027 #define pte_unmap_unlock(pte, ptl) do { \ 3028 spin_unlock(ptl); \ 3029 pte_unmap(pte); \ 3030 } while (0) 3031 3032 #define pte_alloc(mm, pmd) (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd)) 3033 3034 #define pte_alloc_map(mm, pmd, address) \ 3035 (pte_alloc(mm, pmd) ? NULL : pte_offset_map(pmd, address)) 3036 3037 #define pte_alloc_map_lock(mm, pmd, address, ptlp) \ 3038 (pte_alloc(mm, pmd) ? \ 3039 NULL : pte_offset_map_lock(mm, pmd, address, ptlp)) 3040 3041 #define pte_alloc_kernel(pmd, address) \ 3042 ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd))? \ 3043 NULL: pte_offset_kernel(pmd, address)) 3044 3045 #if USE_SPLIT_PMD_PTLOCKS 3046 3047 static inline struct page *pmd_pgtable_page(pmd_t *pmd) 3048 { 3049 unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1); 3050 return virt_to_page((void *)((unsigned long) pmd & mask)); 3051 } 3052 3053 static inline struct ptdesc *pmd_ptdesc(pmd_t *pmd) 3054 { 3055 return page_ptdesc(pmd_pgtable_page(pmd)); 3056 } 3057 3058 static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd) 3059 { 3060 return ptlock_ptr(pmd_ptdesc(pmd)); 3061 } 3062 3063 static inline bool pmd_ptlock_init(struct ptdesc *ptdesc) 3064 { 3065 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3066 ptdesc->pmd_huge_pte = NULL; 3067 #endif 3068 return ptlock_init(ptdesc); 3069 } 3070 3071 static inline void pmd_ptlock_free(struct ptdesc *ptdesc) 3072 { 3073 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3074 VM_BUG_ON_PAGE(ptdesc->pmd_huge_pte, ptdesc_page(ptdesc)); 3075 #endif 3076 ptlock_free(ptdesc); 3077 } 3078 3079 #define pmd_huge_pte(mm, pmd) (pmd_ptdesc(pmd)->pmd_huge_pte) 3080 3081 #else 3082 3083 static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd) 3084 { 3085 return &mm->page_table_lock; 3086 } 3087 3088 static inline bool pmd_ptlock_init(struct ptdesc *ptdesc) { return true; } 3089 static inline void pmd_ptlock_free(struct ptdesc *ptdesc) {} 3090 3091 #define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte) 3092 3093 #endif 3094 3095 static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd) 3096 { 3097 spinlock_t *ptl = pmd_lockptr(mm, pmd); 3098 spin_lock(ptl); 3099 return ptl; 3100 } 3101 3102 static inline bool pagetable_pmd_ctor(struct ptdesc *ptdesc) 3103 { 3104 struct folio *folio = ptdesc_folio(ptdesc); 3105 3106 if (!pmd_ptlock_init(ptdesc)) 3107 return false; 3108 __folio_set_pgtable(folio); 3109 lruvec_stat_add_folio(folio, NR_PAGETABLE); 3110 return true; 3111 } 3112 3113 static inline void pagetable_pmd_dtor(struct ptdesc *ptdesc) 3114 { 3115 struct folio *folio = ptdesc_folio(ptdesc); 3116 3117 pmd_ptlock_free(ptdesc); 3118 __folio_clear_pgtable(folio); 3119 lruvec_stat_sub_folio(folio, NR_PAGETABLE); 3120 } 3121 3122 /* 3123 * No scalability reason to split PUD locks yet, but follow the same pattern 3124 * as the PMD locks to make it easier if we decide to. The VM should not be 3125 * considered ready to switch to split PUD locks yet; there may be places 3126 * which need to be converted from page_table_lock. 3127 */ 3128 static inline spinlock_t *pud_lockptr(struct mm_struct *mm, pud_t *pud) 3129 { 3130 return &mm->page_table_lock; 3131 } 3132 3133 static inline spinlock_t *pud_lock(struct mm_struct *mm, pud_t *pud) 3134 { 3135 spinlock_t *ptl = pud_lockptr(mm, pud); 3136 3137 spin_lock(ptl); 3138 return ptl; 3139 } 3140 3141 static inline void pagetable_pud_ctor(struct ptdesc *ptdesc) 3142 { 3143 struct folio *folio = ptdesc_folio(ptdesc); 3144 3145 __folio_set_pgtable(folio); 3146 lruvec_stat_add_folio(folio, NR_PAGETABLE); 3147 } 3148 3149 static inline void pagetable_pud_dtor(struct ptdesc *ptdesc) 3150 { 3151 struct folio *folio = ptdesc_folio(ptdesc); 3152 3153 __folio_clear_pgtable(folio); 3154 lruvec_stat_sub_folio(folio, NR_PAGETABLE); 3155 } 3156 3157 extern void __init pagecache_init(void); 3158 extern void free_initmem(void); 3159 3160 /* 3161 * Free reserved pages within range [PAGE_ALIGN(start), end & PAGE_MASK) 3162 * into the buddy system. The freed pages will be poisoned with pattern 3163 * "poison" if it's within range [0, UCHAR_MAX]. 3164 * Return pages freed into the buddy system. 3165 */ 3166 extern unsigned long free_reserved_area(void *start, void *end, 3167 int poison, const char *s); 3168 3169 extern void adjust_managed_page_count(struct page *page, long count); 3170 3171 extern void reserve_bootmem_region(phys_addr_t start, 3172 phys_addr_t end, int nid); 3173 3174 /* Free the reserved page into the buddy system, so it gets managed. */ 3175 static inline void free_reserved_page(struct page *page) 3176 { 3177 if (mem_alloc_profiling_enabled()) { 3178 union codetag_ref *ref = get_page_tag_ref(page); 3179 3180 if (ref) { 3181 set_codetag_empty(ref); 3182 put_page_tag_ref(ref); 3183 } 3184 } 3185 ClearPageReserved(page); 3186 init_page_count(page); 3187 __free_page(page); 3188 adjust_managed_page_count(page, 1); 3189 } 3190 #define free_highmem_page(page) free_reserved_page(page) 3191 3192 static inline void mark_page_reserved(struct page *page) 3193 { 3194 SetPageReserved(page); 3195 adjust_managed_page_count(page, -1); 3196 } 3197 3198 static inline void free_reserved_ptdesc(struct ptdesc *pt) 3199 { 3200 free_reserved_page(ptdesc_page(pt)); 3201 } 3202 3203 /* 3204 * Default method to free all the __init memory into the buddy system. 3205 * The freed pages will be poisoned with pattern "poison" if it's within 3206 * range [0, UCHAR_MAX]. 3207 * Return pages freed into the buddy system. 3208 */ 3209 static inline unsigned long free_initmem_default(int poison) 3210 { 3211 extern char __init_begin[], __init_end[]; 3212 3213 return free_reserved_area(&__init_begin, &__init_end, 3214 poison, "unused kernel image (initmem)"); 3215 } 3216 3217 static inline unsigned long get_num_physpages(void) 3218 { 3219 int nid; 3220 unsigned long phys_pages = 0; 3221 3222 for_each_online_node(nid) 3223 phys_pages += node_present_pages(nid); 3224 3225 return phys_pages; 3226 } 3227 3228 /* 3229 * Using memblock node mappings, an architecture may initialise its 3230 * zones, allocate the backing mem_map and account for memory holes in an 3231 * architecture independent manner. 3232 * 3233 * An architecture is expected to register range of page frames backed by 3234 * physical memory with memblock_add[_node]() before calling 3235 * free_area_init() passing in the PFN each zone ends at. At a basic 3236 * usage, an architecture is expected to do something like 3237 * 3238 * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn, 3239 * max_highmem_pfn}; 3240 * for_each_valid_physical_page_range() 3241 * memblock_add_node(base, size, nid, MEMBLOCK_NONE) 3242 * free_area_init(max_zone_pfns); 3243 */ 3244 void free_area_init(unsigned long *max_zone_pfn); 3245 unsigned long node_map_pfn_alignment(void); 3246 extern unsigned long absent_pages_in_range(unsigned long start_pfn, 3247 unsigned long end_pfn); 3248 extern void get_pfn_range_for_nid(unsigned int nid, 3249 unsigned long *start_pfn, unsigned long *end_pfn); 3250 3251 #ifndef CONFIG_NUMA 3252 static inline int early_pfn_to_nid(unsigned long pfn) 3253 { 3254 return 0; 3255 } 3256 #else 3257 /* please see mm/page_alloc.c */ 3258 extern int __meminit early_pfn_to_nid(unsigned long pfn); 3259 #endif 3260 3261 extern void mem_init(void); 3262 extern void __init mmap_init(void); 3263 3264 extern void __show_mem(unsigned int flags, nodemask_t *nodemask, int max_zone_idx); 3265 static inline void show_mem(void) 3266 { 3267 __show_mem(0, NULL, MAX_NR_ZONES - 1); 3268 } 3269 extern long si_mem_available(void); 3270 extern void si_meminfo(struct sysinfo * val); 3271 extern void si_meminfo_node(struct sysinfo *val, int nid); 3272 3273 extern __printf(3, 4) 3274 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...); 3275 3276 extern void setup_per_cpu_pageset(void); 3277 3278 /* nommu.c */ 3279 extern atomic_long_t mmap_pages_allocated; 3280 extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t); 3281 3282 /* interval_tree.c */ 3283 void vma_interval_tree_insert(struct vm_area_struct *node, 3284 struct rb_root_cached *root); 3285 void vma_interval_tree_insert_after(struct vm_area_struct *node, 3286 struct vm_area_struct *prev, 3287 struct rb_root_cached *root); 3288 void vma_interval_tree_remove(struct vm_area_struct *node, 3289 struct rb_root_cached *root); 3290 struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root_cached *root, 3291 unsigned long start, unsigned long last); 3292 struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node, 3293 unsigned long start, unsigned long last); 3294 3295 #define vma_interval_tree_foreach(vma, root, start, last) \ 3296 for (vma = vma_interval_tree_iter_first(root, start, last); \ 3297 vma; vma = vma_interval_tree_iter_next(vma, start, last)) 3298 3299 void anon_vma_interval_tree_insert(struct anon_vma_chain *node, 3300 struct rb_root_cached *root); 3301 void anon_vma_interval_tree_remove(struct anon_vma_chain *node, 3302 struct rb_root_cached *root); 3303 struct anon_vma_chain * 3304 anon_vma_interval_tree_iter_first(struct rb_root_cached *root, 3305 unsigned long start, unsigned long last); 3306 struct anon_vma_chain *anon_vma_interval_tree_iter_next( 3307 struct anon_vma_chain *node, unsigned long start, unsigned long last); 3308 #ifdef CONFIG_DEBUG_VM_RB 3309 void anon_vma_interval_tree_verify(struct anon_vma_chain *node); 3310 #endif 3311 3312 #define anon_vma_interval_tree_foreach(avc, root, start, last) \ 3313 for (avc = anon_vma_interval_tree_iter_first(root, start, last); \ 3314 avc; avc = anon_vma_interval_tree_iter_next(avc, start, last)) 3315 3316 /* mmap.c */ 3317 extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin); 3318 extern int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma, 3319 unsigned long start, unsigned long end, pgoff_t pgoff, 3320 struct vm_area_struct *next); 3321 extern int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, 3322 unsigned long start, unsigned long end, pgoff_t pgoff); 3323 extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *); 3324 extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *); 3325 extern void unlink_file_vma(struct vm_area_struct *); 3326 extern struct vm_area_struct *copy_vma(struct vm_area_struct **, 3327 unsigned long addr, unsigned long len, pgoff_t pgoff, 3328 bool *need_rmap_locks); 3329 extern void exit_mmap(struct mm_struct *); 3330 struct vm_area_struct *vma_modify(struct vma_iterator *vmi, 3331 struct vm_area_struct *prev, 3332 struct vm_area_struct *vma, 3333 unsigned long start, unsigned long end, 3334 unsigned long vm_flags, 3335 struct mempolicy *policy, 3336 struct vm_userfaultfd_ctx uffd_ctx, 3337 struct anon_vma_name *anon_name); 3338 3339 /* We are about to modify the VMA's flags. */ 3340 static inline struct vm_area_struct 3341 *vma_modify_flags(struct vma_iterator *vmi, 3342 struct vm_area_struct *prev, 3343 struct vm_area_struct *vma, 3344 unsigned long start, unsigned long end, 3345 unsigned long new_flags) 3346 { 3347 return vma_modify(vmi, prev, vma, start, end, new_flags, 3348 vma_policy(vma), vma->vm_userfaultfd_ctx, 3349 anon_vma_name(vma)); 3350 } 3351 3352 /* We are about to modify the VMA's flags and/or anon_name. */ 3353 static inline struct vm_area_struct 3354 *vma_modify_flags_name(struct vma_iterator *vmi, 3355 struct vm_area_struct *prev, 3356 struct vm_area_struct *vma, 3357 unsigned long start, 3358 unsigned long end, 3359 unsigned long new_flags, 3360 struct anon_vma_name *new_name) 3361 { 3362 return vma_modify(vmi, prev, vma, start, end, new_flags, 3363 vma_policy(vma), vma->vm_userfaultfd_ctx, new_name); 3364 } 3365 3366 /* We are about to modify the VMA's memory policy. */ 3367 static inline struct vm_area_struct 3368 *vma_modify_policy(struct vma_iterator *vmi, 3369 struct vm_area_struct *prev, 3370 struct vm_area_struct *vma, 3371 unsigned long start, unsigned long end, 3372 struct mempolicy *new_pol) 3373 { 3374 return vma_modify(vmi, prev, vma, start, end, vma->vm_flags, 3375 new_pol, vma->vm_userfaultfd_ctx, anon_vma_name(vma)); 3376 } 3377 3378 /* We are about to modify the VMA's flags and/or uffd context. */ 3379 static inline struct vm_area_struct 3380 *vma_modify_flags_uffd(struct vma_iterator *vmi, 3381 struct vm_area_struct *prev, 3382 struct vm_area_struct *vma, 3383 unsigned long start, unsigned long end, 3384 unsigned long new_flags, 3385 struct vm_userfaultfd_ctx new_ctx) 3386 { 3387 return vma_modify(vmi, prev, vma, start, end, new_flags, 3388 vma_policy(vma), new_ctx, anon_vma_name(vma)); 3389 } 3390 3391 static inline int check_data_rlimit(unsigned long rlim, 3392 unsigned long new, 3393 unsigned long start, 3394 unsigned long end_data, 3395 unsigned long start_data) 3396 { 3397 if (rlim < RLIM_INFINITY) { 3398 if (((new - start) + (end_data - start_data)) > rlim) 3399 return -ENOSPC; 3400 } 3401 3402 return 0; 3403 } 3404 3405 extern int mm_take_all_locks(struct mm_struct *mm); 3406 extern void mm_drop_all_locks(struct mm_struct *mm); 3407 3408 extern int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file); 3409 extern int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file); 3410 extern struct file *get_mm_exe_file(struct mm_struct *mm); 3411 extern struct file *get_task_exe_file(struct task_struct *task); 3412 3413 extern bool may_expand_vm(struct mm_struct *, vm_flags_t, unsigned long npages); 3414 extern void vm_stat_account(struct mm_struct *, vm_flags_t, long npages); 3415 3416 extern bool vma_is_special_mapping(const struct vm_area_struct *vma, 3417 const struct vm_special_mapping *sm); 3418 extern struct vm_area_struct *_install_special_mapping(struct mm_struct *mm, 3419 unsigned long addr, unsigned long len, 3420 unsigned long flags, 3421 const struct vm_special_mapping *spec); 3422 /* This is an obsolete alternative to _install_special_mapping. */ 3423 extern int install_special_mapping(struct mm_struct *mm, 3424 unsigned long addr, unsigned long len, 3425 unsigned long flags, struct page **pages); 3426 3427 unsigned long randomize_stack_top(unsigned long stack_top); 3428 unsigned long randomize_page(unsigned long start, unsigned long range); 3429 3430 unsigned long 3431 __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, 3432 unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags); 3433 3434 static inline unsigned long 3435 get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, 3436 unsigned long pgoff, unsigned long flags) 3437 { 3438 return __get_unmapped_area(file, addr, len, pgoff, flags, 0); 3439 } 3440 3441 extern unsigned long mmap_region(struct file *file, unsigned long addr, 3442 unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, 3443 struct list_head *uf); 3444 extern unsigned long do_mmap(struct file *file, unsigned long addr, 3445 unsigned long len, unsigned long prot, unsigned long flags, 3446 vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, 3447 struct list_head *uf); 3448 extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, 3449 unsigned long start, size_t len, struct list_head *uf, 3450 bool unlock); 3451 extern int do_munmap(struct mm_struct *, unsigned long, size_t, 3452 struct list_head *uf); 3453 extern int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior); 3454 3455 #ifdef CONFIG_MMU 3456 extern int do_vma_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, 3457 unsigned long start, unsigned long end, 3458 struct list_head *uf, bool unlock); 3459 extern int __mm_populate(unsigned long addr, unsigned long len, 3460 int ignore_errors); 3461 static inline void mm_populate(unsigned long addr, unsigned long len) 3462 { 3463 /* Ignore errors */ 3464 (void) __mm_populate(addr, len, 1); 3465 } 3466 #else 3467 static inline void mm_populate(unsigned long addr, unsigned long len) {} 3468 #endif 3469 3470 /* This takes the mm semaphore itself */ 3471 extern int __must_check vm_brk_flags(unsigned long, unsigned long, unsigned long); 3472 extern int vm_munmap(unsigned long, size_t); 3473 extern unsigned long __must_check vm_mmap(struct file *, unsigned long, 3474 unsigned long, unsigned long, 3475 unsigned long, unsigned long); 3476 3477 struct vm_unmapped_area_info { 3478 #define VM_UNMAPPED_AREA_TOPDOWN 1 3479 unsigned long flags; 3480 unsigned long length; 3481 unsigned long low_limit; 3482 unsigned long high_limit; 3483 unsigned long align_mask; 3484 unsigned long align_offset; 3485 unsigned long start_gap; 3486 }; 3487 3488 extern unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info); 3489 3490 /* truncate.c */ 3491 extern void truncate_inode_pages(struct address_space *, loff_t); 3492 extern void truncate_inode_pages_range(struct address_space *, 3493 loff_t lstart, loff_t lend); 3494 extern void truncate_inode_pages_final(struct address_space *); 3495 3496 /* generic vm_area_ops exported for stackable file systems */ 3497 extern vm_fault_t filemap_fault(struct vm_fault *vmf); 3498 extern vm_fault_t filemap_map_pages(struct vm_fault *vmf, 3499 pgoff_t start_pgoff, pgoff_t end_pgoff); 3500 extern vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf); 3501 3502 extern unsigned long stack_guard_gap; 3503 /* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */ 3504 int expand_stack_locked(struct vm_area_struct *vma, unsigned long address); 3505 struct vm_area_struct *expand_stack(struct mm_struct * mm, unsigned long addr); 3506 3507 /* CONFIG_STACK_GROWSUP still needs to grow downwards at some places */ 3508 int expand_downwards(struct vm_area_struct *vma, unsigned long address); 3509 3510 /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */ 3511 extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr); 3512 extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr, 3513 struct vm_area_struct **pprev); 3514 3515 /* 3516 * Look up the first VMA which intersects the interval [start_addr, end_addr) 3517 * NULL if none. Assume start_addr < end_addr. 3518 */ 3519 struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, 3520 unsigned long start_addr, unsigned long end_addr); 3521 3522 /** 3523 * vma_lookup() - Find a VMA at a specific address 3524 * @mm: The process address space. 3525 * @addr: The user address. 3526 * 3527 * Return: The vm_area_struct at the given address, %NULL otherwise. 3528 */ 3529 static inline 3530 struct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr) 3531 { 3532 return mtree_load(&mm->mm_mt, addr); 3533 } 3534 3535 static inline unsigned long stack_guard_start_gap(struct vm_area_struct *vma) 3536 { 3537 if (vma->vm_flags & VM_GROWSDOWN) 3538 return stack_guard_gap; 3539 3540 /* See reasoning around the VM_SHADOW_STACK definition */ 3541 if (vma->vm_flags & VM_SHADOW_STACK) 3542 return PAGE_SIZE; 3543 3544 return 0; 3545 } 3546 3547 static inline unsigned long vm_start_gap(struct vm_area_struct *vma) 3548 { 3549 unsigned long gap = stack_guard_start_gap(vma); 3550 unsigned long vm_start = vma->vm_start; 3551 3552 vm_start -= gap; 3553 if (vm_start > vma->vm_start) 3554 vm_start = 0; 3555 return vm_start; 3556 } 3557 3558 static inline unsigned long vm_end_gap(struct vm_area_struct *vma) 3559 { 3560 unsigned long vm_end = vma->vm_end; 3561 3562 if (vma->vm_flags & VM_GROWSUP) { 3563 vm_end += stack_guard_gap; 3564 if (vm_end < vma->vm_end) 3565 vm_end = -PAGE_SIZE; 3566 } 3567 return vm_end; 3568 } 3569 3570 static inline unsigned long vma_pages(struct vm_area_struct *vma) 3571 { 3572 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; 3573 } 3574 3575 /* Look up the first VMA which exactly match the interval vm_start ... vm_end */ 3576 static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm, 3577 unsigned long vm_start, unsigned long vm_end) 3578 { 3579 struct vm_area_struct *vma = vma_lookup(mm, vm_start); 3580 3581 if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end)) 3582 vma = NULL; 3583 3584 return vma; 3585 } 3586 3587 static inline bool range_in_vma(struct vm_area_struct *vma, 3588 unsigned long start, unsigned long end) 3589 { 3590 return (vma && vma->vm_start <= start && end <= vma->vm_end); 3591 } 3592 3593 #ifdef CONFIG_MMU 3594 pgprot_t vm_get_page_prot(unsigned long vm_flags); 3595 void vma_set_page_prot(struct vm_area_struct *vma); 3596 #else 3597 static inline pgprot_t vm_get_page_prot(unsigned long vm_flags) 3598 { 3599 return __pgprot(0); 3600 } 3601 static inline void vma_set_page_prot(struct vm_area_struct *vma) 3602 { 3603 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); 3604 } 3605 #endif 3606 3607 void vma_set_file(struct vm_area_struct *vma, struct file *file); 3608 3609 #ifdef CONFIG_NUMA_BALANCING 3610 unsigned long change_prot_numa(struct vm_area_struct *vma, 3611 unsigned long start, unsigned long end); 3612 #endif 3613 3614 struct vm_area_struct *find_extend_vma_locked(struct mm_struct *, 3615 unsigned long addr); 3616 int remap_pfn_range(struct vm_area_struct *, unsigned long addr, 3617 unsigned long pfn, unsigned long size, pgprot_t); 3618 int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr, 3619 unsigned long pfn, unsigned long size, pgprot_t prot); 3620 int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); 3621 int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, 3622 struct page **pages, unsigned long *num); 3623 int vm_map_pages(struct vm_area_struct *vma, struct page **pages, 3624 unsigned long num); 3625 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, 3626 unsigned long num); 3627 vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr, 3628 unsigned long pfn); 3629 vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, 3630 unsigned long pfn, pgprot_t pgprot); 3631 vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr, 3632 pfn_t pfn); 3633 vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma, 3634 unsigned long addr, pfn_t pfn); 3635 int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len); 3636 3637 static inline vm_fault_t vmf_insert_page(struct vm_area_struct *vma, 3638 unsigned long addr, struct page *page) 3639 { 3640 int err = vm_insert_page(vma, addr, page); 3641 3642 if (err == -ENOMEM) 3643 return VM_FAULT_OOM; 3644 if (err < 0 && err != -EBUSY) 3645 return VM_FAULT_SIGBUS; 3646 3647 return VM_FAULT_NOPAGE; 3648 } 3649 3650 #ifndef io_remap_pfn_range 3651 static inline int io_remap_pfn_range(struct vm_area_struct *vma, 3652 unsigned long addr, unsigned long pfn, 3653 unsigned long size, pgprot_t prot) 3654 { 3655 return remap_pfn_range(vma, addr, pfn, size, pgprot_decrypted(prot)); 3656 } 3657 #endif 3658 3659 static inline vm_fault_t vmf_error(int err) 3660 { 3661 if (err == -ENOMEM) 3662 return VM_FAULT_OOM; 3663 else if (err == -EHWPOISON) 3664 return VM_FAULT_HWPOISON; 3665 return VM_FAULT_SIGBUS; 3666 } 3667 3668 /* 3669 * Convert errno to return value for ->page_mkwrite() calls. 3670 * 3671 * This should eventually be merged with vmf_error() above, but will need a 3672 * careful audit of all vmf_error() callers. 3673 */ 3674 static inline vm_fault_t vmf_fs_error(int err) 3675 { 3676 if (err == 0) 3677 return VM_FAULT_LOCKED; 3678 if (err == -EFAULT || err == -EAGAIN) 3679 return VM_FAULT_NOPAGE; 3680 if (err == -ENOMEM) 3681 return VM_FAULT_OOM; 3682 /* -ENOSPC, -EDQUOT, -EIO ... */ 3683 return VM_FAULT_SIGBUS; 3684 } 3685 3686 struct page *follow_page(struct vm_area_struct *vma, unsigned long address, 3687 unsigned int foll_flags); 3688 3689 static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags) 3690 { 3691 if (vm_fault & VM_FAULT_OOM) 3692 return -ENOMEM; 3693 if (vm_fault & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE)) 3694 return (foll_flags & FOLL_HWPOISON) ? -EHWPOISON : -EFAULT; 3695 if (vm_fault & (VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV)) 3696 return -EFAULT; 3697 return 0; 3698 } 3699 3700 /* 3701 * Indicates whether GUP can follow a PROT_NONE mapped page, or whether 3702 * a (NUMA hinting) fault is required. 3703 */ 3704 static inline bool gup_can_follow_protnone(struct vm_area_struct *vma, 3705 unsigned int flags) 3706 { 3707 /* 3708 * If callers don't want to honor NUMA hinting faults, no need to 3709 * determine if we would actually have to trigger a NUMA hinting fault. 3710 */ 3711 if (!(flags & FOLL_HONOR_NUMA_FAULT)) 3712 return true; 3713 3714 /* 3715 * NUMA hinting faults don't apply in inaccessible (PROT_NONE) VMAs. 3716 * 3717 * Requiring a fault here even for inaccessible VMAs would mean that 3718 * FOLL_FORCE cannot make any progress, because handle_mm_fault() 3719 * refuses to process NUMA hinting faults in inaccessible VMAs. 3720 */ 3721 return !vma_is_accessible(vma); 3722 } 3723 3724 typedef int (*pte_fn_t)(pte_t *pte, unsigned long addr, void *data); 3725 extern int apply_to_page_range(struct mm_struct *mm, unsigned long address, 3726 unsigned long size, pte_fn_t fn, void *data); 3727 extern int apply_to_existing_page_range(struct mm_struct *mm, 3728 unsigned long address, unsigned long size, 3729 pte_fn_t fn, void *data); 3730 3731 #ifdef CONFIG_PAGE_POISONING 3732 extern void __kernel_poison_pages(struct page *page, int numpages); 3733 extern void __kernel_unpoison_pages(struct page *page, int numpages); 3734 extern bool _page_poisoning_enabled_early; 3735 DECLARE_STATIC_KEY_FALSE(_page_poisoning_enabled); 3736 static inline bool page_poisoning_enabled(void) 3737 { 3738 return _page_poisoning_enabled_early; 3739 } 3740 /* 3741 * For use in fast paths after init_mem_debugging() has run, or when a 3742 * false negative result is not harmful when called too early. 3743 */ 3744 static inline bool page_poisoning_enabled_static(void) 3745 { 3746 return static_branch_unlikely(&_page_poisoning_enabled); 3747 } 3748 static inline void kernel_poison_pages(struct page *page, int numpages) 3749 { 3750 if (page_poisoning_enabled_static()) 3751 __kernel_poison_pages(page, numpages); 3752 } 3753 static inline void kernel_unpoison_pages(struct page *page, int numpages) 3754 { 3755 if (page_poisoning_enabled_static()) 3756 __kernel_unpoison_pages(page, numpages); 3757 } 3758 #else 3759 static inline bool page_poisoning_enabled(void) { return false; } 3760 static inline bool page_poisoning_enabled_static(void) { return false; } 3761 static inline void __kernel_poison_pages(struct page *page, int nunmpages) { } 3762 static inline void kernel_poison_pages(struct page *page, int numpages) { } 3763 static inline void kernel_unpoison_pages(struct page *page, int numpages) { } 3764 #endif 3765 3766 DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc); 3767 static inline bool want_init_on_alloc(gfp_t flags) 3768 { 3769 if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, 3770 &init_on_alloc)) 3771 return true; 3772 return flags & __GFP_ZERO; 3773 } 3774 3775 DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free); 3776 static inline bool want_init_on_free(void) 3777 { 3778 return static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON, 3779 &init_on_free); 3780 } 3781 3782 extern bool _debug_pagealloc_enabled_early; 3783 DECLARE_STATIC_KEY_FALSE(_debug_pagealloc_enabled); 3784 3785 static inline bool debug_pagealloc_enabled(void) 3786 { 3787 return IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) && 3788 _debug_pagealloc_enabled_early; 3789 } 3790 3791 /* 3792 * For use in fast paths after mem_debugging_and_hardening_init() has run, 3793 * or when a false negative result is not harmful when called too early. 3794 */ 3795 static inline bool debug_pagealloc_enabled_static(void) 3796 { 3797 if (!IS_ENABLED(CONFIG_DEBUG_PAGEALLOC)) 3798 return false; 3799 3800 return static_branch_unlikely(&_debug_pagealloc_enabled); 3801 } 3802 3803 /* 3804 * To support DEBUG_PAGEALLOC architecture must ensure that 3805 * __kernel_map_pages() never fails 3806 */ 3807 extern void __kernel_map_pages(struct page *page, int numpages, int enable); 3808 #ifdef CONFIG_DEBUG_PAGEALLOC 3809 static inline void debug_pagealloc_map_pages(struct page *page, int numpages) 3810 { 3811 if (debug_pagealloc_enabled_static()) 3812 __kernel_map_pages(page, numpages, 1); 3813 } 3814 3815 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) 3816 { 3817 if (debug_pagealloc_enabled_static()) 3818 __kernel_map_pages(page, numpages, 0); 3819 } 3820 3821 extern unsigned int _debug_guardpage_minorder; 3822 DECLARE_STATIC_KEY_FALSE(_debug_guardpage_enabled); 3823 3824 static inline unsigned int debug_guardpage_minorder(void) 3825 { 3826 return _debug_guardpage_minorder; 3827 } 3828 3829 static inline bool debug_guardpage_enabled(void) 3830 { 3831 return static_branch_unlikely(&_debug_guardpage_enabled); 3832 } 3833 3834 static inline bool page_is_guard(struct page *page) 3835 { 3836 if (!debug_guardpage_enabled()) 3837 return false; 3838 3839 return PageGuard(page); 3840 } 3841 3842 bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order); 3843 static inline bool set_page_guard(struct zone *zone, struct page *page, 3844 unsigned int order) 3845 { 3846 if (!debug_guardpage_enabled()) 3847 return false; 3848 return __set_page_guard(zone, page, order); 3849 } 3850 3851 void __clear_page_guard(struct zone *zone, struct page *page, unsigned int order); 3852 static inline void clear_page_guard(struct zone *zone, struct page *page, 3853 unsigned int order) 3854 { 3855 if (!debug_guardpage_enabled()) 3856 return; 3857 __clear_page_guard(zone, page, order); 3858 } 3859 3860 #else /* CONFIG_DEBUG_PAGEALLOC */ 3861 static inline void debug_pagealloc_map_pages(struct page *page, int numpages) {} 3862 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) {} 3863 static inline unsigned int debug_guardpage_minorder(void) { return 0; } 3864 static inline bool debug_guardpage_enabled(void) { return false; } 3865 static inline bool page_is_guard(struct page *page) { return false; } 3866 static inline bool set_page_guard(struct zone *zone, struct page *page, 3867 unsigned int order) { return false; } 3868 static inline void clear_page_guard(struct zone *zone, struct page *page, 3869 unsigned int order) {} 3870 #endif /* CONFIG_DEBUG_PAGEALLOC */ 3871 3872 #ifdef __HAVE_ARCH_GATE_AREA 3873 extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm); 3874 extern int in_gate_area_no_mm(unsigned long addr); 3875 extern int in_gate_area(struct mm_struct *mm, unsigned long addr); 3876 #else 3877 static inline struct vm_area_struct *get_gate_vma(struct mm_struct *mm) 3878 { 3879 return NULL; 3880 } 3881 static inline int in_gate_area_no_mm(unsigned long addr) { return 0; } 3882 static inline int in_gate_area(struct mm_struct *mm, unsigned long addr) 3883 { 3884 return 0; 3885 } 3886 #endif /* __HAVE_ARCH_GATE_AREA */ 3887 3888 extern bool process_shares_mm(struct task_struct *p, struct mm_struct *mm); 3889 3890 #ifdef CONFIG_SYSCTL 3891 extern int sysctl_drop_caches; 3892 int drop_caches_sysctl_handler(struct ctl_table *, int, void *, size_t *, 3893 loff_t *); 3894 #endif 3895 3896 void drop_slab(void); 3897 3898 #ifndef CONFIG_MMU 3899 #define randomize_va_space 0 3900 #else 3901 extern int randomize_va_space; 3902 #endif 3903 3904 const char * arch_vma_name(struct vm_area_struct *vma); 3905 #ifdef CONFIG_MMU 3906 void print_vma_addr(char *prefix, unsigned long rip); 3907 #else 3908 static inline void print_vma_addr(char *prefix, unsigned long rip) 3909 { 3910 } 3911 #endif 3912 3913 void *sparse_buffer_alloc(unsigned long size); 3914 struct page * __populate_section_memmap(unsigned long pfn, 3915 unsigned long nr_pages, int nid, struct vmem_altmap *altmap, 3916 struct dev_pagemap *pgmap); 3917 void pmd_init(void *addr); 3918 void pud_init(void *addr); 3919 pgd_t *vmemmap_pgd_populate(unsigned long addr, int node); 3920 p4d_t *vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node); 3921 pud_t *vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node); 3922 pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node); 3923 pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node, 3924 struct vmem_altmap *altmap, struct page *reuse); 3925 void *vmemmap_alloc_block(unsigned long size, int node); 3926 struct vmem_altmap; 3927 void *vmemmap_alloc_block_buf(unsigned long size, int node, 3928 struct vmem_altmap *altmap); 3929 void vmemmap_verify(pte_t *, int, unsigned long, unsigned long); 3930 void vmemmap_set_pmd(pmd_t *pmd, void *p, int node, 3931 unsigned long addr, unsigned long next); 3932 int vmemmap_check_pmd(pmd_t *pmd, int node, 3933 unsigned long addr, unsigned long next); 3934 int vmemmap_populate_basepages(unsigned long start, unsigned long end, 3935 int node, struct vmem_altmap *altmap); 3936 int vmemmap_populate_hugepages(unsigned long start, unsigned long end, 3937 int node, struct vmem_altmap *altmap); 3938 int vmemmap_populate(unsigned long start, unsigned long end, int node, 3939 struct vmem_altmap *altmap); 3940 void vmemmap_populate_print_last(void); 3941 #ifdef CONFIG_MEMORY_HOTPLUG 3942 void vmemmap_free(unsigned long start, unsigned long end, 3943 struct vmem_altmap *altmap); 3944 #endif 3945 3946 #ifdef CONFIG_SPARSEMEM_VMEMMAP 3947 static inline unsigned long vmem_altmap_offset(struct vmem_altmap *altmap) 3948 { 3949 /* number of pfns from base where pfn_to_page() is valid */ 3950 if (altmap) 3951 return altmap->reserve + altmap->free; 3952 return 0; 3953 } 3954 3955 static inline void vmem_altmap_free(struct vmem_altmap *altmap, 3956 unsigned long nr_pfns) 3957 { 3958 altmap->alloc -= nr_pfns; 3959 } 3960 #else 3961 static inline unsigned long vmem_altmap_offset(struct vmem_altmap *altmap) 3962 { 3963 return 0; 3964 } 3965 3966 static inline void vmem_altmap_free(struct vmem_altmap *altmap, 3967 unsigned long nr_pfns) 3968 { 3969 } 3970 #endif 3971 3972 #define VMEMMAP_RESERVE_NR 2 3973 #ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP 3974 static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap, 3975 struct dev_pagemap *pgmap) 3976 { 3977 unsigned long nr_pages; 3978 unsigned long nr_vmemmap_pages; 3979 3980 if (!pgmap || !is_power_of_2(sizeof(struct page))) 3981 return false; 3982 3983 nr_pages = pgmap_vmemmap_nr(pgmap); 3984 nr_vmemmap_pages = ((nr_pages * sizeof(struct page)) >> PAGE_SHIFT); 3985 /* 3986 * For vmemmap optimization with DAX we need minimum 2 vmemmap 3987 * pages. See layout diagram in Documentation/mm/vmemmap_dedup.rst 3988 */ 3989 return !altmap && (nr_vmemmap_pages > VMEMMAP_RESERVE_NR); 3990 } 3991 /* 3992 * If we don't have an architecture override, use the generic rule 3993 */ 3994 #ifndef vmemmap_can_optimize 3995 #define vmemmap_can_optimize __vmemmap_can_optimize 3996 #endif 3997 3998 #else 3999 static inline bool vmemmap_can_optimize(struct vmem_altmap *altmap, 4000 struct dev_pagemap *pgmap) 4001 { 4002 return false; 4003 } 4004 #endif 4005 4006 void register_page_bootmem_memmap(unsigned long section_nr, struct page *map, 4007 unsigned long nr_pages); 4008 4009 enum mf_flags { 4010 MF_COUNT_INCREASED = 1 << 0, 4011 MF_ACTION_REQUIRED = 1 << 1, 4012 MF_MUST_KILL = 1 << 2, 4013 MF_SOFT_OFFLINE = 1 << 3, 4014 MF_UNPOISON = 1 << 4, 4015 MF_SW_SIMULATED = 1 << 5, 4016 MF_NO_RETRY = 1 << 6, 4017 MF_MEM_PRE_REMOVE = 1 << 7, 4018 }; 4019 int mf_dax_kill_procs(struct address_space *mapping, pgoff_t index, 4020 unsigned long count, int mf_flags); 4021 extern int memory_failure(unsigned long pfn, int flags); 4022 extern void memory_failure_queue_kick(int cpu); 4023 extern int unpoison_memory(unsigned long pfn); 4024 extern atomic_long_t num_poisoned_pages __read_mostly; 4025 extern int soft_offline_page(unsigned long pfn, int flags); 4026 #ifdef CONFIG_MEMORY_FAILURE 4027 /* 4028 * Sysfs entries for memory failure handling statistics. 4029 */ 4030 extern const struct attribute_group memory_failure_attr_group; 4031 extern void memory_failure_queue(unsigned long pfn, int flags); 4032 extern int __get_huge_page_for_hwpoison(unsigned long pfn, int flags, 4033 bool *migratable_cleared); 4034 void num_poisoned_pages_inc(unsigned long pfn); 4035 void num_poisoned_pages_sub(unsigned long pfn, long i); 4036 struct task_struct *task_early_kill(struct task_struct *tsk, int force_early); 4037 #else 4038 static inline void memory_failure_queue(unsigned long pfn, int flags) 4039 { 4040 } 4041 4042 static inline int __get_huge_page_for_hwpoison(unsigned long pfn, int flags, 4043 bool *migratable_cleared) 4044 { 4045 return 0; 4046 } 4047 4048 static inline void num_poisoned_pages_inc(unsigned long pfn) 4049 { 4050 } 4051 4052 static inline void num_poisoned_pages_sub(unsigned long pfn, long i) 4053 { 4054 } 4055 #endif 4056 4057 #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_KSM) 4058 void add_to_kill_ksm(struct task_struct *tsk, struct page *p, 4059 struct vm_area_struct *vma, struct list_head *to_kill, 4060 unsigned long ksm_addr); 4061 #endif 4062 4063 #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG) 4064 extern void memblk_nr_poison_inc(unsigned long pfn); 4065 extern void memblk_nr_poison_sub(unsigned long pfn, long i); 4066 #else 4067 static inline void memblk_nr_poison_inc(unsigned long pfn) 4068 { 4069 } 4070 4071 static inline void memblk_nr_poison_sub(unsigned long pfn, long i) 4072 { 4073 } 4074 #endif 4075 4076 #ifndef arch_memory_failure 4077 static inline int arch_memory_failure(unsigned long pfn, int flags) 4078 { 4079 return -ENXIO; 4080 } 4081 #endif 4082 4083 #ifndef arch_is_platform_page 4084 static inline bool arch_is_platform_page(u64 paddr) 4085 { 4086 return false; 4087 } 4088 #endif 4089 4090 /* 4091 * Error handlers for various types of pages. 4092 */ 4093 enum mf_result { 4094 MF_IGNORED, /* Error: cannot be handled */ 4095 MF_FAILED, /* Error: handling failed */ 4096 MF_DELAYED, /* Will be handled later */ 4097 MF_RECOVERED, /* Successfully recovered */ 4098 }; 4099 4100 enum mf_action_page_type { 4101 MF_MSG_KERNEL, 4102 MF_MSG_KERNEL_HIGH_ORDER, 4103 MF_MSG_SLAB, 4104 MF_MSG_DIFFERENT_COMPOUND, 4105 MF_MSG_HUGE, 4106 MF_MSG_FREE_HUGE, 4107 MF_MSG_UNMAP_FAILED, 4108 MF_MSG_DIRTY_SWAPCACHE, 4109 MF_MSG_CLEAN_SWAPCACHE, 4110 MF_MSG_DIRTY_MLOCKED_LRU, 4111 MF_MSG_CLEAN_MLOCKED_LRU, 4112 MF_MSG_DIRTY_UNEVICTABLE_LRU, 4113 MF_MSG_CLEAN_UNEVICTABLE_LRU, 4114 MF_MSG_DIRTY_LRU, 4115 MF_MSG_CLEAN_LRU, 4116 MF_MSG_TRUNCATED_LRU, 4117 MF_MSG_BUDDY, 4118 MF_MSG_DAX, 4119 MF_MSG_UNSPLIT_THP, 4120 MF_MSG_UNKNOWN, 4121 }; 4122 4123 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS) 4124 extern void clear_huge_page(struct page *page, 4125 unsigned long addr_hint, 4126 unsigned int pages_per_huge_page); 4127 int copy_user_large_folio(struct folio *dst, struct folio *src, 4128 unsigned long addr_hint, 4129 struct vm_area_struct *vma); 4130 long copy_folio_from_user(struct folio *dst_folio, 4131 const void __user *usr_src, 4132 bool allow_pagefault); 4133 4134 /** 4135 * vma_is_special_huge - Are transhuge page-table entries considered special? 4136 * @vma: Pointer to the struct vm_area_struct to consider 4137 * 4138 * Whether transhuge page-table entries are considered "special" following 4139 * the definition in vm_normal_page(). 4140 * 4141 * Return: true if transhuge page-table entries should be considered special, 4142 * false otherwise. 4143 */ 4144 static inline bool vma_is_special_huge(const struct vm_area_struct *vma) 4145 { 4146 return vma_is_dax(vma) || (vma->vm_file && 4147 (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))); 4148 } 4149 4150 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */ 4151 4152 #if MAX_NUMNODES > 1 4153 void __init setup_nr_node_ids(void); 4154 #else 4155 static inline void setup_nr_node_ids(void) {} 4156 #endif 4157 4158 extern int memcmp_pages(struct page *page1, struct page *page2); 4159 4160 static inline int pages_identical(struct page *page1, struct page *page2) 4161 { 4162 return !memcmp_pages(page1, page2); 4163 } 4164 4165 #ifdef CONFIG_MAPPING_DIRTY_HELPERS 4166 unsigned long clean_record_shared_mapping_range(struct address_space *mapping, 4167 pgoff_t first_index, pgoff_t nr, 4168 pgoff_t bitmap_pgoff, 4169 unsigned long *bitmap, 4170 pgoff_t *start, 4171 pgoff_t *end); 4172 4173 unsigned long wp_shared_mapping_range(struct address_space *mapping, 4174 pgoff_t first_index, pgoff_t nr); 4175 #endif 4176 4177 extern int sysctl_nr_trim_pages; 4178 4179 #ifdef CONFIG_PRINTK 4180 void mem_dump_obj(void *object); 4181 #else 4182 static inline void mem_dump_obj(void *object) {} 4183 #endif 4184 4185 /** 4186 * seal_check_write - Check for F_SEAL_WRITE or F_SEAL_FUTURE_WRITE flags and 4187 * handle them. 4188 * @seals: the seals to check 4189 * @vma: the vma to operate on 4190 * 4191 * Check whether F_SEAL_WRITE or F_SEAL_FUTURE_WRITE are set; if so, do proper 4192 * check/handling on the vma flags. Return 0 if check pass, or <0 for errors. 4193 */ 4194 static inline int seal_check_write(int seals, struct vm_area_struct *vma) 4195 { 4196 if (seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) { 4197 /* 4198 * New PROT_WRITE and MAP_SHARED mmaps are not allowed when 4199 * write seals are active. 4200 */ 4201 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE)) 4202 return -EPERM; 4203 4204 /* 4205 * Since an F_SEAL_[FUTURE_]WRITE sealed memfd can be mapped as 4206 * MAP_SHARED and read-only, take care to not allow mprotect to 4207 * revert protections on such mappings. Do this only for shared 4208 * mappings. For private mappings, don't need to mask 4209 * VM_MAYWRITE as we still want them to be COW-writable. 4210 */ 4211 if (vma->vm_flags & VM_SHARED) 4212 vm_flags_clear(vma, VM_MAYWRITE); 4213 } 4214 4215 return 0; 4216 } 4217 4218 #ifdef CONFIG_ANON_VMA_NAME 4219 int madvise_set_anon_name(struct mm_struct *mm, unsigned long start, 4220 unsigned long len_in, 4221 struct anon_vma_name *anon_name); 4222 #else 4223 static inline int 4224 madvise_set_anon_name(struct mm_struct *mm, unsigned long start, 4225 unsigned long len_in, struct anon_vma_name *anon_name) { 4226 return 0; 4227 } 4228 #endif 4229 4230 #ifdef CONFIG_UNACCEPTED_MEMORY 4231 4232 bool range_contains_unaccepted_memory(phys_addr_t start, phys_addr_t end); 4233 void accept_memory(phys_addr_t start, phys_addr_t end); 4234 4235 #else 4236 4237 static inline bool range_contains_unaccepted_memory(phys_addr_t start, 4238 phys_addr_t end) 4239 { 4240 return false; 4241 } 4242 4243 static inline void accept_memory(phys_addr_t start, phys_addr_t end) 4244 { 4245 } 4246 4247 #endif 4248 4249 static inline bool pfn_is_unaccepted_memory(unsigned long pfn) 4250 { 4251 phys_addr_t paddr = pfn << PAGE_SHIFT; 4252 4253 return range_contains_unaccepted_memory(paddr, paddr + PAGE_SIZE); 4254 } 4255 4256 void vma_pgtable_walk_begin(struct vm_area_struct *vma); 4257 void vma_pgtable_walk_end(struct vm_area_struct *vma); 4258 4259 #endif /* _LINUX_MM_H */ 4260