1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_MM_H 3 #define _LINUX_MM_H 4 5 #include <linux/args.h> 6 #include <linux/errno.h> 7 #include <linux/mmdebug.h> 8 #include <linux/gfp.h> 9 #include <linux/pgalloc_tag.h> 10 #include <linux/bug.h> 11 #include <linux/list.h> 12 #include <linux/mmzone.h> 13 #include <linux/rbtree.h> 14 #include <linux/atomic.h> 15 #include <linux/debug_locks.h> 16 #include <linux/compiler.h> 17 #include <linux/mm_types.h> 18 #include <linux/mmap_lock.h> 19 #include <linux/range.h> 20 #include <linux/pfn.h> 21 #include <linux/percpu-refcount.h> 22 #include <linux/bit_spinlock.h> 23 #include <linux/shrinker.h> 24 #include <linux/resource.h> 25 #include <linux/page_ext.h> 26 #include <linux/err.h> 27 #include <linux/page-flags.h> 28 #include <linux/page_ref.h> 29 #include <linux/overflow.h> 30 #include <linux/sched.h> 31 #include <linux/pgtable.h> 32 #include <linux/kasan.h> 33 #include <linux/memremap.h> 34 #include <linux/slab.h> 35 #include <linux/cacheinfo.h> 36 #include <linux/rcuwait.h> 37 #include <linux/bitmap.h> 38 #include <linux/bitops.h> 39 #include <linux/iommu-debug-pagealloc.h> 40 #include <linux/kcsan-checks.h> 41 #include <linux/vmemmap-optimization.h> 42 43 struct mempolicy; 44 struct anon_vma; 45 struct anon_vma_chain; 46 struct user_struct; 47 struct pt_regs; 48 struct folio_batch; 49 50 void arch_mm_preinit(void); 51 void mm_core_init_early(void); 52 void mm_core_init(void); 53 void init_mm_internals(void); 54 55 extern atomic_long_t _totalram_pages; 56 static inline unsigned long totalram_pages(void) 57 { 58 return (unsigned long)atomic_long_read(&_totalram_pages); 59 } 60 61 static inline void totalram_pages_add(long count) 62 { 63 atomic_long_add(count, &_totalram_pages); 64 } 65 66 extern void * high_memory; 67 68 /* 69 * Convert between pages and MB 70 * 20 is the shift for 1MB (2^20 = 1MB) 71 * PAGE_SHIFT is the shift for page size (e.g., 12 for 4KB pages) 72 * So (20 - PAGE_SHIFT) converts between pages and MB 73 */ 74 #define PAGES_TO_MB(pages) ((pages) >> (20 - PAGE_SHIFT)) 75 #define MB_TO_PAGES(mb) ((mb) << (20 - PAGE_SHIFT)) 76 77 #ifdef CONFIG_SYSCTL 78 extern int sysctl_legacy_va_layout; 79 #else 80 #define sysctl_legacy_va_layout 0 81 #endif 82 83 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS 84 extern const int mmap_rnd_bits_min; 85 extern int mmap_rnd_bits_max __ro_after_init; 86 extern int mmap_rnd_bits __read_mostly; 87 #endif 88 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS 89 extern const int mmap_rnd_compat_bits_min; 90 extern const int mmap_rnd_compat_bits_max; 91 extern int mmap_rnd_compat_bits __read_mostly; 92 #endif 93 94 #ifndef DIRECT_MAP_PHYSMEM_END 95 # ifdef MAX_PHYSMEM_BITS 96 # define DIRECT_MAP_PHYSMEM_END ((1ULL << MAX_PHYSMEM_BITS) - 1) 97 # else 98 # define DIRECT_MAP_PHYSMEM_END (((phys_addr_t)-1)&~(1ULL<<63)) 99 # endif 100 #endif 101 102 #define INVALID_PHYS_ADDR (~(phys_addr_t)0) 103 104 #include <asm/page.h> 105 #include <asm/processor.h> 106 107 #ifndef __pa_symbol 108 #define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0)) 109 #endif 110 111 #ifndef page_to_virt 112 #define page_to_virt(x) __va(PFN_PHYS(page_to_pfn(x))) 113 #endif 114 115 #ifndef lm_alias 116 #define lm_alias(x) __va(__pa_symbol(x)) 117 #endif 118 119 /* 120 * To prevent common memory management code establishing 121 * a zero page mapping on a read fault. 122 * This macro should be defined within <asm/pgtable.h>. 123 * s390 does this to prevent multiplexing of hardware bits 124 * related to the physical page in case of virtualization. 125 */ 126 #ifndef mm_forbids_zeropage 127 #define mm_forbids_zeropage(X) (0) 128 #endif 129 130 /* 131 * On some architectures it is expensive to call memset() for small sizes. 132 * If an architecture decides to implement their own version of 133 * mm_zero_struct_page they should wrap the defines below in a #ifndef and 134 * define their own version of this macro in <asm/pgtable.h> 135 */ 136 #if BITS_PER_LONG == 64 137 /* This function must be updated when the size of struct page grows above 96 138 * or reduces below 56. The idea that compiler optimizes out switch() 139 * statement, and only leaves move/store instructions. Also the compiler can 140 * combine write statements if they are both assignments and can be reordered, 141 * this can result in several of the writes here being dropped. 142 */ 143 #define mm_zero_struct_page(pp) __mm_zero_struct_page(pp) 144 static inline void __mm_zero_struct_page(struct page *page) 145 { 146 unsigned long *_pp = (void *)page; 147 148 /* Check that struct page is either 56, 64, 72, 80, 88 or 96 bytes */ 149 BUILD_BUG_ON(sizeof(struct page) & 7); 150 BUILD_BUG_ON(sizeof(struct page) < 56); 151 BUILD_BUG_ON(sizeof(struct page) > 96); 152 153 switch (sizeof(struct page)) { 154 case 96: 155 _pp[11] = 0; 156 fallthrough; 157 case 88: 158 _pp[10] = 0; 159 fallthrough; 160 case 80: 161 _pp[9] = 0; 162 fallthrough; 163 case 72: 164 _pp[8] = 0; 165 fallthrough; 166 case 64: 167 _pp[7] = 0; 168 fallthrough; 169 case 56: 170 _pp[6] = 0; 171 _pp[5] = 0; 172 _pp[4] = 0; 173 _pp[3] = 0; 174 _pp[2] = 0; 175 _pp[1] = 0; 176 _pp[0] = 0; 177 } 178 } 179 #else 180 #define mm_zero_struct_page(pp) ((void)memset((pp), 0, sizeof(struct page))) 181 #endif 182 183 /* 184 * Default maximum number of active map areas, this limits the number of vmas 185 * per mm struct. Users can overwrite this number by sysctl but there is a 186 * problem. 187 * 188 * When a program's coredump is generated as ELF format, a section is created 189 * per a vma. In ELF, the number of sections is represented in unsigned short. 190 * This means the number of sections should be smaller than 65535 at coredump. 191 * Because the kernel adds some informative sections to a image of program at 192 * generating coredump, we need some margin. The number of extra sections is 193 * 1-3 now and depends on arch. We use "5" as safe margin, here. 194 * 195 * ELF extended numbering allows more than 65535 sections, so 16-bit bound is 196 * not a hard limit any more. Although some userspace tools can be surprised by 197 * that. 198 */ 199 #define MAPCOUNT_ELF_CORE_MARGIN (5) 200 #define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN) 201 202 extern unsigned long sysctl_user_reserve_kbytes; 203 extern unsigned long sysctl_admin_reserve_kbytes; 204 205 #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) 206 bool page_range_contiguous(const struct page *page, unsigned long nr_pages); 207 #else 208 static inline bool page_range_contiguous(const struct page *page, 209 unsigned long nr_pages) 210 { 211 return true; 212 } 213 #endif 214 215 /* to align the pointer to the (next) page boundary */ 216 #define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE) 217 218 /* to align the pointer to the (prev) page boundary */ 219 #define PAGE_ALIGN_DOWN(addr) ALIGN_DOWN(addr, PAGE_SIZE) 220 221 /* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */ 222 #define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), PAGE_SIZE) 223 224 /** 225 * folio_page_idx - Return the number of a page in a folio. 226 * @folio: The folio. 227 * @page: The folio page. 228 * 229 * This function expects that the page is actually part of the folio. 230 * The returned number is relative to the start of the folio. 231 */ 232 static inline unsigned long folio_page_idx(const struct folio *folio, 233 const struct page *page) 234 { 235 return page - &folio->page; 236 } 237 238 static inline struct folio *lru_to_folio(struct list_head *head) 239 { 240 return list_entry((head)->prev, struct folio, lru); 241 } 242 243 void setup_initial_init_mm(void *start_code, void *end_code, 244 void *end_data, void *brk); 245 246 /* 247 * Linux kernel virtual memory manager primitives. 248 * The idea being to have a "virtual" mm in the same way 249 * we have a virtual fs - giving a cleaner interface to the 250 * mm details, and allowing different kinds of memory mappings 251 * (from shared memory to executable loading to arbitrary 252 * mmap() functions). 253 */ 254 255 struct vm_area_struct *vm_area_alloc(struct mm_struct *); 256 struct vm_area_struct *vm_area_dup(struct vm_area_struct *); 257 void vm_area_free(struct vm_area_struct *); 258 259 #ifndef CONFIG_MMU 260 extern struct rb_root nommu_region_tree; 261 extern struct rw_semaphore nommu_region_sem; 262 263 extern unsigned int kobjsize(const void *objp); 264 #endif 265 266 /* 267 * vm_flags in vm_area_struct, see mm_types.h. 268 * When changing, update also include/trace/events/mmflags.h 269 */ 270 271 #define VM_NONE 0x00000000 272 273 /** 274 * typedef vma_flag_t - specifies an individual VMA flag by bit number. 275 * 276 * This value is made type safe by sparse to avoid passing invalid flag values 277 * around. 278 */ 279 typedef int __bitwise vma_flag_t; 280 281 #define DECLARE_VMA_BIT(name, bitnum) \ 282 VMA_ ## name ## _BIT = ((__force vma_flag_t)bitnum) 283 #define DECLARE_VMA_BIT_ALIAS(name, aliased) \ 284 VMA_ ## name ## _BIT = (VMA_ ## aliased ## _BIT) 285 enum { 286 DECLARE_VMA_BIT(READ, 0), 287 DECLARE_VMA_BIT(WRITE, 1), 288 DECLARE_VMA_BIT(EXEC, 2), 289 DECLARE_VMA_BIT(SHARED, 3), 290 /* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */ 291 DECLARE_VMA_BIT(MAYREAD, 4), /* limits for mprotect() etc. */ 292 DECLARE_VMA_BIT(MAYWRITE, 5), 293 DECLARE_VMA_BIT(MAYEXEC, 6), 294 DECLARE_VMA_BIT(MAYSHARE, 7), 295 DECLARE_VMA_BIT(GROWSDOWN, 8), /* general info on the segment */ 296 #ifdef CONFIG_MMU 297 DECLARE_VMA_BIT(UFFD_MISSING, 9),/* missing pages tracking */ 298 #else 299 /* nommu: R/O MAP_PRIVATE mapping that might overlay a file mapping */ 300 DECLARE_VMA_BIT(MAYOVERLAY, 9), 301 #endif /* CONFIG_MMU */ 302 /* Page-ranges managed without "struct page", just pure PFN */ 303 DECLARE_VMA_BIT(PFNMAP, 10), 304 DECLARE_VMA_BIT(MAYBE_GUARD, 11), 305 DECLARE_VMA_BIT(UFFD_WP, 12), /* wrprotect pages tracking */ 306 DECLARE_VMA_BIT(LOCKED, 13), 307 DECLARE_VMA_BIT(IO, 14), /* Memory mapped I/O or similar */ 308 DECLARE_VMA_BIT(SEQ_READ, 15), /* App will access data sequentially */ 309 DECLARE_VMA_BIT(RAND_READ, 16), /* App will not benefit from clustered reads */ 310 DECLARE_VMA_BIT(DONTCOPY, 17), /* Do not copy this vma on fork */ 311 DECLARE_VMA_BIT(DONTEXPAND, 18),/* Cannot expand with mremap() */ 312 DECLARE_VMA_BIT(LOCKONFAULT, 19),/* Lock pages covered when faulted in */ 313 DECLARE_VMA_BIT(ACCOUNT, 20), /* Is a VM accounted object */ 314 DECLARE_VMA_BIT(NORESERVE, 21), /* should the VM suppress accounting */ 315 DECLARE_VMA_BIT(HUGETLB, 22), /* Huge TLB Page VM */ 316 DECLARE_VMA_BIT(SYNC, 23), /* Synchronous page faults */ 317 DECLARE_VMA_BIT(ARCH_1, 24), /* Architecture-specific flag */ 318 DECLARE_VMA_BIT(WIPEONFORK, 25),/* Wipe VMA contents in child. */ 319 DECLARE_VMA_BIT(DONTDUMP, 26), /* Do not include in the core dump */ 320 DECLARE_VMA_BIT(SOFTDIRTY, 27), /* NOT soft dirty clean area */ 321 DECLARE_VMA_BIT(MIXEDMAP, 28), /* Can contain struct page and pure PFN pages */ 322 DECLARE_VMA_BIT(HUGEPAGE, 29), /* MADV_HUGEPAGE marked this vma */ 323 DECLARE_VMA_BIT(NOHUGEPAGE, 30),/* MADV_NOHUGEPAGE marked this vma */ 324 DECLARE_VMA_BIT(MERGEABLE, 31), /* KSM may merge identical pages */ 325 /* These bits are reused, we define specific uses below. */ 326 DECLARE_VMA_BIT(HIGH_ARCH_0, 32), 327 DECLARE_VMA_BIT(HIGH_ARCH_1, 33), 328 DECLARE_VMA_BIT(HIGH_ARCH_2, 34), 329 DECLARE_VMA_BIT(HIGH_ARCH_3, 35), 330 DECLARE_VMA_BIT(HIGH_ARCH_4, 36), 331 DECLARE_VMA_BIT(HIGH_ARCH_5, 37), 332 DECLARE_VMA_BIT(HIGH_ARCH_6, 38), 333 /* 334 * This flag is used to connect VFIO to arch specific KVM code. It 335 * indicates that the memory under this VMA is safe for use with any 336 * non-cachable memory type inside KVM. Some VFIO devices, on some 337 * platforms, are thought to be unsafe and can cause machine crashes 338 * if KVM does not lock down the memory type. 339 */ 340 DECLARE_VMA_BIT(ALLOW_ANY_UNCACHED, 39), 341 #if defined(CONFIG_PPC32) 342 DECLARE_VMA_BIT_ALIAS(DROPPABLE, ARCH_1), 343 #elif defined(CONFIG_64BIT) 344 DECLARE_VMA_BIT(DROPPABLE, 40), 345 #endif 346 DECLARE_VMA_BIT(UFFD_MINOR, 41), 347 DECLARE_VMA_BIT(SEALED, 42), 348 DECLARE_VMA_BIT(UFFD_RWP, 43), 349 /* Flags that reuse flags above. */ 350 DECLARE_VMA_BIT_ALIAS(PKEY_BIT0, HIGH_ARCH_0), 351 DECLARE_VMA_BIT_ALIAS(PKEY_BIT1, HIGH_ARCH_1), 352 DECLARE_VMA_BIT_ALIAS(PKEY_BIT2, HIGH_ARCH_2), 353 DECLARE_VMA_BIT_ALIAS(PKEY_BIT3, HIGH_ARCH_3), 354 DECLARE_VMA_BIT_ALIAS(PKEY_BIT4, HIGH_ARCH_4), 355 #if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_RISCV_USER_CFI) 356 /* 357 * VM_SHADOW_STACK should not be set with VM_SHARED because of lack of 358 * support core mm. 359 * 360 * These VMAs will get a single end guard page. This helps userspace 361 * protect itself from attacks. A single page is enough for current 362 * shadow stack archs (x86). See the comments near alloc_shstk() in 363 * arch/x86/kernel/shstk.c for more details on the guard size. 364 */ 365 DECLARE_VMA_BIT_ALIAS(SHADOW_STACK, HIGH_ARCH_5), 366 #elif defined(CONFIG_ARM64_GCS) 367 /* 368 * arm64's Guarded Control Stack implements similar functionality and 369 * has similar constraints to shadow stacks. 370 */ 371 DECLARE_VMA_BIT_ALIAS(SHADOW_STACK, HIGH_ARCH_6), 372 #endif 373 DECLARE_VMA_BIT_ALIAS(SAO, ARCH_1), /* Strong Access Ordering (powerpc) */ 374 DECLARE_VMA_BIT_ALIAS(GROWSUP, ARCH_1), /* parisc */ 375 DECLARE_VMA_BIT_ALIAS(SPARC_ADI, ARCH_1), /* sparc64 */ 376 DECLARE_VMA_BIT_ALIAS(ARM64_BTI, ARCH_1), /* arm64 */ 377 DECLARE_VMA_BIT_ALIAS(ARCH_CLEAR, ARCH_1), /* sparc64, arm64 */ 378 DECLARE_VMA_BIT_ALIAS(MAPPED_COPY, ARCH_1), /* !CONFIG_MMU */ 379 DECLARE_VMA_BIT_ALIAS(MTE, HIGH_ARCH_4), /* arm64 */ 380 DECLARE_VMA_BIT_ALIAS(MTE_ALLOWED, HIGH_ARCH_5),/* arm64 */ 381 #ifdef CONFIG_STACK_GROWSUP 382 DECLARE_VMA_BIT_ALIAS(STACK, GROWSUP), 383 DECLARE_VMA_BIT_ALIAS(STACK_EARLY, GROWSDOWN), 384 #else 385 DECLARE_VMA_BIT_ALIAS(STACK, GROWSDOWN), 386 #endif 387 }; 388 #undef DECLARE_VMA_BIT 389 #undef DECLARE_VMA_BIT_ALIAS 390 391 #define INIT_VM_FLAG(name) BIT((__force int) VMA_ ## name ## _BIT) 392 #define VM_READ INIT_VM_FLAG(READ) 393 #define VM_WRITE INIT_VM_FLAG(WRITE) 394 #define VM_EXEC INIT_VM_FLAG(EXEC) 395 #define VM_SHARED INIT_VM_FLAG(SHARED) 396 #define VM_MAYREAD INIT_VM_FLAG(MAYREAD) 397 #define VM_MAYWRITE INIT_VM_FLAG(MAYWRITE) 398 #define VM_MAYEXEC INIT_VM_FLAG(MAYEXEC) 399 #define VM_MAYSHARE INIT_VM_FLAG(MAYSHARE) 400 #define VM_GROWSDOWN INIT_VM_FLAG(GROWSDOWN) 401 #ifdef CONFIG_MMU 402 #define VM_UFFD_MISSING INIT_VM_FLAG(UFFD_MISSING) 403 #else 404 #define VM_UFFD_MISSING VM_NONE 405 #define VM_MAYOVERLAY INIT_VM_FLAG(MAYOVERLAY) 406 #endif 407 #define VM_PFNMAP INIT_VM_FLAG(PFNMAP) 408 #define VM_MAYBE_GUARD INIT_VM_FLAG(MAYBE_GUARD) 409 #define VM_UFFD_WP INIT_VM_FLAG(UFFD_WP) 410 #define VM_LOCKED INIT_VM_FLAG(LOCKED) 411 #define VM_IO INIT_VM_FLAG(IO) 412 #define VM_SEQ_READ INIT_VM_FLAG(SEQ_READ) 413 #define VM_RAND_READ INIT_VM_FLAG(RAND_READ) 414 #define VM_DONTCOPY INIT_VM_FLAG(DONTCOPY) 415 #define VM_DONTEXPAND INIT_VM_FLAG(DONTEXPAND) 416 #define VM_LOCKONFAULT INIT_VM_FLAG(LOCKONFAULT) 417 #define VM_ACCOUNT INIT_VM_FLAG(ACCOUNT) 418 #define VM_NORESERVE INIT_VM_FLAG(NORESERVE) 419 #define VM_HUGETLB INIT_VM_FLAG(HUGETLB) 420 #define VM_SYNC INIT_VM_FLAG(SYNC) 421 #define VM_ARCH_1 INIT_VM_FLAG(ARCH_1) 422 #define VM_WIPEONFORK INIT_VM_FLAG(WIPEONFORK) 423 #define VM_DONTDUMP INIT_VM_FLAG(DONTDUMP) 424 #ifdef CONFIG_MEM_SOFT_DIRTY 425 #define VM_SOFTDIRTY INIT_VM_FLAG(SOFTDIRTY) 426 #else 427 #define VM_SOFTDIRTY VM_NONE 428 #endif 429 #define VM_MIXEDMAP INIT_VM_FLAG(MIXEDMAP) 430 #define VM_HUGEPAGE INIT_VM_FLAG(HUGEPAGE) 431 #define VM_NOHUGEPAGE INIT_VM_FLAG(NOHUGEPAGE) 432 #define VM_MERGEABLE INIT_VM_FLAG(MERGEABLE) 433 #define VM_STACK INIT_VM_FLAG(STACK) 434 #ifdef CONFIG_STACK_GROWSUP 435 #define VM_STACK_EARLY INIT_VM_FLAG(STACK_EARLY) 436 #define VMA_STACK_EARLY mk_vma_flags(VMA_STACK_EARLY_BIT) 437 #else 438 #define VM_STACK_EARLY VM_NONE 439 #define VMA_STACK_EARLY EMPTY_VMA_FLAGS 440 #endif 441 #ifdef CONFIG_ARCH_HAS_PKEYS 442 #define VM_PKEY_SHIFT ((__force int)VMA_HIGH_ARCH_0_BIT) 443 /* Despite the naming, these are FLAGS not bits. */ 444 #define VM_PKEY_BIT0 INIT_VM_FLAG(PKEY_BIT0) 445 #define VM_PKEY_BIT1 INIT_VM_FLAG(PKEY_BIT1) 446 #define VM_PKEY_BIT2 INIT_VM_FLAG(PKEY_BIT2) 447 #if CONFIG_ARCH_PKEY_BITS > 3 448 #define VM_PKEY_BIT3 INIT_VM_FLAG(PKEY_BIT3) 449 #else 450 #define VM_PKEY_BIT3 VM_NONE 451 #endif /* CONFIG_ARCH_PKEY_BITS > 3 */ 452 #if CONFIG_ARCH_PKEY_BITS > 4 453 #define VM_PKEY_BIT4 INIT_VM_FLAG(PKEY_BIT4) 454 #else 455 #define VM_PKEY_BIT4 VM_NONE 456 #endif /* CONFIG_ARCH_PKEY_BITS > 4 */ 457 #endif /* CONFIG_ARCH_HAS_PKEYS */ 458 #if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_ARM64_GCS) || \ 459 defined(CONFIG_RISCV_USER_CFI) 460 #define VM_SHADOW_STACK INIT_VM_FLAG(SHADOW_STACK) 461 #define VMA_SHADOW_STACK mk_vma_flags(VMA_SHADOW_STACK_BIT) 462 #define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT, VMA_SHADOW_STACK_BIT) 463 #else 464 #define VM_SHADOW_STACK VM_NONE 465 #define VMA_SHADOW_STACK EMPTY_VMA_FLAGS 466 #define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT) 467 #endif 468 #if defined(CONFIG_PPC64) 469 #define VM_SAO INIT_VM_FLAG(SAO) 470 #elif defined(CONFIG_PARISC) 471 #define VM_GROWSUP INIT_VM_FLAG(GROWSUP) 472 #define VMA_GROWSUP mk_vma_flags(VMA_GROWSUP_BIT) 473 #elif defined(CONFIG_SPARC64) 474 #define VM_SPARC_ADI INIT_VM_FLAG(SPARC_ADI) 475 #define VM_ARCH_CLEAR INIT_VM_FLAG(ARCH_CLEAR) 476 #elif defined(CONFIG_ARM64) 477 #define VM_ARM64_BTI INIT_VM_FLAG(ARM64_BTI) 478 #define VM_ARCH_CLEAR INIT_VM_FLAG(ARCH_CLEAR) 479 #elif !defined(CONFIG_MMU) 480 #define VM_MAPPED_COPY INIT_VM_FLAG(MAPPED_COPY) 481 #endif 482 #ifndef VM_GROWSUP 483 #define VM_GROWSUP VM_NONE 484 #define VMA_GROWSUP EMPTY_VMA_FLAGS 485 #endif 486 #ifdef CONFIG_ARM64_MTE 487 #define VM_MTE INIT_VM_FLAG(MTE) 488 #define VM_MTE_ALLOWED INIT_VM_FLAG(MTE_ALLOWED) 489 #else 490 #define VM_MTE VM_NONE 491 #define VM_MTE_ALLOWED VM_NONE 492 #endif 493 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR 494 #define VM_UFFD_MINOR INIT_VM_FLAG(UFFD_MINOR) 495 #else 496 #define VM_UFFD_MINOR VM_NONE 497 #endif 498 #ifdef CONFIG_USERFAULTFD_RWP 499 #define VM_UFFD_RWP INIT_VM_FLAG(UFFD_RWP) 500 #else 501 #define VM_UFFD_RWP VM_NONE 502 #endif 503 504 /* 505 * vma_flags_t masks for the userfaultfd VMA flags. The two high-bit modes are 506 * gated on the same configs as their VM_* flags above -- both of which imply 507 * 64BIT -- so an out-of-range bit is never fed to mk_vma_flags() on a build 508 * whose bitmap cannot hold it. 509 */ 510 #define VMA_UFFD_MISSING mk_vma_flags(VMA_UFFD_MISSING_BIT) 511 #define VMA_UFFD_WP mk_vma_flags(VMA_UFFD_WP_BIT) 512 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR 513 #define VMA_UFFD_MINOR mk_vma_flags(VMA_UFFD_MINOR_BIT) 514 #else 515 #define VMA_UFFD_MINOR EMPTY_VMA_FLAGS 516 #endif 517 #ifdef CONFIG_USERFAULTFD_RWP 518 #define VMA_UFFD_RWP mk_vma_flags(VMA_UFFD_RWP_BIT) 519 #else 520 #define VMA_UFFD_RWP EMPTY_VMA_FLAGS 521 #endif 522 523 #ifdef CONFIG_64BIT 524 #define VM_ALLOW_ANY_UNCACHED INIT_VM_FLAG(ALLOW_ANY_UNCACHED) 525 #define VM_SEALED INIT_VM_FLAG(SEALED) 526 #else 527 #define VM_ALLOW_ANY_UNCACHED VM_NONE 528 #define VM_SEALED VM_NONE 529 #endif 530 #if defined(CONFIG_64BIT) || defined(CONFIG_PPC32) 531 #define VM_DROPPABLE INIT_VM_FLAG(DROPPABLE) 532 #define VMA_DROPPABLE mk_vma_flags(VMA_DROPPABLE_BIT) 533 #else 534 #define VM_DROPPABLE VM_NONE 535 #define VMA_DROPPABLE EMPTY_VMA_FLAGS 536 #endif 537 538 /* Bits set in the VMA until the stack is in its final location */ 539 #define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY) 540 #define VMA_STACK_INCOMPLETE_SETUP append_vma_flags( \ 541 VMA_STACK_EARLY, VMA_RAND_READ_BIT, VMA_SEQ_READ_BIT) 542 543 #define TASK_EXEC_BIT ((current->personality & READ_IMPLIES_EXEC) ? \ 544 VMA_EXEC_BIT : VMA_READ_BIT) 545 546 /* Common data flag combinations */ 547 #define VMA_DATA_FLAGS_TSK_EXEC mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, \ 548 TASK_EXEC_BIT, VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, \ 549 VMA_MAYEXEC_BIT) 550 #define VMA_DATA_FLAGS_NON_EXEC mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, \ 551 VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, VMA_MAYEXEC_BIT) 552 #define VMA_DATA_FLAGS_EXEC mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, \ 553 VMA_EXEC_BIT, VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, \ 554 VMA_MAYEXEC_BIT) 555 556 #ifndef VMA_DATA_DEFAULT_FLAGS /* arch can override this */ 557 #define VMA_DATA_DEFAULT_FLAGS VMA_DATA_FLAGS_EXEC 558 #endif 559 560 #ifndef VMA_STACK_DEFAULT_FLAGS /* arch can override this */ 561 #define VMA_STACK_DEFAULT_FLAGS VMA_DATA_DEFAULT_FLAGS 562 #endif 563 564 #define VMA_STACK_FLAGS append_vma_flags(VMA_STACK_DEFAULT_FLAGS, \ 565 VMA_STACK_BIT, VMA_ACCOUNT_BIT) 566 567 /* Temporary until VMA flags conversion complete. */ 568 #define VM_STACK_FLAGS vma_flags_to_legacy(VMA_STACK_FLAGS) 569 570 #ifdef CONFIG_MSEAL_SYSTEM_MAPPINGS 571 #define VM_SEALED_SYSMAP VM_SEALED 572 #else 573 #define VM_SEALED_SYSMAP VM_NONE 574 #endif 575 576 /* VMA basic access permission flags */ 577 #define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC) 578 #define VMA_ACCESS_FLAGS mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT) 579 580 /* 581 * Physically remapped pages are special. Tell the 582 * rest of the world about it: 583 * IO tells people not to look at these pages 584 * (accesses can have side effects). 585 * PFNMAP tells the core MM that the base pages are just 586 * raw PFN mappings, and do not have a "struct page" associated 587 * with them. 588 * DONTEXPAND 589 * Disable vma merging and expanding with mremap(). 590 * DONTDUMP 591 * Omit vma from core dump, even when VM_IO turned off. 592 */ 593 #define VMA_REMAP_FLAGS mk_vma_flags(VMA_IO_BIT, VMA_PFNMAP_BIT, \ 594 VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT) 595 596 /* This mask defines which mm->def_flags a process can inherit its parent */ 597 #define VM_INIT_DEF_MASK VM_NOHUGEPAGE 598 599 /* This mask represents all the VMA flag bits used by mlock */ 600 #define VM_LOCKED_MASK (VM_LOCKED | VM_LOCKONFAULT) 601 602 #define VMA_LOCKED_MASK mk_vma_flags(VMA_LOCKED_BIT, VMA_LOCKONFAULT_BIT) 603 604 /* These flags can be updated atomically via VMA/mmap read lock. */ 605 #define VM_ATOMIC_SET_ALLOWED VM_MAYBE_GUARD 606 607 /* Arch-specific flags to clear when updating VM flags on protection change */ 608 #ifndef VM_ARCH_CLEAR 609 #define VM_ARCH_CLEAR VM_NONE 610 #endif 611 #define VM_FLAGS_CLEAR (ARCH_VM_PKEY_FLAGS | VM_ARCH_CLEAR) 612 613 /* 614 * Flags which should be 'sticky' on merge - that is, flags which, when one VMA 615 * possesses it but the other does not, the merged VMA should nonetheless have 616 * applied to it: 617 * 618 * VMA_SOFTDIRTY_BIT - if a VMA is marked soft-dirty, that is has not had its 619 * references cleared via /proc/$pid/clear_refs, any 620 * merged VMA should be considered soft-dirty also as it 621 * operates at a VMA granularity. 622 * 623 * VMA_MAYBE_GUARD_BIT - If a VMA may have guard regions in place it implies 624 * that mapped page tables may contain metadata not 625 * described by the VMA and thus any merged VMA may also 626 * contain this metadata, and thus we must make this flag 627 * sticky. 628 */ 629 #ifdef CONFIG_MEM_SOFT_DIRTY 630 #define VMA_STICKY_FLAGS mk_vma_flags(VMA_SOFTDIRTY_BIT, VMA_MAYBE_GUARD_BIT) 631 #else 632 #define VMA_STICKY_FLAGS mk_vma_flags(VMA_MAYBE_GUARD_BIT) 633 #endif 634 635 /* 636 * VMA flags we ignore for the purposes of merge, i.e. one VMA possessing one 637 * of these flags and the other not does not preclude a merge. 638 * 639 * VMA_STICKY_FLAGS - When merging VMAs, VMA flags must match, unless they 640 * are 'sticky'. If any sticky flags exist in either VMA, 641 * we simply set all of them on the merged VMA. 642 */ 643 #define VMA_IGNORE_MERGE_FLAGS VMA_STICKY_FLAGS 644 645 /* 646 * Flags which should result in page tables being copied on fork. These are 647 * flags which indicate that the VMA maps page tables which cannot be 648 * reconsistuted upon page fault, so necessitate page table copying upon fork. 649 * 650 * Note that these flags should be compared with the DESTINATION VMA not the 651 * source: VM_UFFD_WP and VM_UFFD_RWP may be cleared on the destination 652 * (dup_userfaultfd() -> userfaultfd_reset_ctx() when the parent context did 653 * not negotiate UFFD_FEATURE_EVENT_FORK), while all other flags propagate. 654 * 655 * VM_PFNMAP / VM_MIXEDMAP - These contain kernel-mapped data which cannot be 656 * reasonably reconstructed on page fault. 657 * 658 * VM_UFFD_WP - Encodes metadata about an installed uffd 659 * VM_UFFD_RWP write- or read-write-protect handler, which 660 * cannot be reconstructed on page fault. 661 * 662 * We always copy pgtables when dst_vma has the 663 * uffd PTE bit in use even if it's file-backed 664 * (e.g. shmem). Because when the uffd bit is 665 * in use, the pgtable contains the protection 666 * information, that's something we can't 667 * retrieve from page cache, and skip copying 668 * will lose those info. 669 * 670 * VM_MAYBE_GUARD - Could contain page guard region markers which 671 * by design are a property of the page tables 672 * only and thus cannot be reconstructed on page 673 * fault. 674 */ 675 #define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_UFFD_RWP | \ 676 VM_MAYBE_GUARD) 677 678 /* 679 * mapping from the currently active vm_flags protection bits (the 680 * low four bits) to a page protection mask.. 681 */ 682 683 /* 684 * The default fault flags that should be used by most of the 685 * arch-specific page fault handlers. 686 */ 687 #define FAULT_FLAG_DEFAULT (FAULT_FLAG_ALLOW_RETRY | \ 688 FAULT_FLAG_KILLABLE | \ 689 FAULT_FLAG_INTERRUPTIBLE) 690 691 /** 692 * fault_flag_allow_retry_first - check ALLOW_RETRY the first time 693 * @flags: Fault flags. 694 * 695 * This is mostly used for places where we want to try to avoid taking 696 * the mmap_lock for too long a time when waiting for another condition 697 * to change, in which case we can try to be polite to release the 698 * mmap_lock in the first round to avoid potential starvation of other 699 * processes that would also want the mmap_lock. 700 * 701 * Return: true if the page fault allows retry and this is the first 702 * attempt of the fault handling; false otherwise. 703 */ 704 static inline bool fault_flag_allow_retry_first(enum fault_flag flags) 705 { 706 return (flags & FAULT_FLAG_ALLOW_RETRY) && 707 (!(flags & FAULT_FLAG_TRIED)); 708 } 709 710 #define FAULT_FLAG_TRACE \ 711 { FAULT_FLAG_WRITE, "WRITE" }, \ 712 { FAULT_FLAG_MKWRITE, "MKWRITE" }, \ 713 { FAULT_FLAG_ALLOW_RETRY, "ALLOW_RETRY" }, \ 714 { FAULT_FLAG_RETRY_NOWAIT, "RETRY_NOWAIT" }, \ 715 { FAULT_FLAG_KILLABLE, "KILLABLE" }, \ 716 { FAULT_FLAG_TRIED, "TRIED" }, \ 717 { FAULT_FLAG_USER, "USER" }, \ 718 { FAULT_FLAG_REMOTE, "REMOTE" }, \ 719 { FAULT_FLAG_INSTRUCTION, "INSTRUCTION" }, \ 720 { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }, \ 721 { FAULT_FLAG_VMA_LOCK, "VMA_LOCK" } 722 723 /* 724 * vm_fault is filled by the pagefault handler and passed to the vma's 725 * ->fault function. The vma's ->fault is responsible for returning a bitmask 726 * of VM_FAULT_xxx flags that give details about how the fault was handled. 727 * 728 * MM layer fills up gfp_mask for page allocations but fault handler might 729 * alter it if its implementation requires a different allocation context. 730 * 731 * pgoff should be used in favour of virtual_address, if possible. 732 */ 733 struct vm_fault { 734 const struct { 735 struct vm_area_struct *vma; /* Target VMA */ 736 gfp_t gfp_mask; /* gfp mask to be used for allocations */ 737 pgoff_t pgoff; /* Logical page offset based on vma */ 738 unsigned long address; /* Faulting virtual address - masked */ 739 unsigned long real_address; /* Faulting virtual address - unmasked */ 740 }; 741 enum fault_flag flags; /* FAULT_FLAG_xxx flags 742 * XXX: should really be 'const' */ 743 pmd_t *pmd; /* Pointer to pmd entry matching 744 * the 'address' */ 745 pud_t *pud; /* Pointer to pud entry matching 746 * the 'address' 747 */ 748 union { 749 pte_t orig_pte; /* Value of PTE at the time of fault */ 750 pmd_t orig_pmd; /* Value of PMD at the time of fault, 751 * used by PMD fault only. 752 */ 753 }; 754 755 struct page *cow_page; /* Page handler may use for COW fault */ 756 struct page *page; /* ->fault handlers should return a 757 * page here, unless VM_FAULT_NOPAGE 758 * is set (which is also implied by 759 * VM_FAULT_ERROR). 760 */ 761 /* These three entries are valid only while holding ptl lock */ 762 pte_t *pte; /* Pointer to pte entry matching 763 * the 'address'. NULL if the page 764 * table hasn't been allocated. 765 */ 766 spinlock_t *ptl; /* Page table lock. 767 * Protects pte page table if 'pte' 768 * is not NULL, otherwise pmd. 769 */ 770 pgtable_t prealloc_pte; /* Pre-allocated pte page table. 771 * vm_ops->map_pages() sets up a page 772 * table from atomic context. 773 * do_fault_around() pre-allocates 774 * page table to avoid allocation from 775 * atomic context. 776 */ 777 }; 778 779 struct vm_uffd_ops; 780 781 /* 782 * These are the virtual MM functions - opening of an area, closing and 783 * unmapping it (needed to keep files on disk up-to-date etc), pointer 784 * to the functions called when a no-page or a wp-page exception occurs. 785 */ 786 struct vm_operations_struct { 787 /** 788 * @open: Called when a VMA is remapped, split or forked. Not called 789 * upon first mapping a VMA. 790 * Context: User context. May sleep. Caller holds mmap_lock. 791 */ 792 void (*open)(struct vm_area_struct *vma); 793 /** 794 * @close: Called when the VMA is being removed from the MM. 795 * Context: User context. May sleep. Caller holds mmap_lock. 796 */ 797 void (*close)(struct vm_area_struct *vma); 798 /** 799 * @mapped: Called when the VMA is first mapped in the MM. Not called if 800 * the new VMA is merged with an adjacent VMA. 801 * 802 * The @vm_private_data field is an output field allowing the user to 803 * modify vma->vm_private_data as necessary. 804 * 805 * ONLY valid if set from f_op->mmap_prepare. Will result in an error if 806 * set from f_op->mmap. 807 * 808 * Returns %0 on success, or an error otherwise. On error, the VMA will 809 * be unmapped. 810 * 811 * Context: User context. May sleep. Caller holds mmap_lock. 812 */ 813 int (*mapped)(unsigned long start, unsigned long end, pgoff_t pgoff, 814 const struct file *file, void **vm_private_data); 815 /* Called any time before splitting to check if it's allowed */ 816 int (*may_split)(struct vm_area_struct *vma, unsigned long addr); 817 int (*mremap)(struct vm_area_struct *vma); 818 /* 819 * Called by mprotect() to make driver-specific permission 820 * checks before mprotect() is finalised. The VMA must not 821 * be modified. Returns 0 if mprotect() can proceed. 822 */ 823 int (*mprotect)(struct vm_area_struct *vma, unsigned long start, 824 unsigned long end, unsigned long newflags); 825 vm_fault_t (*fault)(struct vm_fault *vmf); 826 vm_fault_t (*huge_fault)(struct vm_fault *vmf, unsigned int order); 827 vm_fault_t (*map_pages)(struct vm_fault *vmf, 828 pgoff_t start_pgoff, pgoff_t end_pgoff); 829 unsigned long (*pagesize)(struct vm_area_struct *vma); 830 831 /* notification that a previously read-only page is about to become 832 * writable, if an error is returned it will cause a SIGBUS */ 833 vm_fault_t (*page_mkwrite)(struct vm_fault *vmf); 834 835 /* same as page_mkwrite when using VM_PFNMAP|VM_MIXEDMAP */ 836 vm_fault_t (*pfn_mkwrite)(struct vm_fault *vmf); 837 838 /* called by access_process_vm when get_user_pages() fails, typically 839 * for use by special VMAs. See also generic_access_phys() for a generic 840 * implementation useful for any iomem mapping. 841 */ 842 int (*access)(struct vm_area_struct *vma, unsigned long addr, 843 void *buf, int len, int write); 844 845 /* Called by the /proc/PID/maps code to ask the vma whether it 846 * has a special name. Returning non-NULL will also cause this 847 * vma to be dumped unconditionally. */ 848 const char *(*name)(struct vm_area_struct *vma); 849 850 #ifdef CONFIG_NUMA 851 /* 852 * set_policy() op must add a reference to any non-NULL @new mempolicy 853 * to hold the policy upon return. Caller should pass NULL @new to 854 * remove a policy and fall back to surrounding context--i.e. do not 855 * install a MPOL_DEFAULT policy, nor the task or system default 856 * mempolicy. 857 */ 858 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new); 859 860 /* 861 * get_policy() op must add reference [mpol_get()] to any policy at 862 * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure 863 * in mm/mempolicy.c will do this automatically. 864 * get_policy() must NOT add a ref if the policy at (vma,addr) is not 865 * marked as MPOL_SHARED. vma policies are protected by the mmap_lock. 866 * If no [shared/vma] mempolicy exists at the addr, get_policy() op 867 * must return NULL--i.e., do not "fallback" to task or system default 868 * policy. 869 */ 870 struct mempolicy *(*get_policy)(struct vm_area_struct *vma, 871 unsigned long addr, pgoff_t *ilx); 872 #endif 873 #ifdef CONFIG_FIND_NORMAL_PAGE 874 /* 875 * Called by vm_normal_page() for special PTEs in @vma at @addr. This 876 * allows for returning a "normal" page from vm_normal_page() even 877 * though the PTE indicates that the "struct page" either does not exist 878 * or should not be touched: "special". 879 * 880 * Do not add new users: this really only works when a "normal" page 881 * was mapped, but then the PTE got changed to something weird (+ 882 * marked special) that would not make pte_pfn() identify the originally 883 * inserted page. 884 */ 885 struct page *(*find_normal_page)(struct vm_area_struct *vma, 886 unsigned long addr); 887 #endif /* CONFIG_FIND_NORMAL_PAGE */ 888 #ifdef CONFIG_USERFAULTFD 889 const struct vm_uffd_ops *uffd_ops; 890 #endif 891 }; 892 893 #ifdef CONFIG_NUMA_BALANCING 894 static inline void vma_numab_state_init(struct vm_area_struct *vma) 895 { 896 vma->numab_state = NULL; 897 } 898 static inline void vma_numab_state_free(struct vm_area_struct *vma) 899 { 900 kfree(vma->numab_state); 901 } 902 #else 903 static inline void vma_numab_state_init(struct vm_area_struct *vma) {} 904 static inline void vma_numab_state_free(struct vm_area_struct *vma) {} 905 #endif /* CONFIG_NUMA_BALANCING */ 906 907 /* 908 * These must be here rather than mmap_lock.h as dependent on vm_fault type, 909 * declared in this header. 910 */ 911 static inline void release_fault_lock(struct vm_fault *vmf) 912 { 913 if (vmf->flags & FAULT_FLAG_VMA_LOCK) 914 vma_end_read(vmf->vma); 915 else 916 mmap_read_unlock(vmf->vma->vm_mm); 917 } 918 919 static inline void assert_fault_locked(const struct vm_fault *vmf) 920 { 921 if (vmf->flags & FAULT_FLAG_VMA_LOCK) 922 vma_assert_locked(vmf->vma); 923 else 924 mmap_assert_locked(vmf->vma->vm_mm); 925 } 926 927 static inline bool mm_flags_test(int flag, const struct mm_struct *mm) 928 { 929 return test_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags)); 930 } 931 932 static inline bool mm_flags_test_and_set(int flag, struct mm_struct *mm) 933 { 934 return test_and_set_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags)); 935 } 936 937 static inline bool mm_flags_test_and_clear(int flag, struct mm_struct *mm) 938 { 939 return test_and_clear_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags)); 940 } 941 942 static inline void mm_flags_set(int flag, struct mm_struct *mm) 943 { 944 set_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags)); 945 } 946 947 static inline void mm_flags_clear(int flag, struct mm_struct *mm) 948 { 949 clear_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags)); 950 } 951 952 static inline void mm_flags_clear_all(struct mm_struct *mm) 953 { 954 bitmap_zero(ACCESS_PRIVATE(&mm->flags, __mm_flags), NUM_MM_FLAG_BITS); 955 } 956 957 extern const struct vm_operations_struct vma_dummy_vm_ops; 958 959 static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm) 960 { 961 memset(vma, 0, sizeof(*vma)); 962 vma->vm_mm = mm; 963 vma->vm_ops = &vma_dummy_vm_ops; 964 INIT_LIST_HEAD(&vma->anon_vma_chain); 965 vma_lock_init(vma, false); 966 } 967 968 /* Use when VMA is not part of the VMA tree and needs no locking */ 969 static inline void vm_flags_init(struct vm_area_struct *vma, 970 vm_flags_t flags) 971 { 972 VM_WARN_ON_ONCE(!pgtable_supports_soft_dirty() && (flags & VM_SOFTDIRTY)); 973 vma_flags_clear_all(&vma->flags); 974 vma_flags_overwrite_word(&vma->flags, flags); 975 } 976 977 /* 978 * Use when VMA is part of the VMA tree and modifications need coordination 979 * Note: vm_flags_reset and vm_flags_reset_once do not lock the vma and 980 * it should be locked explicitly beforehand. 981 */ 982 static inline void vm_flags_reset(struct vm_area_struct *vma, 983 vm_flags_t flags) 984 { 985 VM_WARN_ON_ONCE(!pgtable_supports_soft_dirty() && (flags & VM_SOFTDIRTY)); 986 vma_assert_write_locked(vma); 987 vm_flags_init(vma, flags); 988 } 989 990 static inline void vma_flags_reset_once(struct vm_area_struct *vma, 991 vma_flags_t *flags) 992 { 993 const unsigned long word = flags->__vma_flags[0]; 994 995 /* It is assumed only the first system word must be written once. */ 996 vma_flags_overwrite_word_once(&vma->flags, word); 997 /* The remainder can be copied normally. */ 998 if (NUM_VMA_FLAG_BITS > BITS_PER_LONG) { 999 unsigned long *dst = &vma->flags.__vma_flags[1]; 1000 const unsigned long *src = &flags->__vma_flags[1]; 1001 1002 bitmap_copy(dst, src, NUM_VMA_FLAG_BITS - BITS_PER_LONG); 1003 } 1004 } 1005 1006 static inline void vm_flags_set(struct vm_area_struct *vma, 1007 vm_flags_t flags) 1008 { 1009 vma_start_write(vma); 1010 vma_flags_set_word(&vma->flags, flags); 1011 } 1012 1013 static inline void vm_flags_clear(struct vm_area_struct *vma, 1014 vm_flags_t flags) 1015 { 1016 VM_WARN_ON_ONCE(!pgtable_supports_soft_dirty() && (flags & VM_SOFTDIRTY)); 1017 vma_start_write(vma); 1018 vma_flags_clear_word(&vma->flags, flags); 1019 } 1020 1021 /* 1022 * Use only if VMA is not part of the VMA tree or has no other users and 1023 * therefore needs no locking. 1024 */ 1025 static inline void __vm_flags_mod(struct vm_area_struct *vma, 1026 vm_flags_t set, vm_flags_t clear) 1027 { 1028 vm_flags_init(vma, (vma->vm_flags | set) & ~clear); 1029 } 1030 1031 /* 1032 * Use only when the order of set/clear operations is unimportant, otherwise 1033 * use vm_flags_{set|clear} explicitly. 1034 */ 1035 static inline void vm_flags_mod(struct vm_area_struct *vma, 1036 vm_flags_t set, vm_flags_t clear) 1037 { 1038 vma_start_write(vma); 1039 __vm_flags_mod(vma, set, clear); 1040 } 1041 1042 static __always_inline bool __vma_atomic_valid_flag(struct vm_area_struct *vma, 1043 vma_flag_t bit) 1044 { 1045 const vm_flags_t mask = BIT((__force int)bit); 1046 1047 /* Only specific flags are permitted */ 1048 if (WARN_ON_ONCE(!(mask & VM_ATOMIC_SET_ALLOWED))) 1049 return false; 1050 1051 return true; 1052 } 1053 1054 /* 1055 * Set VMA flag atomically. Requires only VMA/mmap read lock. Only specific 1056 * valid flags are allowed to do this. 1057 */ 1058 static __always_inline void vma_set_atomic_flag(struct vm_area_struct *vma, 1059 vma_flag_t bit) 1060 { 1061 unsigned long *bitmap = vma->flags.__vma_flags; 1062 1063 vma_assert_stabilised(vma); 1064 if (__vma_atomic_valid_flag(vma, bit)) 1065 set_bit((__force int)bit, bitmap); 1066 } 1067 1068 /* 1069 * Test for VMA flag atomically. Requires no locks. Only specific valid flags 1070 * are allowed to do this. 1071 * 1072 * This is necessarily racey, so callers must ensure that serialisation is 1073 * achieved through some other means, or that races are permissible. 1074 */ 1075 static __always_inline bool vma_test_atomic_flag(struct vm_area_struct *vma, 1076 vma_flag_t bit) 1077 { 1078 if (__vma_atomic_valid_flag(vma, bit)) 1079 return test_bit((__force int)bit, &vma->vm_flags); 1080 1081 return false; 1082 } 1083 1084 /* Set an individual VMA flag in flags, non-atomically. */ 1085 static __always_inline void vma_flags_set_flag(vma_flags_t *flags, 1086 vma_flag_t bit) 1087 { 1088 unsigned long *bitmap = flags->__vma_flags; 1089 1090 __set_bit((__force int)bit, bitmap); 1091 } 1092 1093 static __always_inline vma_flags_t __mk_vma_flags(vma_flags_t flags, 1094 size_t count, const vma_flag_t *bits) 1095 { 1096 int i; 1097 1098 for (i = 0; i < count; i++) 1099 vma_flags_set_flag(&flags, bits[i]); 1100 return flags; 1101 } 1102 1103 /* 1104 * Helper macro which bitwise-or combines the specified input flags into a 1105 * vma_flags_t bitmap value. E.g.: 1106 * 1107 * vma_flags_t flags = mk_vma_flags(VMA_IO_BIT, VMA_PFNMAP_BIT, 1108 * VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT); 1109 * 1110 * The compiler cleverly optimises away all of the work and this ends up being 1111 * equivalent to aggregating the values manually. 1112 */ 1113 #define mk_vma_flags(...) __mk_vma_flags(EMPTY_VMA_FLAGS, \ 1114 COUNT_ARGS(__VA_ARGS__), (const vma_flag_t []){__VA_ARGS__}) 1115 1116 /* 1117 * Helper macro which acts like mk_vma_flags, only appending to a copy of the 1118 * specified flags rather than establishing new flags. E.g.: 1119 * 1120 * vma_flags_t flags = append_vma_flags(VMA_STACK_DEFAULT_FLAGS, VMA_STACK_BIT, 1121 * VMA_ACCOUNT_BIT); 1122 */ 1123 #define append_vma_flags(flags, ...) __mk_vma_flags(flags, \ 1124 COUNT_ARGS(__VA_ARGS__), (const vma_flag_t []){__VA_ARGS__}) 1125 1126 /* Calculates the number of set bits in the specified VMA flags. */ 1127 static __always_inline int vma_flags_count(const vma_flags_t *flags) 1128 { 1129 const unsigned long *bitmap = flags->__vma_flags; 1130 1131 return bitmap_weight(bitmap, NUM_VMA_FLAG_BITS); 1132 } 1133 1134 /* 1135 * Test whether a specific VMA flag is set, e.g.: 1136 * 1137 * if (vma_flags_test(flags, VMA_READ_BIT)) { ... } 1138 */ 1139 static __always_inline bool vma_flags_test(const vma_flags_t *flags, 1140 vma_flag_t bit) 1141 { 1142 const unsigned long *bitmap = flags->__vma_flags; 1143 1144 return test_bit((__force int)bit, bitmap); 1145 } 1146 1147 /* 1148 * Obtain a set of VMA flags which contain the overlapping flags contained 1149 * within flags and to_and. 1150 */ 1151 static __always_inline vma_flags_t vma_flags_and_mask(const vma_flags_t *flags, 1152 vma_flags_t to_and) 1153 { 1154 vma_flags_t dst; 1155 unsigned long *bitmap_dst = dst.__vma_flags; 1156 const unsigned long *bitmap = flags->__vma_flags; 1157 const unsigned long *bitmap_to_and = to_and.__vma_flags; 1158 1159 bitmap_and(bitmap_dst, bitmap, bitmap_to_and, NUM_VMA_FLAG_BITS); 1160 return dst; 1161 } 1162 1163 /* 1164 * Obtain a set of VMA flags which contains the specified overlapping flags, 1165 * e.g.: 1166 * 1167 * vma_flags_t read_flags = vma_flags_and(&flags, VMA_READ_BIT, 1168 * VMA_MAY_READ_BIT); 1169 */ 1170 #define vma_flags_and(flags, ...) \ 1171 vma_flags_and_mask(flags, mk_vma_flags(__VA_ARGS__)) 1172 1173 /* Test each of to_test flags in flags, non-atomically. */ 1174 static __always_inline bool vma_flags_test_any_mask(const vma_flags_t *flags, 1175 vma_flags_t to_test) 1176 { 1177 const unsigned long *bitmap = flags->__vma_flags; 1178 const unsigned long *bitmap_to_test = to_test.__vma_flags; 1179 1180 return bitmap_intersects(bitmap_to_test, bitmap, NUM_VMA_FLAG_BITS); 1181 } 1182 1183 /* 1184 * Test whether any specified VMA flag is set, e.g.: 1185 * 1186 * if (vma_flags_test_any(flags, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... } 1187 */ 1188 #define vma_flags_test_any(flags, ...) \ 1189 vma_flags_test_any_mask(flags, mk_vma_flags(__VA_ARGS__)) 1190 1191 /* Test that ALL of the to_test flags are set, non-atomically. */ 1192 static __always_inline bool vma_flags_test_all_mask(const vma_flags_t *flags, 1193 vma_flags_t to_test) 1194 { 1195 const unsigned long *bitmap = flags->__vma_flags; 1196 const unsigned long *bitmap_to_test = to_test.__vma_flags; 1197 1198 return bitmap_subset(bitmap_to_test, bitmap, NUM_VMA_FLAG_BITS); 1199 } 1200 1201 /* 1202 * Test whether ALL specified VMA flags are set, e.g.: 1203 * 1204 * if (vma_flags_test_all(flags, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... } 1205 */ 1206 #define vma_flags_test_all(flags, ...) \ 1207 vma_flags_test_all_mask(flags, mk_vma_flags(__VA_ARGS__)) 1208 1209 /* 1210 * Helper to test that a flag mask of type vma_flags_t has a SINGLE flag set 1211 * (returning false if flagmask has no flags set). 1212 * 1213 * This is defined to make the semantics clearer when testing an optionally 1214 * defined VMA flags mask, e.g.: 1215 * 1216 * if (vma_flags_test_single_mask(&flags, VMA_DROPPABLE)) { ... } 1217 * 1218 * When VMA_DROPPABLE is defined if available, or set to EMPTY_VMA_FLAGS 1219 * otherwise. 1220 */ 1221 static __always_inline bool vma_flags_test_single_mask(const vma_flags_t *flags, 1222 vma_flags_t flagmask) 1223 { 1224 VM_WARN_ON_ONCE(vma_flags_count(&flagmask) > 1); 1225 1226 return vma_flags_test_any_mask(flags, flagmask); 1227 } 1228 1229 /* Set each of the to_set flags in flags, non-atomically. */ 1230 static __always_inline void vma_flags_set_mask(vma_flags_t *flags, 1231 vma_flags_t to_set) 1232 { 1233 unsigned long *bitmap = flags->__vma_flags; 1234 const unsigned long *bitmap_to_set = to_set.__vma_flags; 1235 1236 bitmap_or(bitmap, bitmap, bitmap_to_set, NUM_VMA_FLAG_BITS); 1237 } 1238 1239 /* 1240 * Set all specified VMA flags, e.g.: 1241 * 1242 * vma_flags_set(&flags, VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT); 1243 */ 1244 #define vma_flags_set(flags, ...) \ 1245 vma_flags_set_mask(flags, mk_vma_flags(__VA_ARGS__)) 1246 1247 static __always_inline vma_flags_t __mk_vma_flags_from_masks(size_t count, 1248 const vma_flags_t *masks) 1249 { 1250 vma_flags_t flags = EMPTY_VMA_FLAGS; 1251 size_t i; 1252 1253 for (i = 0; i < count; i++) 1254 vma_flags_set_mask(&flags, masks[i]); 1255 return flags; 1256 } 1257 1258 /* 1259 * Combine pre-computed vma_flags_t masks into one value, e.g.: 1260 * 1261 * vma_flags_t flags = mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_MINOR); 1262 * 1263 * Unlike mk_vma_flags(), which takes bit numbers, this takes whole masks -- 1264 * each of which may be EMPTY_VMA_FLAGS when its feature is unavailable -- so a 1265 * bit that does not exist on the current build is never materialised. 1266 */ 1267 #define mk_vma_flags_from_masks(...) \ 1268 __mk_vma_flags_from_masks(COUNT_ARGS(__VA_ARGS__), \ 1269 (const vma_flags_t []){__VA_ARGS__}) 1270 1271 /* Clear all of the to-clear flags in flags, non-atomically. */ 1272 static __always_inline void vma_flags_clear_mask(vma_flags_t *flags, 1273 vma_flags_t to_clear) 1274 { 1275 unsigned long *bitmap = flags->__vma_flags; 1276 const unsigned long *bitmap_to_clear = to_clear.__vma_flags; 1277 1278 bitmap_andnot(bitmap, bitmap, bitmap_to_clear, NUM_VMA_FLAG_BITS); 1279 } 1280 1281 /* 1282 * Clear all specified individual flags, e.g.: 1283 * 1284 * vma_flags_clear(&flags, VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT); 1285 */ 1286 #define vma_flags_clear(flags, ...) \ 1287 vma_flags_clear_mask(flags, mk_vma_flags(__VA_ARGS__)) 1288 1289 /* 1290 * Obtain a VMA flags value containing those flags that are present in flags or 1291 * flags_other but not in both. 1292 */ 1293 static __always_inline vma_flags_t vma_flags_diff_pair(const vma_flags_t *flags, 1294 const vma_flags_t *flags_other) 1295 { 1296 vma_flags_t dst; 1297 const unsigned long *bitmap_other = flags_other->__vma_flags; 1298 const unsigned long *bitmap = flags->__vma_flags; 1299 unsigned long *bitmap_dst = dst.__vma_flags; 1300 1301 bitmap_xor(bitmap_dst, bitmap, bitmap_other, NUM_VMA_FLAG_BITS); 1302 return dst; 1303 } 1304 1305 /* Determine if flags and flags_other have precisely the same flags set. */ 1306 static __always_inline bool vma_flags_same_pair(const vma_flags_t *flags, 1307 const vma_flags_t *flags_other) 1308 { 1309 const unsigned long *bitmap = flags->__vma_flags; 1310 const unsigned long *bitmap_other = flags_other->__vma_flags; 1311 1312 return bitmap_equal(bitmap, bitmap_other, NUM_VMA_FLAG_BITS); 1313 } 1314 1315 /* Determine if flags and flags_other have precisely the same flags set. */ 1316 static __always_inline bool vma_flags_same_mask(const vma_flags_t *flags, 1317 vma_flags_t flags_other) 1318 { 1319 const unsigned long *bitmap = flags->__vma_flags; 1320 const unsigned long *bitmap_other = flags_other.__vma_flags; 1321 1322 return bitmap_equal(bitmap, bitmap_other, NUM_VMA_FLAG_BITS); 1323 } 1324 1325 /* 1326 * Helper macro to determine if only the specific flags are set, e.g.: 1327 * 1328 * if (vma_flags_same(&flags, VMA_WRITE_BIT) { ... } 1329 */ 1330 #define vma_flags_same(flags, ...) \ 1331 vma_flags_same_mask(flags, mk_vma_flags(__VA_ARGS__)) 1332 1333 /* 1334 * Test whether a specific flag in the VMA is set, e.g.: 1335 * 1336 * if (vma_test(vma, VMA_READ_BIT)) { ... } 1337 */ 1338 static __always_inline bool vma_test(const struct vm_area_struct *vma, 1339 vma_flag_t bit) 1340 { 1341 return vma_flags_test(&vma->flags, bit); 1342 } 1343 1344 /* Helper to test any VMA flags in a VMA . */ 1345 static __always_inline bool vma_test_any_mask(const struct vm_area_struct *vma, 1346 vma_flags_t flags) 1347 { 1348 return vma_flags_test_any_mask(&vma->flags, flags); 1349 } 1350 1351 /* 1352 * Helper macro for testing whether any VMA flags are set in a VMA, 1353 * e.g.: 1354 * 1355 * if (vma_test_any(vma, VMA_IO_BIT, VMA_PFNMAP_BIT, 1356 * VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT)) { ... } 1357 */ 1358 #define vma_test_any(vma, ...) \ 1359 vma_test_any_mask(vma, mk_vma_flags(__VA_ARGS__)) 1360 1361 /* 1362 * Helper to test that ALL specified flags are set in a VMA. 1363 * 1364 * Note: appropriate locks must be held, this function does not acquire them for 1365 * you. 1366 */ 1367 static __always_inline bool vma_test_all_mask(const struct vm_area_struct *vma, 1368 vma_flags_t flags) 1369 { 1370 return vma_flags_test_all_mask(&vma->flags, flags); 1371 } 1372 1373 /* 1374 * Helper macro for checking that ALL specified flags are set in a VMA, e.g.: 1375 * 1376 * if (vma_test_all(vma, VMA_READ_BIT, VMA_MAYREAD_BIT) { ... } 1377 */ 1378 #define vma_test_all(vma, ...) \ 1379 vma_test_all_mask(vma, mk_vma_flags(__VA_ARGS__)) 1380 1381 /* 1382 * Helper to test that a flag mask of type vma_flags_t has a SINGLE flag set 1383 * (returning false if flagmask has no flags set). 1384 * 1385 * This is useful when a flag needs to be either defined or not depending upon 1386 * kernel configuration, e.g.: 1387 * 1388 * if (vma_test_single_mask(vma, VMA_DROPPABLE)) { ... } 1389 * 1390 * When VMA_DROPPABLE is defined if available, or set to EMPTY_VMA_FLAGS 1391 * otherwise. 1392 */ 1393 static __always_inline bool 1394 vma_test_single_mask(const struct vm_area_struct *vma, vma_flags_t flagmask) 1395 { 1396 return vma_flags_test_single_mask(&vma->flags, flagmask); 1397 } 1398 1399 /* 1400 * Helper to set all VMA flags in a VMA. 1401 * 1402 * Note: appropriate locks must be held, this function does not acquire them for 1403 * you. 1404 */ 1405 static __always_inline void vma_set_flags_mask(struct vm_area_struct *vma, 1406 vma_flags_t flags) 1407 { 1408 vma_flags_set_mask(&vma->flags, flags); 1409 } 1410 1411 /* 1412 * Helper macro for specifying VMA flags in a VMA, e.g.: 1413 * 1414 * vma_set_flags(vma, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT, 1415 * VMA_DONTDUMP_BIT); 1416 * 1417 * Note: appropriate locks must be held, this function does not acquire them for 1418 * you. 1419 */ 1420 #define vma_set_flags(vma, ...) \ 1421 vma_set_flags_mask(vma, mk_vma_flags(__VA_ARGS__)) 1422 1423 /* Helper to clear all VMA flags in a VMA. */ 1424 static __always_inline void vma_clear_flags_mask(struct vm_area_struct *vma, 1425 vma_flags_t flags) 1426 { 1427 vma_flags_clear_mask(&vma->flags, flags); 1428 } 1429 1430 /* 1431 * Helper macro for clearing VMA flags, e.g.: 1432 * 1433 * vma_clear_flags(vma, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT, 1434 * VMA_DONTDUMP_BIT); 1435 */ 1436 #define vma_clear_flags(vma, ...) \ 1437 vma_clear_flags_mask(vma, mk_vma_flags(__VA_ARGS__)) 1438 1439 /* 1440 * Test whether a specific VMA flag is set in a VMA descriptor, e.g.: 1441 * 1442 * if (vma_desc_test(desc, VMA_READ_BIT)) { ... } 1443 */ 1444 static __always_inline bool vma_desc_test(const struct vm_area_desc *desc, 1445 vma_flag_t bit) 1446 { 1447 return vma_flags_test(&desc->vma_flags, bit); 1448 } 1449 1450 /* Helper to test any VMA flags in a VMA descriptor. */ 1451 static __always_inline bool vma_desc_test_any_mask(const struct vm_area_desc *desc, 1452 vma_flags_t flags) 1453 { 1454 return vma_flags_test_any_mask(&desc->vma_flags, flags); 1455 } 1456 1457 /* 1458 * Helper macro for testing whether any VMA flags are set in a VMA descriptor, 1459 * e.g.: 1460 * 1461 * if (vma_desc_test_any(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, 1462 * VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT)) { ... } 1463 */ 1464 #define vma_desc_test_any(desc, ...) \ 1465 vma_desc_test_any_mask(desc, mk_vma_flags(__VA_ARGS__)) 1466 1467 /* Helper to test all VMA flags in a VMA descriptor. */ 1468 static __always_inline bool vma_desc_test_all_mask(const struct vm_area_desc *desc, 1469 vma_flags_t flags) 1470 { 1471 return vma_flags_test_all_mask(&desc->vma_flags, flags); 1472 } 1473 1474 /* 1475 * Helper macro for testing whether ALL VMA flags are set in a VMA descriptor, 1476 * e.g.: 1477 * 1478 * if (vma_desc_test_all(desc, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... } 1479 */ 1480 #define vma_desc_test_all(desc, ...) \ 1481 vma_desc_test_all_mask(desc, mk_vma_flags(__VA_ARGS__)) 1482 1483 /* Helper to set all VMA flags in a VMA descriptor. */ 1484 static __always_inline void vma_desc_set_flags_mask(struct vm_area_desc *desc, 1485 vma_flags_t flags) 1486 { 1487 vma_flags_set_mask(&desc->vma_flags, flags); 1488 } 1489 1490 /* 1491 * Helper macro for specifying VMA flags for an input pointer to a struct 1492 * vm_area_desc object describing a proposed VMA, e.g.: 1493 * 1494 * vma_desc_set_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT, 1495 * VMA_DONTDUMP_BIT); 1496 */ 1497 #define vma_desc_set_flags(desc, ...) \ 1498 vma_desc_set_flags_mask(desc, mk_vma_flags(__VA_ARGS__)) 1499 1500 /* Helper to clear all VMA flags in a VMA descriptor. */ 1501 static __always_inline void vma_desc_clear_flags_mask(struct vm_area_desc *desc, 1502 vma_flags_t flags) 1503 { 1504 vma_flags_clear_mask(&desc->vma_flags, flags); 1505 } 1506 1507 /* 1508 * Helper macro for clearing VMA flags for an input pointer to a struct 1509 * vm_area_desc object describing a proposed VMA, e.g.: 1510 * 1511 * vma_desc_clear_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT, 1512 * VMA_DONTDUMP_BIT); 1513 */ 1514 #define vma_desc_clear_flags(desc, ...) \ 1515 vma_desc_clear_flags_mask(desc, mk_vma_flags(__VA_ARGS__)) 1516 1517 static inline void vma_set_anonymous(struct vm_area_struct *vma) 1518 { 1519 vma->vm_ops = NULL; 1520 } 1521 1522 static inline bool vma_is_anonymous(const struct vm_area_struct *vma) 1523 { 1524 return !vma->vm_ops; 1525 } 1526 1527 /* 1528 * Indicate if the VMA is a heap for the given task; for 1529 * /proc/PID/maps that is the heap of the main task. 1530 */ 1531 static inline bool vma_is_initial_heap(const struct vm_area_struct *vma) 1532 { 1533 return vma->vm_start < vma->vm_mm->brk && 1534 vma->vm_end > vma->vm_mm->start_brk; 1535 } 1536 1537 /* 1538 * Indicate if the VMA is a stack for the given task; for 1539 * /proc/PID/maps that is the stack of the main task. 1540 */ 1541 static inline bool vma_is_initial_stack(const struct vm_area_struct *vma) 1542 { 1543 /* 1544 * We make no effort to guess what a given thread considers to be 1545 * its "stack". It's not even well-defined for programs written 1546 * languages like Go. 1547 */ 1548 return vma->vm_start <= vma->vm_mm->start_stack && 1549 vma->vm_end >= vma->vm_mm->start_stack; 1550 } 1551 1552 static inline bool vma_flags_can_grow(const vma_flags_t *flags) 1553 { 1554 if (vma_flags_test_single_mask(flags, VMA_GROWSUP)) 1555 return true; 1556 if (vma_flags_test(flags, VMA_GROWSDOWN_BIT)) 1557 return true; 1558 1559 return false; 1560 } 1561 1562 static inline bool vma_can_grow(const struct vm_area_struct *vma) 1563 { 1564 return vma_flags_can_grow(&vma->flags); 1565 } 1566 1567 static inline bool vma_is_temporary_stack(const struct vm_area_struct *vma) 1568 { 1569 if (!vma_can_grow(vma)) 1570 return false; 1571 1572 if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == 1573 VM_STACK_INCOMPLETE_SETUP) 1574 return true; 1575 1576 return false; 1577 } 1578 1579 static inline bool vma_is_foreign(const struct vm_area_struct *vma) 1580 { 1581 if (!current->mm) 1582 return true; 1583 1584 if (current->mm != vma->vm_mm) 1585 return true; 1586 1587 return false; 1588 } 1589 1590 static inline bool vma_is_accessible(const struct vm_area_struct *vma) 1591 { 1592 return vma->vm_flags & VM_ACCESS_FLAGS; 1593 } 1594 1595 static inline bool is_shared_maywrite(const vma_flags_t *flags) 1596 { 1597 return vma_flags_test_all(flags, VMA_SHARED_BIT, VMA_MAYWRITE_BIT); 1598 } 1599 1600 static inline bool vma_is_shared_maywrite(const struct vm_area_struct *vma) 1601 { 1602 return is_shared_maywrite(&vma->flags); 1603 } 1604 1605 /** 1606 * vma_flags_is_hugetlb() - Do the specified VMA flags indicate that the 1607 * VMA is a hugetlb mapping? 1608 * @flags: The VMA flags to test. 1609 * 1610 * Returns: true if the flags indicate a hugetlb mapping, false otherwise. 1611 */ 1612 static inline bool vma_flags_is_hugetlb(const vma_flags_t *flags) 1613 { 1614 return IS_ENABLED(CONFIG_HUGETLB_PAGE) && 1615 vma_flags_test(flags, VMA_HUGETLB_BIT); 1616 } 1617 1618 /** 1619 * vma_is_hugetlb() - Is @vma a hugetlb mapping? 1620 * @vma: The VMA to test. 1621 * 1622 * Returns: true if @vma is a hugetlb mapping, false otherwise. 1623 */ 1624 static inline bool vma_is_hugetlb(const struct vm_area_struct *vma) 1625 { 1626 return vma_flags_is_hugetlb(&vma->flags); 1627 } 1628 1629 /** 1630 * vma_flags_is_mm_managed() - Do the specified VMA flags indicate that the 1631 * contents of the VMA are managed by core mm? 1632 * @flags: The VMA flags to test. 1633 * 1634 * Such a mapping is one established by a simple mmap() or brk() with no driver 1635 * or other kernel component involved. 1636 * 1637 * They are rmappable, and core mm services rmap operations including reclaim, 1638 * as well as population, CoW and merge/split (if eligible). 1639 * 1640 * Mappings which fail this test are those where this is not the case - 1641 * e.g. MMIO, user-mapped kernel pages, udmabuf (pfnmap shmem), etc. 1642 * 1643 * Returns: true if the flags indicate an mm-managed mapping. 1644 */ 1645 static inline bool vma_flags_is_mm_managed(const vma_flags_t *flags) 1646 { 1647 return !vma_flags_test_any(flags, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT); 1648 } 1649 1650 /** 1651 * vma_is_mm_managed() - Are the contents of @vma managed by core mm? 1652 * @vma: The VMA to test. 1653 * 1654 * See vma_flags_is_mm_managed() for a description of this property. 1655 * 1656 * Returns: true if the VMA is mm-managed. 1657 */ 1658 static inline bool vma_is_mm_managed(const struct vm_area_struct *vma) 1659 { 1660 return vma_flags_is_mm_managed(&vma->flags); 1661 } 1662 1663 /** 1664 * vma_flags_is_fixed_mapping() - Do the specified VMA flags indicate that this 1665 * is a fixed mapping that cannot be expanded or merged? 1666 * @flags: The VMA flags to test. 1667 * 1668 * Fixed mappings are those whose size is set at the point of mmap (for 1669 * instance, a mapping of a fixed range of memory established by a driver), 1670 * and thus cannot be expanded or merged. 1671 * 1672 * Returns: true if the flags indicate a fixed mapping. 1673 */ 1674 static inline bool vma_flags_is_fixed_mapping(const vma_flags_t *flags) 1675 { 1676 /* 1677 * VMA_PFNMAP_BIT should imply VMA_DONTEXPAND_BIT, but some callers set 1678 * only the former. 1679 */ 1680 return vma_flags_test_any(flags, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT); 1681 } 1682 1683 /** 1684 * vma_is_fixed_mapping() - Is this VMA a fixed mapping that cannot be 1685 * expanded or merged? 1686 * @vma: The VMA to test. 1687 * 1688 * See vma_flags_is_fixed_mapping() for a description of this property. 1689 * 1690 * Returns: true if the VMA maps a fixed mapping. 1691 */ 1692 static inline bool vma_is_fixed_mapping(const struct vm_area_struct *vma) 1693 { 1694 return vma_flags_is_fixed_mapping(&vma->flags); 1695 } 1696 1697 /** 1698 * vma_flags_is_mm_backed() - Do the specified VMA flags imply the mapping is 1699 * backed by core mm? 1700 * @flags: The VMA flags to test. 1701 * 1702 * An mm-backed mapping is one managed by core mm (see 1703 * vma_flags_is_mm_managed()) whose contents are established by core mm's own 1704 * fault path rather than a custom ->fault handler. 1705 * 1706 * Mappings not managed by core mm are populated by their owner, and fixed 1707 * mappings may be mm-managed but have their contents established by a custom 1708 * ->fault handler, so neither are mm-backed. hugetlb mappings are fixed, but 1709 * core mm owns them entirely, so they are. 1710 * 1711 * These are the only mappings which can sensibly be merged, locked, dumped or 1712 * be subject to uffd faulting, as all of these assume core mm and core mm 1713 * alone manages their contents. 1714 * 1715 * Returns: true if the flags imply an mm-backed mapping, otherwise false. 1716 */ 1717 static inline bool vma_flags_is_mm_backed(const vma_flags_t *flags) 1718 { 1719 /* hugetlb is a fixed mapping, but core mm owns it entirely. */ 1720 if (vma_flags_is_hugetlb(flags)) 1721 return true; 1722 /* 1723 * Mappings not managed by core mm are populated by their owner. 1724 * 1725 * Fixed mappings may be mm-managed, but their contents are established 1726 * by a custom ->fault handler, so core mm cannot assume ordinary fault 1727 * semantics over them. 1728 */ 1729 return vma_flags_is_mm_managed(flags) && !vma_flags_is_fixed_mapping(flags); 1730 } 1731 1732 /** 1733 * vma_is_mm_backed() - Is @vma backed by core mm? 1734 * @vma: The VMA to test. 1735 * 1736 * See vma_flags_is_mm_backed() for details. 1737 * 1738 * Returns: true if the VMA is mm-backed, otherwise false. 1739 */ 1740 static inline bool vma_is_mm_backed(const struct vm_area_struct *vma) 1741 { 1742 return vma_flags_is_mm_backed(&vma->flags); 1743 } 1744 1745 /** 1746 * vma_flags_can_merge() - Do the specified VMA flags permit the VMA to be 1747 * merged with another? 1748 * @flags: The VMA flags to test. 1749 * Returns: true if the flags permit merging, false otherwise. 1750 */ 1751 static inline bool vma_flags_can_merge(const vma_flags_t *flags) 1752 { 1753 /* 1754 * VMA merging assumes that a VMA's flags and fields completely describe 1755 * its state, which is only the case for mm-backed mappings. 1756 * 1757 * Mappings which are not mm-managed may have established state upon 1758 * mapping not embodied in any attribute of the VMA. 1759 * 1760 * Additionally, private (CoW) PFN maps encode the source PFN of the 1761 * range in vma->vm_pgoff, which may otherwise cause spurious merges. 1762 */ 1763 if (!vma_flags_is_mm_backed(flags)) 1764 return false; 1765 /* 1766 * hugetlb mappings contain per-VMA state, so their flags and fields do 1767 * not completely describe them. 1768 */ 1769 return !vma_flags_is_hugetlb(flags); 1770 } 1771 1772 /** 1773 * vma_can_merge() - Do @vma's flags permit it to be merged with another VMA? 1774 * @vma: The VMA to test. 1775 * Returns: true if the flags permit merging, otherwise false. 1776 */ 1777 static inline bool vma_can_merge(const struct vm_area_struct *vma) 1778 { 1779 return vma_flags_can_merge(&vma->flags); 1780 } 1781 1782 /** 1783 * vma_kernel_pagesize - Default page size granularity for this VMA. 1784 * @vma: The user mapping. 1785 * 1786 * The kernel page size specifies in which granularity VMA modifications 1787 * can be performed. Folios in this VMA will be aligned to, and at least 1788 * the size of the number of bytes returned by this function. 1789 * 1790 * The default kernel page size is not affected by Transparent Huge Pages 1791 * being in effect. 1792 * 1793 * Return: The default page size granularity for this VMA. 1794 */ 1795 static inline unsigned long vma_kernel_pagesize(struct vm_area_struct *vma) 1796 { 1797 if (unlikely(vma->vm_ops && vma->vm_ops->pagesize)) 1798 return vma->vm_ops->pagesize(vma); 1799 return PAGE_SIZE; 1800 } 1801 1802 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma); 1803 1804 static inline 1805 struct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max) 1806 { 1807 return mas_find(&vmi->mas, max - 1); 1808 } 1809 1810 static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi) 1811 { 1812 /* 1813 * Uses mas_find() to get the first VMA when the iterator starts. 1814 * Calling mas_next() could skip the first entry. 1815 */ 1816 return mas_find(&vmi->mas, ULONG_MAX); 1817 } 1818 1819 static inline 1820 struct vm_area_struct *vma_iter_next_range(struct vma_iterator *vmi) 1821 { 1822 return mas_next_range(&vmi->mas, ULONG_MAX); 1823 } 1824 1825 1826 static inline struct vm_area_struct *vma_prev(struct vma_iterator *vmi) 1827 { 1828 return mas_prev(&vmi->mas, 0); 1829 } 1830 1831 static inline int vma_iter_clear_gfp(struct vma_iterator *vmi, 1832 unsigned long start, unsigned long end, gfp_t gfp) 1833 { 1834 __mas_set_range(&vmi->mas, start, end - 1); 1835 mas_store_gfp(&vmi->mas, NULL, gfp); 1836 if (unlikely(mas_is_err(&vmi->mas))) 1837 return -ENOMEM; 1838 1839 return 0; 1840 } 1841 1842 /* Free any unused preallocations */ 1843 static inline void vma_iter_free(struct vma_iterator *vmi) 1844 { 1845 mas_destroy(&vmi->mas); 1846 } 1847 1848 static inline int vma_iter_bulk_store(struct vma_iterator *vmi, 1849 struct vm_area_struct *vma) 1850 { 1851 vmi->mas.index = vma->vm_start; 1852 vmi->mas.last = vma->vm_end - 1; 1853 mas_store(&vmi->mas, vma); 1854 if (unlikely(mas_is_err(&vmi->mas))) 1855 return -ENOMEM; 1856 1857 vma_mark_attached(vma); 1858 return 0; 1859 } 1860 1861 static inline void vma_iter_invalidate(struct vma_iterator *vmi) 1862 { 1863 mas_pause(&vmi->mas); 1864 } 1865 1866 static inline void vma_iter_set(struct vma_iterator *vmi, unsigned long addr) 1867 { 1868 mas_set(&vmi->mas, addr); 1869 } 1870 1871 #define for_each_vma(__vmi, __vma) \ 1872 while (((__vma) = vma_next(&(__vmi))) != NULL) 1873 1874 /* The MM code likes to work with exclusive end addresses */ 1875 #define for_each_vma_range(__vmi, __vma, __end) \ 1876 while (((__vma) = vma_find(&(__vmi), (__end))) != NULL) 1877 1878 #ifdef CONFIG_SHMEM 1879 /* 1880 * The vma_is_shmem is not inline because it is used only by slow 1881 * paths in userfault. 1882 */ 1883 bool vma_is_shmem(const struct vm_area_struct *vma); 1884 bool vma_is_anon_shmem(const struct vm_area_struct *vma); 1885 #else 1886 static inline bool vma_is_shmem(const struct vm_area_struct *vma) { return false; } 1887 static inline bool vma_is_anon_shmem(const struct vm_area_struct *vma) { return false; } 1888 #endif 1889 1890 int vma_is_stack_for_current(const struct vm_area_struct *vma); 1891 1892 /* flush_tlb_range() takes a vma, not a mm, and can care about flags */ 1893 #define TLB_FLUSH_VMA(mm,flags) { .vm_mm = (mm), .vm_flags = (flags) } 1894 1895 struct mmu_gather; 1896 struct inode; 1897 1898 extern void prep_compound_page(struct page *page, unsigned int order); 1899 1900 static inline unsigned int folio_large_order(const struct folio *folio) 1901 { 1902 return folio->_flags_1 & 0xff; 1903 } 1904 1905 #ifdef NR_PAGES_IN_LARGE_FOLIO 1906 static inline unsigned long folio_large_nr_pages(const struct folio *folio) 1907 { 1908 return folio->_nr_pages; 1909 } 1910 #else 1911 static inline unsigned long folio_large_nr_pages(const struct folio *folio) 1912 { 1913 return 1L << folio_large_order(folio); 1914 } 1915 #endif 1916 1917 /* 1918 * compound_order() can be called without holding a reference, which means 1919 * that niceties like page_folio() don't work. These callers should be 1920 * prepared to handle wild return values. For example, PG_head may be 1921 * set before the order is initialised, or this may be a tail page. 1922 * See compaction.c for some good examples. 1923 */ 1924 static inline unsigned int compound_order(const struct page *page) 1925 { 1926 const struct folio *folio = (struct folio *)page; 1927 1928 if (!test_bit(PG_head, &folio->flags.f)) 1929 return 0; 1930 return folio_large_order(folio); 1931 } 1932 1933 /** 1934 * folio_order - The allocation order of a folio. 1935 * @folio: The folio. 1936 * 1937 * A folio is composed of 2^order pages. See get_order() for the definition 1938 * of order. 1939 * 1940 * Return: The order of the folio. 1941 */ 1942 static inline unsigned int folio_order(const struct folio *folio) 1943 { 1944 if (!folio_test_large(folio)) 1945 return 0; 1946 return folio_large_order(folio); 1947 } 1948 1949 /** 1950 * folio_reset_order - Reset the folio order and derived _nr_pages 1951 * @folio: The folio. 1952 * 1953 * Reset the order and derived _nr_pages to 0. Must only be used in the 1954 * process of splitting large folios. 1955 */ 1956 static inline void folio_reset_order(struct folio *folio) 1957 { 1958 if (WARN_ON_ONCE(!folio_test_large(folio))) 1959 return; 1960 folio->_flags_1 &= ~0xffUL; 1961 #ifdef NR_PAGES_IN_LARGE_FOLIO 1962 folio->_nr_pages = 0; 1963 #endif 1964 } 1965 1966 #include <linux/huge_mm.h> 1967 1968 /* 1969 * Methods to modify the page usage count. 1970 * 1971 * What counts for a page usage: 1972 * - cache mapping (page->mapping) 1973 * - private data (page->private) 1974 * - page mapped in a task's page tables, each mapping 1975 * is counted separately 1976 * 1977 * Also, many kernel routines increase the page count before a critical 1978 * routine so they can be sure the page doesn't go away from under them. 1979 */ 1980 1981 /* 1982 * Drop a ref, return true if the refcount fell to zero (the page has no users) 1983 */ 1984 static inline int put_page_testzero(struct page *page) 1985 { 1986 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 1987 return page_ref_dec_and_test(page); 1988 } 1989 1990 static inline int folio_put_testzero(struct folio *folio) 1991 { 1992 return put_page_testzero(&folio->page); 1993 } 1994 1995 /* 1996 * Try to grab a ref unless the page has a refcount of zero, return false if 1997 * that is the case. 1998 * This can be called when MMU is off so it must not access 1999 * any of the virtual mappings. 2000 */ 2001 static inline bool get_page_unless_zero(struct page *page) 2002 { 2003 return page_ref_add_unless_zero(page, 1); 2004 } 2005 2006 static inline struct folio *folio_get_nontail_page(struct page *page) 2007 { 2008 if (unlikely(!get_page_unless_zero(page))) 2009 return NULL; 2010 return (struct folio *)page; 2011 } 2012 2013 extern int page_is_ram(unsigned long pfn); 2014 2015 enum { 2016 REGION_INTERSECTS, 2017 REGION_DISJOINT, 2018 REGION_MIXED, 2019 }; 2020 2021 int region_intersects(resource_size_t offset, size_t size, unsigned long flags, 2022 unsigned long desc); 2023 2024 /* Support for virtually mapped pages */ 2025 struct page *vmalloc_to_page(const void *addr); 2026 unsigned long vmalloc_to_pfn(const void *addr); 2027 2028 /* 2029 * Determine if an address is within the vmalloc range 2030 * 2031 * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there 2032 * is no special casing required. 2033 */ 2034 #ifdef CONFIG_MMU 2035 extern bool is_vmalloc_addr(const void *x); 2036 extern int is_vmalloc_or_module_addr(const void *x); 2037 #else 2038 static inline bool is_vmalloc_addr(const void *x) 2039 { 2040 return false; 2041 } 2042 static inline int is_vmalloc_or_module_addr(const void *x) 2043 { 2044 return 0; 2045 } 2046 #endif 2047 2048 /* 2049 * How many times the entire folio is mapped as a single unit (eg by a 2050 * PMD or PUD entry). This is probably not what you want, except for 2051 * debugging purposes or implementation of other core folio_*() primitives. 2052 */ 2053 static inline int folio_entire_mapcount(const struct folio *folio) 2054 { 2055 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 2056 if (!IS_ENABLED(CONFIG_64BIT) && unlikely(folio_large_order(folio) == 1)) 2057 return 0; 2058 return atomic_read(&folio->_entire_mapcount) + 1; 2059 } 2060 2061 static inline int folio_large_mapcount(const struct folio *folio) 2062 { 2063 VM_WARN_ON_FOLIO(!folio_test_large(folio), folio); 2064 return atomic_read(&folio->_large_mapcount) + 1; 2065 } 2066 2067 /** 2068 * folio_mapcount() - Number of mappings of this folio. 2069 * @folio: The folio. 2070 * 2071 * The folio mapcount corresponds to the number of present user page table 2072 * entries that reference any part of a folio. Each such present user page 2073 * table entry must be paired with exactly on folio reference. 2074 * 2075 * For ordindary folios, each user page table entry (PTE/PMD/PUD/...) counts 2076 * exactly once. 2077 * 2078 * For hugetlb folios, each abstracted "hugetlb" user page table entry that 2079 * references the entire folio counts exactly once, even when such special 2080 * page table entries are comprised of multiple ordinary page table entries. 2081 * 2082 * Will report 0 for pages which cannot be mapped into userspace, such as 2083 * slab, page tables and similar. 2084 * 2085 * Return: The number of times this folio is mapped. 2086 */ 2087 static inline int folio_mapcount(const struct folio *folio) 2088 { 2089 int mapcount; 2090 2091 if (likely(!folio_test_large(folio))) { 2092 mapcount = atomic_read(&folio->_mapcount) + 1; 2093 if (page_mapcount_is_type(mapcount)) 2094 mapcount = 0; 2095 return mapcount; 2096 } 2097 return folio_large_mapcount(folio); 2098 } 2099 2100 /** 2101 * folio_mapped - Is this folio mapped into userspace? 2102 * @folio: The folio. 2103 * 2104 * Return: True if any page in this folio is referenced by user page tables. 2105 */ 2106 static inline bool folio_mapped(const struct folio *folio) 2107 { 2108 return folio_mapcount(folio) >= 1; 2109 } 2110 2111 static inline struct page *virt_to_head_page(const void *x) 2112 { 2113 struct page *page = virt_to_page(x); 2114 2115 return compound_head(page); 2116 } 2117 2118 static inline struct folio *virt_to_folio(const void *x) 2119 { 2120 struct page *page = virt_to_page(x); 2121 2122 return page_folio(page); 2123 } 2124 2125 void __folio_put(struct folio *folio); 2126 2127 void split_page(struct page *page, unsigned int order); 2128 void folio_copy(struct folio *dst, struct folio *src); 2129 int folio_mc_copy(struct folio *dst, struct folio *src); 2130 2131 unsigned long nr_free_buffer_pages(void); 2132 2133 /* Returns the number of bytes in this potentially compound page. */ 2134 static inline unsigned long page_size(const struct page *page) 2135 { 2136 return PAGE_SIZE << compound_order(page); 2137 } 2138 2139 /* Returns the number of bits needed for the number of bytes in a page */ 2140 static inline unsigned int page_shift(struct page *page) 2141 { 2142 return PAGE_SHIFT + compound_order(page); 2143 } 2144 2145 /** 2146 * thp_order - Order of a transparent huge page. 2147 * @page: Head page of a transparent huge page. 2148 */ 2149 static inline unsigned int thp_order(struct page *page) 2150 { 2151 VM_BUG_ON_PGFLAGS(PageTail(page), page); 2152 return compound_order(page); 2153 } 2154 2155 /** 2156 * thp_size - Size of a transparent huge page. 2157 * @page: Head page of a transparent huge page. 2158 * 2159 * Return: Number of bytes in this page. 2160 */ 2161 static inline unsigned long thp_size(struct page *page) 2162 { 2163 return PAGE_SIZE << thp_order(page); 2164 } 2165 2166 #ifdef CONFIG_MMU 2167 /* 2168 * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when 2169 * servicing faults for write access. In the normal case, do always want 2170 * pte_mkwrite. But get_user_pages can cause write faults for mappings 2171 * that do not have writing enabled, when used by access_process_vm. 2172 */ 2173 static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) 2174 { 2175 if (likely(vma->vm_flags & VM_WRITE)) 2176 pte = pte_mkwrite(pte, vma); 2177 return pte; 2178 } 2179 2180 vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *page); 2181 void set_pte_range(struct vm_fault *vmf, struct folio *folio, 2182 struct page *page, unsigned int nr, unsigned long addr); 2183 2184 vm_fault_t finish_fault(struct vm_fault *vmf); 2185 #endif 2186 2187 /* 2188 * Multiple processes may "see" the same page. E.g. for untouched 2189 * mappings of /dev/null, all processes see the same page full of 2190 * zeroes, and text pages of executables and shared libraries have 2191 * only one copy in memory, at most, normally. 2192 * 2193 * For the non-reserved pages, page_count(page) denotes a reference count. 2194 * page_count() == 0 means the page is free. page->lru is then used for 2195 * freelist management in the buddy allocator. 2196 * page_count() > 0 means the page has been allocated. 2197 * 2198 * Pages are allocated by the slab allocator in order to provide memory 2199 * to kmalloc and kmem_cache_alloc. In this case, the management of the 2200 * page, and the fields in 'struct page' are the responsibility of mm/slab.c 2201 * unless a particular usage is carefully commented. (the responsibility of 2202 * freeing the kmalloc memory is the caller's, of course). 2203 * 2204 * A page may be used by anyone else who does a __get_free_page(). 2205 * In this case, page_count still tracks the references, and should only 2206 * be used through the normal accessor functions. The top bits of page->flags 2207 * and page->virtual store page management information, but all other fields 2208 * are unused and could be used privately, carefully. The management of this 2209 * page is the responsibility of the one who allocated it, and those who have 2210 * subsequently been given references to it. 2211 * 2212 * The other pages (we may call them "pagecache pages") are completely 2213 * managed by the Linux memory manager: I/O, buffers, swapping etc. 2214 * The following discussion applies only to them. 2215 * 2216 * A pagecache page contains an opaque `private' member, which belongs to the 2217 * page's address_space. Usually, this is the address of a circular list of 2218 * the page's disk buffers. It tells the VM to call into the filesystem to 2219 * release these pages. 2220 * 2221 * A folio may belong to an inode's memory mapping. In this case, 2222 * folio->mapping points to the inode, and folio->index is the file 2223 * offset of the folio, in units of PAGE_SIZE. 2224 * 2225 * If pagecache folios are not associated with an inode, they are said to be 2226 * anonymous folios. These may become associated with the swapcache, and in that 2227 * case PG_swapcache is set, and folio->private is an offset into the swapcache. 2228 * 2229 * In either case (swapcache or inode backed), the pagecache itself holds one 2230 * reference to the folio. Attaching filesystem private data via 2231 * folio_attach_private() also increments the refcount. Each user mapping also 2232 * has a reference to the folio. 2233 * 2234 * The pagecache pages are stored in a per-mapping radix tree, which is 2235 * rooted at mapping->i_pages, and indexed by offset. 2236 * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space 2237 * lists, we instead now tag pages as dirty/writeback in the radix tree. 2238 * 2239 * All pagecache pages may be subject to I/O: 2240 * - inode pages may need to be read from disk, 2241 * - inode pages which have been modified and are MAP_SHARED may need 2242 * to be written back to the inode on disk, 2243 * - anonymous pages (including MAP_PRIVATE file mappings) which have been 2244 * modified may need to be swapped out to swap space and (later) to be read 2245 * back into memory. 2246 */ 2247 2248 /* 127: arbitrary random number, small enough to assemble well */ 2249 #define folio_ref_zero_or_close_to_overflow(folio) \ 2250 ((unsigned int) folio_ref_count(folio) + 127u <= 127u) 2251 2252 /** 2253 * folio_get - Increment the reference count on a folio. 2254 * @folio: The folio. 2255 * 2256 * Context: May be called in any context, as long as you know that 2257 * you have a refcount on the folio. If you do not already have one, 2258 * folio_try_get() may be the right interface for you to use. 2259 */ 2260 static inline void folio_get(struct folio *folio) 2261 { 2262 VM_BUG_ON_FOLIO(folio_ref_zero_or_close_to_overflow(folio), folio); 2263 folio_ref_inc(folio); 2264 } 2265 2266 static inline void get_page(struct page *page) 2267 { 2268 struct folio *folio = page_folio(page); 2269 if (WARN_ON_ONCE(folio_test_slab(folio))) 2270 return; 2271 if (WARN_ON_ONCE(folio_test_large_kmalloc(folio))) 2272 return; 2273 folio_get(folio); 2274 } 2275 2276 static inline __must_check bool try_get_page(struct page *page) 2277 { 2278 page = compound_head(page); 2279 if (WARN_ON_ONCE(page_ref_count(page) <= 0)) 2280 return false; 2281 page_ref_inc(page); 2282 return true; 2283 } 2284 2285 /** 2286 * folio_put - Decrement the reference count on a folio. 2287 * @folio: The folio. 2288 * 2289 * If the folio's reference count reaches zero, the memory will be 2290 * released back to the page allocator and may be used by another 2291 * allocation immediately. Do not access the memory or the struct folio 2292 * after calling folio_put() unless you can be sure that it wasn't the 2293 * last reference. 2294 * 2295 * Context: May be called in process or interrupt context, but not in NMI 2296 * context. May be called while holding a spinlock. 2297 */ 2298 static inline void folio_put(struct folio *folio) 2299 { 2300 if (folio_put_testzero(folio)) 2301 __folio_put(folio); 2302 } 2303 2304 /** 2305 * folio_put_refs - Reduce the reference count on a folio. 2306 * @folio: The folio. 2307 * @refs: The amount to subtract from the folio's reference count. 2308 * 2309 * If the folio's reference count reaches zero, the memory will be 2310 * released back to the page allocator and may be used by another 2311 * allocation immediately. Do not access the memory or the struct folio 2312 * after calling folio_put_refs() unless you can be sure that these weren't 2313 * the last references. 2314 * 2315 * Context: May be called in process or interrupt context, but not in NMI 2316 * context. May be called while holding a spinlock. 2317 */ 2318 static inline void folio_put_refs(struct folio *folio, int refs) 2319 { 2320 if (folio_ref_sub_and_test(folio, refs)) 2321 __folio_put(folio); 2322 } 2323 2324 void folios_put_refs(struct folio_batch *folios, unsigned int *refs); 2325 2326 /* 2327 * union release_pages_arg - an array of pages or folios 2328 * 2329 * release_pages() releases a simple array of multiple pages, and 2330 * accepts various different forms of said page array: either 2331 * a regular old boring array of pages, an array of folios, or 2332 * an array of encoded page pointers. 2333 * 2334 * The transparent union syntax for this kind of "any of these 2335 * argument types" is all kinds of ugly, so look away. 2336 */ 2337 typedef union { 2338 struct page **pages; 2339 struct folio **folios; 2340 struct encoded_page **encoded_pages; 2341 } release_pages_arg __attribute__ ((__transparent_union__)); 2342 2343 void release_pages(release_pages_arg, int nr); 2344 2345 /** 2346 * folios_put - Decrement the reference count on an array of folios. 2347 * @folios: The folios. 2348 * 2349 * Like folio_put(), but for a batch of folios. This is more efficient 2350 * than writing the loop yourself as it will optimise the locks which need 2351 * to be taken if the folios are freed. The folios batch is returned 2352 * empty and ready to be reused for another batch; there is no need to 2353 * reinitialise it. 2354 * 2355 * Context: May be called in process or interrupt context, but not in NMI 2356 * context. May be called while holding a spinlock. 2357 */ 2358 static inline void folios_put(struct folio_batch *folios) 2359 { 2360 folios_put_refs(folios, NULL); 2361 } 2362 2363 static inline void put_page(struct page *page) 2364 { 2365 struct folio *folio = page_folio(page); 2366 2367 if (folio_test_slab(folio) || folio_test_large_kmalloc(folio)) 2368 return; 2369 2370 folio_put(folio); 2371 } 2372 2373 /* 2374 * GUP_PIN_COUNTING_BIAS, and the associated functions that use it, overload 2375 * the page's refcount so that two separate items are tracked: the original page 2376 * reference count, and also a new count of how many pin_user_pages() calls were 2377 * made against the page. ("gup-pinned" is another term for the latter). 2378 * 2379 * With this scheme, pin_user_pages() becomes special: such pages are marked as 2380 * distinct from normal pages. As such, the unpin_user_page() call (and its 2381 * variants) must be used in order to release gup-pinned pages. 2382 * 2383 * Choice of value: 2384 * 2385 * By making GUP_PIN_COUNTING_BIAS a power of two, debugging of page reference 2386 * counts with respect to pin_user_pages() and unpin_user_page() becomes 2387 * simpler, due to the fact that adding an even power of two to the page 2388 * refcount has the effect of using only the upper N bits, for the code that 2389 * counts up using the bias value. This means that the lower bits are left for 2390 * the exclusive use of the original code that increments and decrements by one 2391 * (or at least, by much smaller values than the bias value). 2392 * 2393 * Of course, once the lower bits overflow into the upper bits (and this is 2394 * OK, because subtraction recovers the original values), then visual inspection 2395 * no longer suffices to directly view the separate counts. However, for normal 2396 * applications that don't have huge page reference counts, this won't be an 2397 * issue. 2398 * 2399 * Locking: the lockless algorithm described in folio_try_get_rcu() 2400 * provides safe operation for get_user_pages(), folio_mkclean() and 2401 * other calls that race to set up page table entries. 2402 */ 2403 #define GUP_PIN_COUNTING_BIAS (1U << 10) 2404 2405 void unpin_user_page(struct page *page); 2406 void unpin_folio(struct folio *folio); 2407 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages, 2408 bool make_dirty); 2409 void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages, 2410 bool make_dirty); 2411 void unpin_user_pages(struct page **pages, unsigned long npages); 2412 void unpin_user_folio(struct folio *folio, unsigned long npages); 2413 void unpin_folios(struct folio **folios, unsigned long nfolios); 2414 2415 /** 2416 * vma_flags_is_cow_mapping() - Do these VMA flags imply a CoW mapping? 2417 * @flags: The VMA flags to check. 2418 * 2419 * Mappings which could be CoW'd (subject to Copy-On-Write faults) are 2420 * described as CoW mappings. 2421 * 2422 * All mappings backed by anonymous folios (all anonymous mappings and most 2423 * MAP_PRIVATE-file backed ranges) are CoW mappings. 2424 * 2425 * All other mappings (including all MAP_SHARED mappings) are non-CoW. 2426 * 2427 * The criteria are !VMA_SHARED_BIT, VMA_MAYWRITE_BIT. 2428 * 2429 * VMA_MAYWRITE_BIT is checked instead of VMA_WRITE_BIT to account for both 2430 * future mprotect() calls which can render a read-only mapping writable, and 2431 * GUP with FOLL_FORCE (e.g. ptrace) which can CoW a read-only mapping. 2432 * 2433 * - No anonymous mapping can ever clear VMA_MAYWRITE_BIT. 2434 * 2435 * - Writes to anonymous mappings do not immediately result in CoW faults but 2436 * may do so after the process is forked or if a read is followed by a 2437 * write. 2438 * 2439 * - Writes to MAP_PRIVATE file-backed mappings result in CoW faults and may 2440 * do so again after fork. 2441 * 2442 * - MAP_SHARED mappings of a file opened read-only are transformed into 2443 * VMA_MAYSHARE_BIT, !VMA_SHARED_BIT, !VMA_MAYWRITE_BIT mappings, so remain 2444 * non-CoW. 2445 * 2446 * - Drivers may clear VMA_MAYWRITE_BIT but do so at mmap() time and cannot 2447 * mark themselves anonymous. Having cleared this flag it is not valid for 2448 * them to leave the VMA_WRITE_BIT flag set. 2449 * 2450 * As a consequence, the anonymous reverse mapping only tracks CoW mappings. 2451 * 2452 * Returns: true if the flags indicate a CoW mapping, otherwise false. 2453 */ 2454 static inline bool vma_flags_is_cow_mapping(const vma_flags_t *flags) 2455 { 2456 return vma_flags_test(flags, VMA_MAYWRITE_BIT) && 2457 !vma_flags_test(flags, VMA_SHARED_BIT); 2458 } 2459 2460 /** 2461 * vma_is_cow_mapping() - Is this VMA a CoW mapping? 2462 * @vma: The VMA to check. 2463 * 2464 * See vma_flags_is_cow_mapping() for details. 2465 * 2466 * Returns: true if the VMA is a CoW mapping, otherwise false. 2467 */ 2468 static inline bool vma_is_cow_mapping(const struct vm_area_struct *vma) 2469 { 2470 return vma_flags_is_cow_mapping(&vma->flags); 2471 } 2472 2473 /** 2474 * vma_desc_is_cow_mapping() - Is this VMA descriptor a CoW mapping? 2475 * @desc: The VMA descriptor to check. 2476 * 2477 * See vma_flags_is_cow_mapping() for details. 2478 * 2479 * Returns: true if the VMA descriptor describes a CoW mapping, otherwise 2480 * false. 2481 */ 2482 static inline bool vma_desc_is_cow_mapping(struct vm_area_desc *desc) 2483 { 2484 return vma_flags_is_cow_mapping(&desc->vma_flags); 2485 } 2486 2487 #ifndef CONFIG_MMU 2488 static inline bool is_nommu_shared_mapping(vm_flags_t flags) 2489 { 2490 /* 2491 * NOMMU shared mappings are ordinary MAP_SHARED mappings and selected 2492 * R/O MAP_PRIVATE file mappings that are an effective R/O overlay of 2493 * a file mapping. R/O MAP_PRIVATE mappings might still modify 2494 * underlying memory if ptrace is active, so this is only possible if 2495 * ptrace does not apply. Note that there is no mprotect() to upgrade 2496 * write permissions later. 2497 */ 2498 return flags & (VM_MAYSHARE | VM_MAYOVERLAY); 2499 } 2500 2501 static inline bool is_nommu_shared_vma_flags(const vma_flags_t *flags) 2502 { 2503 return vma_flags_test_any(flags, VMA_MAYSHARE_BIT, VMA_MAYOVERLAY_BIT); 2504 } 2505 #endif 2506 2507 #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) 2508 #define SECTION_IN_PAGE_FLAGS 2509 #endif 2510 2511 /* 2512 * The identification function is mainly used by the buddy allocator for 2513 * determining if two pages could be buddies. We are not really identifying 2514 * the zone since we could be using the section number id if we do not have 2515 * node id available in page flags. 2516 * We only guarantee that it will return the same value for two combinable 2517 * pages in a zone. 2518 */ 2519 static inline int page_zone_id(struct page *page) 2520 { 2521 return (page->flags.f >> ZONEID_PGSHIFT) & ZONEID_MASK; 2522 } 2523 2524 #ifdef NODE_NOT_IN_PAGE_FLAGS 2525 int memdesc_nid(const memdesc_flags_t *mdf); 2526 #else 2527 #ifdef CONFIG_NUMA 2528 static inline int memdesc_nid(const memdesc_flags_t *mdf) 2529 { 2530 ASSERT_EXCLUSIVE_BITS(mdf->f, NODES_MASK << NODES_PGSHIFT); 2531 return (mdf->f >> NODES_PGSHIFT) & NODES_MASK; 2532 } 2533 #else 2534 static inline int memdesc_nid(const memdesc_flags_t *mdf) 2535 { 2536 return 0; 2537 } 2538 #endif 2539 #endif 2540 2541 static inline int page_to_nid(const struct page *page) 2542 { 2543 return memdesc_nid(&(PF_POISONED_CHECK(page)->flags)); 2544 } 2545 2546 static inline int folio_nid(const struct folio *folio) 2547 { 2548 return memdesc_nid(&folio->flags); 2549 } 2550 2551 #ifdef CONFIG_NUMA_BALANCING 2552 /* page access time bits needs to hold at least 4 seconds */ 2553 #define PAGE_ACCESS_TIME_MIN_BITS 12 2554 #if LAST_CPUPID_SHIFT < PAGE_ACCESS_TIME_MIN_BITS 2555 #define PAGE_ACCESS_TIME_BUCKETS \ 2556 (PAGE_ACCESS_TIME_MIN_BITS - LAST_CPUPID_SHIFT) 2557 #else 2558 #define PAGE_ACCESS_TIME_BUCKETS 0 2559 #endif 2560 2561 #define PAGE_ACCESS_TIME_MASK \ 2562 (LAST_CPUPID_MASK << PAGE_ACCESS_TIME_BUCKETS) 2563 2564 static inline int cpu_pid_to_cpupid(int cpu, int pid) 2565 { 2566 return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK); 2567 } 2568 2569 static inline int cpupid_to_pid(int cpupid) 2570 { 2571 return cpupid & LAST__PID_MASK; 2572 } 2573 2574 static inline int cpupid_to_cpu(int cpupid) 2575 { 2576 return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK; 2577 } 2578 2579 static inline int cpupid_to_nid(int cpupid) 2580 { 2581 return cpu_to_node(cpupid_to_cpu(cpupid)); 2582 } 2583 2584 static inline bool cpupid_pid_unset(int cpupid) 2585 { 2586 return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK); 2587 } 2588 2589 static inline bool cpupid_cpu_unset(int cpupid) 2590 { 2591 return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK); 2592 } 2593 2594 static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid) 2595 { 2596 return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid); 2597 } 2598 2599 #define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid) 2600 #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS 2601 static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid) 2602 { 2603 return xchg(&folio->_last_cpupid, cpupid & LAST_CPUPID_MASK); 2604 } 2605 2606 static inline int folio_last_cpupid(struct folio *folio) 2607 { 2608 return folio->_last_cpupid; 2609 } 2610 static inline void page_cpupid_reset_last(struct page *page) 2611 { 2612 page->_last_cpupid = -1 & LAST_CPUPID_MASK; 2613 } 2614 #else 2615 static inline int folio_last_cpupid(struct folio *folio) 2616 { 2617 return (folio->flags.f >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK; 2618 } 2619 2620 int folio_xchg_last_cpupid(struct folio *folio, int cpupid); 2621 2622 static inline void page_cpupid_reset_last(struct page *page) 2623 { 2624 page->flags.f |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT; 2625 } 2626 #endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */ 2627 2628 static inline int folio_xchg_access_time(struct folio *folio, int time) 2629 { 2630 int last_time; 2631 2632 last_time = folio_xchg_last_cpupid(folio, 2633 time >> PAGE_ACCESS_TIME_BUCKETS); 2634 return last_time << PAGE_ACCESS_TIME_BUCKETS; 2635 } 2636 2637 static inline void vma_set_access_pid_bit(struct vm_area_struct *vma) 2638 { 2639 unsigned int pid_bit; 2640 2641 pid_bit = hash_32(current->pid, ilog2(BITS_PER_LONG)); 2642 if (vma->numab_state && !test_bit(pid_bit, &vma->numab_state->pids_active[1])) { 2643 __set_bit(pid_bit, &vma->numab_state->pids_active[1]); 2644 } 2645 } 2646 2647 bool folio_use_access_time(struct folio *folio); 2648 #else /* !CONFIG_NUMA_BALANCING */ 2649 static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid) 2650 { 2651 return folio_nid(folio); /* XXX */ 2652 } 2653 2654 static inline int folio_xchg_access_time(struct folio *folio, int time) 2655 { 2656 return 0; 2657 } 2658 2659 static inline int folio_last_cpupid(struct folio *folio) 2660 { 2661 return folio_nid(folio); /* XXX */ 2662 } 2663 2664 static inline int cpupid_to_nid(int cpupid) 2665 { 2666 return -1; 2667 } 2668 2669 static inline int cpupid_to_pid(int cpupid) 2670 { 2671 return -1; 2672 } 2673 2674 static inline int cpupid_to_cpu(int cpupid) 2675 { 2676 return -1; 2677 } 2678 2679 static inline int cpu_pid_to_cpupid(int nid, int pid) 2680 { 2681 return -1; 2682 } 2683 2684 static inline bool cpupid_pid_unset(int cpupid) 2685 { 2686 return true; 2687 } 2688 2689 static inline void page_cpupid_reset_last(struct page *page) 2690 { 2691 } 2692 2693 static inline bool cpupid_match_pid(struct task_struct *task, int cpupid) 2694 { 2695 return false; 2696 } 2697 2698 static inline void vma_set_access_pid_bit(struct vm_area_struct *vma) 2699 { 2700 } 2701 static inline bool folio_use_access_time(struct folio *folio) 2702 { 2703 return false; 2704 } 2705 #endif /* CONFIG_NUMA_BALANCING */ 2706 2707 #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) 2708 2709 /* 2710 * KASAN per-page tags are stored xor'ed with 0xff. This allows to avoid 2711 * setting tags for all pages to native kernel tag value 0xff, as the default 2712 * value 0x00 maps to 0xff. 2713 */ 2714 2715 static inline u8 page_kasan_tag(const struct page *page) 2716 { 2717 u8 tag = KASAN_TAG_KERNEL; 2718 2719 if (kasan_enabled()) { 2720 tag = (page->flags.f >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK; 2721 tag ^= 0xff; 2722 } 2723 2724 return tag; 2725 } 2726 2727 static inline void page_kasan_tag_set(struct page *page, u8 tag) 2728 { 2729 unsigned long old_flags, flags; 2730 2731 if (!kasan_enabled()) 2732 return; 2733 2734 tag ^= 0xff; 2735 old_flags = READ_ONCE(page->flags.f); 2736 do { 2737 flags = old_flags; 2738 flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT); 2739 flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT; 2740 } while (unlikely(!try_cmpxchg(&page->flags.f, &old_flags, flags))); 2741 } 2742 2743 static inline void page_kasan_tag_reset(struct page *page) 2744 { 2745 if (kasan_enabled()) 2746 page_kasan_tag_set(page, KASAN_TAG_KERNEL); 2747 } 2748 2749 #else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */ 2750 2751 static inline u8 page_kasan_tag(const struct page *page) 2752 { 2753 return 0xff; 2754 } 2755 2756 static inline void page_kasan_tag_set(struct page *page, u8 tag) { } 2757 static inline void page_kasan_tag_reset(struct page *page) { } 2758 2759 #endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */ 2760 2761 static inline struct zone *page_zone(const struct page *page) 2762 { 2763 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)]; 2764 } 2765 2766 static inline pg_data_t *page_pgdat(const struct page *page) 2767 { 2768 return NODE_DATA(page_to_nid(page)); 2769 } 2770 2771 static inline pg_data_t *folio_pgdat(const struct folio *folio) 2772 { 2773 return NODE_DATA(folio_nid(folio)); 2774 } 2775 2776 static inline struct zone *folio_zone(const struct folio *folio) 2777 { 2778 return &folio_pgdat(folio)->node_zones[folio_zonenum(folio)]; 2779 } 2780 2781 #ifdef SECTION_IN_PAGE_FLAGS 2782 static inline void set_page_section(struct page *page, unsigned long section) 2783 { 2784 page->flags.f &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT); 2785 page->flags.f |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; 2786 } 2787 2788 static inline void set_page_section_from_pfn(struct page *page, 2789 unsigned long pfn) 2790 { 2791 set_page_section(page, pfn_to_section_nr(pfn)); 2792 } 2793 2794 static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) 2795 { 2796 ASSERT_EXCLUSIVE_BITS(mdf->f, SECTIONS_MASK << SECTIONS_PGSHIFT); 2797 return (mdf->f >> SECTIONS_PGSHIFT) & SECTIONS_MASK; 2798 } 2799 #else /* !SECTION_IN_PAGE_FLAGS */ 2800 static inline void set_page_section_from_pfn(struct page *page, 2801 unsigned long pfn) 2802 { 2803 } 2804 2805 static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) 2806 { 2807 return 0; 2808 } 2809 #endif /* SECTION_IN_PAGE_FLAGS */ 2810 2811 /** 2812 * folio_pfn - Return the Page Frame Number of a folio. 2813 * @folio: The folio. 2814 * 2815 * A folio may contain multiple pages. The pages have consecutive 2816 * Page Frame Numbers. 2817 * 2818 * Return: The Page Frame Number of the first page in the folio. 2819 */ 2820 static inline unsigned long folio_pfn(const struct folio *folio) 2821 { 2822 return page_to_pfn(&folio->page); 2823 } 2824 2825 static inline struct folio *pfn_folio(unsigned long pfn) 2826 { 2827 return page_folio(pfn_to_page(pfn)); 2828 } 2829 2830 #ifdef CONFIG_MMU 2831 static inline pte_t mk_pte(const struct page *page, pgprot_t pgprot) 2832 { 2833 return pfn_pte(page_to_pfn(page), pgprot); 2834 } 2835 2836 /** 2837 * folio_mk_pte - Create a PTE for this folio 2838 * @folio: The folio to create a PTE for 2839 * @pgprot: The page protection bits to use 2840 * 2841 * Create a page table entry for the first page of this folio. 2842 * This is suitable for passing to set_ptes(). 2843 * 2844 * Return: A page table entry suitable for mapping this folio. 2845 */ 2846 static inline pte_t folio_mk_pte(const struct folio *folio, pgprot_t pgprot) 2847 { 2848 return pfn_pte(folio_pfn(folio), pgprot); 2849 } 2850 2851 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2852 /** 2853 * folio_mk_pmd - Create a PMD for this folio 2854 * @folio: The folio to create a PMD for 2855 * @pgprot: The page protection bits to use 2856 * 2857 * Create a page table entry for the first page of this folio. 2858 * This is suitable for passing to set_pmd_at(). 2859 * 2860 * Return: A page table entry suitable for mapping this folio. 2861 */ 2862 static inline pmd_t folio_mk_pmd(const struct folio *folio, pgprot_t pgprot) 2863 { 2864 return pmd_mkhuge(pfn_pmd(folio_pfn(folio), pgprot)); 2865 } 2866 2867 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 2868 /** 2869 * folio_mk_pud - Create a PUD for this folio 2870 * @folio: The folio to create a PUD for 2871 * @pgprot: The page protection bits to use 2872 * 2873 * Create a page table entry for the first page of this folio. 2874 * This is suitable for passing to set_pud_at(). 2875 * 2876 * Return: A page table entry suitable for mapping this folio. 2877 */ 2878 static inline pud_t folio_mk_pud(const struct folio *folio, pgprot_t pgprot) 2879 { 2880 return pud_mkhuge(pfn_pud(folio_pfn(folio), pgprot)); 2881 } 2882 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 2883 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 2884 #endif /* CONFIG_MMU */ 2885 2886 static inline bool folio_has_pincount(const struct folio *folio) 2887 { 2888 if (IS_ENABLED(CONFIG_64BIT)) 2889 return folio_test_large(folio); 2890 return folio_order(folio) > 1; 2891 } 2892 2893 /** 2894 * folio_maybe_dma_pinned - Report if a folio may be pinned for DMA. 2895 * @folio: The folio. 2896 * 2897 * This function checks if a folio has been pinned via a call to 2898 * a function in the pin_user_pages() family. 2899 * 2900 * For small folios, the return value is partially fuzzy: false is not fuzzy, 2901 * because it means "definitely not pinned for DMA", but true means "probably 2902 * pinned for DMA, but possibly a false positive due to having at least 2903 * GUP_PIN_COUNTING_BIAS worth of normal folio references". 2904 * 2905 * False positives are OK, because: a) it's unlikely for a folio to 2906 * get that many refcounts, and b) all the callers of this routine are 2907 * expected to be able to deal gracefully with a false positive. 2908 * 2909 * For most large folios, the result will be exactly correct. That's because 2910 * we have more tracking data available: the _pincount field is used 2911 * instead of the GUP_PIN_COUNTING_BIAS scheme. 2912 * 2913 * For more information, please see Documentation/core-api/pin_user_pages.rst. 2914 * 2915 * Return: True, if it is likely that the folio has been "dma-pinned". 2916 * False, if the folio is definitely not dma-pinned. 2917 */ 2918 static inline bool folio_maybe_dma_pinned(struct folio *folio) 2919 { 2920 if (folio_has_pincount(folio)) 2921 return atomic_read(&folio->_pincount) > 0; 2922 2923 /* 2924 * folio_ref_count() is signed. If that refcount overflows, then 2925 * folio_ref_count() returns a negative value, and callers will avoid 2926 * further incrementing the refcount. 2927 * 2928 * Here, for that overflow case, use the sign bit to count a little 2929 * bit higher via unsigned math, and thus still get an accurate result. 2930 */ 2931 return ((unsigned int)folio_ref_count(folio)) >= 2932 GUP_PIN_COUNTING_BIAS; 2933 } 2934 2935 /* 2936 * This should most likely only be called during fork() to see whether we 2937 * should break the cow immediately for an anon page on the src mm. 2938 * 2939 * The caller has to hold the PT lock and the vma->vm_mm->->write_protect_seq. 2940 */ 2941 static inline bool folio_needs_cow_for_dma(struct vm_area_struct *vma, 2942 struct folio *folio) 2943 { 2944 VM_BUG_ON(!(raw_read_seqcount(&vma->vm_mm->write_protect_seq) & 1)); 2945 2946 if (!mm_flags_test(MMF_HAS_PINNED, vma->vm_mm)) 2947 return false; 2948 2949 return folio_maybe_dma_pinned(folio); 2950 } 2951 2952 /** 2953 * is_zero_page - Query if a page is a zero page 2954 * @page: The page to query 2955 * 2956 * This returns true if @page is one of the permanent zero pages. 2957 */ 2958 static inline bool is_zero_page(const struct page *page) 2959 { 2960 return is_zero_pfn(page_to_pfn(page)); 2961 } 2962 2963 /** 2964 * is_zero_folio - Query if a folio is a zero page 2965 * @folio: The folio to query 2966 * 2967 * This returns true if @folio is one of the permanent zero pages. 2968 */ 2969 static inline bool is_zero_folio(const struct folio *folio) 2970 { 2971 return is_zero_page(&folio->page); 2972 } 2973 2974 /* MIGRATE_CMA and ZONE_MOVABLE do not allow pin folios */ 2975 #ifdef CONFIG_MIGRATION 2976 static inline bool folio_is_longterm_pinnable(struct folio *folio) 2977 { 2978 #ifdef CONFIG_CMA 2979 int mt = folio_migratetype(folio); 2980 2981 if (mt == MIGRATE_CMA || mt == MIGRATE_ISOLATE) 2982 return false; 2983 #endif 2984 /* The zero page can be "pinned" but gets special handling. */ 2985 if (is_zero_folio(folio)) 2986 return true; 2987 2988 /* Coherent device memory must always allow eviction. */ 2989 if (folio_is_device_coherent(folio)) 2990 return false; 2991 2992 /* 2993 * Filesystems can only tolerate transient delays to truncate and 2994 * hole-punch operations 2995 */ 2996 if (folio_is_fsdax(folio)) 2997 return false; 2998 2999 /* Otherwise, non-movable zone folios can be pinned. */ 3000 return !folio_is_zone_movable(folio); 3001 3002 } 3003 #else 3004 static inline bool folio_is_longterm_pinnable(struct folio *folio) 3005 { 3006 return true; 3007 } 3008 #endif 3009 3010 static inline void set_page_zone(struct page *page, enum zone_type zone) 3011 { 3012 page->flags.f &= ~(ZONES_MASK << ZONES_PGSHIFT); 3013 page->flags.f |= (zone & ZONES_MASK) << ZONES_PGSHIFT; 3014 } 3015 3016 static inline void set_page_node(struct page *page, unsigned long node) 3017 { 3018 page->flags.f &= ~(NODES_MASK << NODES_PGSHIFT); 3019 page->flags.f |= (node & NODES_MASK) << NODES_PGSHIFT; 3020 } 3021 3022 static inline void set_page_links(struct page *page, enum zone_type zone, 3023 unsigned long node, unsigned long pfn) 3024 { 3025 set_page_zone(page, zone); 3026 set_page_node(page, node); 3027 set_page_section_from_pfn(page, pfn); 3028 } 3029 3030 /** 3031 * folio_nr_pages - The number of pages in the folio. 3032 * @folio: The folio. 3033 * 3034 * Return: A positive power of two. 3035 */ 3036 static inline unsigned long folio_nr_pages(const struct folio *folio) 3037 { 3038 if (!folio_test_large(folio)) 3039 return 1; 3040 return folio_large_nr_pages(folio); 3041 } 3042 3043 /* 3044 * compound_nr() returns the number of pages in this potentially compound 3045 * page. compound_nr() can be called on a tail page, and is defined to 3046 * return 1 in that case. 3047 */ 3048 static inline unsigned long compound_nr(const struct page *page) 3049 { 3050 const struct folio *folio = (struct folio *)page; 3051 3052 if (!test_bit(PG_head, &folio->flags.f)) 3053 return 1; 3054 return folio_large_nr_pages(folio); 3055 } 3056 3057 /** 3058 * folio_next - Move to the next physical folio. 3059 * @folio: The folio we're currently operating on. 3060 * 3061 * If you have physically contiguous memory which may span more than 3062 * one folio (eg a &struct bio_vec), use this function to move from one 3063 * folio to the next. Do not use it if the memory is only virtually 3064 * contiguous as the folios are almost certainly not adjacent to each 3065 * other. This is the folio equivalent to writing ``page++``. 3066 * 3067 * Context: We assume that the folios are refcounted and/or locked at a 3068 * higher level and do not adjust the reference counts. 3069 * Return: The next struct folio. 3070 */ 3071 static inline struct folio *folio_next(struct folio *folio) 3072 { 3073 return (struct folio *)folio_page(folio, folio_nr_pages(folio)); 3074 } 3075 3076 /** 3077 * folio_shift - The size of the memory described by this folio. 3078 * @folio: The folio. 3079 * 3080 * A folio represents a number of bytes which is a power-of-two in size. 3081 * This function tells you which power-of-two the folio is. See also 3082 * folio_size() and folio_order(). 3083 * 3084 * Context: The caller should have a reference on the folio to prevent 3085 * it from being split. It is not necessary for the folio to be locked. 3086 * Return: The base-2 logarithm of the size of this folio. 3087 */ 3088 static inline unsigned int folio_shift(const struct folio *folio) 3089 { 3090 return PAGE_SHIFT + folio_order(folio); 3091 } 3092 3093 /** 3094 * folio_size - The number of bytes in a folio. 3095 * @folio: The folio. 3096 * 3097 * Context: The caller should have a reference on the folio to prevent 3098 * it from being split. It is not necessary for the folio to be locked. 3099 * Return: The number of bytes in this folio. 3100 */ 3101 static inline size_t folio_size(const struct folio *folio) 3102 { 3103 return PAGE_SIZE << folio_order(folio); 3104 } 3105 3106 /** 3107 * folio_maybe_mapped_shared - Whether the folio is mapped into the page 3108 * tables of more than one MM 3109 * @folio: The folio. 3110 * 3111 * This function checks if the folio maybe currently mapped into more than one 3112 * MM ("maybe mapped shared"), or if the folio is certainly mapped into a single 3113 * MM ("mapped exclusively"). 3114 * 3115 * For KSM folios, this function also returns "mapped shared" when a folio is 3116 * mapped multiple times into the same MM, because the individual page mappings 3117 * are independent. 3118 * 3119 * For small anonymous folios and anonymous hugetlb folios, the return 3120 * value will be exactly correct: non-KSM folios can only be mapped at most once 3121 * into an MM, and they cannot be partially mapped. KSM folios are 3122 * considered shared even if mapped multiple times into the same MM. 3123 * 3124 * For other folios, the result can be fuzzy: 3125 * #. For partially-mappable large folios (THP), the return value can wrongly 3126 * indicate "mapped shared" (false positive) if a folio was mapped by 3127 * more than two MMs at one point in time. 3128 * #. For pagecache folios (including hugetlb), the return value can wrongly 3129 * indicate "mapped shared" (false positive) when two VMAs in the same MM 3130 * cover the same file range. 3131 * 3132 * Further, this function only considers current page table mappings that 3133 * are tracked using the folio mapcount(s). 3134 * 3135 * This function does not consider: 3136 * #. If the folio might get mapped in the (near) future (e.g., swapcache, 3137 * pagecache, temporary unmapping for migration). 3138 * #. If the folio is mapped differently (VM_PFNMAP). 3139 * #. If hugetlb page table sharing applies. Callers might want to check 3140 * hugetlb_pmd_shared(). 3141 * 3142 * Return: Whether the folio is estimated to be mapped into more than one MM. 3143 */ 3144 static inline bool folio_maybe_mapped_shared(struct folio *folio) 3145 { 3146 int mapcount = folio_mapcount(folio); 3147 3148 /* Only partially-mappable folios require more care. */ 3149 if (!folio_test_large(folio) || unlikely(folio_test_hugetlb(folio))) 3150 return mapcount > 1; 3151 3152 /* 3153 * vm_insert_page() without CONFIG_TRANSPARENT_HUGEPAGE ... 3154 * simply assume "mapped shared", nobody should really care 3155 * about this for arbitrary kernel allocations. 3156 */ 3157 if (!IS_ENABLED(CONFIG_MM_ID)) 3158 return true; 3159 3160 /* 3161 * A single mapping implies "mapped exclusively", even if the 3162 * folio flag says something different: it's easier to handle this 3163 * case here instead of on the RMAP hot path. 3164 */ 3165 if (mapcount <= 1) 3166 return false; 3167 return test_bit(FOLIO_MM_IDS_SHARED_BITNUM, &folio->_mm_ids); 3168 } 3169 3170 /** 3171 * folio_expected_ref_count - calculate the expected folio refcount 3172 * @folio: the folio 3173 * 3174 * Calculate the expected folio refcount, taking references from the pagecache, 3175 * swapcache, private data (folio->private != NULL) and page table mappings into 3176 * account. Useful in combination with folio_ref_count() to detect unexpected 3177 * references (e.g., GUP or other temporary references). 3178 * 3179 * Does currently not consider references from the LRU cache. If the folio 3180 * was isolated from the LRU (which is the case during migration or split), 3181 * the LRU cache does not apply. 3182 * 3183 * Calling this function on an unmapped folio -- !folio_mapped() -- that is 3184 * locked will return a stable result. 3185 * 3186 * Calling this function on a mapped folio will not result in a stable result, 3187 * because nothing stops additional page table mappings from coming (e.g., 3188 * fork()) or going (e.g., munmap()). 3189 * 3190 * Calling this function without the folio lock will also not result in a 3191 * stable result: for example, the folio might get dropped from the swapcache 3192 * concurrently. 3193 * 3194 * However, even when called without the folio lock or on a mapped folio, 3195 * this function can be used to detect unexpected references early (for example, 3196 * if it makes sense to even lock the folio and unmap it). 3197 * 3198 * The caller must add any reference (e.g., from folio_try_get()) it might be 3199 * holding itself to the result. 3200 * 3201 * Returns: the expected folio refcount. 3202 */ 3203 static inline int folio_expected_ref_count(const struct folio *folio) 3204 { 3205 const int order = folio_order(folio); 3206 int ref_count = 0; 3207 3208 if (WARN_ON_ONCE(page_has_type(&folio->page) && !folio_test_hugetlb(folio))) 3209 return 0; 3210 3211 /* One reference per page from the swapcache. */ 3212 ref_count += folio_test_swapcache(folio) << order; 3213 3214 if (!folio_test_anon(folio)) { 3215 /* 3216 * One reference per page from the pagecache. 3217 * Use data_race() since folio might not be locked. 3218 */ 3219 ref_count += !!data_race(folio->mapping) << order; 3220 /* 3221 * One reference from filesystem private data. 3222 */ 3223 ref_count += folio_has_attached_private(folio); 3224 } 3225 3226 /* One reference per page table mapping. */ 3227 return ref_count + folio_mapcount(folio); 3228 } 3229 3230 #ifndef HAVE_ARCH_MAKE_FOLIO_ACCESSIBLE 3231 static inline int arch_make_folio_accessible(struct folio *folio) 3232 { 3233 return 0; 3234 } 3235 #endif 3236 3237 /* 3238 * Some inline functions in vmstat.h depend on page_zone() 3239 */ 3240 #include <linux/vmstat.h> 3241 3242 #if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL) 3243 #define HASHED_PAGE_VIRTUAL 3244 #endif 3245 3246 #if defined(WANT_PAGE_VIRTUAL) 3247 static inline void *page_address(const struct page *page) 3248 { 3249 return page->virtual; 3250 } 3251 static inline void set_page_address(struct page *page, void *address) 3252 { 3253 page->virtual = address; 3254 } 3255 #define page_address_init() do { } while(0) 3256 #endif 3257 3258 #if defined(HASHED_PAGE_VIRTUAL) 3259 void *page_address(const struct page *page); 3260 void set_page_address(struct page *page, void *virtual); 3261 void page_address_init(void); 3262 #endif 3263 3264 static __always_inline void *lowmem_page_address(const struct page *page) 3265 { 3266 return page_to_virt(page); 3267 } 3268 3269 #if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL) 3270 #define page_address(page) lowmem_page_address(page) 3271 #define set_page_address(page, address) do { } while(0) 3272 #define page_address_init() do { } while(0) 3273 #endif 3274 3275 static inline void *folio_address(const struct folio *folio) 3276 { 3277 return page_address(&folio->page); 3278 } 3279 3280 /* 3281 * Return true only if the page has been allocated with 3282 * ALLOC_NO_WATERMARKS and the low watermark was not 3283 * met implying that the system is under some pressure. 3284 */ 3285 static inline bool page_is_pfmemalloc(const struct page *page) 3286 { 3287 /* 3288 * lru.next has bit 1 set if the page is allocated from the 3289 * pfmemalloc reserves. Callers may simply overwrite it if 3290 * they do not need to preserve that information. 3291 */ 3292 return (uintptr_t)page->lru.next & BIT(1); 3293 } 3294 3295 /* 3296 * Return true only if the folio has been allocated with 3297 * ALLOC_NO_WATERMARKS and the low watermark was not 3298 * met implying that the system is under some pressure. 3299 */ 3300 static inline bool folio_is_pfmemalloc(const struct folio *folio) 3301 { 3302 /* 3303 * lru.next has bit 1 set if the page is allocated from the 3304 * pfmemalloc reserves. Callers may simply overwrite it if 3305 * they do not need to preserve that information. 3306 */ 3307 return (uintptr_t)folio->lru.next & BIT(1); 3308 } 3309 3310 /* 3311 * Only to be called by the page allocator on a freshly allocated 3312 * page. 3313 */ 3314 static inline void set_page_pfmemalloc(struct page *page) 3315 { 3316 page->lru.next = (void *)BIT(1); 3317 } 3318 3319 static inline void clear_page_pfmemalloc(struct page *page) 3320 { 3321 page->lru.next = NULL; 3322 } 3323 3324 /* 3325 * Can be called by the pagefault handler when it gets a VM_FAULT_OOM. 3326 */ 3327 extern void pagefault_out_of_memory(void); 3328 3329 #define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK) 3330 #define offset_in_folio(folio, p) ((unsigned long)(p) & (folio_size(folio) - 1)) 3331 3332 /* 3333 * Parameter block passed down to zap_pte_range in exceptional cases. 3334 */ 3335 struct zap_details { 3336 struct folio *single_folio; /* Locked folio to be unmapped */ 3337 bool skip_cows; /* Do not zap COWed private pages */ 3338 bool reclaim_pt; /* Need reclaim page tables? */ 3339 bool reaping; /* Reaping, do not block. */ 3340 zap_flags_t zap_flags; /* Extra flags for zapping */ 3341 }; 3342 3343 /* 3344 * Whether to drop the pte markers, for example, the uffd-wp information for 3345 * file-backed memory. This should only be specified when we will completely 3346 * drop the page in the mm, either by truncation or unmapping of the vma. By 3347 * default, the flag is not set. 3348 */ 3349 #define ZAP_FLAG_DROP_MARKER ((__force zap_flags_t) BIT(0)) 3350 /* Set in unmap_vmas() to indicate a final unmap call. Only used by hugetlb */ 3351 #define ZAP_FLAG_UNMAP ((__force zap_flags_t) BIT(1)) 3352 3353 #ifdef CONFIG_MMU 3354 extern bool can_do_mlock(void); 3355 #else 3356 static inline bool can_do_mlock(void) { return false; } 3357 #endif 3358 extern int user_shm_lock(size_t, struct ucounts *); 3359 extern void user_shm_unlock(size_t, struct ucounts *); 3360 3361 struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr, 3362 pte_t pte); 3363 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, 3364 pte_t pte); 3365 struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma, 3366 unsigned long addr, pmd_t pmd); 3367 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr, 3368 pmd_t pmd); 3369 struct page *vm_normal_page_pud(struct vm_area_struct *vma, unsigned long addr, 3370 pud_t pud); 3371 3372 void zap_special_vma_range(struct vm_area_struct *vma, unsigned long address, 3373 unsigned long size); 3374 void zap_vma_range(struct vm_area_struct *vma, unsigned long address, 3375 unsigned long size); 3376 /** 3377 * zap_vma - zap all page table entries in a vma 3378 * @vma: The vma to zap. 3379 */ 3380 static inline void zap_vma(struct vm_area_struct *vma) 3381 { 3382 zap_vma_range(vma, vma->vm_start, vma->vm_end - vma->vm_start); 3383 } 3384 struct mmu_notifier_range; 3385 3386 void free_pgd_range(struct mmu_gather *tlb, unsigned long addr, 3387 unsigned long end, unsigned long floor, unsigned long ceiling); 3388 int 3389 copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma); 3390 int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, 3391 void *buf, int len, int write); 3392 3393 struct follow_pfnmap_args { 3394 /** 3395 * Inputs: 3396 * @vma: Pointer to @vm_area_struct struct 3397 * @address: the virtual address to walk 3398 */ 3399 struct vm_area_struct *vma; 3400 unsigned long address; 3401 /** 3402 * Internals: 3403 * 3404 * The caller shouldn't touch any of these. 3405 */ 3406 spinlock_t *lock; 3407 pte_t *ptep; 3408 /** 3409 * Outputs: 3410 * 3411 * @pfn: the PFN of the address 3412 * @addr_mask: address mask covering pfn 3413 * @pgprot: the pgprot_t of the mapping 3414 * @writable: whether the mapping is writable 3415 * @special: whether the mapping is a special mapping (real PFN maps) 3416 */ 3417 unsigned long pfn; 3418 unsigned long addr_mask; 3419 pgprot_t pgprot; 3420 bool writable; 3421 bool special; 3422 }; 3423 int follow_pfnmap_start(struct follow_pfnmap_args *args); 3424 void follow_pfnmap_end(struct follow_pfnmap_args *args); 3425 3426 extern void truncate_pagecache(struct inode *inode, loff_t new); 3427 extern void truncate_setsize(struct inode *inode, loff_t newsize); 3428 void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to); 3429 void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end); 3430 int generic_error_remove_folio(struct address_space *mapping, 3431 struct folio *folio); 3432 3433 struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm, 3434 unsigned long address, struct pt_regs *regs); 3435 3436 #ifdef CONFIG_MMU 3437 extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma, 3438 unsigned long address, unsigned int flags, 3439 struct pt_regs *regs); 3440 extern int fixup_user_fault(struct mm_struct *mm, 3441 unsigned long address, unsigned int fault_flags, 3442 bool *unlocked); 3443 void unmap_mapping_pages(struct address_space *mapping, 3444 pgoff_t start, pgoff_t nr, bool even_cows); 3445 void unmap_mapping_range(struct address_space *mapping, 3446 loff_t const holebegin, loff_t const holelen, int even_cows); 3447 #else 3448 static inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma, 3449 unsigned long address, unsigned int flags, 3450 struct pt_regs *regs) 3451 { 3452 /* should never happen if there's no MMU */ 3453 BUG(); 3454 return VM_FAULT_SIGBUS; 3455 } 3456 static inline int fixup_user_fault(struct mm_struct *mm, unsigned long address, 3457 unsigned int fault_flags, bool *unlocked) 3458 { 3459 /* should never happen if there's no MMU */ 3460 BUG(); 3461 return -EFAULT; 3462 } 3463 static inline void unmap_mapping_pages(struct address_space *mapping, 3464 pgoff_t start, pgoff_t nr, bool even_cows) { } 3465 static inline void unmap_mapping_range(struct address_space *mapping, 3466 loff_t const holebegin, loff_t const holelen, int even_cows) { } 3467 #endif 3468 3469 static inline void unmap_shared_mapping_range(struct address_space *mapping, 3470 loff_t const holebegin, loff_t const holelen) 3471 { 3472 unmap_mapping_range(mapping, holebegin, holelen, 0); 3473 } 3474 3475 static inline struct vm_area_struct *vma_lookup(struct mm_struct *mm, 3476 unsigned long addr); 3477 3478 extern int access_process_vm(struct task_struct *tsk, unsigned long addr, 3479 void *buf, int len, unsigned int gup_flags); 3480 extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, 3481 void *buf, int len, unsigned int gup_flags); 3482 3483 int copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, 3484 void *buf, int len, unsigned int gup_flags); 3485 int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, 3486 void *buf, int len, unsigned int gup_flags); 3487 3488 long get_user_pages_remote(struct mm_struct *mm, 3489 unsigned long start, unsigned long nr_pages, 3490 unsigned int gup_flags, struct page **pages, 3491 int *locked); 3492 long pin_user_pages_remote(struct mm_struct *mm, 3493 unsigned long start, unsigned long nr_pages, 3494 unsigned int gup_flags, struct page **pages, 3495 int *locked); 3496 3497 /* 3498 * Retrieves a single page alongside its VMA. Does not support FOLL_NOWAIT. 3499 */ 3500 static inline struct page *get_user_page_vma_remote(struct mm_struct *mm, 3501 unsigned long addr, 3502 int gup_flags, 3503 struct vm_area_struct **vmap) 3504 { 3505 struct page *page; 3506 struct vm_area_struct *vma; 3507 int got; 3508 3509 if (WARN_ON_ONCE(unlikely(gup_flags & FOLL_NOWAIT))) 3510 return ERR_PTR(-EINVAL); 3511 3512 got = get_user_pages_remote(mm, addr, 1, gup_flags, &page, NULL); 3513 3514 if (got < 0) 3515 return ERR_PTR(got); 3516 3517 vma = vma_lookup(mm, addr); 3518 if (WARN_ON_ONCE(!vma)) { 3519 put_page(page); 3520 return ERR_PTR(-EINVAL); 3521 } 3522 3523 *vmap = vma; 3524 return page; 3525 } 3526 3527 long get_user_pages(unsigned long start, unsigned long nr_pages, 3528 unsigned int gup_flags, struct page **pages); 3529 long pin_user_pages(unsigned long start, unsigned long nr_pages, 3530 unsigned int gup_flags, struct page **pages); 3531 long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, 3532 struct page **pages, unsigned int gup_flags); 3533 long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages, 3534 struct page **pages, unsigned int gup_flags); 3535 long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end, 3536 struct folio **folios, unsigned int max_folios, 3537 pgoff_t *offset); 3538 int folio_add_pins(struct folio *folio, unsigned int pins); 3539 3540 int get_user_pages_fast(unsigned long start, int nr_pages, 3541 unsigned int gup_flags, struct page **pages); 3542 int pin_user_pages_fast(unsigned long start, int nr_pages, 3543 unsigned int gup_flags, struct page **pages); 3544 void folio_add_pin(struct folio *folio); 3545 3546 int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc); 3547 int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, 3548 const struct task_struct *task, bool bypass_rlim); 3549 3550 struct kvec; 3551 struct page *get_dump_page(unsigned long addr, int *locked); 3552 3553 bool folio_mark_dirty(struct folio *folio); 3554 bool folio_mark_dirty_lock(struct folio *folio); 3555 bool set_page_dirty(struct page *page); 3556 int set_page_dirty_lock(struct page *page); 3557 3558 int get_cmdline(struct task_struct *task, char *buffer, int buflen); 3559 3560 /* 3561 * Flags used by change_protection(). For now we make it a bitmap so 3562 * that we can pass in multiple flags just like parameters. However 3563 * for now all the callers are only use one of the flags at the same 3564 * time. 3565 */ 3566 /* 3567 * Whether we should manually check if we can map individual PTEs writable, 3568 * because something (e.g., COW, uffd-wp) blocks that from happening for all 3569 * PTEs automatically in a writable mapping. 3570 */ 3571 #define MM_CP_TRY_CHANGE_WRITABLE (1UL << 0) 3572 /* Whether this protection change is for NUMA hints */ 3573 #define MM_CP_PROT_NUMA (1UL << 1) 3574 /* Whether this change is for write protecting */ 3575 #define MM_CP_UFFD_WP (1UL << 2) /* do wp */ 3576 #define MM_CP_UFFD_WP_RESOLVE (1UL << 3) /* Resolve wp */ 3577 #define MM_CP_UFFD_WP_ALL (MM_CP_UFFD_WP | \ 3578 MM_CP_UFFD_WP_RESOLVE) 3579 /* Whether this change is for uffd RWP */ 3580 #define MM_CP_UFFD_RWP (1UL << 4) /* do rwp */ 3581 #define MM_CP_UFFD_RWP_RESOLVE (1UL << 5) /* resolve rwp */ 3582 #define MM_CP_UFFD_RWP_ALL (MM_CP_UFFD_RWP | \ 3583 MM_CP_UFFD_RWP_RESOLVE) 3584 3585 bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, 3586 pte_t pte); 3587 extern long change_protection(struct mmu_gather *tlb, 3588 struct vm_area_struct *vma, unsigned long start, 3589 unsigned long end, unsigned long cp_flags); 3590 extern int mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, 3591 struct vm_area_struct *vma, struct vm_area_struct **pprev, 3592 unsigned long start, unsigned long end, vm_flags_t newflags); 3593 3594 /* 3595 * doesn't attempt to fault and will return short. 3596 */ 3597 int get_user_pages_fast_only(unsigned long start, int nr_pages, 3598 unsigned int gup_flags, struct page **pages); 3599 3600 static inline bool get_user_page_fast_only(unsigned long addr, 3601 unsigned int gup_flags, struct page **pagep) 3602 { 3603 return get_user_pages_fast_only(addr, 1, gup_flags, pagep) == 1; 3604 } 3605 /* 3606 * per-process(per-mm_struct) statistics. 3607 */ 3608 static inline unsigned long get_mm_counter(struct mm_struct *mm, int member) 3609 { 3610 return percpu_counter_read_positive(&mm->rss_stat[member]); 3611 } 3612 3613 static inline unsigned long get_mm_counter_sum(struct mm_struct *mm, int member) 3614 { 3615 return percpu_counter_sum_positive(&mm->rss_stat[member]); 3616 } 3617 3618 void mm_trace_rss_stat(struct mm_struct *mm, int member); 3619 3620 static inline void add_mm_counter(struct mm_struct *mm, int member, long value) 3621 { 3622 percpu_counter_add(&mm->rss_stat[member], value); 3623 3624 mm_trace_rss_stat(mm, member); 3625 } 3626 3627 static inline void inc_mm_counter(struct mm_struct *mm, int member) 3628 { 3629 percpu_counter_inc(&mm->rss_stat[member]); 3630 3631 mm_trace_rss_stat(mm, member); 3632 } 3633 3634 static inline void dec_mm_counter(struct mm_struct *mm, int member) 3635 { 3636 percpu_counter_dec(&mm->rss_stat[member]); 3637 3638 mm_trace_rss_stat(mm, member); 3639 } 3640 3641 /* Optimized variant when folio is already known not to be anon */ 3642 static inline int mm_counter_file(struct folio *folio) 3643 { 3644 if (folio_test_swapbacked(folio)) 3645 return MM_SHMEMPAGES; 3646 return MM_FILEPAGES; 3647 } 3648 3649 static inline int mm_counter(struct folio *folio) 3650 { 3651 if (folio_test_anon(folio)) 3652 return MM_ANONPAGES; 3653 return mm_counter_file(folio); 3654 } 3655 3656 static inline unsigned long get_mm_rss(struct mm_struct *mm) 3657 { 3658 return get_mm_counter(mm, MM_FILEPAGES) + 3659 get_mm_counter(mm, MM_ANONPAGES) + 3660 get_mm_counter(mm, MM_SHMEMPAGES); 3661 } 3662 3663 static inline unsigned long get_mm_rss_sum(struct mm_struct *mm) 3664 { 3665 return get_mm_counter_sum(mm, MM_FILEPAGES) + 3666 get_mm_counter_sum(mm, MM_ANONPAGES) + 3667 get_mm_counter_sum(mm, MM_SHMEMPAGES); 3668 } 3669 3670 static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm) 3671 { 3672 return max(mm->hiwater_rss, get_mm_rss(mm)); 3673 } 3674 3675 static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm) 3676 { 3677 return max(mm->hiwater_vm, mm->total_vm); 3678 } 3679 3680 static inline void update_hiwater_rss(struct mm_struct *mm) 3681 { 3682 unsigned long _rss = get_mm_rss(mm); 3683 3684 if (data_race(mm->hiwater_rss) < _rss) 3685 data_race(mm->hiwater_rss = _rss); 3686 } 3687 3688 static inline void update_hiwater_vm(struct mm_struct *mm) 3689 { 3690 if (mm->hiwater_vm < mm->total_vm) 3691 mm->hiwater_vm = mm->total_vm; 3692 } 3693 3694 static inline void reset_mm_hiwater_rss(struct mm_struct *mm) 3695 { 3696 mm->hiwater_rss = get_mm_rss(mm); 3697 } 3698 3699 static inline void setmax_mm_hiwater_rss(unsigned long *maxrss, 3700 struct mm_struct *mm) 3701 { 3702 unsigned long hiwater_rss = get_mm_hiwater_rss(mm); 3703 3704 if (*maxrss < hiwater_rss) 3705 *maxrss = hiwater_rss; 3706 } 3707 3708 #ifndef CONFIG_ARCH_HAS_PTE_SPECIAL 3709 static inline int pte_special(pte_t pte) 3710 { 3711 return 0; 3712 } 3713 3714 static inline pte_t pte_mkspecial(pte_t pte) 3715 { 3716 return pte; 3717 } 3718 #endif 3719 3720 #ifndef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP 3721 static inline bool pmd_special(pmd_t pmd) 3722 { 3723 return false; 3724 } 3725 3726 static inline pmd_t pmd_mkspecial(pmd_t pmd) 3727 { 3728 return pmd; 3729 } 3730 #endif /* CONFIG_ARCH_SUPPORTS_PMD_PFNMAP */ 3731 3732 #ifndef CONFIG_ARCH_SUPPORTS_PUD_PFNMAP 3733 static inline bool pud_special(pud_t pud) 3734 { 3735 return false; 3736 } 3737 3738 static inline pud_t pud_mkspecial(pud_t pud) 3739 { 3740 return pud; 3741 } 3742 #endif /* CONFIG_ARCH_SUPPORTS_PUD_PFNMAP */ 3743 3744 extern pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, 3745 spinlock_t **ptl); 3746 3747 #ifdef __PAGETABLE_P4D_FOLDED 3748 static inline int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, 3749 unsigned long address) 3750 { 3751 return 0; 3752 } 3753 #else 3754 int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address); 3755 #endif 3756 3757 #if defined(__PAGETABLE_PUD_FOLDED) || !defined(CONFIG_MMU) 3758 static inline int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, 3759 unsigned long address) 3760 { 3761 return 0; 3762 } 3763 static inline void mm_inc_nr_puds(struct mm_struct *mm) {} 3764 static inline void mm_dec_nr_puds(struct mm_struct *mm) {} 3765 3766 #else 3767 int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address); 3768 3769 static inline void mm_inc_nr_puds(struct mm_struct *mm) 3770 { 3771 if (mm_pud_folded(mm)) 3772 return; 3773 atomic_long_add(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes); 3774 } 3775 3776 static inline void mm_dec_nr_puds(struct mm_struct *mm) 3777 { 3778 if (mm_pud_folded(mm)) 3779 return; 3780 atomic_long_sub(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes); 3781 } 3782 #endif 3783 3784 #if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU) 3785 static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud, 3786 unsigned long address) 3787 { 3788 return 0; 3789 } 3790 3791 static inline void mm_inc_nr_pmds(struct mm_struct *mm) {} 3792 static inline void mm_dec_nr_pmds(struct mm_struct *mm) {} 3793 3794 #else 3795 int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address); 3796 3797 static inline void mm_inc_nr_pmds(struct mm_struct *mm) 3798 { 3799 if (mm_pmd_folded(mm)) 3800 return; 3801 atomic_long_add(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes); 3802 } 3803 3804 static inline void mm_dec_nr_pmds(struct mm_struct *mm) 3805 { 3806 if (mm_pmd_folded(mm)) 3807 return; 3808 atomic_long_sub(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes); 3809 } 3810 #endif 3811 3812 #ifdef CONFIG_MMU 3813 static inline void mm_pgtables_bytes_init(struct mm_struct *mm) 3814 { 3815 atomic_long_set(&mm->pgtables_bytes, 0); 3816 } 3817 3818 static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm) 3819 { 3820 return atomic_long_read(&mm->pgtables_bytes); 3821 } 3822 3823 static inline void mm_inc_nr_ptes(struct mm_struct *mm) 3824 { 3825 atomic_long_add(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes); 3826 } 3827 3828 static inline void mm_dec_nr_ptes(struct mm_struct *mm) 3829 { 3830 atomic_long_sub(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes); 3831 } 3832 #else 3833 3834 static inline void mm_pgtables_bytes_init(struct mm_struct *mm) {} 3835 static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm) 3836 { 3837 return 0; 3838 } 3839 3840 static inline void mm_inc_nr_ptes(struct mm_struct *mm) {} 3841 static inline void mm_dec_nr_ptes(struct mm_struct *mm) {} 3842 #endif 3843 3844 int __pte_alloc(struct mm_struct *mm, pmd_t *pmd); 3845 int __pte_alloc_kernel(pmd_t *pmd); 3846 3847 #if defined(CONFIG_MMU) 3848 3849 static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd, 3850 unsigned long address) 3851 { 3852 return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ? 3853 NULL : p4d_offset(pgd, address); 3854 } 3855 3856 static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d, 3857 unsigned long address) 3858 { 3859 return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ? 3860 NULL : pud_offset(p4d, address); 3861 } 3862 3863 static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) 3864 { 3865 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))? 3866 NULL: pmd_offset(pud, address); 3867 } 3868 #endif /* CONFIG_MMU */ 3869 3870 enum pt_flags { 3871 PT_kernel = PG_referenced, 3872 PT_reserved = PG_reserved, 3873 /* High bits are used for zone/node/section */ 3874 }; 3875 3876 static inline struct ptdesc *virt_to_ptdesc(const void *x) 3877 { 3878 return page_ptdesc(virt_to_page(x)); 3879 } 3880 3881 /** 3882 * ptdesc_address - Virtual address of page table. 3883 * @pt: Page table descriptor. 3884 * 3885 * Return: The first byte of the page table described by @pt. 3886 */ 3887 static inline void *ptdesc_address(const struct ptdesc *pt) 3888 { 3889 return folio_address(ptdesc_folio(pt)); 3890 } 3891 3892 static inline bool pagetable_is_reserved(struct ptdesc *pt) 3893 { 3894 return test_bit(PT_reserved, &pt->pt_flags.f); 3895 } 3896 3897 /** 3898 * ptdesc_set_kernel - Mark a ptdesc used to map the kernel 3899 * @ptdesc: The ptdesc to be marked 3900 * 3901 * Kernel page tables often need special handling. Set a flag so that 3902 * the handling code knows this ptdesc will not be used for userspace. 3903 */ 3904 static inline void ptdesc_set_kernel(struct ptdesc *ptdesc) 3905 { 3906 set_bit(PT_kernel, &ptdesc->pt_flags.f); 3907 } 3908 3909 /** 3910 * ptdesc_clear_kernel - Mark a ptdesc as no longer used to map the kernel 3911 * @ptdesc: The ptdesc to be unmarked 3912 * 3913 * Use when the ptdesc is no longer used to map the kernel and no longer 3914 * needs special handling. 3915 */ 3916 static inline void ptdesc_clear_kernel(struct ptdesc *ptdesc) 3917 { 3918 /* 3919 * Note: the 'PG_referenced' bit does not strictly need to be 3920 * cleared before freeing the page. But this is nice for 3921 * symmetry. 3922 */ 3923 clear_bit(PT_kernel, &ptdesc->pt_flags.f); 3924 } 3925 3926 /** 3927 * ptdesc_test_kernel - Check if a ptdesc is used to map the kernel 3928 * @ptdesc: The ptdesc being tested 3929 * 3930 * Call to tell if the ptdesc used to map the kernel. 3931 */ 3932 static inline bool ptdesc_test_kernel(const struct ptdesc *ptdesc) 3933 { 3934 return test_bit(PT_kernel, &ptdesc->pt_flags.f); 3935 } 3936 3937 /** 3938 * pagetable_alloc - Allocate pagetables 3939 * @gfp: GFP flags 3940 * @order: desired pagetable order 3941 * 3942 * pagetable_alloc allocates memory for page tables as well as a page table 3943 * descriptor to describe that memory. 3944 * 3945 * Return: The ptdesc describing the allocated page tables. 3946 */ 3947 static inline struct ptdesc *pagetable_alloc_noprof(gfp_t gfp, unsigned int order) 3948 { 3949 struct page *page = alloc_pages_noprof(gfp | __GFP_COMP, order); 3950 3951 return page_ptdesc(page); 3952 } 3953 #define pagetable_alloc(...) alloc_hooks(pagetable_alloc_noprof(__VA_ARGS__)) 3954 3955 static inline void __pagetable_free(struct ptdesc *pt) 3956 { 3957 struct page *page = ptdesc_page(pt); 3958 3959 __free_pages(page, compound_order(page)); 3960 } 3961 3962 #ifdef CONFIG_ASYNC_KERNEL_PGTABLE_FREE 3963 void pagetable_free_kernel(struct ptdesc *pt); 3964 #else 3965 static inline void pagetable_free_kernel(struct ptdesc *pt) 3966 { 3967 __pagetable_free(pt); 3968 } 3969 #endif 3970 /** 3971 * pagetable_free - Free pagetables 3972 * @pt: The page table descriptor 3973 * 3974 * pagetable_free frees the memory of all page tables described by a page 3975 * table descriptor and the memory for the descriptor itself. 3976 */ 3977 static inline void pagetable_free(struct ptdesc *pt) 3978 { 3979 if (ptdesc_test_kernel(pt)) { 3980 ptdesc_clear_kernel(pt); 3981 pagetable_free_kernel(pt); 3982 } else { 3983 __pagetable_free(pt); 3984 } 3985 } 3986 3987 #if defined(CONFIG_SPLIT_PTE_PTLOCKS) 3988 #if ALLOC_SPLIT_PTLOCKS 3989 void __init ptlock_cache_init(void); 3990 bool ptlock_alloc(struct ptdesc *ptdesc); 3991 void ptlock_free(struct ptdesc *ptdesc); 3992 3993 static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc) 3994 { 3995 return ptdesc->ptl; 3996 } 3997 #else /* ALLOC_SPLIT_PTLOCKS */ 3998 static inline void ptlock_cache_init(void) 3999 { 4000 } 4001 4002 static inline bool ptlock_alloc(struct ptdesc *ptdesc) 4003 { 4004 return true; 4005 } 4006 4007 static inline void ptlock_free(struct ptdesc *ptdesc) 4008 { 4009 } 4010 4011 static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc) 4012 { 4013 return &ptdesc->ptl; 4014 } 4015 #endif /* ALLOC_SPLIT_PTLOCKS */ 4016 4017 static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd) 4018 { 4019 return ptlock_ptr(page_ptdesc(pmd_page(*pmd))); 4020 } 4021 4022 static inline spinlock_t *ptep_lockptr(struct mm_struct *mm, pte_t *pte) 4023 { 4024 BUILD_BUG_ON(IS_ENABLED(CONFIG_HIGHPTE)); 4025 BUILD_BUG_ON(MAX_PTRS_PER_PTE * sizeof(pte_t) > PAGE_SIZE); 4026 return ptlock_ptr(virt_to_ptdesc(pte)); 4027 } 4028 4029 static inline bool ptlock_init(struct ptdesc *ptdesc) 4030 { 4031 /* 4032 * prep_new_page() initialize page->private (and therefore page->ptl) 4033 * with 0. Make sure nobody took it in use in between. 4034 * 4035 * It can happen if arch try to use slab for page table allocation: 4036 * slab code uses page->slab_cache, which share storage with page->ptl. 4037 */ 4038 VM_BUG_ON_PAGE(*(unsigned long *)&ptdesc->ptl, ptdesc_page(ptdesc)); 4039 if (!ptlock_alloc(ptdesc)) 4040 return false; 4041 spin_lock_init(ptlock_ptr(ptdesc)); 4042 return true; 4043 } 4044 4045 #else /* !defined(CONFIG_SPLIT_PTE_PTLOCKS) */ 4046 /* 4047 * We use mm->page_table_lock to guard all pagetable pages of the mm. 4048 */ 4049 static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd) 4050 { 4051 return &mm->page_table_lock; 4052 } 4053 static inline spinlock_t *ptep_lockptr(struct mm_struct *mm, pte_t *pte) 4054 { 4055 return &mm->page_table_lock; 4056 } 4057 static inline void ptlock_cache_init(void) {} 4058 static inline bool ptlock_init(struct ptdesc *ptdesc) { return true; } 4059 static inline void ptlock_free(struct ptdesc *ptdesc) {} 4060 #endif /* defined(CONFIG_SPLIT_PTE_PTLOCKS) */ 4061 4062 static inline void __pagetable_ctor(struct ptdesc *ptdesc) 4063 { 4064 struct folio *folio = ptdesc_folio(ptdesc); 4065 4066 __folio_set_pgtable(folio); 4067 lruvec_stat_add_folio(folio, NR_PAGETABLE); 4068 } 4069 4070 static inline void pagetable_dtor(struct ptdesc *ptdesc) 4071 { 4072 struct folio *folio = ptdesc_folio(ptdesc); 4073 4074 ptlock_free(ptdesc); 4075 __folio_clear_pgtable(folio); 4076 lruvec_stat_sub_folio(folio, NR_PAGETABLE); 4077 } 4078 4079 static inline void pagetable_dtor_free(struct ptdesc *ptdesc) 4080 { 4081 pagetable_dtor(ptdesc); 4082 pagetable_free(ptdesc); 4083 } 4084 4085 static inline bool pagetable_pte_ctor(struct mm_struct *mm, 4086 struct ptdesc *ptdesc) 4087 { 4088 if (mm != &init_mm && !ptlock_init(ptdesc)) 4089 return false; 4090 __pagetable_ctor(ptdesc); 4091 return true; 4092 } 4093 4094 pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp); 4095 4096 static inline pte_t *pte_offset_map(pmd_t *pmd, unsigned long addr) 4097 { 4098 return __pte_offset_map(pmd, addr, NULL); 4099 } 4100 4101 pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, 4102 unsigned long addr, spinlock_t **ptlp); 4103 4104 pte_t *pte_offset_map_ro_nolock(struct mm_struct *mm, pmd_t *pmd, 4105 unsigned long addr, spinlock_t **ptlp); 4106 pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, 4107 unsigned long addr, pmd_t *pmdvalp, 4108 spinlock_t **ptlp); 4109 4110 #define pte_unmap_unlock(pte, ptl) do { \ 4111 spin_unlock(ptl); \ 4112 pte_unmap(pte); \ 4113 } while (0) 4114 4115 #define pte_alloc(mm, pmd) (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd)) 4116 4117 #define pte_alloc_map(mm, pmd, address) \ 4118 (pte_alloc(mm, pmd) ? NULL : pte_offset_map(pmd, address)) 4119 4120 #define pte_alloc_map_lock(mm, pmd, address, ptlp) \ 4121 (pte_alloc(mm, pmd) ? \ 4122 NULL : pte_offset_map_lock(mm, pmd, address, ptlp)) 4123 4124 #define pte_alloc_kernel(pmd, address) \ 4125 ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd))? \ 4126 NULL: pte_offset_kernel(pmd, address)) 4127 4128 #if defined(CONFIG_SPLIT_PMD_PTLOCKS) 4129 4130 static inline struct page *pmd_pgtable_page(pmd_t *pmd) 4131 { 4132 unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1); 4133 return virt_to_page((void *)((unsigned long) pmd & mask)); 4134 } 4135 4136 static inline struct ptdesc *pmd_ptdesc(pmd_t *pmd) 4137 { 4138 return page_ptdesc(pmd_pgtable_page(pmd)); 4139 } 4140 4141 static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd) 4142 { 4143 return ptlock_ptr(pmd_ptdesc(pmd)); 4144 } 4145 4146 static inline bool pmd_ptlock_init(struct ptdesc *ptdesc) 4147 { 4148 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4149 ptdesc->pmd_huge_pte = NULL; 4150 #endif 4151 return ptlock_init(ptdesc); 4152 } 4153 4154 #define pmd_huge_pte(mm, pmd) (pmd_ptdesc(pmd)->pmd_huge_pte) 4155 4156 #else 4157 4158 static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd) 4159 { 4160 return &mm->page_table_lock; 4161 } 4162 4163 static inline bool pmd_ptlock_init(struct ptdesc *ptdesc) { return true; } 4164 4165 #define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte) 4166 4167 #endif 4168 4169 static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd) 4170 { 4171 spinlock_t *ptl = pmd_lockptr(mm, pmd); 4172 spin_lock(ptl); 4173 return ptl; 4174 } 4175 4176 static inline bool pagetable_pmd_ctor(struct mm_struct *mm, 4177 struct ptdesc *ptdesc) 4178 { 4179 if (mm != &init_mm && !pmd_ptlock_init(ptdesc)) 4180 return false; 4181 ptdesc_pmd_pts_init(ptdesc); 4182 __pagetable_ctor(ptdesc); 4183 return true; 4184 } 4185 4186 /* 4187 * No scalability reason to split PUD locks yet, but follow the same pattern 4188 * as the PMD locks to make it easier if we decide to. The VM should not be 4189 * considered ready to switch to split PUD locks yet; there may be places 4190 * which need to be converted from page_table_lock. 4191 */ 4192 static inline spinlock_t *pud_lockptr(struct mm_struct *mm, pud_t *pud) 4193 { 4194 return &mm->page_table_lock; 4195 } 4196 4197 static inline spinlock_t *pud_lock(struct mm_struct *mm, pud_t *pud) 4198 { 4199 spinlock_t *ptl = pud_lockptr(mm, pud); 4200 4201 spin_lock(ptl); 4202 return ptl; 4203 } 4204 4205 static inline void pagetable_pud_ctor(struct ptdesc *ptdesc) 4206 { 4207 __pagetable_ctor(ptdesc); 4208 } 4209 4210 static inline void pagetable_p4d_ctor(struct ptdesc *ptdesc) 4211 { 4212 __pagetable_ctor(ptdesc); 4213 } 4214 4215 static inline void pagetable_pgd_ctor(struct ptdesc *ptdesc) 4216 { 4217 __pagetable_ctor(ptdesc); 4218 } 4219 4220 extern void __init pagecache_init(void); 4221 extern void free_initmem(void); 4222 4223 /* 4224 * Free reserved pages within range [PAGE_ALIGN(start), end & PAGE_MASK) 4225 * into the buddy system. The freed pages will be poisoned with pattern 4226 * "poison" if it's within range [0, UCHAR_MAX]. 4227 * Return pages freed into the buddy system. 4228 */ 4229 extern unsigned long free_reserved_area(void *start, void *end, 4230 int poison, const char *s); 4231 4232 extern void adjust_managed_page_count(struct page *page, long count); 4233 4234 void free_reserved_pages(struct page *page, unsigned int order); 4235 4236 static inline void free_reserved_page(struct page *page) 4237 { 4238 free_reserved_pages(page, 0); 4239 } 4240 4241 static inline void free_reserved_ptdesc(struct ptdesc *pt) 4242 { 4243 free_reserved_page(ptdesc_page(pt)); 4244 } 4245 4246 /* 4247 * Default method to free all the __init memory into the buddy system. 4248 * The freed pages will be poisoned with pattern "poison" if it's within 4249 * range [0, UCHAR_MAX]. 4250 * Return pages freed into the buddy system. 4251 */ 4252 static inline unsigned long free_initmem_default(int poison) 4253 { 4254 extern char __init_begin[], __init_end[]; 4255 4256 return free_reserved_area(&__init_begin, &__init_end, 4257 poison, "unused kernel image (initmem)"); 4258 } 4259 4260 static inline unsigned long get_num_physpages(void) 4261 { 4262 int nid; 4263 unsigned long phys_pages = 0; 4264 4265 for_each_online_node(nid) 4266 phys_pages += node_present_pages(nid); 4267 4268 return phys_pages; 4269 } 4270 4271 /* 4272 * FIXME: Using memblock node mappings, an architecture may initialise its 4273 * zones, allocate the backing mem_map and account for memory holes in an 4274 * architecture independent manner. 4275 * 4276 * An architecture is expected to register range of page frames backed by 4277 * physical memory with memblock_add[_node]() before calling 4278 * free_area_init() passing in the PFN each zone ends at. At a basic 4279 * usage, an architecture is expected to do something like 4280 * 4281 * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn, 4282 * max_highmem_pfn}; 4283 * for_each_valid_physical_page_range() 4284 * memblock_add_node(base, size, nid, MEMBLOCK_NONE) 4285 * free_area_init(max_zone_pfns); 4286 */ 4287 void arch_zone_limits_init(unsigned long *max_zone_pfn); 4288 unsigned long node_map_pfn_alignment(void); 4289 extern unsigned long absent_pages_in_range(unsigned long start_pfn, 4290 unsigned long end_pfn); 4291 extern void get_pfn_range_for_nid(unsigned int nid, 4292 unsigned long *start_pfn, unsigned long *end_pfn); 4293 4294 #ifndef CONFIG_NUMA 4295 static inline int early_pfn_to_nid(unsigned long pfn) 4296 { 4297 return 0; 4298 } 4299 #else 4300 /* please see mm/page_alloc.c */ 4301 extern int __meminit early_pfn_to_nid(unsigned long pfn); 4302 #endif 4303 4304 extern void mem_init(void); 4305 extern void __init mmap_init(void); 4306 4307 extern void __show_mem(unsigned int flags, const nodemask_t *nodemask, int max_zone_idx); 4308 static inline void show_mem(void) 4309 { 4310 __show_mem(0, NULL, MAX_NR_ZONES - 1); 4311 } 4312 extern long si_mem_available(void); 4313 extern void si_meminfo(struct sysinfo * val); 4314 extern void si_meminfo_node(struct sysinfo *val, int nid); 4315 4316 extern __printf(3, 4) 4317 void warn_alloc(gfp_t gfp_mask, const nodemask_t *nodemask, const char *fmt, ...); 4318 4319 extern void setup_per_cpu_pageset(void); 4320 4321 /* nommu.c */ 4322 extern atomic_long_t mmap_pages_allocated; 4323 extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t); 4324 4325 /* interval_tree.c */ 4326 void mapping_rmap_tree_insert(struct vm_area_struct *vma, 4327 struct address_space *mapping); 4328 void mapping_rmap_tree_insert_after(struct vm_area_struct *vma, 4329 struct vm_area_struct *prev, 4330 struct address_space *mapping); 4331 void mapping_rmap_tree_remove(struct vm_area_struct *vma, 4332 struct address_space *mapping); 4333 void mapping_rmap_tree_pre_update(struct vm_area_struct *vma, 4334 struct address_space *mapping, 4335 bool pgoff_unchanged); 4336 void mapping_rmap_tree_post_update(struct vm_area_struct *vma, 4337 struct address_space *mapping, 4338 bool pgoff_unchanged); 4339 struct vm_area_struct * 4340 mapping_rmap_tree_iter_first(struct address_space *mapping, 4341 pgoff_t pgoff_start, pgoff_t pgoff_last); 4342 struct vm_area_struct * 4343 mapping_rmap_tree_iter_next(struct vm_area_struct *vma, 4344 pgoff_t pgoff_start, pgoff_t pgoff_last); 4345 4346 #define mapping_rmap_tree_foreach(vma, mapping, pgoff_start, pgoff_last) \ 4347 for (vma = mapping_rmap_tree_iter_first(mapping, pgoff_start, \ 4348 pgoff_last); \ 4349 vma; vma = mapping_rmap_tree_iter_next(vma, pgoff_start, \ 4350 pgoff_last)) 4351 4352 void anon_rmap_tree_insert(struct anon_vma_chain *avc, 4353 struct anon_vma *anon_vma); 4354 void anon_rmap_tree_remove(struct anon_vma_chain *avc, 4355 struct anon_vma *anon_vma); 4356 void anon_rmap_tree_pre_update_vma(struct vm_area_struct *vma, 4357 bool anon_pgoff_unchanged); 4358 void anon_rmap_tree_post_update_vma(struct vm_area_struct *vma, 4359 bool anon_pgoff_unchanged); 4360 struct anon_vma_chain * 4361 anon_rmap_tree_iter_first(struct anon_vma *anon_vma, 4362 pgoff_t pgoff_start, pgoff_t pgoff_last); 4363 struct anon_vma_chain * 4364 anon_rmap_tree_iter_next(struct anon_vma_chain *avc, 4365 pgoff_t pgoff_start, pgoff_t pgoff_last); 4366 #ifdef CONFIG_DEBUG_VM_RB 4367 void anon_rmap_tree_verify(struct anon_vma_chain *avc); 4368 #endif 4369 4370 #define anon_rmap_tree_foreach(avc, anon_vma, pgoff_start, pgoff_last) \ 4371 for (avc = anon_rmap_tree_iter_first(anon_vma, pgoff_start, pgoff_last); \ 4372 avc; avc = anon_rmap_tree_iter_next(avc, pgoff_start, pgoff_last)) 4373 4374 /* mmap.c */ 4375 extern int __vm_enough_memory(const struct mm_struct *mm, long pages, int cap_sys_admin); 4376 extern void exit_mmap(struct mm_struct *); 4377 bool mmap_read_lock_maybe_expand(struct mm_struct *mm, struct vm_area_struct *vma, 4378 unsigned long addr, bool write); 4379 4380 static inline int check_data_rlimit(unsigned long rlim, 4381 unsigned long new, 4382 unsigned long start, 4383 unsigned long end_data, 4384 unsigned long start_data) 4385 { 4386 if (rlim < RLIM_INFINITY) { 4387 if (((new - start) + (end_data - start_data)) > rlim) 4388 return -ENOSPC; 4389 } 4390 4391 return 0; 4392 } 4393 4394 extern int mm_take_all_locks(struct mm_struct *mm); 4395 extern void mm_drop_all_locks(struct mm_struct *mm); 4396 4397 extern int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file); 4398 extern int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file); 4399 extern struct file *get_mm_exe_file(struct mm_struct *mm); 4400 extern struct file *get_task_exe_file(struct task_struct *task); 4401 4402 extern void vm_stat_account(struct mm_struct *, vm_flags_t, long npages); 4403 4404 extern bool vma_is_special_mapping(const struct vm_area_struct *vma, 4405 const struct vm_special_mapping *sm); 4406 struct vm_area_struct *_install_special_mapping(struct mm_struct *mm, 4407 unsigned long addr, unsigned long len, 4408 vm_flags_t vm_flags, 4409 const struct vm_special_mapping *spec); 4410 4411 unsigned long randomize_stack_top(unsigned long stack_top); 4412 unsigned long randomize_page(unsigned long start, unsigned long range); 4413 4414 unsigned long 4415 __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, 4416 unsigned long pgoff, unsigned long flags, 4417 vma_flags_t vma_flags); 4418 4419 static inline unsigned long 4420 get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, 4421 unsigned long pgoff, unsigned long flags) 4422 { 4423 return __get_unmapped_area(file, addr, len, pgoff, flags, 4424 EMPTY_VMA_FLAGS); 4425 } 4426 4427 unsigned long do_mmap(struct file *file, unsigned long addr, 4428 unsigned long len, unsigned long prot, unsigned long flags, 4429 vma_flags_t vma_flags, unsigned long pgoff, unsigned long *populate, 4430 struct list_head *uf); 4431 extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, 4432 unsigned long start, size_t len, struct list_head *uf, 4433 bool unlock); 4434 int do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, 4435 struct mm_struct *mm, unsigned long start, 4436 unsigned long end, struct list_head *uf, bool unlock); 4437 extern int do_munmap(struct mm_struct *, unsigned long, size_t, 4438 struct list_head *uf); 4439 extern int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior); 4440 4441 #ifdef CONFIG_MMU 4442 extern int __mm_populate(unsigned long addr, unsigned long len, 4443 int ignore_errors); 4444 static inline void mm_populate(unsigned long addr, unsigned long len) 4445 { 4446 /* Ignore errors */ 4447 (void) __mm_populate(addr, len, 1); 4448 } 4449 #else 4450 static inline void mm_populate(unsigned long addr, unsigned long len) {} 4451 #endif 4452 4453 /* This takes the mm semaphore itself */ 4454 int __must_check vm_brk_flags(unsigned long addr, unsigned long request, bool is_exec); 4455 int vm_munmap(unsigned long start, size_t len); 4456 unsigned long __must_check vm_mmap(struct file *file, unsigned long addr, 4457 unsigned long len, unsigned long prot, 4458 unsigned long flag, unsigned long offset); 4459 unsigned long __must_check vm_mmap_shadow_stack(unsigned long addr, 4460 unsigned long len, unsigned long flags); 4461 4462 struct vm_unmapped_area_info { 4463 #define VM_UNMAPPED_AREA_TOPDOWN 1 4464 unsigned long flags; 4465 unsigned long length; 4466 unsigned long low_limit; 4467 unsigned long high_limit; 4468 unsigned long align_mask; 4469 unsigned long align_offset; 4470 unsigned long start_gap; 4471 }; 4472 4473 extern unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info); 4474 4475 /* truncate.c */ 4476 void truncate_inode_pages(struct address_space *mapping, loff_t lstart); 4477 void truncate_inode_pages_range(struct address_space *mapping, loff_t lstart, 4478 uoff_t lend); 4479 void truncate_inode_pages_final(struct address_space *mapping); 4480 4481 /* generic vm_area_ops exported for stackable file systems */ 4482 extern vm_fault_t filemap_fault(struct vm_fault *vmf); 4483 extern vm_fault_t filemap_map_pages(struct vm_fault *vmf, 4484 pgoff_t start_pgoff, pgoff_t end_pgoff); 4485 extern vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf); 4486 4487 extern unsigned long stack_guard_gap; 4488 /* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */ 4489 int expand_stack_locked(struct vm_area_struct *vma, unsigned long address); 4490 struct vm_area_struct *expand_stack(struct mm_struct * mm, unsigned long addr); 4491 4492 /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */ 4493 extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr); 4494 extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr, 4495 struct vm_area_struct **pprev); 4496 4497 /* 4498 * Look up the first VMA which intersects the interval [start_addr, end_addr) 4499 * NULL if none. Assume start_addr < end_addr. 4500 */ 4501 struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, 4502 unsigned long start_addr, unsigned long end_addr); 4503 4504 /** 4505 * vma_lookup() - Find a VMA at a specific address 4506 * @mm: The process address space. 4507 * @addr: The user address. 4508 * 4509 * Return: The vm_area_struct at the given address, %NULL otherwise. 4510 */ 4511 static inline 4512 struct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr) 4513 { 4514 return mtree_load(&mm->mm_mt, addr); 4515 } 4516 4517 static inline unsigned long stack_guard_start_gap(const struct vm_area_struct *vma) 4518 { 4519 if (vma->vm_flags & VM_GROWSDOWN) 4520 return stack_guard_gap; 4521 4522 /* See reasoning around the VM_SHADOW_STACK definition */ 4523 if (vma->vm_flags & VM_SHADOW_STACK) 4524 return PAGE_SIZE; 4525 4526 return 0; 4527 } 4528 4529 static inline unsigned long vm_start_gap(const struct vm_area_struct *vma) 4530 { 4531 unsigned long gap = stack_guard_start_gap(vma); 4532 unsigned long vm_start = vma->vm_start; 4533 4534 vm_start -= gap; 4535 if (vm_start > vma->vm_start) 4536 vm_start = 0; 4537 return vm_start; 4538 } 4539 4540 static inline unsigned long vm_end_gap(const struct vm_area_struct *vma) 4541 { 4542 unsigned long vm_end = vma->vm_end; 4543 4544 if (vma->vm_flags & VM_GROWSUP) { 4545 vm_end += stack_guard_gap; 4546 if (vm_end < vma->vm_end) 4547 vm_end = -PAGE_SIZE; 4548 } 4549 return vm_end; 4550 } 4551 4552 static inline unsigned long vma_pages(const struct vm_area_struct *vma) 4553 { 4554 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; 4555 } 4556 4557 /** 4558 * vma_start_pgoff() - Get the page offset of the start of @vma 4559 * @vma: The VMA whose page offset is required. 4560 * 4561 * If the VMA is file-backed, this is the page offset into the file. 4562 * 4563 * If @vma is anonymous, this is the virtual page offset of the start of the 4564 * VMA - if unfaulted, then vma->vm_start >> PAGE_SHIFT, if faulted then the 4565 * virtual page offset at the time of first fault. 4566 * 4567 * If @vma is a MAP_PRIVATE file-backed mapping, then this returns the 4568 * page offset within the file. 4569 * 4570 * Edge cases: nommu does not abide by these and CoW MAP_PRIVATE-pfnmap regions 4571 * have their page offset set to the first PFN in the range. 4572 * 4573 * Returns: The page offset of the start of @vma. 4574 */ 4575 static inline pgoff_t vma_start_pgoff(const struct vm_area_struct *vma) 4576 { 4577 return vma->vm_pgoff; 4578 } 4579 4580 /** 4581 * vma_end_pgoff() - Get the page offset of the exclusive end of @vma 4582 * @vma: The VMA whose end page offset is required. 4583 * 4584 * This returns the exclusive end page offset of @vma, which is useful for 4585 * expressing page offset ranges. 4586 * 4587 * See the description of vma_start_pgoff() for a description of VMA page 4588 * offsets. 4589 * 4590 * Returns: The exclusive end page offset of @vma. 4591 */ 4592 static inline pgoff_t vma_end_pgoff(const struct vm_area_struct *vma) 4593 { 4594 return vma_start_pgoff(vma) + vma_pages(vma); 4595 } 4596 4597 /** 4598 * vma_last_pgoff() - Get the page offset of the last page in @vma 4599 * @vma: The VMA whose last page offset is required. 4600 * 4601 * This returns the last page offset contained within @vma. 4602 * 4603 * See the description of vma_start_pgoff() for a description of VMA page 4604 * offsets. 4605 * 4606 * Returns: The last page offset of @vma. 4607 */ 4608 static inline pgoff_t vma_last_pgoff(const struct vm_area_struct *vma) 4609 { 4610 return vma_end_pgoff(vma) - 1; 4611 } 4612 4613 /** 4614 * vma_start_anon_pgoff() - Get the anonymous page offset of the start of @vma 4615 * @vma: The VMA whose anonymous page offset is required. 4616 * 4617 * If unfaulted, then this is vma->vm_start >> PAGE_SHIFT, if faulted then the 4618 * anonymous page offset at the time of first fault. 4619 * 4620 * If the VMA is anonymous, this returns the same value as vma_start_pgoff(). 4621 * 4622 * This value is used for tracking MAP_PRIVATE file-backed mappings by their 4623 * anonymous page offset. 4624 * 4625 * Returns: The anonymous page offset of the start of @vma. 4626 */ 4627 static inline pgoff_t vma_start_anon_pgoff(const struct vm_area_struct *vma) 4628 { 4629 pgoff_t pgoff = 0; 4630 4631 #ifdef CONFIG_64BIT 4632 pgoff += vma->__vm_anon_pgoff_hi; 4633 pgoff <<= 32; 4634 #endif 4635 pgoff += vma->__vm_anon_pgoff_lo; 4636 return pgoff; 4637 } 4638 4639 /** 4640 * vma_end_anon_pgoff() - Get the anonymous page offset of the exclusive end of 4641 * @vma. 4642 * @vma: The VMA whose end anonymous page offset is required. 4643 * 4644 * This returns the anonymous exclusive end page offset of @vma, which is useful 4645 * for expressing page offset ranges. 4646 * 4647 * See the description of vma_start_anon_pgoff() for a description of VMA 4648 * anonymous page offsets. 4649 * 4650 * Returns: The exclusive end anonymous page offset of @vma. 4651 */ 4652 static inline pgoff_t vma_end_anon_pgoff(const struct vm_area_struct *vma) 4653 { 4654 return vma_start_anon_pgoff(vma) + vma_pages(vma); 4655 } 4656 4657 /** 4658 * vma_last_anon_pgoff() - Get the anonymous page offset of the last page in 4659 * @vma. 4660 * @vma: The VMA whose last anonymous page offset is required. 4661 * 4662 * See the description of vma_start_anon_pgoff() for a description of VMA 4663 * anonymous page offsets. 4664 * 4665 * Returns: The last anonymous page offset of @vma. 4666 */ 4667 static inline pgoff_t vma_last_anon_pgoff(const struct vm_area_struct *vma) 4668 { 4669 return vma_end_anon_pgoff(vma) - 1; 4670 } 4671 4672 static inline unsigned long vma_desc_size(const struct vm_area_desc *desc) 4673 { 4674 return desc->end - desc->start; 4675 } 4676 4677 static inline unsigned long vma_desc_pages(const struct vm_area_desc *desc) 4678 { 4679 return vma_desc_size(desc) >> PAGE_SHIFT; 4680 } 4681 4682 /** 4683 * mmap_action_remap - helper for mmap_prepare hook to specify that a pure PFN 4684 * remap is required. 4685 * @desc: The VMA descriptor for the VMA requiring remap. 4686 * @start: The virtual address to start the remap from, must be within the VMA. 4687 * @start_pfn: The first PFN in the range to remap. 4688 * @size: The size of the range to remap, in bytes, at most spanning to the end 4689 * of the VMA. 4690 */ 4691 static inline void mmap_action_remap(struct vm_area_desc *desc, 4692 unsigned long start, 4693 unsigned long start_pfn, 4694 unsigned long size) 4695 { 4696 struct mmap_action *action = &desc->action; 4697 4698 /* [start, start + size) must be within the VMA. */ 4699 WARN_ON_ONCE(start < desc->start || start >= desc->end); 4700 WARN_ON_ONCE(start + size > desc->end); 4701 4702 action->type = MMAP_REMAP_PFN; 4703 action->remap.start = start; 4704 action->remap.start_pfn = start_pfn; 4705 action->remap.size = size; 4706 action->remap.pgprot = desc->page_prot; 4707 } 4708 4709 /** 4710 * mmap_action_remap_full - helper for mmap_prepare hook to specify that the 4711 * entirety of a VMA should be PFN remapped. 4712 * @desc: The VMA descriptor for the VMA requiring remap. 4713 * @start_pfn: The first PFN in the range to remap. 4714 */ 4715 static inline void mmap_action_remap_full(struct vm_area_desc *desc, 4716 unsigned long start_pfn) 4717 { 4718 mmap_action_remap(desc, desc->start, start_pfn, vma_desc_size(desc)); 4719 } 4720 4721 /** 4722 * mmap_action_ioremap - helper for mmap_prepare hook to specify that a pure PFN 4723 * I/O remap is required. 4724 * @desc: The VMA descriptor for the VMA requiring remap. 4725 * @start: The virtual address to start the remap from, must be within the VMA. 4726 * @start_pfn: The first PFN in the range to remap. 4727 * @size: The size of the range to remap, in bytes, at most spanning to the end 4728 * of the VMA. 4729 */ 4730 static inline void mmap_action_ioremap(struct vm_area_desc *desc, 4731 unsigned long start, 4732 unsigned long start_pfn, 4733 unsigned long size) 4734 { 4735 mmap_action_remap(desc, start, start_pfn, size); 4736 desc->action.type = MMAP_IO_REMAP_PFN; 4737 } 4738 4739 /** 4740 * mmap_action_ioremap_full - helper for mmap_prepare hook to specify that the 4741 * entirety of a VMA should be PFN I/O remapped. 4742 * @desc: The VMA descriptor for the VMA requiring remap. 4743 * @start_pfn: The first PFN in the range to remap. 4744 */ 4745 static inline void mmap_action_ioremap_full(struct vm_area_desc *desc, 4746 unsigned long start_pfn) 4747 { 4748 mmap_action_ioremap(desc, desc->start, start_pfn, vma_desc_size(desc)); 4749 } 4750 4751 /** 4752 * mmap_action_simple_ioremap - helper for mmap_prepare hook to specify that the 4753 * physical range in [start_phys_addr, start_phys_addr + size) should be I/O 4754 * remapped. 4755 * @desc: The VMA descriptor for the VMA requiring remap. 4756 * @start_phys_addr: Start of the physical memory to be mapped. 4757 * @size: Size of the area to map. 4758 * 4759 * NOTE: Some drivers might want to tweak desc->page_prot for purposes of 4760 * write-combine or similar. 4761 */ 4762 static inline void mmap_action_simple_ioremap(struct vm_area_desc *desc, 4763 phys_addr_t start_phys_addr, 4764 unsigned long size) 4765 { 4766 struct mmap_action *action = &desc->action; 4767 4768 action->simple_ioremap.start_phys_addr = start_phys_addr; 4769 action->simple_ioremap.size = size; 4770 action->type = MMAP_SIMPLE_IO_REMAP; 4771 } 4772 4773 /** 4774 * mmap_action_map_kernel_pages - helper for mmap_prepare hook to specify that 4775 * @num kernel pages contained in the @pages array should be mapped to userland 4776 * starting at virtual address @start. 4777 * @desc: The VMA descriptor for the VMA requiring kernel pags to be mapped. 4778 * @start: The virtual address from which to map them. 4779 * @pages: An array of struct page pointers describing the memory to map. 4780 * @nr_pages: The number of entries in the @pages array. 4781 */ 4782 static inline void mmap_action_map_kernel_pages(struct vm_area_desc *desc, 4783 unsigned long start, struct page **pages, 4784 unsigned long nr_pages) 4785 { 4786 struct mmap_action *action = &desc->action; 4787 4788 action->type = MMAP_KERNEL_PAGES; 4789 action->map_kernel.start = start; 4790 action->map_kernel.pages = pages; 4791 action->map_kernel.nr_pages = nr_pages; 4792 action->map_kernel.pgoff = desc->pgoff; 4793 } 4794 4795 /** 4796 * mmap_action_map_kernel_pages_full - helper for mmap_prepare hook to specify that 4797 * kernel pages contained in the @pages array should be mapped to userland 4798 * from @desc->start to @desc->end. 4799 * @desc: The VMA descriptor for the VMA requiring kernel pags to be mapped. 4800 * @pages: An array of struct page pointers describing the memory to map. 4801 * 4802 * The caller must ensure that @pages contains sufficient entries to cover the 4803 * entire range described by @desc. 4804 */ 4805 static inline void mmap_action_map_kernel_pages_full(struct vm_area_desc *desc, 4806 struct page **pages) 4807 { 4808 mmap_action_map_kernel_pages(desc, desc->start, pages, 4809 vma_desc_pages(desc)); 4810 } 4811 4812 static inline 4813 void mmap_action_map_discontig_kernel_pages(struct vm_area_desc *desc, 4814 void *init_private, const struct discontig_kernel_page_ops *ops) 4815 { 4816 struct mmap_action *action = &desc->action; 4817 4818 action->type = MMAP_DISCONTIG_KERNEL_PAGES; 4819 action->map_kernel_discontig.init_private = init_private; 4820 action->map_kernel_discontig.ops = ops; 4821 } 4822 4823 int mmap_action_prepare(struct vm_area_desc *desc); 4824 int mmap_action_complete(struct vm_area_struct *vma, 4825 struct mmap_action *action, bool is_compat); 4826 4827 static inline void 4828 discontig_kernel_map_abort(struct discontig_kernel_page_state *state) 4829 { 4830 state->action = DISCONTIG_KERNEL_PAGE_ABORT; 4831 } 4832 4833 static inline void 4834 discontig_kernel_map_page(struct discontig_kernel_page_state *state, 4835 struct page *page) 4836 { 4837 struct folio *folio = page_folio(page); 4838 unsigned long nr_pages = 1; 4839 4840 if (folio_test_large(folio)) { 4841 VM_WARN_ON_ONCE(page != folio_page(folio, 0)); 4842 nr_pages = folio_nr_pages(folio); 4843 } 4844 4845 state->action = DISCONTIG_KERNEL_PAGE_MAP_CONTIG_PAGES; 4846 state->__page = page; 4847 state->__nr_pages = min(nr_pages, state->nr_pages_remain); 4848 } 4849 4850 static inline void 4851 discontig_kernel_map_pages(struct discontig_kernel_page_state *state, 4852 struct page **page_arr, unsigned long nr_pages) 4853 { 4854 state->action = DISCONTIG_KERNEL_PAGE_MAP_PAGES; 4855 state->__page_arr = page_arr; 4856 state->__nr_pages = min(nr_pages, state->nr_pages_remain); 4857 } 4858 4859 /* Look up the first VMA which exactly match the interval vm_start ... vm_end */ 4860 static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm, 4861 unsigned long vm_start, unsigned long vm_end) 4862 { 4863 struct vm_area_struct *vma = vma_lookup(mm, vm_start); 4864 4865 if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end)) 4866 vma = NULL; 4867 4868 return vma; 4869 } 4870 4871 /** 4872 * range_is_subset - Is the specified inner range a subset of the outer range? 4873 * @outer_start: The start of the outer range. 4874 * @outer_end: The exclusive end of the outer range. 4875 * @inner_start: The start of the inner range. 4876 * @inner_end: The exclusive end of the inner range. 4877 * 4878 * Returns: %true if [inner_start, inner_end) is a subset of [outer_start, 4879 * outer_end), otherwise %false. 4880 */ 4881 static inline bool range_is_subset(unsigned long outer_start, 4882 unsigned long outer_end, 4883 unsigned long inner_start, 4884 unsigned long inner_end) 4885 { 4886 return outer_start <= inner_start && inner_end <= outer_end; 4887 } 4888 4889 /** 4890 * range_in_vma - is the specified [@start, @end) range a subset of the VMA? 4891 * @vma: The VMA against which we want to check [@start, @end). 4892 * @start: The start of the range we wish to check. 4893 * @end: The exclusive end of the range we wish to check. 4894 * 4895 * Returns: %true if [@start, @end) is a subset of [@vma->vm_start, 4896 * @vma->vm_end), %false otherwise. 4897 */ 4898 static inline bool range_in_vma(const struct vm_area_struct *vma, 4899 unsigned long start, unsigned long end) 4900 { 4901 if (!vma) 4902 return false; 4903 4904 return range_is_subset(vma->vm_start, vma->vm_end, start, end); 4905 } 4906 4907 /** 4908 * range_in_vma_desc - is the specified [@start, @end) range a subset of the VMA 4909 * described by @desc, a VMA descriptor? 4910 * @desc: The VMA descriptor against which we want to check [@start, @end). 4911 * @start: The start of the range we wish to check. 4912 * @end: The exclusive end of the range we wish to check. 4913 * 4914 * Returns: %true if [@start, @end) is a subset of [@desc->start, @desc->end), 4915 * %false otherwise. 4916 */ 4917 static inline bool range_in_vma_desc(const struct vm_area_desc *desc, 4918 unsigned long start, unsigned long end) 4919 { 4920 if (!desc) 4921 return false; 4922 4923 return range_is_subset(desc->start, desc->end, start, end); 4924 } 4925 4926 #ifdef CONFIG_MMU 4927 pgprot_t vm_get_page_prot(vm_flags_t vm_flags); 4928 4929 static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags) 4930 { 4931 const vm_flags_t vm_flags = vma_flags_to_legacy(vma_flags); 4932 4933 return vm_get_page_prot(vm_flags); 4934 } 4935 4936 static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma) 4937 { 4938 return vma_flags_to_page_prot(vma->flags); 4939 } 4940 4941 void vma_set_page_prot(struct vm_area_struct *vma); 4942 #else 4943 static inline pgprot_t vm_get_page_prot(vm_flags_t vm_flags) 4944 { 4945 return __pgprot(0); 4946 } 4947 static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags) 4948 { 4949 return __pgprot(0); 4950 } 4951 static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma) 4952 { 4953 return __pgprot(0); 4954 } 4955 static inline void vma_set_page_prot(struct vm_area_struct *vma) 4956 { 4957 vma->vm_page_prot = vma_get_page_prot(vma); 4958 } 4959 #endif 4960 4961 void vma_set_file(struct vm_area_struct *vma, struct file *file); 4962 4963 #ifdef CONFIG_NUMA_BALANCING 4964 unsigned long change_prot_numa(struct vm_area_struct *vma, 4965 unsigned long start, unsigned long end); 4966 #endif 4967 4968 struct vm_area_struct *find_extend_vma_locked(struct mm_struct *, 4969 unsigned long addr); 4970 int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, 4971 unsigned long pfn, unsigned long size, pgprot_t pgprot); 4972 4973 int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); 4974 int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, 4975 struct page **pages, unsigned long *num); 4976 int vm_map_pages(struct vm_area_struct *vma, struct page **pages, 4977 unsigned long num); 4978 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, 4979 unsigned long num); 4980 vm_fault_t vmf_insert_page_mkwrite(struct vm_fault *vmf, struct page *page, 4981 bool write); 4982 vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr, 4983 unsigned long pfn); 4984 vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, 4985 unsigned long pfn, pgprot_t pgprot); 4986 vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr, 4987 unsigned long pfn); 4988 vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma, 4989 unsigned long addr, unsigned long pfn); 4990 int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len); 4991 4992 static inline vm_fault_t vmf_insert_page(struct vm_area_struct *vma, 4993 unsigned long addr, struct page *page) 4994 { 4995 int err = vm_insert_page(vma, addr, page); 4996 4997 if (err == -ENOMEM) 4998 return VM_FAULT_OOM; 4999 if (err < 0 && err != -EBUSY) 5000 return VM_FAULT_SIGBUS; 5001 5002 return VM_FAULT_NOPAGE; 5003 } 5004 5005 #ifndef io_remap_pfn_range_pfn 5006 static inline unsigned long io_remap_pfn_range_pfn(unsigned long pfn, 5007 unsigned long size) 5008 { 5009 return pfn; 5010 } 5011 #endif 5012 5013 static inline int io_remap_pfn_range(struct vm_area_struct *vma, 5014 unsigned long addr, unsigned long orig_pfn, 5015 unsigned long size, pgprot_t orig_prot) 5016 { 5017 const unsigned long pfn = io_remap_pfn_range_pfn(orig_pfn, size); 5018 const pgprot_t prot = pgprot_decrypted(orig_prot); 5019 5020 return remap_pfn_range(vma, addr, pfn, size, prot); 5021 } 5022 5023 static inline vm_fault_t vmf_error(int err) 5024 { 5025 if (err == -ENOMEM) 5026 return VM_FAULT_OOM; 5027 else if (err == -EHWPOISON) 5028 return VM_FAULT_HWPOISON; 5029 return VM_FAULT_SIGBUS; 5030 } 5031 5032 /* 5033 * Convert errno to return value for ->page_mkwrite() calls. 5034 * 5035 * This should eventually be merged with vmf_error() above, but will need a 5036 * careful audit of all vmf_error() callers. 5037 */ 5038 static inline vm_fault_t vmf_fs_error(int err) 5039 { 5040 if (err == 0) 5041 return VM_FAULT_LOCKED; 5042 if (err == -EFAULT || err == -EAGAIN) 5043 return VM_FAULT_NOPAGE; 5044 if (err == -ENOMEM) 5045 return VM_FAULT_OOM; 5046 /* -ENOSPC, -EDQUOT, -EIO ... */ 5047 return VM_FAULT_SIGBUS; 5048 } 5049 5050 static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags) 5051 { 5052 if (vm_fault & VM_FAULT_OOM) 5053 return -ENOMEM; 5054 if (vm_fault & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE)) 5055 return (foll_flags & FOLL_HWPOISON) ? -EHWPOISON : -EFAULT; 5056 if (vm_fault & (VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV)) 5057 return -EFAULT; 5058 return 0; 5059 } 5060 5061 /* 5062 * Indicates whether GUP can follow a PROT_NONE mapped page, or whether 5063 * a (NUMA hinting or userfaultfd RWP) fault is required. 5064 */ 5065 static inline bool gup_can_follow_protnone(const struct vm_area_struct *vma, 5066 unsigned int flags) 5067 { 5068 /* 5069 * VM_UFFD_RWP uses protnone as an access-tracking marker, not for 5070 * NUMA hinting. GUP must always take a fault so the access is 5071 * delivered to userfaultfd, regardless of FOLL_HONOR_NUMA_FAULT. 5072 * 5073 * Only do so while the VMA is accessible. If it has been made 5074 * inaccessible (e.g. mprotect(PROT_NONE)), fall through to the guard 5075 * below: forcing a fault there would loop, as handle_mm_fault() makes 5076 * no progress on protnone in an inaccessible VMA, and the access is 5077 * denied regardless of RWP anyway. 5078 */ 5079 if (vma_test_single_mask(vma, VMA_UFFD_RWP) && vma_is_accessible(vma)) 5080 return false; 5081 5082 /* 5083 * If callers don't want to honor NUMA hinting faults, no need to 5084 * determine if we would actually have to trigger a NUMA hinting fault. 5085 */ 5086 if (!(flags & FOLL_HONOR_NUMA_FAULT)) 5087 return true; 5088 5089 /* 5090 * NUMA hinting faults don't apply in inaccessible (PROT_NONE) VMAs. 5091 * 5092 * Requiring a fault here even for inaccessible VMAs would mean that 5093 * FOLL_FORCE cannot make any progress, because handle_mm_fault() 5094 * refuses to process NUMA hinting faults in inaccessible VMAs. 5095 */ 5096 return !vma_is_accessible(vma); 5097 } 5098 5099 typedef int (*pte_fn_t)(pte_t *pte, unsigned long addr, void *data); 5100 extern int apply_to_page_range(struct mm_struct *mm, unsigned long address, 5101 unsigned long size, pte_fn_t fn, void *data); 5102 extern int apply_to_existing_page_range(struct mm_struct *mm, 5103 unsigned long address, unsigned long size, 5104 pte_fn_t fn, void *data); 5105 5106 #ifdef CONFIG_PAGE_POISONING 5107 extern void __kernel_poison_pages(struct page *page, int numpages); 5108 extern void __kernel_unpoison_pages(struct page *page, int numpages); 5109 extern bool _page_poisoning_enabled_early; 5110 DECLARE_STATIC_KEY_FALSE(_page_poisoning_enabled); 5111 static inline bool page_poisoning_enabled(void) 5112 { 5113 return _page_poisoning_enabled_early; 5114 } 5115 /* 5116 * For use in fast paths after init_mem_debugging() has run, or when a 5117 * false negative result is not harmful when called too early. 5118 */ 5119 static inline bool page_poisoning_enabled_static(void) 5120 { 5121 return static_branch_unlikely(&_page_poisoning_enabled); 5122 } 5123 static inline void kernel_poison_pages(struct page *page, int numpages) 5124 { 5125 if (page_poisoning_enabled_static()) 5126 __kernel_poison_pages(page, numpages); 5127 } 5128 static inline void kernel_unpoison_pages(struct page *page, int numpages) 5129 { 5130 if (page_poisoning_enabled_static()) 5131 __kernel_unpoison_pages(page, numpages); 5132 } 5133 #else 5134 static inline bool page_poisoning_enabled(void) { return false; } 5135 static inline bool page_poisoning_enabled_static(void) { return false; } 5136 static inline void __kernel_poison_pages(struct page *page, int nunmpages) { } 5137 static inline void kernel_poison_pages(struct page *page, int numpages) { } 5138 static inline void kernel_unpoison_pages(struct page *page, int numpages) { } 5139 #endif 5140 5141 DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc); 5142 static inline bool want_init_on_alloc(gfp_t flags) 5143 { 5144 if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, 5145 &init_on_alloc)) 5146 return true; 5147 return flags & __GFP_ZERO; 5148 } 5149 5150 DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free); 5151 static inline bool want_init_on_free(void) 5152 { 5153 return static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON, 5154 &init_on_free); 5155 } 5156 5157 extern bool _debug_pagealloc_enabled_early; 5158 DECLARE_STATIC_KEY_FALSE(_debug_pagealloc_enabled); 5159 5160 static inline bool debug_pagealloc_enabled(void) 5161 { 5162 return IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) && 5163 _debug_pagealloc_enabled_early; 5164 } 5165 5166 /* 5167 * For use in fast paths after mem_debugging_and_hardening_init() has run, 5168 * or when a false negative result is not harmful when called too early. 5169 */ 5170 static inline bool debug_pagealloc_enabled_static(void) 5171 { 5172 if (!IS_ENABLED(CONFIG_DEBUG_PAGEALLOC)) 5173 return false; 5174 5175 return static_branch_unlikely(&_debug_pagealloc_enabled); 5176 } 5177 5178 /* 5179 * To support DEBUG_PAGEALLOC architecture must ensure that 5180 * __kernel_map_pages() never fails 5181 */ 5182 extern void __kernel_map_pages(struct page *page, int numpages, int enable); 5183 #ifdef CONFIG_DEBUG_PAGEALLOC 5184 static inline void debug_pagealloc_map_pages(struct page *page, int numpages) 5185 { 5186 iommu_debug_check_unmapped(page, numpages); 5187 5188 if (debug_pagealloc_enabled_static()) 5189 __kernel_map_pages(page, numpages, 1); 5190 } 5191 5192 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) 5193 { 5194 iommu_debug_check_unmapped(page, numpages); 5195 5196 if (debug_pagealloc_enabled_static()) 5197 __kernel_map_pages(page, numpages, 0); 5198 } 5199 5200 extern unsigned int _debug_guardpage_minorder; 5201 DECLARE_STATIC_KEY_FALSE(_debug_guardpage_enabled); 5202 5203 static inline unsigned int debug_guardpage_minorder(void) 5204 { 5205 return _debug_guardpage_minorder; 5206 } 5207 5208 static inline bool debug_guardpage_enabled(void) 5209 { 5210 return static_branch_unlikely(&_debug_guardpage_enabled); 5211 } 5212 5213 static inline bool page_is_guard(const struct page *page) 5214 { 5215 if (!debug_guardpage_enabled()) 5216 return false; 5217 5218 return PageGuard(page); 5219 } 5220 5221 bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order); 5222 static inline bool set_page_guard(struct zone *zone, struct page *page, 5223 unsigned int order) 5224 { 5225 if (!debug_guardpage_enabled()) 5226 return false; 5227 return __set_page_guard(zone, page, order); 5228 } 5229 5230 void __clear_page_guard(struct zone *zone, struct page *page, unsigned int order); 5231 static inline void clear_page_guard(struct zone *zone, struct page *page, 5232 unsigned int order) 5233 { 5234 if (!debug_guardpage_enabled()) 5235 return; 5236 __clear_page_guard(zone, page, order); 5237 } 5238 5239 #else /* CONFIG_DEBUG_PAGEALLOC */ 5240 static inline void debug_pagealloc_map_pages(struct page *page, int numpages) {} 5241 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) {} 5242 static inline unsigned int debug_guardpage_minorder(void) { return 0; } 5243 static inline bool debug_guardpage_enabled(void) { return false; } 5244 static inline bool page_is_guard(const struct page *page) { return false; } 5245 static inline bool set_page_guard(struct zone *zone, struct page *page, 5246 unsigned int order) { return false; } 5247 static inline void clear_page_guard(struct zone *zone, struct page *page, 5248 unsigned int order) {} 5249 #endif /* CONFIG_DEBUG_PAGEALLOC */ 5250 5251 #ifndef clear_pages 5252 /** 5253 * clear_pages() - clear a page range for kernel-internal use. 5254 * @addr: start address 5255 * @npages: number of pages 5256 * 5257 * Use clear_user_pages() instead when clearing a page range to be 5258 * mapped to user space. 5259 * 5260 * Does absolutely no exception handling. 5261 * 5262 * Note that even though the clearing operation is preemptible, clear_pages() 5263 * does not (and on architectures where it reduces to a few long-running 5264 * instructions, might not be able to) call cond_resched() to check if 5265 * rescheduling is required. 5266 * 5267 * When running under preemptible models this is not a problem. Under 5268 * cooperatively scheduled models, however, the caller is expected to 5269 * limit @npages to no more than PROCESS_PAGES_NON_PREEMPT_BATCH. 5270 */ 5271 static inline void clear_pages(void *addr, unsigned int npages) 5272 { 5273 do { 5274 clear_page(addr); 5275 addr += PAGE_SIZE; 5276 } while (--npages); 5277 } 5278 #endif 5279 5280 #ifndef PROCESS_PAGES_NON_PREEMPT_BATCH 5281 #ifdef clear_pages 5282 /* 5283 * The architecture defines clear_pages(), and we assume that it is 5284 * generally "fast". So choose a batch size large enough to allow the processor 5285 * headroom for optimizing the operation and yet small enough that we see 5286 * reasonable preemption latency for when this optimization is not possible 5287 * (ex. slow microarchitectures, memory bandwidth saturation.) 5288 * 5289 * With a value of 32MB and assuming a memory bandwidth of ~10GBps, this should 5290 * result in worst case preemption latency of around 3ms when clearing pages. 5291 * 5292 * (See comment above clear_pages() for why preemption latency is a concern 5293 * here.) 5294 */ 5295 #define PROCESS_PAGES_NON_PREEMPT_BATCH (SZ_32M >> PAGE_SHIFT) 5296 #else /* !clear_pages */ 5297 /* 5298 * The architecture does not provide a clear_pages() implementation. Assume 5299 * that clear_page() -- which clear_pages() will fallback to -- is relatively 5300 * slow and choose a small value for PROCESS_PAGES_NON_PREEMPT_BATCH. 5301 */ 5302 #define PROCESS_PAGES_NON_PREEMPT_BATCH 1 5303 #endif 5304 #endif 5305 5306 #ifdef __HAVE_ARCH_GATE_AREA 5307 extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm); 5308 extern int in_gate_area_no_mm(unsigned long addr); 5309 extern int in_gate_area(struct mm_struct *mm, unsigned long addr); 5310 #else 5311 static inline struct vm_area_struct *get_gate_vma(struct mm_struct *mm) 5312 { 5313 return NULL; 5314 } 5315 static inline int in_gate_area_no_mm(unsigned long addr) { return 0; } 5316 static inline int in_gate_area(struct mm_struct *mm, unsigned long addr) 5317 { 5318 return 0; 5319 } 5320 #endif /* __HAVE_ARCH_GATE_AREA */ 5321 5322 bool process_shares_mm(const struct task_struct *p, const struct mm_struct *mm); 5323 5324 void drop_slab(void); 5325 5326 #ifndef CONFIG_MMU 5327 #define randomize_va_space 0 5328 #else 5329 extern int randomize_va_space; 5330 #endif 5331 5332 const char * arch_vma_name(struct vm_area_struct *vma); 5333 #ifdef CONFIG_MMU 5334 void print_vma_addr(char *prefix, unsigned long rip); 5335 #else 5336 static inline void print_vma_addr(char *prefix, unsigned long rip) 5337 { 5338 } 5339 #endif 5340 5341 struct page * __populate_section_memmap(unsigned long pfn, 5342 unsigned long nr_pages, int nid, struct vmem_altmap *altmap, 5343 struct dev_pagemap *pgmap); 5344 void *vmemmap_alloc_block(unsigned long size, int node); 5345 struct vmem_altmap; 5346 void *vmemmap_alloc_block_buf(unsigned long size, int node, 5347 struct vmem_altmap *altmap); 5348 void vmemmap_verify(pte_t *, int, unsigned long, unsigned long); 5349 void vmemmap_set_pmd(pmd_t *pmd, void *p, int node, 5350 unsigned long addr, unsigned long next); 5351 int vmemmap_check_pmd(pmd_t *pmd, int node, 5352 unsigned long addr, unsigned long next); 5353 int vmemmap_populate_basepages(unsigned long start, unsigned long end, 5354 int node, struct vmem_altmap *altmap); 5355 int vmemmap_populate_hugepages(unsigned long start, unsigned long end, 5356 int node, struct vmem_altmap *altmap); 5357 int vmemmap_populate(unsigned long start, unsigned long end, int node, 5358 struct vmem_altmap *altmap); 5359 void vmemmap_wrprotect_hvo(unsigned long start, unsigned long end, int node, 5360 unsigned long headsize); 5361 void vmemmap_populate_print_last(void); 5362 #ifdef CONFIG_MEMORY_HOTPLUG 5363 void vmemmap_free(unsigned long start, unsigned long end, 5364 struct vmem_altmap *altmap); 5365 #endif 5366 5367 #ifdef CONFIG_SPARSEMEM_VMEMMAP 5368 static inline unsigned long vmem_altmap_offset(const struct vmem_altmap *altmap) 5369 { 5370 /* number of pfns from base where pfn_to_page() is valid */ 5371 if (altmap) 5372 return altmap->reserve + altmap->free; 5373 return 0; 5374 } 5375 5376 static inline void vmem_altmap_free(struct vmem_altmap *altmap, 5377 unsigned long nr_pfns) 5378 { 5379 altmap->alloc -= nr_pfns; 5380 } 5381 #else 5382 static inline unsigned long vmem_altmap_offset(const struct vmem_altmap *altmap) 5383 { 5384 return 0; 5385 } 5386 5387 static inline void vmem_altmap_free(struct vmem_altmap *altmap, 5388 unsigned long nr_pfns) 5389 { 5390 } 5391 #endif 5392 5393 #ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP 5394 static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap, 5395 struct dev_pagemap *pgmap) 5396 { 5397 unsigned long nr_pages; 5398 unsigned long nr_vmemmap_pages; 5399 5400 if (!IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION)) 5401 return false; 5402 5403 if (!pgmap || !is_power_of_2(sizeof(struct page))) 5404 return false; 5405 5406 nr_pages = pgmap_vmemmap_nr(pgmap); 5407 nr_vmemmap_pages = ((nr_pages * sizeof(struct page)) >> PAGE_SHIFT); 5408 /* 5409 * For vmemmap optimization with DAX we need minimum 2 vmemmap 5410 * pages. See layout diagram in Documentation/mm/vmemmap_dedup.rst 5411 */ 5412 return !altmap && (nr_vmemmap_pages > VMEMMAP_OPTIMIZATION_PAGES); 5413 } 5414 /* 5415 * If we don't have an architecture override, use the generic rule 5416 */ 5417 #ifndef vmemmap_can_optimize 5418 #define vmemmap_can_optimize __vmemmap_can_optimize 5419 #endif 5420 5421 #else 5422 static inline bool vmemmap_can_optimize(struct vmem_altmap *altmap, 5423 struct dev_pagemap *pgmap) 5424 { 5425 return false; 5426 } 5427 #endif 5428 5429 enum mf_flags { 5430 MF_COUNT_INCREASED = 1 << 0, 5431 MF_ACTION_REQUIRED = 1 << 1, 5432 MF_MUST_KILL = 1 << 2, 5433 MF_SOFT_OFFLINE = 1 << 3, 5434 MF_UNPOISON = 1 << 4, 5435 MF_SW_SIMULATED = 1 << 5, 5436 MF_NO_RETRY = 1 << 6, 5437 MF_MEM_PRE_REMOVE = 1 << 7, 5438 }; 5439 int mf_dax_kill_procs(struct address_space *mapping, pgoff_t index, 5440 unsigned long count, int mf_flags); 5441 extern int memory_failure(unsigned long pfn, int flags); 5442 extern int unpoison_memory(unsigned long pfn); 5443 extern atomic_long_t num_poisoned_pages __read_mostly; 5444 extern int soft_offline_page(unsigned long pfn, int flags); 5445 #ifdef CONFIG_MEMORY_FAILURE 5446 /* 5447 * Sysfs entries for memory failure handling statistics. 5448 */ 5449 extern const struct attribute_group memory_failure_attr_group; 5450 extern void memory_failure_queue(unsigned long pfn, int flags); 5451 void num_poisoned_pages_inc(unsigned long pfn); 5452 void num_poisoned_pages_sub(unsigned long pfn, long i); 5453 phys_addr_t range_first_hwpoison(phys_addr_t start, unsigned long size); 5454 phys_addr_t range_last_hwpoison(phys_addr_t start, unsigned long size); 5455 #else 5456 static inline void memory_failure_queue(unsigned long pfn, int flags) 5457 { 5458 } 5459 5460 static inline void num_poisoned_pages_inc(unsigned long pfn) 5461 { 5462 } 5463 5464 static inline void num_poisoned_pages_sub(unsigned long pfn, long i) 5465 { 5466 } 5467 5468 static inline phys_addr_t range_first_hwpoison(phys_addr_t start, 5469 unsigned long size) 5470 { 5471 return PHYS_ADDR_MAX; 5472 } 5473 5474 static inline phys_addr_t range_last_hwpoison(phys_addr_t start, 5475 unsigned long size) 5476 { 5477 return PHYS_ADDR_MAX; 5478 } 5479 #endif 5480 5481 #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG) 5482 extern void memblk_nr_poison_inc(unsigned long pfn); 5483 extern void memblk_nr_poison_sub(unsigned long pfn, long i); 5484 #else 5485 static inline void memblk_nr_poison_inc(unsigned long pfn) 5486 { 5487 } 5488 5489 static inline void memblk_nr_poison_sub(unsigned long pfn, long i) 5490 { 5491 } 5492 #endif 5493 5494 #ifndef arch_memory_failure 5495 static inline int arch_memory_failure(unsigned long pfn, int flags) 5496 { 5497 return -ENXIO; 5498 } 5499 #endif 5500 5501 #ifndef arch_is_platform_page 5502 static inline bool arch_is_platform_page(u64 paddr) 5503 { 5504 return false; 5505 } 5506 #endif 5507 5508 /* 5509 * Error handlers for various types of pages. 5510 */ 5511 enum mf_result { 5512 MF_IGNORED, /* Error: cannot be handled */ 5513 MF_FAILED, /* Error: handling failed */ 5514 MF_DELAYED, /* Will be handled later */ 5515 MF_RECOVERED, /* Successfully recovered */ 5516 }; 5517 5518 enum mf_action_page_type { 5519 MF_MSG_KERNEL, 5520 MF_MSG_KERNEL_HIGH_ORDER, 5521 MF_MSG_DIFFERENT_COMPOUND, 5522 MF_MSG_HUGE, 5523 MF_MSG_FREE_HUGE, 5524 MF_MSG_GET_HWPOISON, 5525 MF_MSG_UNMAP_FAILED, 5526 MF_MSG_DIRTY_SWAPCACHE, 5527 MF_MSG_CLEAN_SWAPCACHE, 5528 MF_MSG_DIRTY_MLOCKED_LRU, 5529 MF_MSG_CLEAN_MLOCKED_LRU, 5530 MF_MSG_DIRTY_UNEVICTABLE_LRU, 5531 MF_MSG_CLEAN_UNEVICTABLE_LRU, 5532 MF_MSG_DIRTY_LRU, 5533 MF_MSG_CLEAN_LRU, 5534 MF_MSG_TRUNCATED_LRU, 5535 MF_MSG_BUDDY, 5536 MF_MSG_DAX, 5537 MF_MSG_UNSPLIT_THP, 5538 MF_MSG_ALREADY_POISONED, 5539 MF_MSG_PFN_MAP, 5540 MF_MSG_UNKNOWN, 5541 }; 5542 5543 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS) 5544 void folio_zero_user(struct folio *folio, unsigned long addr_hint); 5545 int copy_user_large_folio(struct folio *dst, struct folio *src, 5546 unsigned long addr_hint, 5547 struct vm_area_struct *vma); 5548 long copy_folio_from_user(struct folio *dst_folio, 5549 const void __user *usr_src, 5550 bool allow_pagefault); 5551 5552 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */ 5553 5554 #if MAX_NUMNODES > 1 5555 void __init setup_nr_node_ids(void); 5556 #else 5557 static inline void setup_nr_node_ids(void) {} 5558 #endif 5559 5560 extern int memcmp_pages(struct page *page1, struct page *page2); 5561 5562 static inline int pages_identical(struct page *page1, struct page *page2) 5563 { 5564 return !memcmp_pages(page1, page2); 5565 } 5566 5567 #ifdef CONFIG_MAPPING_DIRTY_HELPERS 5568 unsigned long clean_record_shared_mapping_range(struct address_space *mapping, 5569 pgoff_t first_index, pgoff_t nr, 5570 pgoff_t bitmap_pgoff, 5571 unsigned long *bitmap, 5572 pgoff_t *start, 5573 pgoff_t *end); 5574 5575 unsigned long wp_shared_mapping_range(struct address_space *mapping, 5576 pgoff_t first_index, pgoff_t nr); 5577 #endif 5578 5579 #ifdef CONFIG_ANON_VMA_NAME 5580 int set_anon_vma_name(unsigned long addr, unsigned long size, 5581 const char __user *uname); 5582 #else 5583 static inline 5584 int set_anon_vma_name(unsigned long addr, unsigned long size, 5585 const char __user *uname) 5586 { 5587 return -EINVAL; 5588 } 5589 #endif 5590 5591 #ifdef CONFIG_UNACCEPTED_MEMORY 5592 5593 bool range_contains_unaccepted_memory(phys_addr_t start, unsigned long size); 5594 void accept_memory(phys_addr_t start, unsigned long size); 5595 5596 #else 5597 5598 static inline bool range_contains_unaccepted_memory(phys_addr_t start, 5599 unsigned long size) 5600 { 5601 return false; 5602 } 5603 5604 static inline void accept_memory(phys_addr_t start, unsigned long size) 5605 { 5606 } 5607 5608 #endif 5609 5610 static inline bool pfn_is_unaccepted_memory(unsigned long pfn) 5611 { 5612 return range_contains_unaccepted_memory(pfn << PAGE_SHIFT, PAGE_SIZE); 5613 } 5614 5615 void vma_pgtable_walk_begin(struct vm_area_struct *vma); 5616 void vma_pgtable_walk_end(struct vm_area_struct *vma); 5617 5618 int reserve_mem_find_by_name(const char *name, phys_addr_t *start, phys_addr_t *size); 5619 int reserve_mem_release_by_name(const char *name); 5620 5621 #ifdef CONFIG_64BIT 5622 void mseal_mmap_page_zero(void); 5623 #else 5624 static inline void mseal_mmap_page_zero(void) {} 5625 #endif 5626 5627 /* 5628 * user_alloc_needs_zeroing checks if a user folio from page allocator needs to 5629 * be zeroed or not. 5630 */ 5631 static inline bool user_alloc_needs_zeroing(void) 5632 { 5633 /* 5634 * for user folios, arch with cache aliasing requires cache flush and 5635 * arc changes folio->flags to make icache coherent with dcache, so 5636 * always return false to make caller use 5637 * clear_user_page()/clear_user_highpage(). 5638 */ 5639 return cpu_dcache_is_aliasing() || cpu_icache_is_aliasing() || 5640 !static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, 5641 &init_on_alloc); 5642 } 5643 5644 int arch_get_shadow_stack_status(struct task_struct *t, unsigned long __user *status); 5645 int arch_set_shadow_stack_status(struct task_struct *t, unsigned long status); 5646 int arch_lock_shadow_stack_status(struct task_struct *t, unsigned long status); 5647 5648 /* 5649 * DMA mapping IDs for page_pool 5650 * 5651 * When DMA-mapping a page, page_pool allocates an ID (from an xarray) and 5652 * stashes it in the upper bits of page->pp_magic. We always want to be able to 5653 * unambiguously identify page pool pages (using page_pool_page_is_pp()). Non-PP 5654 * pages can have arbitrary kernel pointers stored in the same field as pp_magic 5655 * (since it overlaps with page->lru.next), so we must ensure that we cannot 5656 * mistake a valid kernel pointer with any of the values we write into this 5657 * field. 5658 * 5659 * On architectures that set POISON_POINTER_DELTA, this is already ensured, 5660 * since this value becomes part of PP_SIGNATURE; meaning we can just use the 5661 * space between the PP_SIGNATURE value (without POISON_POINTER_DELTA), and the 5662 * lowest bits of POISON_POINTER_DELTA. On arches where POISON_POINTER_DELTA is 5663 * 0, we use the lowest bit of PAGE_OFFSET as the boundary if that value is 5664 * known at compile-time. 5665 * 5666 * If the value of PAGE_OFFSET is not known at compile time, or if it is too 5667 * small to leave at least 8 bits available above PP_SIGNATURE, we define the 5668 * number of bits to be 0, which turns off the DMA index tracking altogether 5669 * (see page_pool_register_dma_index()). 5670 */ 5671 #define PP_DMA_INDEX_SHIFT (1 + __fls(PP_SIGNATURE - POISON_POINTER_DELTA)) 5672 #if POISON_POINTER_DELTA > 0 5673 /* PP_SIGNATURE includes POISON_POINTER_DELTA, so limit the size of the DMA 5674 * index to not overlap with that if set 5675 */ 5676 #define PP_DMA_INDEX_BITS MIN(32, __ffs(POISON_POINTER_DELTA) - PP_DMA_INDEX_SHIFT) 5677 #else 5678 /* Use the lowest bit of PAGE_OFFSET if there's at least 8 bits available; see above */ 5679 #define PP_DMA_INDEX_MIN_OFFSET (1 << (PP_DMA_INDEX_SHIFT + 8)) 5680 #define PP_DMA_INDEX_BITS ((__builtin_constant_p(PAGE_OFFSET) && \ 5681 PAGE_OFFSET >= PP_DMA_INDEX_MIN_OFFSET && \ 5682 !(PAGE_OFFSET & (PP_DMA_INDEX_MIN_OFFSET - 1))) ? \ 5683 MIN(32, __ffs(PAGE_OFFSET) - PP_DMA_INDEX_SHIFT) : 0) 5684 5685 #endif 5686 5687 #define PP_DMA_INDEX_MASK GENMASK(PP_DMA_INDEX_BITS + PP_DMA_INDEX_SHIFT - 1, \ 5688 PP_DMA_INDEX_SHIFT) 5689 5690 /* Mask used for checking in page_pool_page_is_pp() below. page->pp_magic is 5691 * OR'ed with PP_SIGNATURE after the allocation in order to preserve bit 0 for 5692 * the head page of compound page and bit 1 for pfmemalloc page, as well as the 5693 * bits used for the DMA index. page_is_pfmemalloc() is checked in 5694 * __page_pool_put_page() to avoid recycling the pfmemalloc page. 5695 */ 5696 #define PP_MAGIC_MASK ~(PP_DMA_INDEX_MASK | 0x3UL) 5697 5698 #ifdef CONFIG_PAGE_POOL 5699 static inline bool page_pool_page_is_pp(const struct page *page) 5700 { 5701 return (page->pp_magic & PP_MAGIC_MASK) == PP_SIGNATURE; 5702 } 5703 #else 5704 static inline bool page_pool_page_is_pp(const struct page *page) 5705 { 5706 return false; 5707 } 5708 #endif 5709 5710 #define PAGE_SNAPSHOT_FAITHFUL (1 << 0) 5711 #define PAGE_SNAPSHOT_PG_BUDDY (1 << 1) 5712 #define PAGE_SNAPSHOT_PG_IDLE (1 << 2) 5713 5714 struct page_snapshot { 5715 struct folio folio_snapshot; 5716 struct page page_snapshot; 5717 unsigned long pfn; 5718 unsigned long idx; 5719 unsigned long flags; 5720 }; 5721 5722 static inline bool snapshot_page_is_faithful(const struct page_snapshot *ps) 5723 { 5724 return ps->flags & PAGE_SNAPSHOT_FAITHFUL; 5725 } 5726 5727 void snapshot_page(struct page_snapshot *ps, const struct page *page); 5728 5729 void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, 5730 struct vm_area_struct *vma, unsigned long addr, 5731 bool uffd_wp); 5732 5733 #endif /* _LINUX_MM_H */ 5734