1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* internal.h: mm/ internal definitions 3 * 4 * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 #ifndef __MM_INTERNAL_H 8 #define __MM_INTERNAL_H 9 10 #include <linux/fs.h> 11 #include <linux/mm.h> 12 #include <linux/pagemap.h> 13 #include <linux/rmap.h> 14 #include <linux/swap.h> 15 #include <linux/swapops.h> 16 #include <linux/tracepoint-defs.h> 17 18 struct folio_batch; 19 20 /* 21 * The set of flags that only affect watermark checking and reclaim 22 * behaviour. This is used by the MM to obey the caller constraints 23 * about IO, FS and watermark checking while ignoring placement 24 * hints such as HIGHMEM usage. 25 */ 26 #define GFP_RECLAIM_MASK (__GFP_RECLAIM|__GFP_HIGH|__GFP_IO|__GFP_FS|\ 27 __GFP_NOWARN|__GFP_RETRY_MAYFAIL|__GFP_NOFAIL|\ 28 __GFP_NORETRY|__GFP_MEMALLOC|__GFP_NOMEMALLOC|\ 29 __GFP_NOLOCKDEP) 30 31 /* The GFP flags allowed during early boot */ 32 #define GFP_BOOT_MASK (__GFP_BITS_MASK & ~(__GFP_RECLAIM|__GFP_IO|__GFP_FS)) 33 34 /* Control allocation cpuset and node placement constraints */ 35 #define GFP_CONSTRAINT_MASK (__GFP_HARDWALL|__GFP_THISNODE) 36 37 /* Do not use these with a slab allocator */ 38 #define GFP_SLAB_BUG_MASK (__GFP_DMA32|__GFP_HIGHMEM|~__GFP_BITS_MASK) 39 40 /* 41 * Different from WARN_ON_ONCE(), no warning will be issued 42 * when we specify __GFP_NOWARN. 43 */ 44 #define WARN_ON_ONCE_GFP(cond, gfp) ({ \ 45 static bool __section(".data.once") __warned; \ 46 int __ret_warn_once = !!(cond); \ 47 \ 48 if (unlikely(!(gfp & __GFP_NOWARN) && __ret_warn_once && !__warned)) { \ 49 __warned = true; \ 50 WARN_ON(1); \ 51 } \ 52 unlikely(__ret_warn_once); \ 53 }) 54 55 void page_writeback_init(void); 56 57 /* 58 * If a 16GB hugetlb folio were mapped by PTEs of all of its 4kB pages, 59 * its nr_pages_mapped would be 0x400000: choose the ENTIRELY_MAPPED bit 60 * above that range, instead of 2*(PMD_SIZE/PAGE_SIZE). Hugetlb currently 61 * leaves nr_pages_mapped at 0, but avoid surprise if it participates later. 62 */ 63 #define ENTIRELY_MAPPED 0x800000 64 #define FOLIO_PAGES_MAPPED (ENTIRELY_MAPPED - 1) 65 66 /* 67 * Flags passed to __show_mem() and show_free_areas() to suppress output in 68 * various contexts. 69 */ 70 #define SHOW_MEM_FILTER_NODES (0x0001u) /* disallowed nodes */ 71 72 /* 73 * How many individual pages have an elevated _mapcount. Excludes 74 * the folio's entire_mapcount. 75 * 76 * Don't use this function outside of debugging code. 77 */ 78 static inline int folio_nr_pages_mapped(const struct folio *folio) 79 { 80 return atomic_read(&folio->_nr_pages_mapped) & FOLIO_PAGES_MAPPED; 81 } 82 83 /* 84 * Retrieve the first entry of a folio based on a provided entry within the 85 * folio. We cannot rely on folio->swap as there is no guarantee that it has 86 * been initialized. Used for calling arch_swap_restore() 87 */ 88 static inline swp_entry_t folio_swap(swp_entry_t entry, 89 const struct folio *folio) 90 { 91 swp_entry_t swap = { 92 .val = ALIGN_DOWN(entry.val, folio_nr_pages(folio)), 93 }; 94 95 return swap; 96 } 97 98 static inline void *folio_raw_mapping(const struct folio *folio) 99 { 100 unsigned long mapping = (unsigned long)folio->mapping; 101 102 return (void *)(mapping & ~PAGE_MAPPING_FLAGS); 103 } 104 105 #ifdef CONFIG_MMU 106 107 /* Flags for folio_pte_batch(). */ 108 typedef int __bitwise fpb_t; 109 110 /* Compare PTEs after pte_mkclean(), ignoring the dirty bit. */ 111 #define FPB_IGNORE_DIRTY ((__force fpb_t)BIT(0)) 112 113 /* Compare PTEs after pte_clear_soft_dirty(), ignoring the soft-dirty bit. */ 114 #define FPB_IGNORE_SOFT_DIRTY ((__force fpb_t)BIT(1)) 115 116 static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags) 117 { 118 if (flags & FPB_IGNORE_DIRTY) 119 pte = pte_mkclean(pte); 120 if (likely(flags & FPB_IGNORE_SOFT_DIRTY)) 121 pte = pte_clear_soft_dirty(pte); 122 return pte_wrprotect(pte_mkold(pte)); 123 } 124 125 /** 126 * folio_pte_batch - detect a PTE batch for a large folio 127 * @folio: The large folio to detect a PTE batch for. 128 * @addr: The user virtual address the first page is mapped at. 129 * @start_ptep: Page table pointer for the first entry. 130 * @pte: Page table entry for the first page. 131 * @max_nr: The maximum number of table entries to consider. 132 * @flags: Flags to modify the PTE batch semantics. 133 * @any_writable: Optional pointer to indicate whether any entry except the 134 * first one is writable. 135 * @any_young: Optional pointer to indicate whether any entry except the 136 * first one is young. 137 * @any_dirty: Optional pointer to indicate whether any entry except the 138 * first one is dirty. 139 * 140 * Detect a PTE batch: consecutive (present) PTEs that map consecutive 141 * pages of the same large folio. 142 * 143 * All PTEs inside a PTE batch have the same PTE bits set, excluding the PFN, 144 * the accessed bit, writable bit, dirty bit (with FPB_IGNORE_DIRTY) and 145 * soft-dirty bit (with FPB_IGNORE_SOFT_DIRTY). 146 * 147 * start_ptep must map any page of the folio. max_nr must be at least one and 148 * must be limited by the caller so scanning cannot exceed a single page table. 149 * 150 * Return: the number of table entries in the batch. 151 */ 152 static inline int folio_pte_batch(struct folio *folio, unsigned long addr, 153 pte_t *start_ptep, pte_t pte, int max_nr, fpb_t flags, 154 bool *any_writable, bool *any_young, bool *any_dirty) 155 { 156 unsigned long folio_end_pfn = folio_pfn(folio) + folio_nr_pages(folio); 157 const pte_t *end_ptep = start_ptep + max_nr; 158 pte_t expected_pte, *ptep; 159 bool writable, young, dirty; 160 int nr; 161 162 if (any_writable) 163 *any_writable = false; 164 if (any_young) 165 *any_young = false; 166 if (any_dirty) 167 *any_dirty = false; 168 169 VM_WARN_ON_FOLIO(!pte_present(pte), folio); 170 VM_WARN_ON_FOLIO(!folio_test_large(folio) || max_nr < 1, folio); 171 VM_WARN_ON_FOLIO(page_folio(pfn_to_page(pte_pfn(pte))) != folio, folio); 172 173 nr = pte_batch_hint(start_ptep, pte); 174 expected_pte = __pte_batch_clear_ignored(pte_advance_pfn(pte, nr), flags); 175 ptep = start_ptep + nr; 176 177 while (ptep < end_ptep) { 178 pte = ptep_get(ptep); 179 if (any_writable) 180 writable = !!pte_write(pte); 181 if (any_young) 182 young = !!pte_young(pte); 183 if (any_dirty) 184 dirty = !!pte_dirty(pte); 185 pte = __pte_batch_clear_ignored(pte, flags); 186 187 if (!pte_same(pte, expected_pte)) 188 break; 189 190 /* 191 * Stop immediately once we reached the end of the folio. In 192 * corner cases the next PFN might fall into a different 193 * folio. 194 */ 195 if (pte_pfn(pte) >= folio_end_pfn) 196 break; 197 198 if (any_writable) 199 *any_writable |= writable; 200 if (any_young) 201 *any_young |= young; 202 if (any_dirty) 203 *any_dirty |= dirty; 204 205 nr = pte_batch_hint(ptep, pte); 206 expected_pte = pte_advance_pfn(expected_pte, nr); 207 ptep += nr; 208 } 209 210 return min(ptep - start_ptep, max_nr); 211 } 212 213 /** 214 * pte_move_swp_offset - Move the swap entry offset field of a swap pte 215 * forward or backward by delta 216 * @pte: The initial pte state; is_swap_pte(pte) must be true and 217 * non_swap_entry() must be false. 218 * @delta: The direction and the offset we are moving; forward if delta 219 * is positive; backward if delta is negative 220 * 221 * Moves the swap offset, while maintaining all other fields, including 222 * swap type, and any swp pte bits. The resulting pte is returned. 223 */ 224 static inline pte_t pte_move_swp_offset(pte_t pte, long delta) 225 { 226 swp_entry_t entry = pte_to_swp_entry(pte); 227 pte_t new = __swp_entry_to_pte(__swp_entry(swp_type(entry), 228 (swp_offset(entry) + delta))); 229 230 if (pte_swp_soft_dirty(pte)) 231 new = pte_swp_mksoft_dirty(new); 232 if (pte_swp_exclusive(pte)) 233 new = pte_swp_mkexclusive(new); 234 if (pte_swp_uffd_wp(pte)) 235 new = pte_swp_mkuffd_wp(new); 236 237 return new; 238 } 239 240 241 /** 242 * pte_next_swp_offset - Increment the swap entry offset field of a swap pte. 243 * @pte: The initial pte state; is_swap_pte(pte) must be true and 244 * non_swap_entry() must be false. 245 * 246 * Increments the swap offset, while maintaining all other fields, including 247 * swap type, and any swp pte bits. The resulting pte is returned. 248 */ 249 static inline pte_t pte_next_swp_offset(pte_t pte) 250 { 251 return pte_move_swp_offset(pte, 1); 252 } 253 254 /** 255 * swap_pte_batch - detect a PTE batch for a set of contiguous swap entries 256 * @start_ptep: Page table pointer for the first entry. 257 * @max_nr: The maximum number of table entries to consider. 258 * @pte: Page table entry for the first entry. 259 * 260 * Detect a batch of contiguous swap entries: consecutive (non-present) PTEs 261 * containing swap entries all with consecutive offsets and targeting the same 262 * swap type, all with matching swp pte bits. 263 * 264 * max_nr must be at least one and must be limited by the caller so scanning 265 * cannot exceed a single page table. 266 * 267 * Return: the number of table entries in the batch. 268 */ 269 static inline int swap_pte_batch(pte_t *start_ptep, int max_nr, pte_t pte) 270 { 271 pte_t expected_pte = pte_next_swp_offset(pte); 272 const pte_t *end_ptep = start_ptep + max_nr; 273 pte_t *ptep = start_ptep + 1; 274 275 VM_WARN_ON(max_nr < 1); 276 VM_WARN_ON(!is_swap_pte(pte)); 277 VM_WARN_ON(non_swap_entry(pte_to_swp_entry(pte))); 278 279 while (ptep < end_ptep) { 280 pte = ptep_get(ptep); 281 282 if (!pte_same(pte, expected_pte)) 283 break; 284 285 expected_pte = pte_next_swp_offset(expected_pte); 286 ptep++; 287 } 288 289 return ptep - start_ptep; 290 } 291 #endif /* CONFIG_MMU */ 292 293 void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio, 294 int nr_throttled); 295 static inline void acct_reclaim_writeback(struct folio *folio) 296 { 297 pg_data_t *pgdat = folio_pgdat(folio); 298 int nr_throttled = atomic_read(&pgdat->nr_writeback_throttled); 299 300 if (nr_throttled) 301 __acct_reclaim_writeback(pgdat, folio, nr_throttled); 302 } 303 304 static inline void wake_throttle_isolated(pg_data_t *pgdat) 305 { 306 wait_queue_head_t *wqh; 307 308 wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_ISOLATED]; 309 if (waitqueue_active(wqh)) 310 wake_up(wqh); 311 } 312 313 vm_fault_t vmf_anon_prepare(struct vm_fault *vmf); 314 vm_fault_t do_swap_page(struct vm_fault *vmf); 315 void folio_rotate_reclaimable(struct folio *folio); 316 bool __folio_end_writeback(struct folio *folio); 317 void deactivate_file_folio(struct folio *folio); 318 void folio_activate(struct folio *folio); 319 320 void free_pgtables(struct mmu_gather *tlb, struct ma_state *mas, 321 struct vm_area_struct *start_vma, unsigned long floor, 322 unsigned long ceiling, bool mm_wr_locked); 323 void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte); 324 325 struct zap_details; 326 void unmap_page_range(struct mmu_gather *tlb, 327 struct vm_area_struct *vma, 328 unsigned long addr, unsigned long end, 329 struct zap_details *details); 330 331 void page_cache_ra_order(struct readahead_control *, struct file_ra_state *, 332 unsigned int order); 333 void force_page_cache_ra(struct readahead_control *, unsigned long nr); 334 static inline void force_page_cache_readahead(struct address_space *mapping, 335 struct file *file, pgoff_t index, unsigned long nr_to_read) 336 { 337 DEFINE_READAHEAD(ractl, file, &file->f_ra, mapping, index); 338 force_page_cache_ra(&ractl, nr_to_read); 339 } 340 341 unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start, 342 pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices); 343 unsigned find_get_entries(struct address_space *mapping, pgoff_t *start, 344 pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices); 345 void filemap_free_folio(struct address_space *mapping, struct folio *folio); 346 int truncate_inode_folio(struct address_space *mapping, struct folio *folio); 347 bool truncate_inode_partial_folio(struct folio *folio, loff_t start, 348 loff_t end); 349 long mapping_evict_folio(struct address_space *mapping, struct folio *folio); 350 unsigned long mapping_try_invalidate(struct address_space *mapping, 351 pgoff_t start, pgoff_t end, unsigned long *nr_failed); 352 353 /** 354 * folio_evictable - Test whether a folio is evictable. 355 * @folio: The folio to test. 356 * 357 * Test whether @folio is evictable -- i.e., should be placed on 358 * active/inactive lists vs unevictable list. 359 * 360 * Reasons folio might not be evictable: 361 * 1. folio's mapping marked unevictable 362 * 2. One of the pages in the folio is part of an mlocked VMA 363 */ 364 static inline bool folio_evictable(struct folio *folio) 365 { 366 bool ret; 367 368 /* Prevent address_space of inode and swap cache from being freed */ 369 rcu_read_lock(); 370 ret = !mapping_unevictable(folio_mapping(folio)) && 371 !folio_test_mlocked(folio); 372 rcu_read_unlock(); 373 return ret; 374 } 375 376 /* 377 * Turn a non-refcounted page (->_refcount == 0) into refcounted with 378 * a count of one. 379 */ 380 static inline void set_page_refcounted(struct page *page) 381 { 382 VM_BUG_ON_PAGE(PageTail(page), page); 383 VM_BUG_ON_PAGE(page_ref_count(page), page); 384 set_page_count(page, 1); 385 } 386 387 /* 388 * Return true if a folio needs ->release_folio() calling upon it. 389 */ 390 static inline bool folio_needs_release(struct folio *folio) 391 { 392 struct address_space *mapping = folio_mapping(folio); 393 394 return folio_has_private(folio) || 395 (mapping && mapping_release_always(mapping)); 396 } 397 398 extern unsigned long highest_memmap_pfn; 399 400 /* 401 * Maximum number of reclaim retries without progress before the OOM 402 * killer is consider the only way forward. 403 */ 404 #define MAX_RECLAIM_RETRIES 16 405 406 /* 407 * in mm/vmscan.c: 408 */ 409 bool isolate_lru_page(struct page *page); 410 bool folio_isolate_lru(struct folio *folio); 411 void putback_lru_page(struct page *page); 412 void folio_putback_lru(struct folio *folio); 413 extern void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason); 414 415 /* 416 * in mm/rmap.c: 417 */ 418 pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address); 419 420 /* 421 * in mm/page_alloc.c 422 */ 423 #define K(x) ((x) << (PAGE_SHIFT-10)) 424 425 extern char * const zone_names[MAX_NR_ZONES]; 426 427 /* perform sanity checks on struct pages being allocated or freed */ 428 DECLARE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled); 429 430 extern int min_free_kbytes; 431 432 void setup_per_zone_wmarks(void); 433 void calculate_min_free_kbytes(void); 434 int __meminit init_per_zone_wmark_min(void); 435 void page_alloc_sysctl_init(void); 436 437 /* 438 * Structure for holding the mostly immutable allocation parameters passed 439 * between functions involved in allocations, including the alloc_pages* 440 * family of functions. 441 * 442 * nodemask, migratetype and highest_zoneidx are initialized only once in 443 * __alloc_pages() and then never change. 444 * 445 * zonelist, preferred_zone and highest_zoneidx are set first in 446 * __alloc_pages() for the fast path, and might be later changed 447 * in __alloc_pages_slowpath(). All other functions pass the whole structure 448 * by a const pointer. 449 */ 450 struct alloc_context { 451 struct zonelist *zonelist; 452 nodemask_t *nodemask; 453 struct zoneref *preferred_zoneref; 454 int migratetype; 455 456 /* 457 * highest_zoneidx represents highest usable zone index of 458 * the allocation request. Due to the nature of the zone, 459 * memory on lower zone than the highest_zoneidx will be 460 * protected by lowmem_reserve[highest_zoneidx]. 461 * 462 * highest_zoneidx is also used by reclaim/compaction to limit 463 * the target zone since higher zone than this index cannot be 464 * usable for this allocation request. 465 */ 466 enum zone_type highest_zoneidx; 467 bool spread_dirty_pages; 468 }; 469 470 /* 471 * This function returns the order of a free page in the buddy system. In 472 * general, page_zone(page)->lock must be held by the caller to prevent the 473 * page from being allocated in parallel and returning garbage as the order. 474 * If a caller does not hold page_zone(page)->lock, it must guarantee that the 475 * page cannot be allocated or merged in parallel. Alternatively, it must 476 * handle invalid values gracefully, and use buddy_order_unsafe() below. 477 */ 478 static inline unsigned int buddy_order(struct page *page) 479 { 480 /* PageBuddy() must be checked by the caller */ 481 return page_private(page); 482 } 483 484 /* 485 * Like buddy_order(), but for callers who cannot afford to hold the zone lock. 486 * PageBuddy() should be checked first by the caller to minimize race window, 487 * and invalid values must be handled gracefully. 488 * 489 * READ_ONCE is used so that if the caller assigns the result into a local 490 * variable and e.g. tests it for valid range before using, the compiler cannot 491 * decide to remove the variable and inline the page_private(page) multiple 492 * times, potentially observing different values in the tests and the actual 493 * use of the result. 494 */ 495 #define buddy_order_unsafe(page) READ_ONCE(page_private(page)) 496 497 /* 498 * This function checks whether a page is free && is the buddy 499 * we can coalesce a page and its buddy if 500 * (a) the buddy is not in a hole (check before calling!) && 501 * (b) the buddy is in the buddy system && 502 * (c) a page and its buddy have the same order && 503 * (d) a page and its buddy are in the same zone. 504 * 505 * For recording whether a page is in the buddy system, we set PageBuddy. 506 * Setting, clearing, and testing PageBuddy is serialized by zone->lock. 507 * 508 * For recording page's order, we use page_private(page). 509 */ 510 static inline bool page_is_buddy(struct page *page, struct page *buddy, 511 unsigned int order) 512 { 513 if (!page_is_guard(buddy) && !PageBuddy(buddy)) 514 return false; 515 516 if (buddy_order(buddy) != order) 517 return false; 518 519 /* 520 * zone check is done late to avoid uselessly calculating 521 * zone/node ids for pages that could never merge. 522 */ 523 if (page_zone_id(page) != page_zone_id(buddy)) 524 return false; 525 526 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 527 528 return true; 529 } 530 531 /* 532 * Locate the struct page for both the matching buddy in our 533 * pair (buddy1) and the combined O(n+1) page they form (page). 534 * 535 * 1) Any buddy B1 will have an order O twin B2 which satisfies 536 * the following equation: 537 * B2 = B1 ^ (1 << O) 538 * For example, if the starting buddy (buddy2) is #8 its order 539 * 1 buddy is #10: 540 * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10 541 * 542 * 2) Any buddy B will have an order O+1 parent P which 543 * satisfies the following equation: 544 * P = B & ~(1 << O) 545 * 546 * Assumption: *_mem_map is contiguous at least up to MAX_PAGE_ORDER 547 */ 548 static inline unsigned long 549 __find_buddy_pfn(unsigned long page_pfn, unsigned int order) 550 { 551 return page_pfn ^ (1 << order); 552 } 553 554 /* 555 * Find the buddy of @page and validate it. 556 * @page: The input page 557 * @pfn: The pfn of the page, it saves a call to page_to_pfn() when the 558 * function is used in the performance-critical __free_one_page(). 559 * @order: The order of the page 560 * @buddy_pfn: The output pointer to the buddy pfn, it also saves a call to 561 * page_to_pfn(). 562 * 563 * The found buddy can be a non PageBuddy, out of @page's zone, or its order is 564 * not the same as @page. The validation is necessary before use it. 565 * 566 * Return: the found buddy page or NULL if not found. 567 */ 568 static inline struct page *find_buddy_page_pfn(struct page *page, 569 unsigned long pfn, unsigned int order, unsigned long *buddy_pfn) 570 { 571 unsigned long __buddy_pfn = __find_buddy_pfn(pfn, order); 572 struct page *buddy; 573 574 buddy = page + (__buddy_pfn - pfn); 575 if (buddy_pfn) 576 *buddy_pfn = __buddy_pfn; 577 578 if (page_is_buddy(page, buddy, order)) 579 return buddy; 580 return NULL; 581 } 582 583 extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn, 584 unsigned long end_pfn, struct zone *zone); 585 586 static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn, 587 unsigned long end_pfn, struct zone *zone) 588 { 589 if (zone->contiguous) 590 return pfn_to_page(start_pfn); 591 592 return __pageblock_pfn_to_page(start_pfn, end_pfn, zone); 593 } 594 595 void set_zone_contiguous(struct zone *zone); 596 597 static inline void clear_zone_contiguous(struct zone *zone) 598 { 599 zone->contiguous = false; 600 } 601 602 extern int __isolate_free_page(struct page *page, unsigned int order); 603 extern void __putback_isolated_page(struct page *page, unsigned int order, 604 int mt); 605 extern void memblock_free_pages(struct page *page, unsigned long pfn, 606 unsigned int order); 607 extern void __free_pages_core(struct page *page, unsigned int order); 608 609 /* 610 * This will have no effect, other than possibly generating a warning, if the 611 * caller passes in a non-large folio. 612 */ 613 static inline void folio_set_order(struct folio *folio, unsigned int order) 614 { 615 if (WARN_ON_ONCE(!order || !folio_test_large(folio))) 616 return; 617 618 folio->_flags_1 = (folio->_flags_1 & ~0xffUL) | order; 619 #ifdef CONFIG_64BIT 620 folio->_folio_nr_pages = 1U << order; 621 #endif 622 } 623 624 void folio_undo_large_rmappable(struct folio *folio); 625 626 static inline struct folio *page_rmappable_folio(struct page *page) 627 { 628 struct folio *folio = (struct folio *)page; 629 630 if (folio && folio_test_large(folio)) 631 folio_set_large_rmappable(folio); 632 return folio; 633 } 634 635 static inline void prep_compound_head(struct page *page, unsigned int order) 636 { 637 struct folio *folio = (struct folio *)page; 638 639 folio_set_order(folio, order); 640 atomic_set(&folio->_large_mapcount, -1); 641 atomic_set(&folio->_entire_mapcount, -1); 642 atomic_set(&folio->_nr_pages_mapped, 0); 643 atomic_set(&folio->_pincount, 0); 644 if (order > 1) 645 INIT_LIST_HEAD(&folio->_deferred_list); 646 } 647 648 static inline void prep_compound_tail(struct page *head, int tail_idx) 649 { 650 struct page *p = head + tail_idx; 651 652 p->mapping = TAIL_MAPPING; 653 set_compound_head(p, head); 654 set_page_private(p, 0); 655 } 656 657 extern void prep_compound_page(struct page *page, unsigned int order); 658 659 extern void post_alloc_hook(struct page *page, unsigned int order, 660 gfp_t gfp_flags); 661 extern bool free_pages_prepare(struct page *page, unsigned int order); 662 663 extern int user_min_free_kbytes; 664 665 void free_unref_page(struct page *page, unsigned int order); 666 void free_unref_folios(struct folio_batch *fbatch); 667 668 extern void zone_pcp_reset(struct zone *zone); 669 extern void zone_pcp_disable(struct zone *zone); 670 extern void zone_pcp_enable(struct zone *zone); 671 extern void zone_pcp_init(struct zone *zone); 672 673 extern void *memmap_alloc(phys_addr_t size, phys_addr_t align, 674 phys_addr_t min_addr, 675 int nid, bool exact_nid); 676 677 void memmap_init_range(unsigned long, int, unsigned long, unsigned long, 678 unsigned long, enum meminit_context, struct vmem_altmap *, int); 679 680 #if defined CONFIG_COMPACTION || defined CONFIG_CMA 681 682 /* 683 * in mm/compaction.c 684 */ 685 /* 686 * compact_control is used to track pages being migrated and the free pages 687 * they are being migrated to during memory compaction. The free_pfn starts 688 * at the end of a zone and migrate_pfn begins at the start. Movable pages 689 * are moved to the end of a zone during a compaction run and the run 690 * completes when free_pfn <= migrate_pfn 691 */ 692 struct compact_control { 693 struct list_head freepages[NR_PAGE_ORDERS]; /* List of free pages to migrate to */ 694 struct list_head migratepages; /* List of pages being migrated */ 695 unsigned int nr_freepages; /* Number of isolated free pages */ 696 unsigned int nr_migratepages; /* Number of pages to migrate */ 697 unsigned long free_pfn; /* isolate_freepages search base */ 698 /* 699 * Acts as an in/out parameter to page isolation for migration. 700 * isolate_migratepages uses it as a search base. 701 * isolate_migratepages_block will update the value to the next pfn 702 * after the last isolated one. 703 */ 704 unsigned long migrate_pfn; 705 unsigned long fast_start_pfn; /* a pfn to start linear scan from */ 706 struct zone *zone; 707 unsigned long total_migrate_scanned; 708 unsigned long total_free_scanned; 709 unsigned short fast_search_fail;/* failures to use free list searches */ 710 short search_order; /* order to start a fast search at */ 711 const gfp_t gfp_mask; /* gfp mask of a direct compactor */ 712 int order; /* order a direct compactor needs */ 713 int migratetype; /* migratetype of direct compactor */ 714 const unsigned int alloc_flags; /* alloc flags of a direct compactor */ 715 const int highest_zoneidx; /* zone index of a direct compactor */ 716 enum migrate_mode mode; /* Async or sync migration mode */ 717 bool ignore_skip_hint; /* Scan blocks even if marked skip */ 718 bool no_set_skip_hint; /* Don't mark blocks for skipping */ 719 bool ignore_block_suitable; /* Scan blocks considered unsuitable */ 720 bool direct_compaction; /* False from kcompactd or /proc/... */ 721 bool proactive_compaction; /* kcompactd proactive compaction */ 722 bool whole_zone; /* Whole zone should/has been scanned */ 723 bool contended; /* Signal lock contention */ 724 bool finish_pageblock; /* Scan the remainder of a pageblock. Used 725 * when there are potentially transient 726 * isolation or migration failures to 727 * ensure forward progress. 728 */ 729 bool alloc_contig; /* alloc_contig_range allocation */ 730 }; 731 732 /* 733 * Used in direct compaction when a page should be taken from the freelists 734 * immediately when one is created during the free path. 735 */ 736 struct capture_control { 737 struct compact_control *cc; 738 struct page *page; 739 }; 740 741 unsigned long 742 isolate_freepages_range(struct compact_control *cc, 743 unsigned long start_pfn, unsigned long end_pfn); 744 int 745 isolate_migratepages_range(struct compact_control *cc, 746 unsigned long low_pfn, unsigned long end_pfn); 747 748 int __alloc_contig_migrate_range(struct compact_control *cc, 749 unsigned long start, unsigned long end, 750 int migratetype); 751 752 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ 753 void init_cma_reserved_pageblock(struct page *page); 754 755 #endif /* CONFIG_COMPACTION || CONFIG_CMA */ 756 757 int find_suitable_fallback(struct free_area *area, unsigned int order, 758 int migratetype, bool only_stealable, bool *can_steal); 759 760 static inline bool free_area_empty(struct free_area *area, int migratetype) 761 { 762 return list_empty(&area->free_list[migratetype]); 763 } 764 765 /* 766 * These three helpers classifies VMAs for virtual memory accounting. 767 */ 768 769 /* 770 * Executable code area - executable, not writable, not stack 771 */ 772 static inline bool is_exec_mapping(vm_flags_t flags) 773 { 774 return (flags & (VM_EXEC | VM_WRITE | VM_STACK)) == VM_EXEC; 775 } 776 777 /* 778 * Stack area (including shadow stacks) 779 * 780 * VM_GROWSUP / VM_GROWSDOWN VMAs are always private anonymous: 781 * do_mmap() forbids all other combinations. 782 */ 783 static inline bool is_stack_mapping(vm_flags_t flags) 784 { 785 return ((flags & VM_STACK) == VM_STACK) || (flags & VM_SHADOW_STACK); 786 } 787 788 /* 789 * Data area - private, writable, not stack 790 */ 791 static inline bool is_data_mapping(vm_flags_t flags) 792 { 793 return (flags & (VM_WRITE | VM_SHARED | VM_STACK)) == VM_WRITE; 794 } 795 796 /* mm/util.c */ 797 struct anon_vma *folio_anon_vma(struct folio *folio); 798 799 #ifdef CONFIG_MMU 800 void unmap_mapping_folio(struct folio *folio); 801 extern long populate_vma_page_range(struct vm_area_struct *vma, 802 unsigned long start, unsigned long end, int *locked); 803 extern long faultin_page_range(struct mm_struct *mm, unsigned long start, 804 unsigned long end, bool write, int *locked); 805 extern bool mlock_future_ok(struct mm_struct *mm, unsigned long flags, 806 unsigned long bytes); 807 808 /* 809 * NOTE: This function can't tell whether the folio is "fully mapped" in the 810 * range. 811 * "fully mapped" means all the pages of folio is associated with the page 812 * table of range while this function just check whether the folio range is 813 * within the range [start, end). Function caller needs to do page table 814 * check if it cares about the page table association. 815 * 816 * Typical usage (like mlock or madvise) is: 817 * Caller knows at least 1 page of folio is associated with page table of VMA 818 * and the range [start, end) is intersect with the VMA range. Caller wants 819 * to know whether the folio is fully associated with the range. It calls 820 * this function to check whether the folio is in the range first. Then checks 821 * the page table to know whether the folio is fully mapped to the range. 822 */ 823 static inline bool 824 folio_within_range(struct folio *folio, struct vm_area_struct *vma, 825 unsigned long start, unsigned long end) 826 { 827 pgoff_t pgoff, addr; 828 unsigned long vma_pglen = vma_pages(vma); 829 830 VM_WARN_ON_FOLIO(folio_test_ksm(folio), folio); 831 if (start > end) 832 return false; 833 834 if (start < vma->vm_start) 835 start = vma->vm_start; 836 837 if (end > vma->vm_end) 838 end = vma->vm_end; 839 840 pgoff = folio_pgoff(folio); 841 842 /* if folio start address is not in vma range */ 843 if (!in_range(pgoff, vma->vm_pgoff, vma_pglen)) 844 return false; 845 846 addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); 847 848 return !(addr < start || end - addr < folio_size(folio)); 849 } 850 851 static inline bool 852 folio_within_vma(struct folio *folio, struct vm_area_struct *vma) 853 { 854 return folio_within_range(folio, vma, vma->vm_start, vma->vm_end); 855 } 856 857 /* 858 * mlock_vma_folio() and munlock_vma_folio(): 859 * should be called with vma's mmap_lock held for read or write, 860 * under page table lock for the pte/pmd being added or removed. 861 * 862 * mlock is usually called at the end of folio_add_*_rmap_*(), munlock at 863 * the end of folio_remove_rmap_*(); but new anon folios are managed by 864 * folio_add_lru_vma() calling mlock_new_folio(). 865 */ 866 void mlock_folio(struct folio *folio); 867 static inline void mlock_vma_folio(struct folio *folio, 868 struct vm_area_struct *vma) 869 { 870 /* 871 * The VM_SPECIAL check here serves two purposes. 872 * 1) VM_IO check prevents migration from double-counting during mlock. 873 * 2) Although mmap_region() and mlock_fixup() take care that VM_LOCKED 874 * is never left set on a VM_SPECIAL vma, there is an interval while 875 * file->f_op->mmap() is using vm_insert_page(s), when VM_LOCKED may 876 * still be set while VM_SPECIAL bits are added: so ignore it then. 877 */ 878 if (unlikely((vma->vm_flags & (VM_LOCKED|VM_SPECIAL)) == VM_LOCKED)) 879 mlock_folio(folio); 880 } 881 882 void munlock_folio(struct folio *folio); 883 static inline void munlock_vma_folio(struct folio *folio, 884 struct vm_area_struct *vma) 885 { 886 /* 887 * munlock if the function is called. Ideally, we should only 888 * do munlock if any page of folio is unmapped from VMA and 889 * cause folio not fully mapped to VMA. 890 * 891 * But it's not easy to confirm that's the situation. So we 892 * always munlock the folio and page reclaim will correct it 893 * if it's wrong. 894 */ 895 if (unlikely(vma->vm_flags & VM_LOCKED)) 896 munlock_folio(folio); 897 } 898 899 void mlock_new_folio(struct folio *folio); 900 bool need_mlock_drain(int cpu); 901 void mlock_drain_local(void); 902 void mlock_drain_remote(int cpu); 903 904 extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma); 905 906 /** 907 * vma_address - Find the virtual address a page range is mapped at 908 * @vma: The vma which maps this object. 909 * @pgoff: The page offset within its object. 910 * @nr_pages: The number of pages to consider. 911 * 912 * If any page in this range is mapped by this VMA, return the first address 913 * where any of these pages appear. Otherwise, return -EFAULT. 914 */ 915 static inline unsigned long vma_address(struct vm_area_struct *vma, 916 pgoff_t pgoff, unsigned long nr_pages) 917 { 918 unsigned long address; 919 920 if (pgoff >= vma->vm_pgoff) { 921 address = vma->vm_start + 922 ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); 923 /* Check for address beyond vma (or wrapped through 0?) */ 924 if (address < vma->vm_start || address >= vma->vm_end) 925 address = -EFAULT; 926 } else if (pgoff + nr_pages - 1 >= vma->vm_pgoff) { 927 /* Test above avoids possibility of wrap to 0 on 32-bit */ 928 address = vma->vm_start; 929 } else { 930 address = -EFAULT; 931 } 932 return address; 933 } 934 935 /* 936 * Then at what user virtual address will none of the range be found in vma? 937 * Assumes that vma_address() already returned a good starting address. 938 */ 939 static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw) 940 { 941 struct vm_area_struct *vma = pvmw->vma; 942 pgoff_t pgoff; 943 unsigned long address; 944 945 /* Common case, plus ->pgoff is invalid for KSM */ 946 if (pvmw->nr_pages == 1) 947 return pvmw->address + PAGE_SIZE; 948 949 pgoff = pvmw->pgoff + pvmw->nr_pages; 950 address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); 951 /* Check for address beyond vma (or wrapped through 0?) */ 952 if (address < vma->vm_start || address > vma->vm_end) 953 address = vma->vm_end; 954 return address; 955 } 956 957 static inline struct file *maybe_unlock_mmap_for_io(struct vm_fault *vmf, 958 struct file *fpin) 959 { 960 int flags = vmf->flags; 961 962 if (fpin) 963 return fpin; 964 965 /* 966 * FAULT_FLAG_RETRY_NOWAIT means we don't want to wait on page locks or 967 * anything, so we only pin the file and drop the mmap_lock if only 968 * FAULT_FLAG_ALLOW_RETRY is set, while this is the first attempt. 969 */ 970 if (fault_flag_allow_retry_first(flags) && 971 !(flags & FAULT_FLAG_RETRY_NOWAIT)) { 972 fpin = get_file(vmf->vma->vm_file); 973 release_fault_lock(vmf); 974 } 975 return fpin; 976 } 977 #else /* !CONFIG_MMU */ 978 static inline void unmap_mapping_folio(struct folio *folio) { } 979 static inline void mlock_new_folio(struct folio *folio) { } 980 static inline bool need_mlock_drain(int cpu) { return false; } 981 static inline void mlock_drain_local(void) { } 982 static inline void mlock_drain_remote(int cpu) { } 983 static inline void vunmap_range_noflush(unsigned long start, unsigned long end) 984 { 985 } 986 #endif /* !CONFIG_MMU */ 987 988 /* Memory initialisation debug and verification */ 989 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 990 DECLARE_STATIC_KEY_TRUE(deferred_pages); 991 992 bool __init deferred_grow_zone(struct zone *zone, unsigned int order); 993 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 994 995 enum mminit_level { 996 MMINIT_WARNING, 997 MMINIT_VERIFY, 998 MMINIT_TRACE 999 }; 1000 1001 #ifdef CONFIG_DEBUG_MEMORY_INIT 1002 1003 extern int mminit_loglevel; 1004 1005 #define mminit_dprintk(level, prefix, fmt, arg...) \ 1006 do { \ 1007 if (level < mminit_loglevel) { \ 1008 if (level <= MMINIT_WARNING) \ 1009 pr_warn("mminit::" prefix " " fmt, ##arg); \ 1010 else \ 1011 printk(KERN_DEBUG "mminit::" prefix " " fmt, ##arg); \ 1012 } \ 1013 } while (0) 1014 1015 extern void mminit_verify_pageflags_layout(void); 1016 extern void mminit_verify_zonelist(void); 1017 #else 1018 1019 static inline void mminit_dprintk(enum mminit_level level, 1020 const char *prefix, const char *fmt, ...) 1021 { 1022 } 1023 1024 static inline void mminit_verify_pageflags_layout(void) 1025 { 1026 } 1027 1028 static inline void mminit_verify_zonelist(void) 1029 { 1030 } 1031 #endif /* CONFIG_DEBUG_MEMORY_INIT */ 1032 1033 #define NODE_RECLAIM_NOSCAN -2 1034 #define NODE_RECLAIM_FULL -1 1035 #define NODE_RECLAIM_SOME 0 1036 #define NODE_RECLAIM_SUCCESS 1 1037 1038 #ifdef CONFIG_NUMA 1039 extern int node_reclaim(struct pglist_data *, gfp_t, unsigned int); 1040 extern int find_next_best_node(int node, nodemask_t *used_node_mask); 1041 #else 1042 static inline int node_reclaim(struct pglist_data *pgdat, gfp_t mask, 1043 unsigned int order) 1044 { 1045 return NODE_RECLAIM_NOSCAN; 1046 } 1047 static inline int find_next_best_node(int node, nodemask_t *used_node_mask) 1048 { 1049 return NUMA_NO_NODE; 1050 } 1051 #endif 1052 1053 /* 1054 * mm/memory-failure.c 1055 */ 1056 void shake_folio(struct folio *folio); 1057 extern int hwpoison_filter(struct page *p); 1058 1059 extern u32 hwpoison_filter_dev_major; 1060 extern u32 hwpoison_filter_dev_minor; 1061 extern u64 hwpoison_filter_flags_mask; 1062 extern u64 hwpoison_filter_flags_value; 1063 extern u64 hwpoison_filter_memcg; 1064 extern u32 hwpoison_filter_enable; 1065 1066 extern unsigned long __must_check vm_mmap_pgoff(struct file *, unsigned long, 1067 unsigned long, unsigned long, 1068 unsigned long, unsigned long); 1069 1070 extern void set_pageblock_order(void); 1071 unsigned long reclaim_pages(struct list_head *folio_list); 1072 unsigned int reclaim_clean_pages_from_list(struct zone *zone, 1073 struct list_head *folio_list); 1074 /* The ALLOC_WMARK bits are used as an index to zone->watermark */ 1075 #define ALLOC_WMARK_MIN WMARK_MIN 1076 #define ALLOC_WMARK_LOW WMARK_LOW 1077 #define ALLOC_WMARK_HIGH WMARK_HIGH 1078 #define ALLOC_NO_WATERMARKS 0x04 /* don't check watermarks at all */ 1079 1080 /* Mask to get the watermark bits */ 1081 #define ALLOC_WMARK_MASK (ALLOC_NO_WATERMARKS-1) 1082 1083 /* 1084 * Only MMU archs have async oom victim reclaim - aka oom_reaper so we 1085 * cannot assume a reduced access to memory reserves is sufficient for 1086 * !MMU 1087 */ 1088 #ifdef CONFIG_MMU 1089 #define ALLOC_OOM 0x08 1090 #else 1091 #define ALLOC_OOM ALLOC_NO_WATERMARKS 1092 #endif 1093 1094 #define ALLOC_NON_BLOCK 0x10 /* Caller cannot block. Allow access 1095 * to 25% of the min watermark or 1096 * 62.5% if __GFP_HIGH is set. 1097 */ 1098 #define ALLOC_MIN_RESERVE 0x20 /* __GFP_HIGH set. Allow access to 50% 1099 * of the min watermark. 1100 */ 1101 #define ALLOC_CPUSET 0x40 /* check for correct cpuset */ 1102 #define ALLOC_CMA 0x80 /* allow allocations from CMA areas */ 1103 #ifdef CONFIG_ZONE_DMA32 1104 #define ALLOC_NOFRAGMENT 0x100 /* avoid mixing pageblock types */ 1105 #else 1106 #define ALLOC_NOFRAGMENT 0x0 1107 #endif 1108 #define ALLOC_HIGHATOMIC 0x200 /* Allows access to MIGRATE_HIGHATOMIC */ 1109 #define ALLOC_KSWAPD 0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */ 1110 1111 /* Flags that allow allocations below the min watermark. */ 1112 #define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM) 1113 1114 enum ttu_flags; 1115 struct tlbflush_unmap_batch; 1116 1117 1118 /* 1119 * only for MM internal work items which do not depend on 1120 * any allocations or locks which might depend on allocations 1121 */ 1122 extern struct workqueue_struct *mm_percpu_wq; 1123 1124 #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH 1125 void try_to_unmap_flush(void); 1126 void try_to_unmap_flush_dirty(void); 1127 void flush_tlb_batched_pending(struct mm_struct *mm); 1128 #else 1129 static inline void try_to_unmap_flush(void) 1130 { 1131 } 1132 static inline void try_to_unmap_flush_dirty(void) 1133 { 1134 } 1135 static inline void flush_tlb_batched_pending(struct mm_struct *mm) 1136 { 1137 } 1138 #endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */ 1139 1140 extern const struct trace_print_flags pageflag_names[]; 1141 extern const struct trace_print_flags pagetype_names[]; 1142 extern const struct trace_print_flags vmaflag_names[]; 1143 extern const struct trace_print_flags gfpflag_names[]; 1144 1145 static inline bool is_migrate_highatomic(enum migratetype migratetype) 1146 { 1147 return migratetype == MIGRATE_HIGHATOMIC; 1148 } 1149 1150 void setup_zone_pageset(struct zone *zone); 1151 1152 struct migration_target_control { 1153 int nid; /* preferred node id */ 1154 nodemask_t *nmask; 1155 gfp_t gfp_mask; 1156 enum migrate_reason reason; 1157 }; 1158 1159 /* 1160 * mm/filemap.c 1161 */ 1162 size_t splice_folio_into_pipe(struct pipe_inode_info *pipe, 1163 struct folio *folio, loff_t fpos, size_t size); 1164 1165 /* 1166 * mm/vmalloc.c 1167 */ 1168 #ifdef CONFIG_MMU 1169 void __init vmalloc_init(void); 1170 int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end, 1171 pgprot_t prot, struct page **pages, unsigned int page_shift); 1172 #else 1173 static inline void vmalloc_init(void) 1174 { 1175 } 1176 1177 static inline 1178 int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end, 1179 pgprot_t prot, struct page **pages, unsigned int page_shift) 1180 { 1181 return -EINVAL; 1182 } 1183 #endif 1184 1185 int __must_check __vmap_pages_range_noflush(unsigned long addr, 1186 unsigned long end, pgprot_t prot, 1187 struct page **pages, unsigned int page_shift); 1188 1189 void vunmap_range_noflush(unsigned long start, unsigned long end); 1190 1191 void __vunmap_range_noflush(unsigned long start, unsigned long end); 1192 1193 int numa_migrate_prep(struct folio *folio, struct vm_fault *vmf, 1194 unsigned long addr, int page_nid, int *flags); 1195 1196 void free_zone_device_folio(struct folio *folio); 1197 int migrate_device_coherent_page(struct page *page); 1198 1199 /* 1200 * mm/gup.c 1201 */ 1202 struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags); 1203 int __must_check try_grab_page(struct page *page, unsigned int flags); 1204 1205 /* 1206 * mm/huge_memory.c 1207 */ 1208 void touch_pud(struct vm_area_struct *vma, unsigned long addr, 1209 pud_t *pud, bool write); 1210 void touch_pmd(struct vm_area_struct *vma, unsigned long addr, 1211 pmd_t *pmd, bool write); 1212 1213 /* 1214 * mm/mmap.c 1215 */ 1216 struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi, 1217 struct vm_area_struct *vma, 1218 unsigned long delta); 1219 1220 enum { 1221 /* mark page accessed */ 1222 FOLL_TOUCH = 1 << 16, 1223 /* a retry, previous pass started an IO */ 1224 FOLL_TRIED = 1 << 17, 1225 /* we are working on non-current tsk/mm */ 1226 FOLL_REMOTE = 1 << 18, 1227 /* pages must be released via unpin_user_page */ 1228 FOLL_PIN = 1 << 19, 1229 /* gup_fast: prevent fall-back to slow gup */ 1230 FOLL_FAST_ONLY = 1 << 20, 1231 /* allow unlocking the mmap lock */ 1232 FOLL_UNLOCKABLE = 1 << 21, 1233 /* VMA lookup+checks compatible with MADV_POPULATE_(READ|WRITE) */ 1234 FOLL_MADV_POPULATE = 1 << 22, 1235 }; 1236 1237 #define INTERNAL_GUP_FLAGS (FOLL_TOUCH | FOLL_TRIED | FOLL_REMOTE | FOLL_PIN | \ 1238 FOLL_FAST_ONLY | FOLL_UNLOCKABLE | \ 1239 FOLL_MADV_POPULATE) 1240 1241 /* 1242 * Indicates for which pages that are write-protected in the page table, 1243 * whether GUP has to trigger unsharing via FAULT_FLAG_UNSHARE such that the 1244 * GUP pin will remain consistent with the pages mapped into the page tables 1245 * of the MM. 1246 * 1247 * Temporary unmapping of PageAnonExclusive() pages or clearing of 1248 * PageAnonExclusive() has to protect against concurrent GUP: 1249 * * Ordinary GUP: Using the PT lock 1250 * * GUP-fast and fork(): mm->write_protect_seq 1251 * * GUP-fast and KSM or temporary unmapping (swap, migration): see 1252 * folio_try_share_anon_rmap_*() 1253 * 1254 * Must be called with the (sub)page that's actually referenced via the 1255 * page table entry, which might not necessarily be the head page for a 1256 * PTE-mapped THP. 1257 * 1258 * If the vma is NULL, we're coming from the GUP-fast path and might have 1259 * to fallback to the slow path just to lookup the vma. 1260 */ 1261 static inline bool gup_must_unshare(struct vm_area_struct *vma, 1262 unsigned int flags, struct page *page) 1263 { 1264 /* 1265 * FOLL_WRITE is implicitly handled correctly as the page table entry 1266 * has to be writable -- and if it references (part of) an anonymous 1267 * folio, that part is required to be marked exclusive. 1268 */ 1269 if ((flags & (FOLL_WRITE | FOLL_PIN)) != FOLL_PIN) 1270 return false; 1271 /* 1272 * Note: PageAnon(page) is stable until the page is actually getting 1273 * freed. 1274 */ 1275 if (!PageAnon(page)) { 1276 /* 1277 * We only care about R/O long-term pining: R/O short-term 1278 * pinning does not have the semantics to observe successive 1279 * changes through the process page tables. 1280 */ 1281 if (!(flags & FOLL_LONGTERM)) 1282 return false; 1283 1284 /* We really need the vma ... */ 1285 if (!vma) 1286 return true; 1287 1288 /* 1289 * ... because we only care about writable private ("COW") 1290 * mappings where we have to break COW early. 1291 */ 1292 return is_cow_mapping(vma->vm_flags); 1293 } 1294 1295 /* Paired with a memory barrier in folio_try_share_anon_rmap_*(). */ 1296 if (IS_ENABLED(CONFIG_HAVE_GUP_FAST)) 1297 smp_rmb(); 1298 1299 /* 1300 * Note that PageKsm() pages cannot be exclusive, and consequently, 1301 * cannot get pinned. 1302 */ 1303 return !PageAnonExclusive(page); 1304 } 1305 1306 extern bool mirrored_kernelcore; 1307 extern bool memblock_has_mirror(void); 1308 1309 static __always_inline void vma_set_range(struct vm_area_struct *vma, 1310 unsigned long start, unsigned long end, 1311 pgoff_t pgoff) 1312 { 1313 vma->vm_start = start; 1314 vma->vm_end = end; 1315 vma->vm_pgoff = pgoff; 1316 } 1317 1318 static inline bool vma_soft_dirty_enabled(struct vm_area_struct *vma) 1319 { 1320 /* 1321 * NOTE: we must check this before VM_SOFTDIRTY on soft-dirty 1322 * enablements, because when without soft-dirty being compiled in, 1323 * VM_SOFTDIRTY is defined as 0x0, then !(vm_flags & VM_SOFTDIRTY) 1324 * will be constantly true. 1325 */ 1326 if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY)) 1327 return false; 1328 1329 /* 1330 * Soft-dirty is kind of special: its tracking is enabled when the 1331 * vma flags not set. 1332 */ 1333 return !(vma->vm_flags & VM_SOFTDIRTY); 1334 } 1335 1336 static inline void vma_iter_config(struct vma_iterator *vmi, 1337 unsigned long index, unsigned long last) 1338 { 1339 __mas_set_range(&vmi->mas, index, last - 1); 1340 } 1341 1342 static inline void vma_iter_reset(struct vma_iterator *vmi) 1343 { 1344 mas_reset(&vmi->mas); 1345 } 1346 1347 static inline 1348 struct vm_area_struct *vma_iter_prev_range_limit(struct vma_iterator *vmi, unsigned long min) 1349 { 1350 return mas_prev_range(&vmi->mas, min); 1351 } 1352 1353 static inline 1354 struct vm_area_struct *vma_iter_next_range_limit(struct vma_iterator *vmi, unsigned long max) 1355 { 1356 return mas_next_range(&vmi->mas, max); 1357 } 1358 1359 static inline int vma_iter_area_lowest(struct vma_iterator *vmi, unsigned long min, 1360 unsigned long max, unsigned long size) 1361 { 1362 return mas_empty_area(&vmi->mas, min, max - 1, size); 1363 } 1364 1365 static inline int vma_iter_area_highest(struct vma_iterator *vmi, unsigned long min, 1366 unsigned long max, unsigned long size) 1367 { 1368 return mas_empty_area_rev(&vmi->mas, min, max - 1, size); 1369 } 1370 1371 /* 1372 * VMA Iterator functions shared between nommu and mmap 1373 */ 1374 static inline int vma_iter_prealloc(struct vma_iterator *vmi, 1375 struct vm_area_struct *vma) 1376 { 1377 return mas_preallocate(&vmi->mas, vma, GFP_KERNEL); 1378 } 1379 1380 static inline void vma_iter_clear(struct vma_iterator *vmi) 1381 { 1382 mas_store_prealloc(&vmi->mas, NULL); 1383 } 1384 1385 static inline struct vm_area_struct *vma_iter_load(struct vma_iterator *vmi) 1386 { 1387 return mas_walk(&vmi->mas); 1388 } 1389 1390 /* Store a VMA with preallocated memory */ 1391 static inline void vma_iter_store(struct vma_iterator *vmi, 1392 struct vm_area_struct *vma) 1393 { 1394 1395 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE) 1396 if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start && 1397 vmi->mas.index > vma->vm_start)) { 1398 pr_warn("%lx > %lx\n store vma %lx-%lx\n into slot %lx-%lx\n", 1399 vmi->mas.index, vma->vm_start, vma->vm_start, 1400 vma->vm_end, vmi->mas.index, vmi->mas.last); 1401 } 1402 if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start && 1403 vmi->mas.last < vma->vm_start)) { 1404 pr_warn("%lx < %lx\nstore vma %lx-%lx\ninto slot %lx-%lx\n", 1405 vmi->mas.last, vma->vm_start, vma->vm_start, vma->vm_end, 1406 vmi->mas.index, vmi->mas.last); 1407 } 1408 #endif 1409 1410 if (vmi->mas.status != ma_start && 1411 ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start))) 1412 vma_iter_invalidate(vmi); 1413 1414 __mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1); 1415 mas_store_prealloc(&vmi->mas, vma); 1416 } 1417 1418 static inline int vma_iter_store_gfp(struct vma_iterator *vmi, 1419 struct vm_area_struct *vma, gfp_t gfp) 1420 { 1421 if (vmi->mas.status != ma_start && 1422 ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start))) 1423 vma_iter_invalidate(vmi); 1424 1425 __mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1); 1426 mas_store_gfp(&vmi->mas, vma, gfp); 1427 if (unlikely(mas_is_err(&vmi->mas))) 1428 return -ENOMEM; 1429 1430 return 0; 1431 } 1432 1433 /* 1434 * VMA lock generalization 1435 */ 1436 struct vma_prepare { 1437 struct vm_area_struct *vma; 1438 struct vm_area_struct *adj_next; 1439 struct file *file; 1440 struct address_space *mapping; 1441 struct anon_vma *anon_vma; 1442 struct vm_area_struct *insert; 1443 struct vm_area_struct *remove; 1444 struct vm_area_struct *remove2; 1445 }; 1446 1447 void __meminit __init_single_page(struct page *page, unsigned long pfn, 1448 unsigned long zone, int nid); 1449 1450 /* shrinker related functions */ 1451 unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg, 1452 int priority); 1453 1454 #ifdef CONFIG_64BIT 1455 static inline int can_do_mseal(unsigned long flags) 1456 { 1457 if (flags) 1458 return -EINVAL; 1459 1460 return 0; 1461 } 1462 1463 bool can_modify_mm(struct mm_struct *mm, unsigned long start, 1464 unsigned long end); 1465 bool can_modify_mm_madv(struct mm_struct *mm, unsigned long start, 1466 unsigned long end, int behavior); 1467 #else 1468 static inline int can_do_mseal(unsigned long flags) 1469 { 1470 return -EPERM; 1471 } 1472 1473 static inline bool can_modify_mm(struct mm_struct *mm, unsigned long start, 1474 unsigned long end) 1475 { 1476 return true; 1477 } 1478 1479 static inline bool can_modify_mm_madv(struct mm_struct *mm, unsigned long start, 1480 unsigned long end, int behavior) 1481 { 1482 return true; 1483 } 1484 #endif 1485 1486 #ifdef CONFIG_SHRINKER_DEBUG 1487 static inline __printf(2, 0) int shrinker_debugfs_name_alloc( 1488 struct shrinker *shrinker, const char *fmt, va_list ap) 1489 { 1490 shrinker->name = kvasprintf_const(GFP_KERNEL, fmt, ap); 1491 1492 return shrinker->name ? 0 : -ENOMEM; 1493 } 1494 1495 static inline void shrinker_debugfs_name_free(struct shrinker *shrinker) 1496 { 1497 kfree_const(shrinker->name); 1498 shrinker->name = NULL; 1499 } 1500 1501 extern int shrinker_debugfs_add(struct shrinker *shrinker); 1502 extern struct dentry *shrinker_debugfs_detach(struct shrinker *shrinker, 1503 int *debugfs_id); 1504 extern void shrinker_debugfs_remove(struct dentry *debugfs_entry, 1505 int debugfs_id); 1506 #else /* CONFIG_SHRINKER_DEBUG */ 1507 static inline int shrinker_debugfs_add(struct shrinker *shrinker) 1508 { 1509 return 0; 1510 } 1511 static inline int shrinker_debugfs_name_alloc(struct shrinker *shrinker, 1512 const char *fmt, va_list ap) 1513 { 1514 return 0; 1515 } 1516 static inline void shrinker_debugfs_name_free(struct shrinker *shrinker) 1517 { 1518 } 1519 static inline struct dentry *shrinker_debugfs_detach(struct shrinker *shrinker, 1520 int *debugfs_id) 1521 { 1522 *debugfs_id = -1; 1523 return NULL; 1524 } 1525 static inline void shrinker_debugfs_remove(struct dentry *debugfs_entry, 1526 int debugfs_id) 1527 { 1528 } 1529 #endif /* CONFIG_SHRINKER_DEBUG */ 1530 1531 /* Only track the nodes of mappings with shadow entries */ 1532 void workingset_update_node(struct xa_node *node); 1533 extern struct list_lru shadow_nodes; 1534 1535 struct unlink_vma_file_batch { 1536 int count; 1537 struct vm_area_struct *vmas[8]; 1538 }; 1539 1540 void unlink_file_vma_batch_init(struct unlink_vma_file_batch *); 1541 void unlink_file_vma_batch_add(struct unlink_vma_file_batch *, struct vm_area_struct *); 1542 void unlink_file_vma_batch_final(struct unlink_vma_file_batch *); 1543 1544 #endif /* __MM_INTERNAL_H */ 1545