1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* internal.h: mm/ internal definitions 3 * 4 * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 #ifndef __MM_INTERNAL_H 8 #define __MM_INTERNAL_H 9 10 #include <linux/fs.h> 11 #include <linux/khugepaged.h> 12 #include <linux/mm.h> 13 #include <linux/mm_inline.h> 14 #include <linux/mmu_notifier.h> 15 #include <linux/pagemap.h> 16 #include <linux/pagewalk.h> 17 #include <linux/rmap.h> 18 #include <linux/swap.h> 19 #include <linux/leafops.h> 20 #include <linux/tracepoint-defs.h> 21 22 /* Internal core VMA manipulation functions. */ 23 #include "vma.h" 24 25 struct folio_batch; 26 struct hstate; 27 28 struct huge_bootmem_page { 29 struct list_head list; 30 struct hstate *hstate; 31 unsigned long flags; 32 }; 33 34 /* mm/workingset.c */ 35 bool workingset_test_recent(void *shadow, bool file, bool *workingset, 36 bool flush); 37 void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages); 38 void *workingset_eviction(struct folio *folio, 39 struct mem_cgroup *target_memcg); 40 void workingset_refault(struct folio *folio, void *shadow); 41 void workingset_activation(struct folio *folio); 42 43 /* mm/folio.c */ 44 void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma); 45 46 static inline bool folio_may_be_lru_cached(const struct folio *folio) 47 { 48 /* 49 * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting. 50 * Holding small numbers of low-order mTHP folios in per-CPU LRU cache 51 * will be sensible, but nobody has implemented and tested that yet. 52 */ 53 return !folio_test_large(folio); 54 } 55 56 static inline void lru_cache_enable(void) 57 { 58 atomic_dec(&lru_disable_count); 59 } 60 61 void lru_cache_disable(void); 62 void lru_add_drain(void); 63 void lru_add_drain_cpu(int cpu); 64 void lru_add_drain_cpu_zone(struct zone *zone); 65 void folio_deactivate(struct folio *folio); 66 void folio_mark_lazyfree(struct folio *folio); 67 68 /* mm/vmscan.c */ 69 unsigned long zone_reclaimable_pages(struct zone *zone); 70 unsigned long try_to_free_pages(struct zonelist *zonelist, int order, 71 gfp_t gfp_mask, const nodemask_t *mask); 72 unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, 73 int zone_idx); 74 75 #define MEMCG_RECLAIM_MAY_SWAP (1 << 1) 76 #define MEMCG_RECLAIM_PROACTIVE (1 << 2) 77 #define MIN_SWAPPINESS 0 78 #define MAX_SWAPPINESS 200 79 80 /* Just reclaim from anon folios in proactive memory reclaim */ 81 #define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1) 82 83 unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, 84 unsigned long nr_pages, 85 gfp_t gfp_mask, 86 unsigned int reclaim_options, 87 int *swappiness); 88 unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, 89 gfp_t gfp_mask, bool noswap, 90 pg_data_t *pgdat, 91 unsigned long *nr_scanned); 92 93 #ifdef CONFIG_NUMA 94 extern int sysctl_min_unmapped_ratio; 95 extern int sysctl_min_slab_ratio; 96 #endif 97 98 /* 99 * Maintains state across a page table move. The operation assumes both source 100 * and destination VMAs already exist and are specified by the user. 101 * 102 * Partial moves are permitted, but the old and new ranges must both reside 103 * within a VMA. 104 * 105 * mmap lock must be held in write and VMA write locks must be held on any VMA 106 * that is visible. 107 * 108 * Use the PAGETABLE_MOVE() macro to initialise this struct. 109 * 110 * The old_addr and new_addr fields are updated as the page table move is 111 * executed. 112 * 113 * NOTE: The page table move is affected by reading from [old_addr, old_end), 114 * and old_addr may be updated for better page table alignment, so len_in 115 * represents the length of the range being copied as specified by the user. 116 */ 117 struct pagetable_move_control { 118 struct vm_area_struct *old; /* Source VMA. */ 119 struct vm_area_struct *new; /* Destination VMA. */ 120 unsigned long old_addr; /* Address from which the move begins. */ 121 unsigned long old_end; /* Exclusive address at which old range ends. */ 122 unsigned long new_addr; /* Address to move page tables to. */ 123 unsigned long len_in; /* Bytes to remap specified by user. */ 124 125 bool need_rmap_locks; /* Do rmap locks need to be taken? */ 126 bool for_stack; /* Is this an early temp stack being moved? */ 127 }; 128 129 #define PAGETABLE_MOVE(name, old_, new_, old_addr_, new_addr_, len_) \ 130 struct pagetable_move_control name = { \ 131 .old = old_, \ 132 .new = new_, \ 133 .old_addr = old_addr_, \ 134 .old_end = (old_addr_) + (len_), \ 135 .new_addr = new_addr_, \ 136 .len_in = len_, \ 137 } 138 139 /* 140 * The set of flags that only affect watermark checking and reclaim 141 * behaviour. This is used by the MM to obey the caller constraints 142 * about IO, FS and watermark checking while ignoring placement 143 * hints such as HIGHMEM usage. 144 */ 145 #define GFP_RECLAIM_MASK (__GFP_RECLAIM|__GFP_HIGH|__GFP_IO|__GFP_FS|\ 146 __GFP_NOWARN|__GFP_RETRY_MAYFAIL|__GFP_NOFAIL|\ 147 __GFP_NORETRY|__GFP_MEMALLOC|__GFP_NOMEMALLOC|\ 148 __GFP_NOLOCKDEP) 149 150 /* The GFP flags allowed during early boot */ 151 #define GFP_BOOT_MASK (__GFP_BITS_MASK & ~(__GFP_RECLAIM|__GFP_IO|__GFP_FS)) 152 153 /* Control allocation cpuset and node placement constraints */ 154 #define GFP_CONSTRAINT_MASK (__GFP_HARDWALL|__GFP_THISNODE) 155 156 /* Do not use these with a slab allocator */ 157 #define GFP_SLAB_BUG_MASK (__GFP_DMA32|__GFP_HIGHMEM|~__GFP_BITS_MASK) 158 159 /* 160 * Different from WARN_ON_ONCE(), no warning will be issued 161 * when we specify __GFP_NOWARN. 162 */ 163 #define WARN_ON_ONCE_GFP(cond, gfp) ({ \ 164 static bool __section(".data..once") __warned; \ 165 int __ret_warn_once = !!(cond); \ 166 \ 167 if (unlikely(!(gfp & __GFP_NOWARN) && __ret_warn_once && !__warned)) { \ 168 __warned = true; \ 169 WARN_ON(1); \ 170 } \ 171 unlikely(__ret_warn_once); \ 172 }) 173 174 void page_writeback_init(void); 175 176 /* 177 * If a 16GB hugetlb folio were mapped by PTEs of all of its 4kB pages, 178 * its nr_pages_mapped would be 0x400000: choose the ENTIRELY_MAPPED bit 179 * above that range, instead of 2*(PMD_SIZE/PAGE_SIZE). Hugetlb currently 180 * leaves nr_pages_mapped at 0, but avoid surprise if it participates later. 181 */ 182 #define ENTIRELY_MAPPED 0x800000 183 #define FOLIO_PAGES_MAPPED (ENTIRELY_MAPPED - 1) 184 185 /* 186 * Flags passed to __show_mem() and show_free_areas() to suppress output in 187 * various contexts. 188 */ 189 #define SHOW_MEM_FILTER_NODES (0x0001u) /* disallowed nodes */ 190 191 /* 192 * How many individual pages have an elevated _mapcount. Excludes 193 * the folio's entire_mapcount. 194 * 195 * Don't use this function outside of debugging code. 196 */ 197 static inline int folio_nr_pages_mapped(const struct folio *folio) 198 { 199 if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) 200 return -1; 201 return atomic_read(&folio->_nr_pages_mapped) & FOLIO_PAGES_MAPPED; 202 } 203 204 /* 205 * Retrieve the first entry of a folio based on a provided entry within the 206 * folio. We cannot rely on folio->swap as there is no guarantee that it has 207 * been initialized. Used for calling arch_swap_restore() 208 */ 209 static inline swp_entry_t folio_swap(swp_entry_t entry, 210 const struct folio *folio) 211 { 212 swp_entry_t swap = { 213 .val = ALIGN_DOWN(entry.val, folio_nr_pages(folio)), 214 }; 215 216 return swap; 217 } 218 219 static inline void *folio_raw_mapping(const struct folio *folio) 220 { 221 unsigned long mapping = (unsigned long)folio->mapping; 222 223 return (void *)(mapping & ~FOLIO_MAPPING_FLAGS); 224 } 225 226 /* 227 * This is a file-backed mapping, and is about to be memory mapped - invoke its 228 * mmap hook and safely handle error conditions. On error, VMA hooks will be 229 * mutated. 230 * 231 * @file: File which backs the mapping. 232 * @vma: VMA which we are mapping. 233 * 234 * Returns: 0 if success, error otherwise. 235 */ 236 static inline int mmap_file(struct file *file, struct vm_area_struct *vma) 237 { 238 int err = vfs_mmap(file, vma); 239 240 if (likely(!err)) 241 return 0; 242 243 /* 244 * OK, we tried to call the file hook for mmap(), but an error 245 * arose. The mapping is in an inconsistent state and we must not invoke 246 * any further hooks on it. 247 */ 248 vma->vm_ops = &vma_dummy_vm_ops; 249 250 return err; 251 } 252 253 /* 254 * If the VMA has a close hook then close it, and since closing it might leave 255 * it in an inconsistent state which makes the use of any hooks suspect, clear 256 * them down by installing dummy empty hooks. 257 */ 258 static inline void vma_close(struct vm_area_struct *vma) 259 { 260 if (vma->vm_ops && vma->vm_ops->close) { 261 vma->vm_ops->close(vma); 262 263 /* 264 * The mapping is in an inconsistent state, and no further hooks 265 * may be invoked upon it. 266 */ 267 vma->vm_ops = &vma_dummy_vm_ops; 268 } 269 } 270 271 /* unmap_vmas is in mm/memory.c */ 272 void unmap_vmas(struct mmu_gather *tlb, struct unmap_desc *unmap); 273 274 #ifdef CONFIG_MMU 275 276 bool cond_install_uffd_wp_ptes(struct vm_area_struct *vma, 277 unsigned long addr, pte_t *ptep, pte_t pte, 278 unsigned long nr_ptes); 279 280 static inline void get_anon_vma(struct anon_vma *anon_vma) 281 { 282 atomic_inc(&anon_vma->refcount); 283 } 284 285 void __put_anon_vma(struct anon_vma *anon_vma); 286 287 static inline void put_anon_vma(struct anon_vma *anon_vma) 288 { 289 if (atomic_dec_and_test(&anon_vma->refcount)) 290 __put_anon_vma(anon_vma); 291 } 292 293 static inline void anon_vma_lock_write(struct anon_vma *anon_vma) 294 { 295 down_write(&anon_vma->root->rwsem); 296 } 297 298 static inline int anon_vma_trylock_write(struct anon_vma *anon_vma) 299 { 300 return down_write_trylock(&anon_vma->root->rwsem); 301 } 302 303 static inline void anon_vma_unlock_write(struct anon_vma *anon_vma) 304 { 305 up_write(&anon_vma->root->rwsem); 306 } 307 308 static inline void anon_vma_lock_read(struct anon_vma *anon_vma) 309 { 310 down_read(&anon_vma->root->rwsem); 311 } 312 313 static inline int anon_vma_trylock_read(struct anon_vma *anon_vma) 314 { 315 return down_read_trylock(&anon_vma->root->rwsem); 316 } 317 318 static inline void anon_vma_unlock_read(struct anon_vma *anon_vma) 319 { 320 up_read(&anon_vma->root->rwsem); 321 } 322 323 struct anon_vma *folio_get_anon_vma(const struct folio *folio); 324 325 /* Operations which modify VMAs. */ 326 enum vma_operation { 327 VMA_OP_SPLIT, 328 VMA_OP_MERGE_UNFAULTED, 329 VMA_OP_REMAP, 330 VMA_OP_FORK, 331 }; 332 333 int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src, 334 enum vma_operation operation); 335 int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma); 336 int __anon_vma_prepare(struct vm_area_struct *vma); 337 void unlink_anon_vmas(struct vm_area_struct *vma); 338 339 static inline int anon_vma_prepare(struct vm_area_struct *vma) 340 { 341 if (likely(vma->anon_vma)) 342 return 0; 343 344 return __anon_vma_prepare(vma); 345 } 346 347 /* Flags for folio_pte_batch(). */ 348 typedef int __bitwise fpb_t; 349 350 /* Compare PTEs respecting the dirty bit. */ 351 #define FPB_RESPECT_DIRTY ((__force fpb_t)BIT(0)) 352 353 /* Compare PTEs respecting the soft-dirty bit. */ 354 #define FPB_RESPECT_SOFT_DIRTY ((__force fpb_t)BIT(1)) 355 356 /* Compare PTEs respecting the writable bit. */ 357 #define FPB_RESPECT_WRITE ((__force fpb_t)BIT(2)) 358 359 /* 360 * Merge PTE write bits: if any PTE in the batch is writable, modify the 361 * PTE at @ptentp to be writable. 362 */ 363 #define FPB_MERGE_WRITE ((__force fpb_t)BIT(3)) 364 365 /* 366 * Merge PTE young and dirty bits: if any PTE in the batch is young or dirty, 367 * modify the PTE at @ptentp to be young or dirty, respectively. 368 */ 369 #define FPB_MERGE_YOUNG_DIRTY ((__force fpb_t)BIT(4)) 370 371 static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags) 372 { 373 if (!(flags & FPB_RESPECT_DIRTY)) 374 pte = pte_mkclean(pte); 375 if (likely(!(flags & FPB_RESPECT_SOFT_DIRTY))) 376 pte = pte_clear_soft_dirty(pte); 377 if (likely(!(flags & FPB_RESPECT_WRITE))) 378 pte = pte_wrprotect(pte); 379 return pte_mkold(pte); 380 } 381 382 /** 383 * folio_pte_batch_flags - detect a PTE batch for a large folio 384 * @folio: The large folio to detect a PTE batch for. 385 * @vma: The VMA. Only relevant with FPB_MERGE_WRITE, otherwise can be NULL. 386 * @ptep: Page table pointer for the first entry. 387 * @ptentp: Pointer to a COPY of the first page table entry whose flags this 388 * function updates based on @flags if appropriate. 389 * @max_nr: The maximum number of table entries to consider. 390 * @flags: Flags to modify the PTE batch semantics. 391 * 392 * Detect a PTE batch: consecutive (present) PTEs that map consecutive 393 * pages of the same large folio in a single VMA and a single page table. 394 * 395 * All PTEs inside a PTE batch have the same PTE bits set, excluding the PFN, 396 * the accessed bit, writable bit, dirty bit (unless FPB_RESPECT_DIRTY is set) 397 * and soft-dirty bit (unless FPB_RESPECT_SOFT_DIRTY is set). 398 * 399 * @ptep must map any page of the folio. max_nr must be at least one and 400 * must be limited by the caller so scanning cannot exceed a single VMA and 401 * a single page table. 402 * 403 * Depending on the FPB_MERGE_* flags, the pte stored at @ptentp will 404 * be updated: it's crucial that a pointer to a COPY of the first 405 * page table entry, obtained through ptep_get(), is provided as @ptentp. 406 * 407 * This function will be inlined to optimize based on the input parameters; 408 * consider using folio_pte_batch() instead if applicable. 409 * 410 * Return: the number of table entries in the batch. 411 */ 412 static inline unsigned int folio_pte_batch_flags(struct folio *folio, 413 struct vm_area_struct *vma, pte_t *ptep, pte_t *ptentp, 414 unsigned int max_nr, fpb_t flags) 415 { 416 bool any_writable = false, any_young = false, any_dirty = false; 417 pte_t expected_pte, pte = *ptentp; 418 unsigned int nr, cur_nr; 419 420 VM_WARN_ON_FOLIO(!pte_present(pte), folio); 421 VM_WARN_ON_FOLIO(!folio_test_large(folio) || max_nr < 1, folio); 422 VM_WARN_ON_FOLIO(page_folio(pfn_to_page(pte_pfn(pte))) != folio, folio); 423 /* 424 * Ensure this is a pointer to a copy not a pointer into a page table. 425 * If this is a stack value, it won't be a valid virtual address, but 426 * that's fine because it also cannot be pointing into the page table. 427 */ 428 VM_WARN_ON(virt_addr_valid(ptentp) && PageTable(virt_to_page(ptentp))); 429 430 /* Limit max_nr to the actual remaining PFNs in the folio we could batch. */ 431 max_nr = min_t(unsigned long, max_nr, 432 folio_pfn(folio) + folio_nr_pages(folio) - pte_pfn(pte)); 433 434 nr = pte_batch_hint(ptep, pte); 435 expected_pte = __pte_batch_clear_ignored(pte_advance_pfn(pte, nr), flags); 436 ptep = ptep + nr; 437 438 while (nr < max_nr) { 439 pte = ptep_get(ptep); 440 441 if (!pte_same(__pte_batch_clear_ignored(pte, flags), expected_pte)) 442 break; 443 444 if (flags & FPB_MERGE_WRITE) 445 any_writable |= pte_write(pte); 446 if (flags & FPB_MERGE_YOUNG_DIRTY) { 447 any_young |= pte_young(pte); 448 any_dirty |= pte_dirty(pte); 449 } 450 451 cur_nr = pte_batch_hint(ptep, pte); 452 expected_pte = pte_advance_pfn(expected_pte, cur_nr); 453 ptep += cur_nr; 454 nr += cur_nr; 455 } 456 457 if (any_writable) 458 *ptentp = pte_mkwrite(*ptentp, vma); 459 if (any_young) 460 *ptentp = pte_mkyoung(*ptentp); 461 if (any_dirty) 462 *ptentp = pte_mkdirty(*ptentp); 463 464 return min(nr, max_nr); 465 } 466 467 unsigned int folio_pte_batch(struct folio *folio, pte_t *ptep, pte_t pte, 468 unsigned int max_nr); 469 470 /** 471 * pte_move_swp_offset - Move the swap entry offset field of a swap pte 472 * forward or backward by delta 473 * @pte: The initial pte state; must be a swap entry 474 * @delta: The direction and the offset we are moving; forward if delta 475 * is positive; backward if delta is negative 476 * 477 * Moves the swap offset, while maintaining all other fields, including 478 * swap type, and any swp pte bits. The resulting pte is returned. 479 */ 480 static inline pte_t pte_move_swp_offset(pte_t pte, long delta) 481 { 482 const softleaf_t entry = softleaf_from_pte(pte); 483 pte_t new = __swp_entry_to_pte(__swp_entry(swp_type(entry), 484 (swp_offset(entry) + delta))); 485 486 if (pte_swp_soft_dirty(pte)) 487 new = pte_swp_mksoft_dirty(new); 488 if (pte_swp_exclusive(pte)) 489 new = pte_swp_mkexclusive(new); 490 if (pte_swp_uffd(pte)) 491 new = pte_swp_mkuffd(new); 492 493 return new; 494 } 495 496 497 /** 498 * pte_next_swp_offset - Increment the swap entry offset field of a swap pte. 499 * @pte: The initial pte state; must be a swap entry. 500 * 501 * Increments the swap offset, while maintaining all other fields, including 502 * swap type, and any swp pte bits. The resulting pte is returned. 503 */ 504 static inline pte_t pte_next_swp_offset(pte_t pte) 505 { 506 return pte_move_swp_offset(pte, 1); 507 } 508 509 /** 510 * swap_pte_batch - detect a PTE batch for a set of contiguous swap entries 511 * @start_ptep: Page table pointer for the first entry. 512 * @max_nr: The maximum number of table entries to consider. 513 * @pte: Page table entry for the first entry. 514 * 515 * Detect a batch of contiguous swap entries: consecutive (non-present) PTEs 516 * containing swap entries all with consecutive offsets and targeting the same 517 * swap type, all with matching swp pte bits. 518 * 519 * max_nr must be at least one and must be limited by the caller so scanning 520 * cannot exceed a single page table. 521 * 522 * Return: the number of table entries in the batch. 523 */ 524 static inline int swap_pte_batch(pte_t *start_ptep, int max_nr, pte_t pte) 525 { 526 pte_t expected_pte = pte_next_swp_offset(pte); 527 const pte_t *end_ptep = start_ptep + max_nr; 528 pte_t *ptep = start_ptep + 1; 529 530 VM_WARN_ON(max_nr < 1); 531 VM_WARN_ON(!softleaf_is_swap(softleaf_from_pte(pte))); 532 533 while (ptep < end_ptep) { 534 pte = ptep_get(ptep); 535 536 if (!pte_same(pte, expected_pte)) 537 break; 538 expected_pte = pte_next_swp_offset(expected_pte); 539 ptep++; 540 } 541 542 return ptep - start_ptep; 543 } 544 #endif /* CONFIG_MMU */ 545 546 void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio, 547 int nr_throttled); 548 static inline void acct_reclaim_writeback(struct folio *folio) 549 { 550 pg_data_t *pgdat = folio_pgdat(folio); 551 int nr_throttled = atomic_read(&pgdat->nr_writeback_throttled); 552 553 if (nr_throttled) 554 __acct_reclaim_writeback(pgdat, folio, nr_throttled); 555 } 556 557 static inline void wake_throttle_isolated(pg_data_t *pgdat) 558 { 559 wait_queue_head_t *wqh; 560 561 wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_ISOLATED]; 562 if (waitqueue_active(wqh)) 563 wake_up(wqh); 564 } 565 566 vm_fault_t __vmf_anon_prepare(struct vm_fault *vmf); 567 static inline vm_fault_t vmf_anon_prepare(struct vm_fault *vmf) 568 { 569 vm_fault_t ret = __vmf_anon_prepare(vmf); 570 571 if (unlikely(ret & VM_FAULT_RETRY)) 572 vma_end_read(vmf->vma); 573 return ret; 574 } 575 576 vm_fault_t do_swap_page(struct vm_fault *vmf); 577 void folio_rotate_reclaimable(struct folio *folio); 578 bool __folio_end_writeback(struct folio *folio); 579 void deactivate_file_folio(struct folio *folio); 580 void folio_activate(struct folio *folio); 581 582 void free_pgtables(struct mmu_gather *tlb, struct unmap_desc *desc); 583 584 void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte); 585 586 /** 587 * sync_with_folio_pmd_zap - sync with concurrent zapping of a folio PMD 588 * @mm: The mm_struct. 589 * @pmdp: Pointer to the pmd that was found to be pmd_none(). 590 * 591 * When we find a pmd_none() while unmapping a folio without holding the PTL, 592 * zap_huge_pmd() may have cleared the PMD but not yet modified the folio to 593 * indicate that it's unmapped. Skipping the PMD without synchronization could 594 * make folio unmapping code assume that unmapping failed. 595 * 596 * Wait for concurrent zapping to complete by grabbing the PTL. 597 */ 598 static inline void sync_with_folio_pmd_zap(struct mm_struct *mm, pmd_t *pmdp) 599 { 600 spinlock_t *ptl = pmd_lock(mm, pmdp); 601 602 spin_unlock(ptl); 603 } 604 605 struct zap_details; 606 void zap_vma_range_batched(struct mmu_gather *tlb, 607 struct vm_area_struct *vma, unsigned long addr, 608 unsigned long size, struct zap_details *details); 609 int zap_vma_for_reaping(struct vm_area_struct *vma); 610 int folio_unmap_invalidate(struct address_space *mapping, struct folio *folio, 611 gfp_t gfp); 612 613 void page_cache_ra_order(struct readahead_control *, struct file_ra_state *); 614 void force_page_cache_ra(struct readahead_control *, unsigned long nr); 615 static inline void force_page_cache_readahead(struct address_space *mapping, 616 struct file *file, pgoff_t index, unsigned long nr_to_read) 617 { 618 DEFINE_READAHEAD(ractl, file, &file->f_ra, mapping, index); 619 force_page_cache_ra(&ractl, nr_to_read); 620 } 621 622 unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start, 623 pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices); 624 unsigned find_get_entries(struct address_space *mapping, pgoff_t *start, 625 pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices); 626 int truncate_inode_folio(struct address_space *mapping, struct folio *folio); 627 bool truncate_inode_partial_folio(struct folio *folio, loff_t start, 628 loff_t end); 629 long mapping_evict_folio(struct address_space *mapping, struct folio *folio); 630 unsigned long mapping_try_invalidate(struct address_space *mapping, 631 pgoff_t start, pgoff_t end, unsigned long *nr_failed); 632 633 /** 634 * folio_evictable - Test whether a folio is evictable. 635 * @folio: The folio to test. 636 * 637 * Test whether @folio is evictable -- i.e., should be placed on 638 * active/inactive lists vs unevictable list. 639 * 640 * Reasons folio might not be evictable: 641 * 1. folio's mapping marked unevictable 642 * 2. One of the pages in the folio is part of an mlocked VMA 643 */ 644 static inline bool folio_evictable(struct folio *folio) 645 { 646 bool ret; 647 648 /* Prevent address_space of inode and swap cache from being freed */ 649 rcu_read_lock(); 650 ret = !mapping_unevictable(folio_mapping(folio)) && 651 !folio_test_mlocked(folio); 652 rcu_read_unlock(); 653 return ret; 654 } 655 656 /* 657 * Turn a non-refcounted page (->_refcount == 0) into refcounted with 658 * a count of one. 659 */ 660 static inline void set_page_refcounted(struct page *page) 661 { 662 VM_BUG_ON_PAGE(PageTail(page), page); 663 VM_BUG_ON_PAGE(page_ref_count(page), page); 664 set_page_count(page, 1); 665 } 666 667 static inline void set_pages_refcounted(struct page *page, unsigned long nr_pages) 668 { 669 unsigned long pfn = page_to_pfn(page); 670 671 for (; nr_pages--; pfn++) 672 set_page_refcounted(pfn_to_page(pfn)); 673 } 674 675 /* 676 * Return true if a folio needs ->release_folio() calling upon it. 677 */ 678 static inline bool folio_needs_release(struct folio *folio) 679 { 680 struct address_space *mapping = folio_mapping(folio); 681 682 return folio_has_private(folio) || 683 (mapping && mapping_release_always(mapping)); 684 } 685 686 extern unsigned long highest_memmap_pfn; 687 688 /* 689 * Maximum number of reclaim retries without progress before the OOM 690 * killer is consider the only way forward. 691 */ 692 #define MAX_RECLAIM_RETRIES 16 693 694 /* 695 * in mm/vmscan.c: 696 */ 697 bool folio_isolate_lru(struct folio *folio); 698 void folio_putback_lru(struct folio *folio); 699 extern void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason); 700 int user_proactive_reclaim(char *buf, 701 struct mem_cgroup *memcg, pg_data_t *pgdat); 702 703 /* 704 * in mm/rmap.c: 705 */ 706 pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address); 707 708 /* 709 * in mm/khugepaged.c 710 */ 711 void set_recommended_min_free_kbytes(void); 712 713 /* 714 * in mm/page_alloc.c 715 */ 716 #define K(x) ((x) << (PAGE_SHIFT-10)) 717 718 extern char * const zone_names[MAX_NR_ZONES]; 719 720 extern int min_free_kbytes; 721 extern int defrag_mode; 722 723 void setup_per_zone_wmarks(void); 724 void calculate_min_free_kbytes(void); 725 int __meminit init_per_zone_wmark_min(void); 726 727 extern int __isolate_free_page(struct page *page, unsigned int order); 728 extern void __putback_isolated_page(struct page *page, unsigned int order, 729 int mt); 730 731 /* 732 * This will have no effect, other than possibly generating a warning, if the 733 * caller passes in a non-large folio. 734 */ 735 static inline void folio_set_order(struct folio *folio, unsigned int order) 736 { 737 if (WARN_ON_ONCE(!order || !folio_test_large(folio))) 738 return; 739 VM_WARN_ON_ONCE(order > MAX_FOLIO_ORDER); 740 741 folio->_flags_1 = (folio->_flags_1 & ~0xffUL) | order; 742 #ifdef NR_PAGES_IN_LARGE_FOLIO 743 folio->_nr_pages = 1U << order; 744 #endif 745 } 746 747 bool __folio_unqueue_deferred_split(struct folio *folio); 748 static inline bool folio_unqueue_deferred_split(struct folio *folio) 749 { 750 if (folio_order(folio) <= 1 || !folio_test_large_rmappable(folio)) 751 return false; 752 753 /* 754 * At this point, there is no one trying to add the folio to 755 * deferred_list. If folio is not in deferred_list, it's safe 756 * to check without acquiring the list_lru lock. 757 */ 758 if (data_race(list_empty(&folio->_deferred_list))) 759 return false; 760 761 return __folio_unqueue_deferred_split(folio); 762 } 763 764 static inline struct folio *page_rmappable_folio(struct page *page) 765 { 766 struct folio *folio = (struct folio *)page; 767 768 if (folio && folio_test_large(folio)) 769 folio_set_large_rmappable(folio); 770 return folio; 771 } 772 773 static inline void prep_compound_head(struct page *page, unsigned int order) 774 { 775 struct folio *folio = (struct folio *)page; 776 777 folio_set_order(folio, order); 778 atomic_set(&folio->_large_mapcount, -1); 779 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) 780 atomic_set(&folio->_nr_pages_mapped, 0); 781 if (IS_ENABLED(CONFIG_MM_ID)) { 782 folio->_mm_ids = 0; 783 folio->_mm_id_mapcount[0] = -1; 784 folio->_mm_id_mapcount[1] = -1; 785 } 786 if (IS_ENABLED(CONFIG_64BIT) || order > 1) { 787 atomic_set(&folio->_pincount, 0); 788 atomic_set(&folio->_entire_mapcount, -1); 789 } 790 if (order > 1) 791 INIT_LIST_HEAD(&folio->_deferred_list); 792 } 793 794 static inline void prep_compound_tail(struct page *tail, 795 const struct page *head, unsigned int order) 796 { 797 tail->mapping = TAIL_MAPPING; 798 set_compound_head(tail, head, order); 799 VM_WARN_ON_ONCE(tail->private); 800 } 801 802 static inline void init_compound_tail(struct page *tail, 803 const struct page *head, unsigned int order, struct zone *zone) 804 { 805 atomic_set(&tail->_mapcount, -1); 806 set_page_node(tail, zone_to_nid(zone)); 807 set_page_zone(tail, zone_idx(zone)); 808 prep_compound_tail(tail, head, order); 809 } 810 811 #if defined CONFIG_COMPACTION || defined CONFIG_CMA 812 813 /* 814 * in mm/compaction.c 815 */ 816 /* 817 * compact_control is used to track pages being migrated and the free pages 818 * they are being migrated to during memory compaction. The free_pfn starts 819 * at the end of a zone and migrate_pfn begins at the start. Movable pages 820 * are moved to the end of a zone during a compaction run and the run 821 * completes when free_pfn <= migrate_pfn 822 */ 823 struct compact_control { 824 struct list_head freepages[NR_PAGE_ORDERS]; /* List of free pages to migrate to */ 825 struct list_head migratepages; /* List of pages being migrated */ 826 unsigned int nr_freepages; /* Number of isolated free pages */ 827 unsigned int nr_migratepages; /* Number of pages to migrate */ 828 unsigned long free_pfn; /* isolate_freepages search base */ 829 /* 830 * Acts as an in/out parameter to page isolation for migration. 831 * isolate_migratepages uses it as a search base. 832 * isolate_migratepages_block will update the value to the next pfn 833 * after the last isolated one. 834 */ 835 unsigned long migrate_pfn; 836 unsigned long fast_start_pfn; /* a pfn to start linear scan from */ 837 struct zone *zone; 838 unsigned long total_migrate_scanned; 839 unsigned long total_free_scanned; 840 unsigned short fast_search_fail;/* failures to use free list searches */ 841 short search_order; /* order to start a fast search at */ 842 const gfp_t gfp_mask; /* gfp mask of a direct compactor */ 843 int order; /* order a direct compactor needs */ 844 int migratetype; /* migratetype of direct compactor */ 845 const unsigned int alloc_flags; /* alloc flags of a direct compactor */ 846 const int highest_zoneidx; /* zone index of a direct compactor */ 847 enum migrate_mode mode; /* Async or sync migration mode */ 848 bool ignore_skip_hint; /* Scan blocks even if marked skip */ 849 bool no_set_skip_hint; /* Don't mark blocks for skipping */ 850 bool ignore_block_suitable; /* Scan blocks considered unsuitable */ 851 bool direct_compaction; /* False from kcompactd or /proc/... */ 852 bool proactive_compaction; /* kcompactd proactive compaction */ 853 bool whole_zone; /* Whole zone should/has been scanned */ 854 bool contended; /* Signal lock contention */ 855 bool finish_pageblock; /* Scan the remainder of a pageblock. Used 856 * when there are potentially transient 857 * isolation or migration failures to 858 * ensure forward progress. 859 */ 860 bool alloc_contig; /* alloc_contig_range allocation */ 861 }; 862 863 /* 864 * Used in direct compaction when a page should be taken from the freelists 865 * immediately when one is created during the free path. 866 */ 867 struct capture_control { 868 struct zone *zone; 869 int migratetype; 870 /* 871 * Allocation request order. May differ from the compaction 872 * order: defrag_mode promotes sub-block allocations to 873 * pageblock-order compaction; capture still matches at the 874 * original allocation order so prep_new_page() is consistent. 875 */ 876 int order; 877 struct page *page; 878 }; 879 880 unsigned long 881 isolate_freepages_range(struct compact_control *cc, 882 unsigned long start_pfn, unsigned long end_pfn); 883 int 884 isolate_migratepages_range(struct compact_control *cc, 885 unsigned long low_pfn, unsigned long end_pfn); 886 887 #endif /* CONFIG_COMPACTION || CONFIG_CMA */ 888 889 struct cma; 890 891 #ifdef CONFIG_CMA 892 bool cma_validate_zones(struct cma *cma); 893 void *cma_reserve_early(struct cma *cma, unsigned long size); 894 #else 895 static inline bool cma_validate_zones(struct cma *cma) 896 { 897 return false; 898 } 899 static inline void *cma_reserve_early(struct cma *cma, unsigned long size) 900 { 901 return NULL; 902 } 903 #endif 904 905 /* mm/util.c */ 906 struct anon_vma *folio_anon_vma(const struct folio *folio); 907 908 #ifdef CONFIG_MMU 909 void unmap_mapping_folio(struct folio *folio); 910 extern long populate_vma_page_range(struct vm_area_struct *vma, 911 unsigned long start, unsigned long end, int *locked); 912 extern long faultin_page_range(struct mm_struct *mm, unsigned long start, 913 unsigned long end, bool write, int *locked); 914 bool mlock_future_ok(const struct mm_struct *mm, bool is_vma_locked, 915 unsigned long bytes); 916 917 /* 918 * NOTE: This function can't tell whether the folio is "fully mapped" in the 919 * range. 920 * "fully mapped" means all the pages of folio is associated with the page 921 * table of range while this function just check whether the folio range is 922 * within the range [start, end). Function caller needs to do page table 923 * check if it cares about the page table association. 924 * 925 * Typical usage (like mlock or madvise) is: 926 * Caller knows at least 1 page of folio is associated with page table of VMA 927 * and the range [start, end) is intersect with the VMA range. Caller wants 928 * to know whether the folio is fully associated with the range. It calls 929 * this function to check whether the folio is in the range first. Then checks 930 * the page table to know whether the folio is fully mapped to the range. 931 */ 932 static inline bool 933 folio_within_range(struct folio *folio, struct vm_area_struct *vma, 934 unsigned long start, unsigned long end) 935 { 936 const unsigned long vma_pglen = vma_pages(vma); 937 pgoff_t pgoff_folio, pgoff_vma_start; 938 unsigned long addr; 939 940 VM_WARN_ON_FOLIO(folio_test_ksm(folio), folio); 941 if (start > end) 942 return false; 943 944 pgoff_folio = folio_pgoff(folio); 945 pgoff_vma_start = folio_test_anon(folio) ? 946 vma_start_anon_pgoff(vma) : vma_start_pgoff(vma); 947 948 if (start < vma->vm_start) 949 start = vma->vm_start; 950 951 if (end > vma->vm_end) 952 end = vma->vm_end; 953 954 /* if folio start address is not in vma range */ 955 if (!in_range(pgoff_folio, pgoff_vma_start, vma_pglen)) 956 return false; 957 958 addr = vma->vm_start + ((pgoff_folio - pgoff_vma_start) << PAGE_SHIFT); 959 960 return !(addr < start || end - addr < folio_size(folio)); 961 } 962 963 static inline bool 964 folio_within_vma(struct folio *folio, struct vm_area_struct *vma) 965 { 966 return folio_within_range(folio, vma, vma->vm_start, vma->vm_end); 967 } 968 969 /* 970 * mlock_vma_folio() and munlock_vma_folio(): 971 * should be called with vma's mmap_lock held for read or write, 972 * under page table lock for the pte/pmd being added or removed. 973 * 974 * mlock is usually called at the end of folio_add_*_rmap_*(), munlock at 975 * the end of folio_remove_rmap_*(); but new anon folios are managed by 976 * folio_add_lru_vma() calling mlock_new_folio(). 977 */ 978 void mlock_folio(struct folio *folio); 979 static inline void mlock_vma_folio(struct folio *folio, 980 struct vm_area_struct *vma) 981 { 982 /* 983 * The VM_SPECIAL check here serves two purposes. 984 * 1) VM_IO check prevents migration from double-counting during mlock. 985 * 2) Although mmap_region() and mlock_fixup() take care that VM_LOCKED 986 * is never left set on a VM_SPECIAL vma, there is an interval while 987 * file->f_op->mmap() is using vm_insert_page(s), when VM_LOCKED may 988 * still be set while VM_SPECIAL bits are added: so ignore it then. 989 */ 990 if (unlikely((vma->vm_flags & (VM_LOCKED|VM_SPECIAL)) == VM_LOCKED)) 991 mlock_folio(folio); 992 } 993 994 void munlock_folio(struct folio *folio); 995 static inline void munlock_vma_folio(struct folio *folio, 996 struct vm_area_struct *vma) 997 { 998 /* 999 * munlock if the function is called. Ideally, we should only 1000 * do munlock if any page of folio is unmapped from VMA and 1001 * cause folio not fully mapped to VMA. 1002 * 1003 * But it's not easy to confirm that's the situation. So we 1004 * always munlock the folio and page reclaim will correct it 1005 * if it's wrong. 1006 */ 1007 if (unlikely(vma->vm_flags & VM_LOCKED)) 1008 munlock_folio(folio); 1009 } 1010 1011 void mlock_new_folio(struct folio *folio); 1012 bool need_mlock_drain(int cpu); 1013 void mlock_drain_local(void); 1014 void mlock_drain_remote(int cpu); 1015 1016 extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma); 1017 1018 static inline unsigned long __vma_address(const struct vm_area_struct *vma, 1019 pgoff_t pgoff, pgoff_t pgoff_start, unsigned long nr_pages) 1020 { 1021 unsigned long address; 1022 1023 if (pgoff >= pgoff_start) { 1024 address = vma->vm_start + 1025 ((pgoff - pgoff_start) << PAGE_SHIFT); 1026 /* Check for address beyond vma (or wrapped through 0?) */ 1027 if (address < vma->vm_start || address >= vma->vm_end) 1028 address = -EFAULT; 1029 } else if (pgoff + nr_pages - 1 >= pgoff_start) { 1030 /* Test above avoids possibility of wrap to 0 on 32-bit */ 1031 address = vma->vm_start; 1032 } else { 1033 address = -EFAULT; 1034 } 1035 return address; 1036 } 1037 1038 /** 1039 * vma_filebacked_address - Find the virtual address a file-backed page range is 1040 * mapped at. 1041 * @vma: The vma which maps this object. 1042 * @pgoff: The page offset within its object. 1043 * @nr_pages: The number of pages to consider. 1044 * 1045 * Returns: If any page in this range is mapped by this VMA, return the first 1046 * address where any of these pages appear. Otherwise, return -EFAULT. 1047 */ 1048 static inline unsigned long vma_filebacked_address(const struct vm_area_struct *vma, 1049 pgoff_t pgoff, unsigned long nr_pages) 1050 { 1051 VM_WARN_ON_ONCE(vma_is_anonymous(vma)); 1052 1053 return __vma_address(vma, pgoff, vma_start_pgoff(vma), nr_pages); 1054 } 1055 1056 /** 1057 * vma_anon_address - Find the virtual address an anonymous page range is mapped 1058 * at. 1059 * @vma: The vma which maps this object. 1060 * @pgoff_anon: The anonymous page index belonging to the folio. 1061 * @nr_pages: The number of pages to consider. 1062 * 1063 * This is only valid for anonymous or MAP_PRIVATE-mapped file-backed VMAs. 1064 * 1065 * Returns: If any page in this range is mapped by this VMA, return the first 1066 * address where any of these pages appear. Otherwise, return -EFAULT. 1067 */ 1068 static inline unsigned long vma_anon_address(const struct vm_area_struct *vma, 1069 pgoff_t pgoff_anon, unsigned long nr_pages) 1070 { 1071 VM_WARN_ON_ONCE(!vma_is_cow_mapping(vma)); 1072 1073 return __vma_address(vma, pgoff_anon, vma_start_anon_pgoff(vma), nr_pages); 1074 } 1075 1076 /* 1077 * At what user virtual address will none of the range be found in vma? 1078 * Assumes that vma_address() already returned a good starting address. 1079 */ 1080 static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw) 1081 { 1082 const pgoff_t pgoff_end = pvmw->pgoff + pvmw->nr_pages; 1083 const struct vm_area_struct *vma = pvmw->vma; 1084 pgoff_t pgoff_vma_start; 1085 unsigned long address; 1086 1087 /* Common case, plus ->pgoff is invalid for KSM */ 1088 if (pvmw->nr_pages == 1) 1089 return pvmw->address + PAGE_SIZE; 1090 1091 if (pvmw->pgoff_is_anon) 1092 pgoff_vma_start = vma_start_anon_pgoff(vma); 1093 else 1094 pgoff_vma_start = vma_start_pgoff(vma); 1095 1096 address = vma->vm_start + 1097 ((pgoff_end - pgoff_vma_start) << PAGE_SHIFT); 1098 /* Check for address beyond vma (or wrapped through 0?) */ 1099 if (address < vma->vm_start || address > vma->vm_end) 1100 address = vma->vm_end; 1101 return address; 1102 } 1103 1104 static inline struct file *maybe_unlock_mmap_for_io(struct vm_fault *vmf, 1105 struct file *fpin) 1106 { 1107 int flags = vmf->flags; 1108 1109 if (fpin) 1110 return fpin; 1111 1112 /* 1113 * FAULT_FLAG_RETRY_NOWAIT means we don't want to wait on page locks or 1114 * anything, so we only pin the file and drop the mmap_lock if only 1115 * FAULT_FLAG_ALLOW_RETRY is set, while this is the first attempt. 1116 */ 1117 if (fault_flag_allow_retry_first(flags) && 1118 !(flags & FAULT_FLAG_RETRY_NOWAIT)) { 1119 fpin = get_file(vmf->vma->vm_file); 1120 release_fault_lock(vmf); 1121 } 1122 return fpin; 1123 } 1124 1125 static inline bool vma_supports_mlock(const struct vm_area_struct *vma) 1126 { 1127 if (vma_test_any_mask(vma, VMA_SPECIAL_FLAGS)) 1128 return false; 1129 if (vma_test_single_mask(vma, VMA_DROPPABLE)) 1130 return false; 1131 if (vma_is_dax(vma) || is_vm_hugetlb_page(vma)) 1132 return false; 1133 return vma != get_gate_vma(current->mm); 1134 } 1135 1136 #else /* !CONFIG_MMU */ 1137 static inline void unmap_mapping_folio(struct folio *folio) { } 1138 static inline void mlock_new_folio(struct folio *folio) { } 1139 static inline bool need_mlock_drain(int cpu) { return false; } 1140 static inline void mlock_drain_local(void) { } 1141 static inline void mlock_drain_remote(int cpu) { } 1142 #endif /* !CONFIG_MMU */ 1143 1144 #ifdef CONFIG_NUMA 1145 extern int node_reclaim_mode; 1146 1147 extern unsigned long node_reclaim(struct pglist_data *pgdat, 1148 gfp_t gfp_mask, unsigned int order); 1149 extern int find_next_best_node(int node, nodemask_t *used_node_mask); 1150 #else 1151 #define node_reclaim_mode 0 1152 1153 static inline unsigned long node_reclaim(struct pglist_data *pgdat, 1154 gfp_t mask, unsigned int order) 1155 { 1156 return 0; 1157 } 1158 static inline int find_next_best_node(int node, nodemask_t *used_node_mask) 1159 { 1160 return NUMA_NO_NODE; 1161 } 1162 #endif 1163 1164 static inline bool node_reclaim_enabled(void) 1165 { 1166 /* Is any node_reclaim_mode bit set? */ 1167 return node_reclaim_mode & (RECLAIM_ZONE|RECLAIM_WRITE|RECLAIM_UNMAP); 1168 } 1169 1170 /* 1171 * mm/memory-failure.c 1172 */ 1173 #ifdef CONFIG_MEMORY_FAILURE 1174 int unmap_poisoned_folio(struct folio *folio, unsigned long pfn, bool must_kill); 1175 void shake_folio(struct folio *folio); 1176 typedef int hwpoison_filter_func_t(struct page *p); 1177 void hwpoison_filter_register(hwpoison_filter_func_t *filter); 1178 void hwpoison_filter_unregister(void); 1179 1180 #define MAGIC_HWPOISON 0x48575053U /* HWPS */ 1181 void SetPageHWPoisonTakenOff(struct page *page); 1182 void ClearPageHWPoisonTakenOff(struct page *page); 1183 bool take_page_off_buddy(struct page *page); 1184 bool put_page_back_buddy(struct page *page); 1185 struct task_struct *task_early_kill(struct task_struct *tsk, int force_early); 1186 void add_to_kill_ksm(struct task_struct *tsk, const struct page *p, 1187 struct vm_area_struct *vma, struct list_head *to_kill, 1188 unsigned long ksm_addr); 1189 unsigned long page_mapped_in_vma(const struct page *page, 1190 struct vm_area_struct *vma); 1191 1192 #else 1193 static inline int unmap_poisoned_folio(struct folio *folio, unsigned long pfn, bool must_kill) 1194 { 1195 return -EBUSY; 1196 } 1197 #endif 1198 1199 extern unsigned long __must_check vm_mmap_pgoff(struct file *, unsigned long, 1200 unsigned long, unsigned long, 1201 unsigned long, unsigned long); 1202 1203 unsigned long reclaim_pages(struct list_head *folio_list); 1204 unsigned int reclaim_clean_pages_from_list(struct zone *zone, 1205 struct list_head *folio_list); 1206 1207 enum ttu_flags; 1208 struct tlbflush_unmap_batch; 1209 1210 1211 /* 1212 * only for MM internal work items which do not depend on 1213 * any allocations or locks which might depend on allocations 1214 */ 1215 extern struct workqueue_struct *mm_percpu_wq; 1216 1217 #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH 1218 void try_to_unmap_flush(void); 1219 void try_to_unmap_flush_dirty(void); 1220 void flush_tlb_batched_pending(struct mm_struct *mm); 1221 #else 1222 static inline void try_to_unmap_flush(void) 1223 { 1224 } 1225 static inline void try_to_unmap_flush_dirty(void) 1226 { 1227 } 1228 static inline void flush_tlb_batched_pending(struct mm_struct *mm) 1229 { 1230 } 1231 #endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */ 1232 1233 extern const struct trace_print_flags pageflag_names[]; 1234 extern const struct trace_print_flags vmaflag_names[]; 1235 extern const struct trace_print_flags gfpflag_names[]; 1236 1237 void setup_zone_pageset(struct zone *zone); 1238 1239 struct migration_target_control { 1240 int nid; /* preferred node id */ 1241 nodemask_t *nmask; 1242 gfp_t gfp_mask; 1243 enum migrate_reason reason; 1244 }; 1245 1246 /* 1247 * mm/filemap.c 1248 */ 1249 size_t splice_folio_into_pipe(struct pipe_inode_info *pipe, 1250 struct folio *folio, loff_t fpos, size_t size); 1251 1252 static inline bool vma_is_single_threaded_private(struct vm_area_struct *vma) 1253 { 1254 if (vma->vm_flags & VM_SHARED) 1255 return false; 1256 1257 return atomic_read(&vma->vm_mm->mm_users) == 1; 1258 } 1259 1260 #ifdef CONFIG_NUMA_BALANCING 1261 bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma, 1262 bool is_private_single_threaded); 1263 1264 #else 1265 static inline bool folio_can_map_prot_numa(struct folio *folio, 1266 struct vm_area_struct *vma, bool is_private_single_threaded) 1267 { 1268 return false; 1269 } 1270 #endif 1271 1272 int numa_migrate_check(struct folio *folio, struct vm_fault *vmf, 1273 unsigned long addr, int *flags, bool writable, 1274 int *last_cpupid); 1275 1276 void free_zone_device_folio(struct folio *folio); 1277 int migrate_device_coherent_folio(struct folio *folio); 1278 1279 /* 1280 * mm/gup.c 1281 */ 1282 int __must_check try_grab_folio(struct folio *folio, int refs, 1283 unsigned int flags); 1284 1285 /* 1286 * mm/huge_memory.c 1287 */ 1288 void touch_pud(struct vm_area_struct *vma, unsigned long addr, 1289 pud_t *pud, bool write); 1290 bool touch_pmd(struct vm_area_struct *vma, unsigned long addr, 1291 pmd_t *pmd, bool write); 1292 1293 /* 1294 * Parses a string with mem suffixes into its order. Useful to parse kernel 1295 * parameters. 1296 */ 1297 static inline int get_order_from_str(const char *size_str, 1298 unsigned long valid_orders) 1299 { 1300 unsigned long size; 1301 char *endptr; 1302 int order; 1303 1304 size = memparse(size_str, &endptr); 1305 1306 if (!is_power_of_2(size)) 1307 return -EINVAL; 1308 order = get_order(size); 1309 if (BIT(order) & ~valid_orders) 1310 return -EINVAL; 1311 1312 return order; 1313 } 1314 1315 enum { 1316 /* mark page accessed */ 1317 FOLL_TOUCH = 1 << 16, 1318 /* a retry, previous pass started an IO */ 1319 FOLL_TRIED = 1 << 17, 1320 /* we are working on non-current tsk/mm */ 1321 FOLL_REMOTE = 1 << 18, 1322 /* pages must be released via unpin_user_page */ 1323 FOLL_PIN = 1 << 19, 1324 /* gup_fast: prevent fall-back to slow gup */ 1325 FOLL_FAST_ONLY = 1 << 20, 1326 /* allow unlocking the mmap lock */ 1327 FOLL_UNLOCKABLE = 1 << 21, 1328 /* VMA lookup+checks compatible with MADV_POPULATE_(READ|WRITE) */ 1329 FOLL_MADV_POPULATE = 1 << 22, 1330 }; 1331 1332 #define INTERNAL_GUP_FLAGS (FOLL_TOUCH | FOLL_TRIED | FOLL_REMOTE | FOLL_PIN | \ 1333 FOLL_FAST_ONLY | FOLL_UNLOCKABLE | \ 1334 FOLL_MADV_POPULATE) 1335 1336 /* 1337 * Indicates for which pages that are write-protected in the page table, 1338 * whether GUP has to trigger unsharing via FAULT_FLAG_UNSHARE such that the 1339 * GUP pin will remain consistent with the pages mapped into the page tables 1340 * of the MM. 1341 * 1342 * Temporary unmapping of PageAnonExclusive() pages or clearing of 1343 * PageAnonExclusive() has to protect against concurrent GUP: 1344 * * Ordinary GUP: Using the PT lock 1345 * * GUP-fast and fork(): mm->write_protect_seq 1346 * * GUP-fast and KSM or temporary unmapping (swap, migration): see 1347 * folio_try_share_anon_rmap_*() 1348 * 1349 * Must be called with the (sub)page that's actually referenced via the 1350 * page table entry, which might not necessarily be the head page for a 1351 * PTE-mapped THP. 1352 * 1353 * If the vma is NULL, we're coming from the GUP-fast path and might have 1354 * to fallback to the slow path just to lookup the vma. 1355 */ 1356 static inline bool gup_must_unshare(struct vm_area_struct *vma, 1357 unsigned int flags, struct page *page) 1358 { 1359 /* 1360 * FOLL_WRITE is implicitly handled correctly as the page table entry 1361 * has to be writable -- and if it references (part of) an anonymous 1362 * folio, that part is required to be marked exclusive. 1363 */ 1364 if ((flags & (FOLL_WRITE | FOLL_PIN)) != FOLL_PIN) 1365 return false; 1366 /* 1367 * Note: PageAnon(page) is stable until the page is actually getting 1368 * freed. 1369 */ 1370 if (!PageAnon(page)) { 1371 /* 1372 * We only care about R/O long-term pining: R/O short-term 1373 * pinning does not have the semantics to observe successive 1374 * changes through the process page tables. 1375 */ 1376 if (!(flags & FOLL_LONGTERM)) 1377 return false; 1378 1379 /* We really need the vma ... */ 1380 if (!vma) 1381 return true; 1382 1383 /* 1384 * ... because we only care about writable private ("COW") 1385 * mappings where we have to break COW early. 1386 */ 1387 return vma_is_cow_mapping(vma); 1388 } 1389 1390 /* Paired with a memory barrier in folio_try_share_anon_rmap_*(). */ 1391 if (IS_ENABLED(CONFIG_HAVE_GUP_FAST)) 1392 smp_rmb(); 1393 1394 /* 1395 * Note that KSM pages cannot be exclusive, and consequently, 1396 * cannot get pinned. 1397 */ 1398 return !PageAnonExclusive(page); 1399 } 1400 1401 1402 static inline bool vma_soft_dirty_enabled(struct vm_area_struct *vma) 1403 { 1404 /* 1405 * NOTE: we must check this before VM_SOFTDIRTY on soft-dirty 1406 * enablements, because when without soft-dirty being compiled in, 1407 * VM_SOFTDIRTY is defined as 0x0, then !(vm_flags & VM_SOFTDIRTY) 1408 * will be constantly true. 1409 */ 1410 if (!pgtable_supports_soft_dirty()) 1411 return false; 1412 1413 /* 1414 * Soft-dirty is kind of special: its tracking is enabled when the 1415 * vma flags not set. 1416 */ 1417 return !(vma->vm_flags & VM_SOFTDIRTY); 1418 } 1419 1420 static inline bool pmd_needs_soft_dirty_wp(struct vm_area_struct *vma, pmd_t pmd) 1421 { 1422 return vma_soft_dirty_enabled(vma) && !pmd_soft_dirty(pmd); 1423 } 1424 1425 static inline bool pte_needs_soft_dirty_wp(struct vm_area_struct *vma, pte_t pte) 1426 { 1427 return vma_soft_dirty_enabled(vma) && !pte_soft_dirty(pte); 1428 } 1429 1430 /* shrinker related functions */ 1431 unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg, 1432 int priority); 1433 1434 int shmem_add_to_page_cache(struct folio *folio, 1435 struct address_space *mapping, 1436 pgoff_t index, void *expected, gfp_t gfp); 1437 int shmem_inode_acct_blocks(struct inode *inode, long pages); 1438 bool shmem_recalc_inode(struct inode *inode, long alloced, long swapped); 1439 1440 #ifdef CONFIG_SHRINKER_DEBUG 1441 static inline __printf(2, 0) int shrinker_debugfs_name_alloc( 1442 struct shrinker *shrinker, const char *fmt, va_list ap) 1443 { 1444 shrinker->name = kvasprintf_const(GFP_KERNEL, fmt, ap); 1445 1446 return shrinker->name ? 0 : -ENOMEM; 1447 } 1448 1449 static inline void shrinker_debugfs_name_free(struct shrinker *shrinker) 1450 { 1451 kfree_const(shrinker->name); 1452 shrinker->name = NULL; 1453 } 1454 1455 extern int shrinker_debugfs_add(struct shrinker *shrinker); 1456 extern struct dentry *shrinker_debugfs_detach(struct shrinker *shrinker, 1457 int *debugfs_id); 1458 extern void shrinker_debugfs_remove(struct dentry *debugfs_entry, 1459 int debugfs_id); 1460 #else /* CONFIG_SHRINKER_DEBUG */ 1461 static inline int shrinker_debugfs_add(struct shrinker *shrinker) 1462 { 1463 return 0; 1464 } 1465 static inline int shrinker_debugfs_name_alloc(struct shrinker *shrinker, 1466 const char *fmt, va_list ap) 1467 { 1468 return 0; 1469 } 1470 static inline void shrinker_debugfs_name_free(struct shrinker *shrinker) 1471 { 1472 } 1473 static inline struct dentry *shrinker_debugfs_detach(struct shrinker *shrinker, 1474 int *debugfs_id) 1475 { 1476 *debugfs_id = -1; 1477 return NULL; 1478 } 1479 static inline void shrinker_debugfs_remove(struct dentry *debugfs_entry, 1480 int debugfs_id) 1481 { 1482 } 1483 #endif /* CONFIG_SHRINKER_DEBUG */ 1484 1485 /* Only track the nodes of mappings with shadow entries */ 1486 void workingset_update_node(struct xa_node *node); 1487 extern struct list_lru shadow_nodes; 1488 #define mapping_set_update(xas, mapping) do { \ 1489 if (!dax_mapping(mapping) && !shmem_mapping(mapping)) { \ 1490 xas_set_update(xas, workingset_update_node); \ 1491 xas_set_lru(xas, &shadow_nodes); \ 1492 } \ 1493 } while (0) 1494 1495 /* mremap.c */ 1496 unsigned long move_page_tables(struct pagetable_move_control *pmc); 1497 1498 #ifdef CONFIG_UNACCEPTED_MEMORY 1499 void accept_page(struct page *page); 1500 #else /* CONFIG_UNACCEPTED_MEMORY */ 1501 static inline void accept_page(struct page *page) 1502 { 1503 } 1504 #endif /* CONFIG_UNACCEPTED_MEMORY */ 1505 1506 /* pagewalk.c */ 1507 int walk_page_range_mm_unsafe(struct mm_struct *mm, unsigned long start, 1508 unsigned long end, const struct mm_walk_ops *ops, 1509 void *private); 1510 int walk_page_range_vma_unsafe(struct vm_area_struct *vma, unsigned long start, 1511 unsigned long end, const struct mm_walk_ops *ops, 1512 void *private); 1513 int walk_page_range_debug(struct mm_struct *mm, unsigned long start, 1514 unsigned long end, const struct mm_walk_ops *ops, 1515 pgd_t *pgd, void *private); 1516 1517 void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm); 1518 int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm); 1519 1520 int remap_pfn_range_prepare(struct vm_area_desc *desc); 1521 int remap_pfn_range_complete(struct vm_area_struct *vma, 1522 struct mmap_action *action); 1523 int simple_ioremap_prepare(struct vm_area_desc *desc); 1524 1525 static inline int io_remap_pfn_range_prepare(struct vm_area_desc *desc) 1526 { 1527 struct mmap_action *action = &desc->action; 1528 const unsigned long orig_pfn = action->remap.start_pfn; 1529 const pgprot_t orig_pgprot = action->remap.pgprot; 1530 const unsigned long size = action->remap.size; 1531 const unsigned long pfn = io_remap_pfn_range_pfn(orig_pfn, size); 1532 int err; 1533 1534 action->remap.start_pfn = pfn; 1535 action->remap.pgprot = pgprot_decrypted(orig_pgprot); 1536 err = remap_pfn_range_prepare(desc); 1537 if (err) 1538 return err; 1539 1540 /* Remap does the actual work. */ 1541 action->type = MMAP_REMAP_PFN; 1542 return 0; 1543 } 1544 1545 /* 1546 * When we succeed an mmap action or just before we unmap a VMA on error, we 1547 * need to ensure any rmap lock held is released. On unmap it's required to 1548 * avoid a deadlock. 1549 */ 1550 static inline void maybe_rmap_unlock_action(struct vm_area_struct *vma, 1551 struct mmap_action *action) 1552 { 1553 struct file *file; 1554 1555 if (!action->hide_from_rmap_until_complete) 1556 return; 1557 1558 VM_WARN_ON_ONCE(vma_is_anonymous(vma)); 1559 file = vma->vm_file; 1560 i_mmap_unlock_write(file->f_mapping); 1561 action->hide_from_rmap_until_complete = false; 1562 } 1563 1564 #ifdef CONFIG_MMU_NOTIFIER 1565 static inline bool clear_flush_young_ptes_notify(struct vm_area_struct *vma, 1566 unsigned long addr, pte_t *ptep, unsigned int nr) 1567 { 1568 bool young; 1569 1570 young = clear_flush_young_ptes(vma, addr, ptep, nr); 1571 young |= mmu_notifier_clear_flush_young(vma->vm_mm, addr, 1572 addr + nr * PAGE_SIZE); 1573 return young; 1574 } 1575 1576 static inline bool pmdp_clear_flush_young_notify(struct vm_area_struct *vma, 1577 unsigned long addr, pmd_t *pmdp) 1578 { 1579 bool young; 1580 1581 young = pmdp_clear_flush_young(vma, addr, pmdp); 1582 young |= mmu_notifier_clear_flush_young(vma->vm_mm, addr, addr + PMD_SIZE); 1583 return young; 1584 } 1585 1586 static inline bool test_and_clear_young_ptes_notify(struct vm_area_struct *vma, 1587 unsigned long addr, pte_t *ptep, unsigned int nr) 1588 { 1589 bool young; 1590 1591 young = test_and_clear_young_ptes(vma, addr, ptep, nr); 1592 young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + nr * PAGE_SIZE); 1593 return young; 1594 } 1595 1596 static inline bool pmdp_test_and_clear_young_notify(struct vm_area_struct *vma, 1597 unsigned long addr, pmd_t *pmdp) 1598 { 1599 bool young; 1600 1601 young = pmdp_test_and_clear_young(vma, addr, pmdp); 1602 young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + PMD_SIZE); 1603 return young; 1604 } 1605 1606 #else /* CONFIG_MMU_NOTIFIER */ 1607 1608 #define clear_flush_young_ptes_notify clear_flush_young_ptes 1609 #define pmdp_clear_flush_young_notify pmdp_clear_flush_young 1610 #define test_and_clear_young_ptes_notify test_and_clear_young_ptes 1611 #define pmdp_test_and_clear_young_notify pmdp_test_and_clear_young 1612 1613 #endif /* CONFIG_MMU_NOTIFIER */ 1614 1615 extern int sysctl_max_map_count; 1616 static inline int get_sysctl_max_map_count(void) 1617 { 1618 return READ_ONCE(sysctl_max_map_count); 1619 } 1620 1621 bool may_expand_vm(struct mm_struct *mm, const vma_flags_t *vma_flags, 1622 unsigned long npages); 1623 1624 static inline void mm_prepare_for_swap_entries(struct mm_struct *mm) 1625 { 1626 if (list_empty(&mm->mmlist)) { 1627 spin_lock(&mmlist_lock); 1628 if (list_empty(&mm->mmlist)) 1629 list_add(&mm->mmlist, &init_mm.mmlist); 1630 spin_unlock(&mmlist_lock); 1631 } 1632 } 1633 1634 static inline bool can_spin_trylock(void) 1635 { 1636 /* 1637 * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is 1638 * unsafe in NMI. If spin_trylock() is called from hard IRQ the current 1639 * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will 1640 * mark the task as the owner of another rt_spin_lock which will 1641 * confuse PI logic, so return immediately if called from hard IRQ or 1642 * NMI. 1643 * 1644 * Note, irqs_disabled() case is ok. spin_trylock() can be called 1645 * from raw_spin_lock_irqsave region. 1646 */ 1647 if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq())) 1648 return false; 1649 1650 /* On UP, spin_trylock() always succeeds even when it is locked */ 1651 if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) 1652 return false; 1653 1654 return true; 1655 } 1656 1657 #endif /* __MM_INTERNAL_H */ 1658