1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* internal.h: mm/ internal definitions 3 * 4 * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 #ifndef __MM_INTERNAL_H 8 #define __MM_INTERNAL_H 9 10 #include <linux/fs.h> 11 #include <linux/khugepaged.h> 12 #include <linux/mm.h> 13 #include <linux/mm_inline.h> 14 #include <linux/mmu_notifier.h> 15 #include <linux/pagemap.h> 16 #include <linux/pagewalk.h> 17 #include <linux/rmap.h> 18 #include <linux/swap.h> 19 #include <linux/leafops.h> 20 #include <linux/tracepoint-defs.h> 21 22 /* Internal core VMA manipulation functions. */ 23 #include "vma.h" 24 25 struct folio_batch; 26 struct hstate; 27 28 struct huge_bootmem_page { 29 struct list_head list; 30 struct hstate *hstate; 31 unsigned long flags; 32 }; 33 34 /* mm/workingset.c */ 35 bool workingset_test_recent(void *shadow, bool file, bool *workingset, 36 bool flush); 37 void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages); 38 void *workingset_eviction(struct folio *folio, 39 struct mem_cgroup *target_memcg); 40 void workingset_refault(struct folio *folio, void *shadow); 41 void workingset_activation(struct folio *folio); 42 43 /* mm/folio.c */ 44 void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file, 45 unsigned int nr_io, unsigned int nr_rotated); 46 void lru_note_cost_refault(struct folio *folio); 47 void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma); 48 49 static inline bool folio_may_be_lru_cached(struct folio *folio) 50 { 51 /* 52 * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting. 53 * Holding small numbers of low-order mTHP folios in per-CPU LRU cache 54 * will be sensible, but nobody has implemented and tested that yet. 55 */ 56 return !folio_test_large(folio); 57 } 58 59 static inline void lru_cache_enable(void) 60 { 61 atomic_dec(&lru_disable_count); 62 } 63 64 void lru_cache_disable(void); 65 void lru_add_drain(void); 66 void lru_add_drain_cpu(int cpu); 67 void lru_add_drain_cpu_zone(struct zone *zone); 68 void folio_deactivate(struct folio *folio); 69 void folio_mark_lazyfree(struct folio *folio); 70 71 /* mm/vmscan.c */ 72 unsigned long zone_reclaimable_pages(struct zone *zone); 73 unsigned long try_to_free_pages(struct zonelist *zonelist, int order, 74 gfp_t gfp_mask, const nodemask_t *mask); 75 unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, 76 int zone_idx); 77 78 #define MEMCG_RECLAIM_MAY_SWAP (1 << 1) 79 #define MEMCG_RECLAIM_PROACTIVE (1 << 2) 80 #define MIN_SWAPPINESS 0 81 #define MAX_SWAPPINESS 200 82 83 /* Just reclaim from anon folios in proactive memory reclaim */ 84 #define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1) 85 86 unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, 87 unsigned long nr_pages, 88 gfp_t gfp_mask, 89 unsigned int reclaim_options, 90 int *swappiness); 91 unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, 92 gfp_t gfp_mask, bool noswap, 93 pg_data_t *pgdat, 94 unsigned long *nr_scanned); 95 96 #ifdef CONFIG_NUMA 97 extern int sysctl_min_unmapped_ratio; 98 extern int sysctl_min_slab_ratio; 99 #endif 100 101 /* 102 * Maintains state across a page table move. The operation assumes both source 103 * and destination VMAs already exist and are specified by the user. 104 * 105 * Partial moves are permitted, but the old and new ranges must both reside 106 * within a VMA. 107 * 108 * mmap lock must be held in write and VMA write locks must be held on any VMA 109 * that is visible. 110 * 111 * Use the PAGETABLE_MOVE() macro to initialise this struct. 112 * 113 * The old_addr and new_addr fields are updated as the page table move is 114 * executed. 115 * 116 * NOTE: The page table move is affected by reading from [old_addr, old_end), 117 * and old_addr may be updated for better page table alignment, so len_in 118 * represents the length of the range being copied as specified by the user. 119 */ 120 struct pagetable_move_control { 121 struct vm_area_struct *old; /* Source VMA. */ 122 struct vm_area_struct *new; /* Destination VMA. */ 123 unsigned long old_addr; /* Address from which the move begins. */ 124 unsigned long old_end; /* Exclusive address at which old range ends. */ 125 unsigned long new_addr; /* Address to move page tables to. */ 126 unsigned long len_in; /* Bytes to remap specified by user. */ 127 128 bool need_rmap_locks; /* Do rmap locks need to be taken? */ 129 bool for_stack; /* Is this an early temp stack being moved? */ 130 }; 131 132 #define PAGETABLE_MOVE(name, old_, new_, old_addr_, new_addr_, len_) \ 133 struct pagetable_move_control name = { \ 134 .old = old_, \ 135 .new = new_, \ 136 .old_addr = old_addr_, \ 137 .old_end = (old_addr_) + (len_), \ 138 .new_addr = new_addr_, \ 139 .len_in = len_, \ 140 } 141 142 /* 143 * The set of flags that only affect watermark checking and reclaim 144 * behaviour. This is used by the MM to obey the caller constraints 145 * about IO, FS and watermark checking while ignoring placement 146 * hints such as HIGHMEM usage. 147 */ 148 #define GFP_RECLAIM_MASK (__GFP_RECLAIM|__GFP_HIGH|__GFP_IO|__GFP_FS|\ 149 __GFP_NOWARN|__GFP_RETRY_MAYFAIL|__GFP_NOFAIL|\ 150 __GFP_NORETRY|__GFP_MEMALLOC|__GFP_NOMEMALLOC|\ 151 __GFP_NOLOCKDEP) 152 153 /* The GFP flags allowed during early boot */ 154 #define GFP_BOOT_MASK (__GFP_BITS_MASK & ~(__GFP_RECLAIM|__GFP_IO|__GFP_FS)) 155 156 /* Control allocation cpuset and node placement constraints */ 157 #define GFP_CONSTRAINT_MASK (__GFP_HARDWALL|__GFP_THISNODE) 158 159 /* Do not use these with a slab allocator */ 160 #define GFP_SLAB_BUG_MASK (__GFP_DMA32|__GFP_HIGHMEM|~__GFP_BITS_MASK) 161 162 /* 163 * Different from WARN_ON_ONCE(), no warning will be issued 164 * when we specify __GFP_NOWARN. 165 */ 166 #define WARN_ON_ONCE_GFP(cond, gfp) ({ \ 167 static bool __section(".data..once") __warned; \ 168 int __ret_warn_once = !!(cond); \ 169 \ 170 if (unlikely(!(gfp & __GFP_NOWARN) && __ret_warn_once && !__warned)) { \ 171 __warned = true; \ 172 WARN_ON(1); \ 173 } \ 174 unlikely(__ret_warn_once); \ 175 }) 176 177 void page_writeback_init(void); 178 179 /* 180 * If a 16GB hugetlb folio were mapped by PTEs of all of its 4kB pages, 181 * its nr_pages_mapped would be 0x400000: choose the ENTIRELY_MAPPED bit 182 * above that range, instead of 2*(PMD_SIZE/PAGE_SIZE). Hugetlb currently 183 * leaves nr_pages_mapped at 0, but avoid surprise if it participates later. 184 */ 185 #define ENTIRELY_MAPPED 0x800000 186 #define FOLIO_PAGES_MAPPED (ENTIRELY_MAPPED - 1) 187 188 /* 189 * Flags passed to __show_mem() and show_free_areas() to suppress output in 190 * various contexts. 191 */ 192 #define SHOW_MEM_FILTER_NODES (0x0001u) /* disallowed nodes */ 193 194 /* 195 * How many individual pages have an elevated _mapcount. Excludes 196 * the folio's entire_mapcount. 197 * 198 * Don't use this function outside of debugging code. 199 */ 200 static inline int folio_nr_pages_mapped(const struct folio *folio) 201 { 202 if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) 203 return -1; 204 return atomic_read(&folio->_nr_pages_mapped) & FOLIO_PAGES_MAPPED; 205 } 206 207 /* 208 * Retrieve the first entry of a folio based on a provided entry within the 209 * folio. We cannot rely on folio->swap as there is no guarantee that it has 210 * been initialized. Used for calling arch_swap_restore() 211 */ 212 static inline swp_entry_t folio_swap(swp_entry_t entry, 213 const struct folio *folio) 214 { 215 swp_entry_t swap = { 216 .val = ALIGN_DOWN(entry.val, folio_nr_pages(folio)), 217 }; 218 219 return swap; 220 } 221 222 static inline void *folio_raw_mapping(const struct folio *folio) 223 { 224 unsigned long mapping = (unsigned long)folio->mapping; 225 226 return (void *)(mapping & ~FOLIO_MAPPING_FLAGS); 227 } 228 229 /* 230 * This is a file-backed mapping, and is about to be memory mapped - invoke its 231 * mmap hook and safely handle error conditions. On error, VMA hooks will be 232 * mutated. 233 * 234 * @file: File which backs the mapping. 235 * @vma: VMA which we are mapping. 236 * 237 * Returns: 0 if success, error otherwise. 238 */ 239 static inline int mmap_file(struct file *file, struct vm_area_struct *vma) 240 { 241 int err = vfs_mmap(file, vma); 242 243 if (likely(!err)) 244 return 0; 245 246 /* 247 * OK, we tried to call the file hook for mmap(), but an error 248 * arose. The mapping is in an inconsistent state and we must not invoke 249 * any further hooks on it. 250 */ 251 vma->vm_ops = &vma_dummy_vm_ops; 252 253 return err; 254 } 255 256 /* 257 * If the VMA has a close hook then close it, and since closing it might leave 258 * it in an inconsistent state which makes the use of any hooks suspect, clear 259 * them down by installing dummy empty hooks. 260 */ 261 static inline void vma_close(struct vm_area_struct *vma) 262 { 263 if (vma->vm_ops && vma->vm_ops->close) { 264 vma->vm_ops->close(vma); 265 266 /* 267 * The mapping is in an inconsistent state, and no further hooks 268 * may be invoked upon it. 269 */ 270 vma->vm_ops = &vma_dummy_vm_ops; 271 } 272 } 273 274 /* unmap_vmas is in mm/memory.c */ 275 void unmap_vmas(struct mmu_gather *tlb, struct unmap_desc *unmap); 276 277 #ifdef CONFIG_MMU 278 279 bool cond_install_uffd_wp_ptes(struct vm_area_struct *vma, 280 unsigned long addr, pte_t *ptep, pte_t pte, 281 unsigned long nr_ptes); 282 283 static inline void get_anon_vma(struct anon_vma *anon_vma) 284 { 285 atomic_inc(&anon_vma->refcount); 286 } 287 288 void __put_anon_vma(struct anon_vma *anon_vma); 289 290 static inline void put_anon_vma(struct anon_vma *anon_vma) 291 { 292 if (atomic_dec_and_test(&anon_vma->refcount)) 293 __put_anon_vma(anon_vma); 294 } 295 296 static inline void anon_vma_lock_write(struct anon_vma *anon_vma) 297 { 298 down_write(&anon_vma->root->rwsem); 299 } 300 301 static inline int anon_vma_trylock_write(struct anon_vma *anon_vma) 302 { 303 return down_write_trylock(&anon_vma->root->rwsem); 304 } 305 306 static inline void anon_vma_unlock_write(struct anon_vma *anon_vma) 307 { 308 up_write(&anon_vma->root->rwsem); 309 } 310 311 static inline void anon_vma_lock_read(struct anon_vma *anon_vma) 312 { 313 down_read(&anon_vma->root->rwsem); 314 } 315 316 static inline int anon_vma_trylock_read(struct anon_vma *anon_vma) 317 { 318 return down_read_trylock(&anon_vma->root->rwsem); 319 } 320 321 static inline void anon_vma_unlock_read(struct anon_vma *anon_vma) 322 { 323 up_read(&anon_vma->root->rwsem); 324 } 325 326 struct anon_vma *folio_get_anon_vma(const struct folio *folio); 327 328 /* Operations which modify VMAs. */ 329 enum vma_operation { 330 VMA_OP_SPLIT, 331 VMA_OP_MERGE_UNFAULTED, 332 VMA_OP_REMAP, 333 VMA_OP_FORK, 334 }; 335 336 int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src, 337 enum vma_operation operation); 338 int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma); 339 int __anon_vma_prepare(struct vm_area_struct *vma); 340 void unlink_anon_vmas(struct vm_area_struct *vma); 341 342 static inline int anon_vma_prepare(struct vm_area_struct *vma) 343 { 344 if (likely(vma->anon_vma)) 345 return 0; 346 347 return __anon_vma_prepare(vma); 348 } 349 350 /* Flags for folio_pte_batch(). */ 351 typedef int __bitwise fpb_t; 352 353 /* Compare PTEs respecting the dirty bit. */ 354 #define FPB_RESPECT_DIRTY ((__force fpb_t)BIT(0)) 355 356 /* Compare PTEs respecting the soft-dirty bit. */ 357 #define FPB_RESPECT_SOFT_DIRTY ((__force fpb_t)BIT(1)) 358 359 /* Compare PTEs respecting the writable bit. */ 360 #define FPB_RESPECT_WRITE ((__force fpb_t)BIT(2)) 361 362 /* 363 * Merge PTE write bits: if any PTE in the batch is writable, modify the 364 * PTE at @ptentp to be writable. 365 */ 366 #define FPB_MERGE_WRITE ((__force fpb_t)BIT(3)) 367 368 /* 369 * Merge PTE young and dirty bits: if any PTE in the batch is young or dirty, 370 * modify the PTE at @ptentp to be young or dirty, respectively. 371 */ 372 #define FPB_MERGE_YOUNG_DIRTY ((__force fpb_t)BIT(4)) 373 374 static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags) 375 { 376 if (!(flags & FPB_RESPECT_DIRTY)) 377 pte = pte_mkclean(pte); 378 if (likely(!(flags & FPB_RESPECT_SOFT_DIRTY))) 379 pte = pte_clear_soft_dirty(pte); 380 if (likely(!(flags & FPB_RESPECT_WRITE))) 381 pte = pte_wrprotect(pte); 382 return pte_mkold(pte); 383 } 384 385 /** 386 * folio_pte_batch_flags - detect a PTE batch for a large folio 387 * @folio: The large folio to detect a PTE batch for. 388 * @vma: The VMA. Only relevant with FPB_MERGE_WRITE, otherwise can be NULL. 389 * @ptep: Page table pointer for the first entry. 390 * @ptentp: Pointer to a COPY of the first page table entry whose flags this 391 * function updates based on @flags if appropriate. 392 * @max_nr: The maximum number of table entries to consider. 393 * @flags: Flags to modify the PTE batch semantics. 394 * 395 * Detect a PTE batch: consecutive (present) PTEs that map consecutive 396 * pages of the same large folio in a single VMA and a single page table. 397 * 398 * All PTEs inside a PTE batch have the same PTE bits set, excluding the PFN, 399 * the accessed bit, writable bit, dirty bit (unless FPB_RESPECT_DIRTY is set) 400 * and soft-dirty bit (unless FPB_RESPECT_SOFT_DIRTY is set). 401 * 402 * @ptep must map any page of the folio. max_nr must be at least one and 403 * must be limited by the caller so scanning cannot exceed a single VMA and 404 * a single page table. 405 * 406 * Depending on the FPB_MERGE_* flags, the pte stored at @ptentp will 407 * be updated: it's crucial that a pointer to a COPY of the first 408 * page table entry, obtained through ptep_get(), is provided as @ptentp. 409 * 410 * This function will be inlined to optimize based on the input parameters; 411 * consider using folio_pte_batch() instead if applicable. 412 * 413 * Return: the number of table entries in the batch. 414 */ 415 static inline unsigned int folio_pte_batch_flags(struct folio *folio, 416 struct vm_area_struct *vma, pte_t *ptep, pte_t *ptentp, 417 unsigned int max_nr, fpb_t flags) 418 { 419 bool any_writable = false, any_young = false, any_dirty = false; 420 pte_t expected_pte, pte = *ptentp; 421 unsigned int nr, cur_nr; 422 423 VM_WARN_ON_FOLIO(!pte_present(pte), folio); 424 VM_WARN_ON_FOLIO(!folio_test_large(folio) || max_nr < 1, folio); 425 VM_WARN_ON_FOLIO(page_folio(pfn_to_page(pte_pfn(pte))) != folio, folio); 426 /* 427 * Ensure this is a pointer to a copy not a pointer into a page table. 428 * If this is a stack value, it won't be a valid virtual address, but 429 * that's fine because it also cannot be pointing into the page table. 430 */ 431 VM_WARN_ON(virt_addr_valid(ptentp) && PageTable(virt_to_page(ptentp))); 432 433 /* Limit max_nr to the actual remaining PFNs in the folio we could batch. */ 434 max_nr = min_t(unsigned long, max_nr, 435 folio_pfn(folio) + folio_nr_pages(folio) - pte_pfn(pte)); 436 437 nr = pte_batch_hint(ptep, pte); 438 expected_pte = __pte_batch_clear_ignored(pte_advance_pfn(pte, nr), flags); 439 ptep = ptep + nr; 440 441 while (nr < max_nr) { 442 pte = ptep_get(ptep); 443 444 if (!pte_same(__pte_batch_clear_ignored(pte, flags), expected_pte)) 445 break; 446 447 if (flags & FPB_MERGE_WRITE) 448 any_writable |= pte_write(pte); 449 if (flags & FPB_MERGE_YOUNG_DIRTY) { 450 any_young |= pte_young(pte); 451 any_dirty |= pte_dirty(pte); 452 } 453 454 cur_nr = pte_batch_hint(ptep, pte); 455 expected_pte = pte_advance_pfn(expected_pte, cur_nr); 456 ptep += cur_nr; 457 nr += cur_nr; 458 } 459 460 if (any_writable) 461 *ptentp = pte_mkwrite(*ptentp, vma); 462 if (any_young) 463 *ptentp = pte_mkyoung(*ptentp); 464 if (any_dirty) 465 *ptentp = pte_mkdirty(*ptentp); 466 467 return min(nr, max_nr); 468 } 469 470 unsigned int folio_pte_batch(struct folio *folio, pte_t *ptep, pte_t pte, 471 unsigned int max_nr); 472 473 /** 474 * pte_move_swp_offset - Move the swap entry offset field of a swap pte 475 * forward or backward by delta 476 * @pte: The initial pte state; must be a swap entry 477 * @delta: The direction and the offset we are moving; forward if delta 478 * is positive; backward if delta is negative 479 * 480 * Moves the swap offset, while maintaining all other fields, including 481 * swap type, and any swp pte bits. The resulting pte is returned. 482 */ 483 static inline pte_t pte_move_swp_offset(pte_t pte, long delta) 484 { 485 const softleaf_t entry = softleaf_from_pte(pte); 486 pte_t new = __swp_entry_to_pte(__swp_entry(swp_type(entry), 487 (swp_offset(entry) + delta))); 488 489 if (pte_swp_soft_dirty(pte)) 490 new = pte_swp_mksoft_dirty(new); 491 if (pte_swp_exclusive(pte)) 492 new = pte_swp_mkexclusive(new); 493 if (pte_swp_uffd(pte)) 494 new = pte_swp_mkuffd(new); 495 496 return new; 497 } 498 499 500 /** 501 * pte_next_swp_offset - Increment the swap entry offset field of a swap pte. 502 * @pte: The initial pte state; must be a swap entry. 503 * 504 * Increments the swap offset, while maintaining all other fields, including 505 * swap type, and any swp pte bits. The resulting pte is returned. 506 */ 507 static inline pte_t pte_next_swp_offset(pte_t pte) 508 { 509 return pte_move_swp_offset(pte, 1); 510 } 511 512 /** 513 * swap_pte_batch - detect a PTE batch for a set of contiguous swap entries 514 * @start_ptep: Page table pointer for the first entry. 515 * @max_nr: The maximum number of table entries to consider. 516 * @pte: Page table entry for the first entry. 517 * 518 * Detect a batch of contiguous swap entries: consecutive (non-present) PTEs 519 * containing swap entries all with consecutive offsets and targeting the same 520 * swap type, all with matching swp pte bits. 521 * 522 * max_nr must be at least one and must be limited by the caller so scanning 523 * cannot exceed a single page table. 524 * 525 * Return: the number of table entries in the batch. 526 */ 527 static inline int swap_pte_batch(pte_t *start_ptep, int max_nr, pte_t pte) 528 { 529 pte_t expected_pte = pte_next_swp_offset(pte); 530 const pte_t *end_ptep = start_ptep + max_nr; 531 pte_t *ptep = start_ptep + 1; 532 533 VM_WARN_ON(max_nr < 1); 534 VM_WARN_ON(!softleaf_is_swap(softleaf_from_pte(pte))); 535 536 while (ptep < end_ptep) { 537 pte = ptep_get(ptep); 538 539 if (!pte_same(pte, expected_pte)) 540 break; 541 expected_pte = pte_next_swp_offset(expected_pte); 542 ptep++; 543 } 544 545 return ptep - start_ptep; 546 } 547 #endif /* CONFIG_MMU */ 548 549 void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio, 550 int nr_throttled); 551 static inline void acct_reclaim_writeback(struct folio *folio) 552 { 553 pg_data_t *pgdat = folio_pgdat(folio); 554 int nr_throttled = atomic_read(&pgdat->nr_writeback_throttled); 555 556 if (nr_throttled) 557 __acct_reclaim_writeback(pgdat, folio, nr_throttled); 558 } 559 560 static inline void wake_throttle_isolated(pg_data_t *pgdat) 561 { 562 wait_queue_head_t *wqh; 563 564 wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_ISOLATED]; 565 if (waitqueue_active(wqh)) 566 wake_up(wqh); 567 } 568 569 vm_fault_t __vmf_anon_prepare(struct vm_fault *vmf); 570 static inline vm_fault_t vmf_anon_prepare(struct vm_fault *vmf) 571 { 572 vm_fault_t ret = __vmf_anon_prepare(vmf); 573 574 if (unlikely(ret & VM_FAULT_RETRY)) 575 vma_end_read(vmf->vma); 576 return ret; 577 } 578 579 vm_fault_t do_swap_page(struct vm_fault *vmf); 580 void folio_rotate_reclaimable(struct folio *folio); 581 bool __folio_end_writeback(struct folio *folio); 582 void deactivate_file_folio(struct folio *folio); 583 void folio_activate(struct folio *folio); 584 585 void free_pgtables(struct mmu_gather *tlb, struct unmap_desc *desc); 586 587 void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte); 588 589 /** 590 * sync_with_folio_pmd_zap - sync with concurrent zapping of a folio PMD 591 * @mm: The mm_struct. 592 * @pmdp: Pointer to the pmd that was found to be pmd_none(). 593 * 594 * When we find a pmd_none() while unmapping a folio without holding the PTL, 595 * zap_huge_pmd() may have cleared the PMD but not yet modified the folio to 596 * indicate that it's unmapped. Skipping the PMD without synchronization could 597 * make folio unmapping code assume that unmapping failed. 598 * 599 * Wait for concurrent zapping to complete by grabbing the PTL. 600 */ 601 static inline void sync_with_folio_pmd_zap(struct mm_struct *mm, pmd_t *pmdp) 602 { 603 spinlock_t *ptl = pmd_lock(mm, pmdp); 604 605 spin_unlock(ptl); 606 } 607 608 struct zap_details; 609 void zap_vma_range_batched(struct mmu_gather *tlb, 610 struct vm_area_struct *vma, unsigned long addr, 611 unsigned long size, struct zap_details *details); 612 int zap_vma_for_reaping(struct vm_area_struct *vma); 613 int folio_unmap_invalidate(struct address_space *mapping, struct folio *folio, 614 gfp_t gfp); 615 616 void page_cache_ra_order(struct readahead_control *, struct file_ra_state *); 617 void force_page_cache_ra(struct readahead_control *, unsigned long nr); 618 static inline void force_page_cache_readahead(struct address_space *mapping, 619 struct file *file, pgoff_t index, unsigned long nr_to_read) 620 { 621 DEFINE_READAHEAD(ractl, file, &file->f_ra, mapping, index); 622 force_page_cache_ra(&ractl, nr_to_read); 623 } 624 625 unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start, 626 pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices); 627 unsigned find_get_entries(struct address_space *mapping, pgoff_t *start, 628 pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices); 629 int truncate_inode_folio(struct address_space *mapping, struct folio *folio); 630 bool truncate_inode_partial_folio(struct folio *folio, loff_t start, 631 loff_t end); 632 long mapping_evict_folio(struct address_space *mapping, struct folio *folio); 633 unsigned long mapping_try_invalidate(struct address_space *mapping, 634 pgoff_t start, pgoff_t end, unsigned long *nr_failed); 635 636 /** 637 * folio_evictable - Test whether a folio is evictable. 638 * @folio: The folio to test. 639 * 640 * Test whether @folio is evictable -- i.e., should be placed on 641 * active/inactive lists vs unevictable list. 642 * 643 * Reasons folio might not be evictable: 644 * 1. folio's mapping marked unevictable 645 * 2. One of the pages in the folio is part of an mlocked VMA 646 */ 647 static inline bool folio_evictable(struct folio *folio) 648 { 649 bool ret; 650 651 /* Prevent address_space of inode and swap cache from being freed */ 652 rcu_read_lock(); 653 ret = !mapping_unevictable(folio_mapping(folio)) && 654 !folio_test_mlocked(folio); 655 rcu_read_unlock(); 656 return ret; 657 } 658 659 /* 660 * Turn a non-refcounted page (->_refcount == 0) into refcounted with 661 * a count of one. 662 */ 663 static inline void set_page_refcounted(struct page *page) 664 { 665 VM_BUG_ON_PAGE(PageTail(page), page); 666 VM_BUG_ON_PAGE(page_ref_count(page), page); 667 set_page_count(page, 1); 668 } 669 670 static inline void set_pages_refcounted(struct page *page, unsigned long nr_pages) 671 { 672 unsigned long pfn = page_to_pfn(page); 673 674 for (; nr_pages--; pfn++) 675 set_page_refcounted(pfn_to_page(pfn)); 676 } 677 678 /* 679 * Return true if a folio needs ->release_folio() calling upon it. 680 */ 681 static inline bool folio_needs_release(struct folio *folio) 682 { 683 struct address_space *mapping = folio_mapping(folio); 684 685 return folio_has_private(folio) || 686 (mapping && mapping_release_always(mapping)); 687 } 688 689 extern unsigned long highest_memmap_pfn; 690 691 /* 692 * Maximum number of reclaim retries without progress before the OOM 693 * killer is consider the only way forward. 694 */ 695 #define MAX_RECLAIM_RETRIES 16 696 697 /* 698 * in mm/vmscan.c: 699 */ 700 bool folio_isolate_lru(struct folio *folio); 701 void folio_putback_lru(struct folio *folio); 702 extern void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason); 703 int user_proactive_reclaim(char *buf, 704 struct mem_cgroup *memcg, pg_data_t *pgdat); 705 706 /* 707 * in mm/rmap.c: 708 */ 709 pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address); 710 711 /* 712 * in mm/khugepaged.c 713 */ 714 void set_recommended_min_free_kbytes(void); 715 716 /* 717 * in mm/page_alloc.c 718 */ 719 #define K(x) ((x) << (PAGE_SHIFT-10)) 720 721 extern char * const zone_names[MAX_NR_ZONES]; 722 723 extern int min_free_kbytes; 724 extern int defrag_mode; 725 726 void setup_per_zone_wmarks(void); 727 void calculate_min_free_kbytes(void); 728 int __meminit init_per_zone_wmark_min(void); 729 730 extern int __isolate_free_page(struct page *page, unsigned int order); 731 extern void __putback_isolated_page(struct page *page, unsigned int order, 732 int mt); 733 734 /* 735 * This will have no effect, other than possibly generating a warning, if the 736 * caller passes in a non-large folio. 737 */ 738 static inline void folio_set_order(struct folio *folio, unsigned int order) 739 { 740 if (WARN_ON_ONCE(!order || !folio_test_large(folio))) 741 return; 742 VM_WARN_ON_ONCE(order > MAX_FOLIO_ORDER); 743 744 folio->_flags_1 = (folio->_flags_1 & ~0xffUL) | order; 745 #ifdef NR_PAGES_IN_LARGE_FOLIO 746 folio->_nr_pages = 1U << order; 747 #endif 748 } 749 750 bool __folio_unqueue_deferred_split(struct folio *folio); 751 static inline bool folio_unqueue_deferred_split(struct folio *folio) 752 { 753 if (folio_order(folio) <= 1 || !folio_test_large_rmappable(folio)) 754 return false; 755 756 /* 757 * At this point, there is no one trying to add the folio to 758 * deferred_list. If folio is not in deferred_list, it's safe 759 * to check without acquiring the list_lru lock. 760 */ 761 if (data_race(list_empty(&folio->_deferred_list))) 762 return false; 763 764 return __folio_unqueue_deferred_split(folio); 765 } 766 767 static inline struct folio *page_rmappable_folio(struct page *page) 768 { 769 struct folio *folio = (struct folio *)page; 770 771 if (folio && folio_test_large(folio)) 772 folio_set_large_rmappable(folio); 773 return folio; 774 } 775 776 static inline void prep_compound_head(struct page *page, unsigned int order) 777 { 778 struct folio *folio = (struct folio *)page; 779 780 folio_set_order(folio, order); 781 atomic_set(&folio->_large_mapcount, -1); 782 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) 783 atomic_set(&folio->_nr_pages_mapped, 0); 784 if (IS_ENABLED(CONFIG_MM_ID)) { 785 folio->_mm_ids = 0; 786 folio->_mm_id_mapcount[0] = -1; 787 folio->_mm_id_mapcount[1] = -1; 788 } 789 if (IS_ENABLED(CONFIG_64BIT) || order > 1) { 790 atomic_set(&folio->_pincount, 0); 791 atomic_set(&folio->_entire_mapcount, -1); 792 } 793 if (order > 1) 794 INIT_LIST_HEAD(&folio->_deferred_list); 795 } 796 797 static inline void prep_compound_tail(struct page *tail, 798 const struct page *head, unsigned int order) 799 { 800 tail->mapping = TAIL_MAPPING; 801 set_compound_head(tail, head, order); 802 VM_WARN_ON_ONCE(tail->private); 803 } 804 805 static inline void init_compound_tail(struct page *tail, 806 const struct page *head, unsigned int order, struct zone *zone) 807 { 808 atomic_set(&tail->_mapcount, -1); 809 set_page_node(tail, zone_to_nid(zone)); 810 set_page_zone(tail, zone_idx(zone)); 811 prep_compound_tail(tail, head, order); 812 } 813 814 #if defined CONFIG_COMPACTION || defined CONFIG_CMA 815 816 /* 817 * in mm/compaction.c 818 */ 819 /* 820 * compact_control is used to track pages being migrated and the free pages 821 * they are being migrated to during memory compaction. The free_pfn starts 822 * at the end of a zone and migrate_pfn begins at the start. Movable pages 823 * are moved to the end of a zone during a compaction run and the run 824 * completes when free_pfn <= migrate_pfn 825 */ 826 struct compact_control { 827 struct list_head freepages[NR_PAGE_ORDERS]; /* List of free pages to migrate to */ 828 struct list_head migratepages; /* List of pages being migrated */ 829 unsigned int nr_freepages; /* Number of isolated free pages */ 830 unsigned int nr_migratepages; /* Number of pages to migrate */ 831 unsigned long free_pfn; /* isolate_freepages search base */ 832 /* 833 * Acts as an in/out parameter to page isolation for migration. 834 * isolate_migratepages uses it as a search base. 835 * isolate_migratepages_block will update the value to the next pfn 836 * after the last isolated one. 837 */ 838 unsigned long migrate_pfn; 839 unsigned long fast_start_pfn; /* a pfn to start linear scan from */ 840 struct zone *zone; 841 unsigned long total_migrate_scanned; 842 unsigned long total_free_scanned; 843 unsigned short fast_search_fail;/* failures to use free list searches */ 844 short search_order; /* order to start a fast search at */ 845 const gfp_t gfp_mask; /* gfp mask of a direct compactor */ 846 int order; /* order a direct compactor needs */ 847 int migratetype; /* migratetype of direct compactor */ 848 const unsigned int alloc_flags; /* alloc flags of a direct compactor */ 849 const int highest_zoneidx; /* zone index of a direct compactor */ 850 enum migrate_mode mode; /* Async or sync migration mode */ 851 bool ignore_skip_hint; /* Scan blocks even if marked skip */ 852 bool no_set_skip_hint; /* Don't mark blocks for skipping */ 853 bool ignore_block_suitable; /* Scan blocks considered unsuitable */ 854 bool direct_compaction; /* False from kcompactd or /proc/... */ 855 bool proactive_compaction; /* kcompactd proactive compaction */ 856 bool whole_zone; /* Whole zone should/has been scanned */ 857 bool contended; /* Signal lock contention */ 858 bool finish_pageblock; /* Scan the remainder of a pageblock. Used 859 * when there are potentially transient 860 * isolation or migration failures to 861 * ensure forward progress. 862 */ 863 bool alloc_contig; /* alloc_contig_range allocation */ 864 }; 865 866 /* 867 * Used in direct compaction when a page should be taken from the freelists 868 * immediately when one is created during the free path. 869 */ 870 struct capture_control { 871 struct zone *zone; 872 int migratetype; 873 /* 874 * Allocation request order. May differ from the compaction 875 * order: defrag_mode promotes sub-block allocations to 876 * pageblock-order compaction; capture still matches at the 877 * original allocation order so prep_new_page() is consistent. 878 */ 879 int order; 880 struct page *page; 881 }; 882 883 unsigned long 884 isolate_freepages_range(struct compact_control *cc, 885 unsigned long start_pfn, unsigned long end_pfn); 886 int 887 isolate_migratepages_range(struct compact_control *cc, 888 unsigned long low_pfn, unsigned long end_pfn); 889 890 #endif /* CONFIG_COMPACTION || CONFIG_CMA */ 891 892 struct cma; 893 894 #ifdef CONFIG_CMA 895 bool cma_validate_zones(struct cma *cma); 896 void *cma_reserve_early(struct cma *cma, unsigned long size); 897 #else 898 static inline bool cma_validate_zones(struct cma *cma) 899 { 900 return false; 901 } 902 static inline void *cma_reserve_early(struct cma *cma, unsigned long size) 903 { 904 return NULL; 905 } 906 #endif 907 908 /* mm/util.c */ 909 struct anon_vma *folio_anon_vma(const struct folio *folio); 910 911 #ifdef CONFIG_MMU 912 void unmap_mapping_folio(struct folio *folio); 913 extern long populate_vma_page_range(struct vm_area_struct *vma, 914 unsigned long start, unsigned long end, int *locked); 915 extern long faultin_page_range(struct mm_struct *mm, unsigned long start, 916 unsigned long end, bool write, int *locked); 917 bool mlock_future_ok(const struct mm_struct *mm, bool is_vma_locked, 918 unsigned long bytes); 919 920 /* 921 * NOTE: This function can't tell whether the folio is "fully mapped" in the 922 * range. 923 * "fully mapped" means all the pages of folio is associated with the page 924 * table of range while this function just check whether the folio range is 925 * within the range [start, end). Function caller needs to do page table 926 * check if it cares about the page table association. 927 * 928 * Typical usage (like mlock or madvise) is: 929 * Caller knows at least 1 page of folio is associated with page table of VMA 930 * and the range [start, end) is intersect with the VMA range. Caller wants 931 * to know whether the folio is fully associated with the range. It calls 932 * this function to check whether the folio is in the range first. Then checks 933 * the page table to know whether the folio is fully mapped to the range. 934 */ 935 static inline bool 936 folio_within_range(struct folio *folio, struct vm_area_struct *vma, 937 unsigned long start, unsigned long end) 938 { 939 const unsigned long vma_pglen = vma_pages(vma); 940 pgoff_t pgoff_folio, pgoff_vma_start; 941 unsigned long addr; 942 943 VM_WARN_ON_FOLIO(folio_test_ksm(folio), folio); 944 if (start > end) 945 return false; 946 947 pgoff_folio = folio_pgoff(folio); 948 pgoff_vma_start = vma_start_pgoff(vma); 949 950 if (start < vma->vm_start) 951 start = vma->vm_start; 952 953 if (end > vma->vm_end) 954 end = vma->vm_end; 955 956 /* if folio start address is not in vma range */ 957 if (!in_range(pgoff_folio, pgoff_vma_start, vma_pglen)) 958 return false; 959 960 addr = vma->vm_start + ((pgoff_folio - pgoff_vma_start) << PAGE_SHIFT); 961 962 return !(addr < start || end - addr < folio_size(folio)); 963 } 964 965 static inline bool 966 folio_within_vma(struct folio *folio, struct vm_area_struct *vma) 967 { 968 return folio_within_range(folio, vma, vma->vm_start, vma->vm_end); 969 } 970 971 /* 972 * mlock_vma_folio() and munlock_vma_folio(): 973 * should be called with vma's mmap_lock held for read or write, 974 * under page table lock for the pte/pmd being added or removed. 975 * 976 * mlock is usually called at the end of folio_add_*_rmap_*(), munlock at 977 * the end of folio_remove_rmap_*(); but new anon folios are managed by 978 * folio_add_lru_vma() calling mlock_new_folio(). 979 */ 980 void mlock_folio(struct folio *folio); 981 static inline void mlock_vma_folio(struct folio *folio, 982 struct vm_area_struct *vma) 983 { 984 /* 985 * The VM_SPECIAL check here serves two purposes. 986 * 1) VM_IO check prevents migration from double-counting during mlock. 987 * 2) Although mmap_region() and mlock_fixup() take care that VM_LOCKED 988 * is never left set on a VM_SPECIAL vma, there is an interval while 989 * file->f_op->mmap() is using vm_insert_page(s), when VM_LOCKED may 990 * still be set while VM_SPECIAL bits are added: so ignore it then. 991 */ 992 if (unlikely((vma->vm_flags & (VM_LOCKED|VM_SPECIAL)) == VM_LOCKED)) 993 mlock_folio(folio); 994 } 995 996 void munlock_folio(struct folio *folio); 997 static inline void munlock_vma_folio(struct folio *folio, 998 struct vm_area_struct *vma) 999 { 1000 /* 1001 * munlock if the function is called. Ideally, we should only 1002 * do munlock if any page of folio is unmapped from VMA and 1003 * cause folio not fully mapped to VMA. 1004 * 1005 * But it's not easy to confirm that's the situation. So we 1006 * always munlock the folio and page reclaim will correct it 1007 * if it's wrong. 1008 */ 1009 if (unlikely(vma->vm_flags & VM_LOCKED)) 1010 munlock_folio(folio); 1011 } 1012 1013 void mlock_new_folio(struct folio *folio); 1014 bool need_mlock_drain(int cpu); 1015 void mlock_drain_local(void); 1016 void mlock_drain_remote(int cpu); 1017 1018 extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma); 1019 1020 /** 1021 * vma_address - Find the virtual address a page range is mapped at 1022 * @vma: The vma which maps this object. 1023 * @pgoff: The page offset within its object. 1024 * @nr_pages: The number of pages to consider. 1025 * 1026 * If any page in this range is mapped by this VMA, return the first address 1027 * where any of these pages appear. Otherwise, return -EFAULT. 1028 */ 1029 static inline unsigned long vma_address(const struct vm_area_struct *vma, 1030 pgoff_t pgoff, unsigned long nr_pages) 1031 { 1032 const pgoff_t pgoff_start = vma_start_pgoff(vma); 1033 unsigned long address; 1034 1035 if (pgoff >= pgoff_start) { 1036 address = vma->vm_start + 1037 ((pgoff - pgoff_start) << PAGE_SHIFT); 1038 /* Check for address beyond vma (or wrapped through 0?) */ 1039 if (address < vma->vm_start || address >= vma->vm_end) 1040 address = -EFAULT; 1041 } else if (pgoff + nr_pages - 1 >= pgoff_start) { 1042 /* Test above avoids possibility of wrap to 0 on 32-bit */ 1043 address = vma->vm_start; 1044 } else { 1045 address = -EFAULT; 1046 } 1047 return address; 1048 } 1049 1050 /* 1051 * Then at what user virtual address will none of the range be found in vma? 1052 * Assumes that vma_address() already returned a good starting address. 1053 */ 1054 static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw) 1055 { 1056 struct vm_area_struct *vma = pvmw->vma; 1057 pgoff_t pgoff; 1058 unsigned long address; 1059 1060 /* Common case, plus ->pgoff is invalid for KSM */ 1061 if (pvmw->nr_pages == 1) 1062 return pvmw->address + PAGE_SIZE; 1063 1064 pgoff = pvmw->pgoff + pvmw->nr_pages; 1065 address = vma->vm_start + 1066 ((pgoff - vma_start_pgoff(vma)) << PAGE_SHIFT); 1067 /* Check for address beyond vma (or wrapped through 0?) */ 1068 if (address < vma->vm_start || address > vma->vm_end) 1069 address = vma->vm_end; 1070 return address; 1071 } 1072 1073 static inline struct file *maybe_unlock_mmap_for_io(struct vm_fault *vmf, 1074 struct file *fpin) 1075 { 1076 int flags = vmf->flags; 1077 1078 if (fpin) 1079 return fpin; 1080 1081 /* 1082 * FAULT_FLAG_RETRY_NOWAIT means we don't want to wait on page locks or 1083 * anything, so we only pin the file and drop the mmap_lock if only 1084 * FAULT_FLAG_ALLOW_RETRY is set, while this is the first attempt. 1085 */ 1086 if (fault_flag_allow_retry_first(flags) && 1087 !(flags & FAULT_FLAG_RETRY_NOWAIT)) { 1088 fpin = get_file(vmf->vma->vm_file); 1089 release_fault_lock(vmf); 1090 } 1091 return fpin; 1092 } 1093 1094 static inline bool vma_supports_mlock(const struct vm_area_struct *vma) 1095 { 1096 if (vma_test_any_mask(vma, VMA_SPECIAL_FLAGS)) 1097 return false; 1098 if (vma_test_single_mask(vma, VMA_DROPPABLE)) 1099 return false; 1100 if (vma_is_dax(vma) || is_vm_hugetlb_page(vma)) 1101 return false; 1102 return vma != get_gate_vma(current->mm); 1103 } 1104 1105 #else /* !CONFIG_MMU */ 1106 static inline void unmap_mapping_folio(struct folio *folio) { } 1107 static inline void mlock_new_folio(struct folio *folio) { } 1108 static inline bool need_mlock_drain(int cpu) { return false; } 1109 static inline void mlock_drain_local(void) { } 1110 static inline void mlock_drain_remote(int cpu) { } 1111 #endif /* !CONFIG_MMU */ 1112 1113 #ifdef CONFIG_NUMA 1114 extern int node_reclaim_mode; 1115 1116 extern unsigned long node_reclaim(struct pglist_data *pgdat, 1117 gfp_t gfp_mask, unsigned int order); 1118 extern int find_next_best_node(int node, nodemask_t *used_node_mask); 1119 #else 1120 #define node_reclaim_mode 0 1121 1122 static inline unsigned long node_reclaim(struct pglist_data *pgdat, 1123 gfp_t mask, unsigned int order) 1124 { 1125 return 0; 1126 } 1127 static inline int find_next_best_node(int node, nodemask_t *used_node_mask) 1128 { 1129 return NUMA_NO_NODE; 1130 } 1131 #endif 1132 1133 static inline bool node_reclaim_enabled(void) 1134 { 1135 /* Is any node_reclaim_mode bit set? */ 1136 return node_reclaim_mode & (RECLAIM_ZONE|RECLAIM_WRITE|RECLAIM_UNMAP); 1137 } 1138 1139 /* 1140 * mm/memory-failure.c 1141 */ 1142 #ifdef CONFIG_MEMORY_FAILURE 1143 int unmap_poisoned_folio(struct folio *folio, unsigned long pfn, bool must_kill); 1144 void shake_folio(struct folio *folio); 1145 typedef int hwpoison_filter_func_t(struct page *p); 1146 void hwpoison_filter_register(hwpoison_filter_func_t *filter); 1147 void hwpoison_filter_unregister(void); 1148 1149 #define MAGIC_HWPOISON 0x48575053U /* HWPS */ 1150 void SetPageHWPoisonTakenOff(struct page *page); 1151 void ClearPageHWPoisonTakenOff(struct page *page); 1152 bool take_page_off_buddy(struct page *page); 1153 bool put_page_back_buddy(struct page *page); 1154 struct task_struct *task_early_kill(struct task_struct *tsk, int force_early); 1155 void add_to_kill_ksm(struct task_struct *tsk, const struct page *p, 1156 struct vm_area_struct *vma, struct list_head *to_kill, 1157 unsigned long ksm_addr); 1158 unsigned long page_mapped_in_vma(const struct page *page, 1159 struct vm_area_struct *vma); 1160 1161 #else 1162 static inline int unmap_poisoned_folio(struct folio *folio, unsigned long pfn, bool must_kill) 1163 { 1164 return -EBUSY; 1165 } 1166 #endif 1167 1168 extern unsigned long __must_check vm_mmap_pgoff(struct file *, unsigned long, 1169 unsigned long, unsigned long, 1170 unsigned long, unsigned long); 1171 1172 unsigned long reclaim_pages(struct list_head *folio_list); 1173 unsigned int reclaim_clean_pages_from_list(struct zone *zone, 1174 struct list_head *folio_list); 1175 1176 enum ttu_flags; 1177 struct tlbflush_unmap_batch; 1178 1179 1180 /* 1181 * only for MM internal work items which do not depend on 1182 * any allocations or locks which might depend on allocations 1183 */ 1184 extern struct workqueue_struct *mm_percpu_wq; 1185 1186 #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH 1187 void try_to_unmap_flush(void); 1188 void try_to_unmap_flush_dirty(void); 1189 void flush_tlb_batched_pending(struct mm_struct *mm); 1190 #else 1191 static inline void try_to_unmap_flush(void) 1192 { 1193 } 1194 static inline void try_to_unmap_flush_dirty(void) 1195 { 1196 } 1197 static inline void flush_tlb_batched_pending(struct mm_struct *mm) 1198 { 1199 } 1200 #endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */ 1201 1202 extern const struct trace_print_flags pageflag_names[]; 1203 extern const struct trace_print_flags vmaflag_names[]; 1204 extern const struct trace_print_flags gfpflag_names[]; 1205 1206 void setup_zone_pageset(struct zone *zone); 1207 1208 struct migration_target_control { 1209 int nid; /* preferred node id */ 1210 nodemask_t *nmask; 1211 gfp_t gfp_mask; 1212 enum migrate_reason reason; 1213 }; 1214 1215 /* 1216 * mm/filemap.c 1217 */ 1218 size_t splice_folio_into_pipe(struct pipe_inode_info *pipe, 1219 struct folio *folio, loff_t fpos, size_t size); 1220 1221 static inline bool vma_is_single_threaded_private(struct vm_area_struct *vma) 1222 { 1223 if (vma->vm_flags & VM_SHARED) 1224 return false; 1225 1226 return atomic_read(&vma->vm_mm->mm_users) == 1; 1227 } 1228 1229 #ifdef CONFIG_NUMA_BALANCING 1230 bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma, 1231 bool is_private_single_threaded); 1232 1233 #else 1234 static inline bool folio_can_map_prot_numa(struct folio *folio, 1235 struct vm_area_struct *vma, bool is_private_single_threaded) 1236 { 1237 return false; 1238 } 1239 #endif 1240 1241 int numa_migrate_check(struct folio *folio, struct vm_fault *vmf, 1242 unsigned long addr, int *flags, bool writable, 1243 int *last_cpupid); 1244 1245 void free_zone_device_folio(struct folio *folio); 1246 int migrate_device_coherent_folio(struct folio *folio); 1247 1248 /* 1249 * mm/gup.c 1250 */ 1251 int __must_check try_grab_folio(struct folio *folio, int refs, 1252 unsigned int flags); 1253 1254 /* 1255 * mm/huge_memory.c 1256 */ 1257 void touch_pud(struct vm_area_struct *vma, unsigned long addr, 1258 pud_t *pud, bool write); 1259 bool touch_pmd(struct vm_area_struct *vma, unsigned long addr, 1260 pmd_t *pmd, bool write); 1261 1262 /* 1263 * Parses a string with mem suffixes into its order. Useful to parse kernel 1264 * parameters. 1265 */ 1266 static inline int get_order_from_str(const char *size_str, 1267 unsigned long valid_orders) 1268 { 1269 unsigned long size; 1270 char *endptr; 1271 int order; 1272 1273 size = memparse(size_str, &endptr); 1274 1275 if (!is_power_of_2(size)) 1276 return -EINVAL; 1277 order = get_order(size); 1278 if (BIT(order) & ~valid_orders) 1279 return -EINVAL; 1280 1281 return order; 1282 } 1283 1284 enum { 1285 /* mark page accessed */ 1286 FOLL_TOUCH = 1 << 16, 1287 /* a retry, previous pass started an IO */ 1288 FOLL_TRIED = 1 << 17, 1289 /* we are working on non-current tsk/mm */ 1290 FOLL_REMOTE = 1 << 18, 1291 /* pages must be released via unpin_user_page */ 1292 FOLL_PIN = 1 << 19, 1293 /* gup_fast: prevent fall-back to slow gup */ 1294 FOLL_FAST_ONLY = 1 << 20, 1295 /* allow unlocking the mmap lock */ 1296 FOLL_UNLOCKABLE = 1 << 21, 1297 /* VMA lookup+checks compatible with MADV_POPULATE_(READ|WRITE) */ 1298 FOLL_MADV_POPULATE = 1 << 22, 1299 }; 1300 1301 #define INTERNAL_GUP_FLAGS (FOLL_TOUCH | FOLL_TRIED | FOLL_REMOTE | FOLL_PIN | \ 1302 FOLL_FAST_ONLY | FOLL_UNLOCKABLE | \ 1303 FOLL_MADV_POPULATE) 1304 1305 /* 1306 * Indicates for which pages that are write-protected in the page table, 1307 * whether GUP has to trigger unsharing via FAULT_FLAG_UNSHARE such that the 1308 * GUP pin will remain consistent with the pages mapped into the page tables 1309 * of the MM. 1310 * 1311 * Temporary unmapping of PageAnonExclusive() pages or clearing of 1312 * PageAnonExclusive() has to protect against concurrent GUP: 1313 * * Ordinary GUP: Using the PT lock 1314 * * GUP-fast and fork(): mm->write_protect_seq 1315 * * GUP-fast and KSM or temporary unmapping (swap, migration): see 1316 * folio_try_share_anon_rmap_*() 1317 * 1318 * Must be called with the (sub)page that's actually referenced via the 1319 * page table entry, which might not necessarily be the head page for a 1320 * PTE-mapped THP. 1321 * 1322 * If the vma is NULL, we're coming from the GUP-fast path and might have 1323 * to fallback to the slow path just to lookup the vma. 1324 */ 1325 static inline bool gup_must_unshare(struct vm_area_struct *vma, 1326 unsigned int flags, struct page *page) 1327 { 1328 /* 1329 * FOLL_WRITE is implicitly handled correctly as the page table entry 1330 * has to be writable -- and if it references (part of) an anonymous 1331 * folio, that part is required to be marked exclusive. 1332 */ 1333 if ((flags & (FOLL_WRITE | FOLL_PIN)) != FOLL_PIN) 1334 return false; 1335 /* 1336 * Note: PageAnon(page) is stable until the page is actually getting 1337 * freed. 1338 */ 1339 if (!PageAnon(page)) { 1340 /* 1341 * We only care about R/O long-term pining: R/O short-term 1342 * pinning does not have the semantics to observe successive 1343 * changes through the process page tables. 1344 */ 1345 if (!(flags & FOLL_LONGTERM)) 1346 return false; 1347 1348 /* We really need the vma ... */ 1349 if (!vma) 1350 return true; 1351 1352 /* 1353 * ... because we only care about writable private ("COW") 1354 * mappings where we have to break COW early. 1355 */ 1356 return is_cow_mapping(vma->vm_flags); 1357 } 1358 1359 /* Paired with a memory barrier in folio_try_share_anon_rmap_*(). */ 1360 if (IS_ENABLED(CONFIG_HAVE_GUP_FAST)) 1361 smp_rmb(); 1362 1363 /* 1364 * Note that KSM pages cannot be exclusive, and consequently, 1365 * cannot get pinned. 1366 */ 1367 return !PageAnonExclusive(page); 1368 } 1369 1370 1371 static inline bool vma_soft_dirty_enabled(struct vm_area_struct *vma) 1372 { 1373 /* 1374 * NOTE: we must check this before VM_SOFTDIRTY on soft-dirty 1375 * enablements, because when without soft-dirty being compiled in, 1376 * VM_SOFTDIRTY is defined as 0x0, then !(vm_flags & VM_SOFTDIRTY) 1377 * will be constantly true. 1378 */ 1379 if (!pgtable_supports_soft_dirty()) 1380 return false; 1381 1382 /* 1383 * Soft-dirty is kind of special: its tracking is enabled when the 1384 * vma flags not set. 1385 */ 1386 return !(vma->vm_flags & VM_SOFTDIRTY); 1387 } 1388 1389 static inline bool pmd_needs_soft_dirty_wp(struct vm_area_struct *vma, pmd_t pmd) 1390 { 1391 return vma_soft_dirty_enabled(vma) && !pmd_soft_dirty(pmd); 1392 } 1393 1394 static inline bool pte_needs_soft_dirty_wp(struct vm_area_struct *vma, pte_t pte) 1395 { 1396 return vma_soft_dirty_enabled(vma) && !pte_soft_dirty(pte); 1397 } 1398 1399 /* shrinker related functions */ 1400 unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg, 1401 int priority); 1402 1403 int shmem_add_to_page_cache(struct folio *folio, 1404 struct address_space *mapping, 1405 pgoff_t index, void *expected, gfp_t gfp); 1406 int shmem_inode_acct_blocks(struct inode *inode, long pages); 1407 bool shmem_recalc_inode(struct inode *inode, long alloced, long swapped); 1408 1409 #ifdef CONFIG_SHRINKER_DEBUG 1410 static inline __printf(2, 0) int shrinker_debugfs_name_alloc( 1411 struct shrinker *shrinker, const char *fmt, va_list ap) 1412 { 1413 shrinker->name = kvasprintf_const(GFP_KERNEL, fmt, ap); 1414 1415 return shrinker->name ? 0 : -ENOMEM; 1416 } 1417 1418 static inline void shrinker_debugfs_name_free(struct shrinker *shrinker) 1419 { 1420 kfree_const(shrinker->name); 1421 shrinker->name = NULL; 1422 } 1423 1424 extern int shrinker_debugfs_add(struct shrinker *shrinker); 1425 extern struct dentry *shrinker_debugfs_detach(struct shrinker *shrinker, 1426 int *debugfs_id); 1427 extern void shrinker_debugfs_remove(struct dentry *debugfs_entry, 1428 int debugfs_id); 1429 #else /* CONFIG_SHRINKER_DEBUG */ 1430 static inline int shrinker_debugfs_add(struct shrinker *shrinker) 1431 { 1432 return 0; 1433 } 1434 static inline int shrinker_debugfs_name_alloc(struct shrinker *shrinker, 1435 const char *fmt, va_list ap) 1436 { 1437 return 0; 1438 } 1439 static inline void shrinker_debugfs_name_free(struct shrinker *shrinker) 1440 { 1441 } 1442 static inline struct dentry *shrinker_debugfs_detach(struct shrinker *shrinker, 1443 int *debugfs_id) 1444 { 1445 *debugfs_id = -1; 1446 return NULL; 1447 } 1448 static inline void shrinker_debugfs_remove(struct dentry *debugfs_entry, 1449 int debugfs_id) 1450 { 1451 } 1452 #endif /* CONFIG_SHRINKER_DEBUG */ 1453 1454 /* Only track the nodes of mappings with shadow entries */ 1455 void workingset_update_node(struct xa_node *node); 1456 extern struct list_lru shadow_nodes; 1457 #define mapping_set_update(xas, mapping) do { \ 1458 if (!dax_mapping(mapping) && !shmem_mapping(mapping)) { \ 1459 xas_set_update(xas, workingset_update_node); \ 1460 xas_set_lru(xas, &shadow_nodes); \ 1461 } \ 1462 } while (0) 1463 1464 /* mremap.c */ 1465 unsigned long move_page_tables(struct pagetable_move_control *pmc); 1466 1467 #ifdef CONFIG_UNACCEPTED_MEMORY 1468 void accept_page(struct page *page); 1469 #else /* CONFIG_UNACCEPTED_MEMORY */ 1470 static inline void accept_page(struct page *page) 1471 { 1472 } 1473 #endif /* CONFIG_UNACCEPTED_MEMORY */ 1474 1475 /* pagewalk.c */ 1476 int walk_page_range_mm_unsafe(struct mm_struct *mm, unsigned long start, 1477 unsigned long end, const struct mm_walk_ops *ops, 1478 void *private); 1479 int walk_page_range_vma_unsafe(struct vm_area_struct *vma, unsigned long start, 1480 unsigned long end, const struct mm_walk_ops *ops, 1481 void *private); 1482 int walk_page_range_debug(struct mm_struct *mm, unsigned long start, 1483 unsigned long end, const struct mm_walk_ops *ops, 1484 pgd_t *pgd, void *private); 1485 1486 void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm); 1487 int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm); 1488 1489 int remap_pfn_range_prepare(struct vm_area_desc *desc); 1490 int remap_pfn_range_complete(struct vm_area_struct *vma, 1491 struct mmap_action *action); 1492 int simple_ioremap_prepare(struct vm_area_desc *desc); 1493 1494 static inline int io_remap_pfn_range_prepare(struct vm_area_desc *desc) 1495 { 1496 struct mmap_action *action = &desc->action; 1497 const unsigned long orig_pfn = action->remap.start_pfn; 1498 const pgprot_t orig_pgprot = action->remap.pgprot; 1499 const unsigned long size = action->remap.size; 1500 const unsigned long pfn = io_remap_pfn_range_pfn(orig_pfn, size); 1501 int err; 1502 1503 action->remap.start_pfn = pfn; 1504 action->remap.pgprot = pgprot_decrypted(orig_pgprot); 1505 err = remap_pfn_range_prepare(desc); 1506 if (err) 1507 return err; 1508 1509 /* Remap does the actual work. */ 1510 action->type = MMAP_REMAP_PFN; 1511 return 0; 1512 } 1513 1514 /* 1515 * When we succeed an mmap action or just before we unmap a VMA on error, we 1516 * need to ensure any rmap lock held is released. On unmap it's required to 1517 * avoid a deadlock. 1518 */ 1519 static inline void maybe_rmap_unlock_action(struct vm_area_struct *vma, 1520 struct mmap_action *action) 1521 { 1522 struct file *file; 1523 1524 if (!action->hide_from_rmap_until_complete) 1525 return; 1526 1527 VM_WARN_ON_ONCE(vma_is_anonymous(vma)); 1528 file = vma->vm_file; 1529 i_mmap_unlock_write(file->f_mapping); 1530 action->hide_from_rmap_until_complete = false; 1531 } 1532 1533 #ifdef CONFIG_MMU_NOTIFIER 1534 static inline bool clear_flush_young_ptes_notify(struct vm_area_struct *vma, 1535 unsigned long addr, pte_t *ptep, unsigned int nr) 1536 { 1537 bool young; 1538 1539 young = clear_flush_young_ptes(vma, addr, ptep, nr); 1540 young |= mmu_notifier_clear_flush_young(vma->vm_mm, addr, 1541 addr + nr * PAGE_SIZE); 1542 return young; 1543 } 1544 1545 static inline bool pmdp_clear_flush_young_notify(struct vm_area_struct *vma, 1546 unsigned long addr, pmd_t *pmdp) 1547 { 1548 bool young; 1549 1550 young = pmdp_clear_flush_young(vma, addr, pmdp); 1551 young |= mmu_notifier_clear_flush_young(vma->vm_mm, addr, addr + PMD_SIZE); 1552 return young; 1553 } 1554 1555 static inline bool test_and_clear_young_ptes_notify(struct vm_area_struct *vma, 1556 unsigned long addr, pte_t *ptep, unsigned int nr) 1557 { 1558 bool young; 1559 1560 young = test_and_clear_young_ptes(vma, addr, ptep, nr); 1561 young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + nr * PAGE_SIZE); 1562 return young; 1563 } 1564 1565 static inline bool pmdp_test_and_clear_young_notify(struct vm_area_struct *vma, 1566 unsigned long addr, pmd_t *pmdp) 1567 { 1568 bool young; 1569 1570 young = pmdp_test_and_clear_young(vma, addr, pmdp); 1571 young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + PMD_SIZE); 1572 return young; 1573 } 1574 1575 #else /* CONFIG_MMU_NOTIFIER */ 1576 1577 #define clear_flush_young_ptes_notify clear_flush_young_ptes 1578 #define pmdp_clear_flush_young_notify pmdp_clear_flush_young 1579 #define test_and_clear_young_ptes_notify test_and_clear_young_ptes 1580 #define pmdp_test_and_clear_young_notify pmdp_test_and_clear_young 1581 1582 #endif /* CONFIG_MMU_NOTIFIER */ 1583 1584 extern int sysctl_max_map_count; 1585 static inline int get_sysctl_max_map_count(void) 1586 { 1587 return READ_ONCE(sysctl_max_map_count); 1588 } 1589 1590 bool may_expand_vm(struct mm_struct *mm, const vma_flags_t *vma_flags, 1591 unsigned long npages); 1592 1593 static inline void mm_prepare_for_swap_entries(struct mm_struct *mm) 1594 { 1595 if (list_empty(&mm->mmlist)) { 1596 spin_lock(&mmlist_lock); 1597 if (list_empty(&mm->mmlist)) 1598 list_add(&mm->mmlist, &init_mm.mmlist); 1599 spin_unlock(&mmlist_lock); 1600 } 1601 } 1602 1603 static inline bool can_spin_trylock(void) 1604 { 1605 /* 1606 * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is 1607 * unsafe in NMI. If spin_trylock() is called from hard IRQ the current 1608 * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will 1609 * mark the task as the owner of another rt_spin_lock which will 1610 * confuse PI logic, so return immediately if called from hard IRQ or 1611 * NMI. 1612 * 1613 * Note, irqs_disabled() case is ok. spin_trylock() can be called 1614 * from raw_spin_lock_irqsave region. 1615 */ 1616 if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq())) 1617 return false; 1618 1619 /* On UP, spin_trylock() always succeeds even when it is locked */ 1620 if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) 1621 return false; 1622 1623 return true; 1624 } 1625 1626 #endif /* __MM_INTERNAL_H */ 1627