44 void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma);
236 static inline int mmap_file(struct file *file, struct vm_area_struct *vma)  in mmap_file()  argument
238 	int err = vfs_mmap(file, vma);  in mmap_file()
248 	vma->vm_ops = &vma_dummy_vm_ops;  in mmap_file()
258 static inline void vma_close(struct vm_area_struct *vma)  in vma_close()  argument
260 	if (vma->vm_ops && vma->vm_ops->close) {  in vma_close()
261 		vma->vm_ops->close(vma);  in vma_close()
267 		vma->vm_ops = &vma_dummy_vm_ops;  in vma_close()
276 bool cond_install_uffd_wp_ptes(struct vm_area_struct *vma,
335 int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma);
336 int  __anon_vma_prepare(struct vm_area_struct *vma);
337 void unlink_anon_vmas(struct vm_area_struct *vma);
339 static inline int anon_vma_prepare(struct vm_area_struct *vma)  in anon_vma_prepare()  argument
341 	if (likely(vma->anon_vma))  in anon_vma_prepare()
344 	return __anon_vma_prepare(vma);  in anon_vma_prepare()
413 		struct vm_area_struct *vma, pte_t *ptep, pte_t *ptentp,  in folio_pte_batch_flags()  argument
458 		*ptentp = pte_mkwrite(*ptentp, vma);  in folio_pte_batch_flags()
572 		vma_end_read(vmf->vma);  in vmf_anon_prepare()
607 		struct vm_area_struct *vma, unsigned long addr,
609 int zap_vma_for_reaping(struct vm_area_struct *vma);
910 extern long populate_vma_page_range(struct vm_area_struct *vma,
933 folio_within_range(struct folio *folio, struct vm_area_struct *vma,  in folio_within_range()  argument
936 	const unsigned long vma_pglen = vma_pages(vma);  in folio_within_range()
946 		vma_start_anon_pgoff(vma) : vma_start_pgoff(vma);  in folio_within_range()
948 	if (start < vma->vm_start)  in folio_within_range()
949 		start = vma->vm_start;  in folio_within_range()
951 	if (end > vma->vm_end)  in folio_within_range()
952 		end = vma->vm_end;  in folio_within_range()
958 	addr = vma->vm_start + ((pgoff_folio - pgoff_vma_start) << PAGE_SHIFT);  in folio_within_range()
964 folio_within_vma(struct folio *folio, struct vm_area_struct *vma)  in folio_within_vma()  argument
966 	return folio_within_range(folio, vma, vma->vm_start, vma->vm_end);  in folio_within_vma()
980 				struct vm_area_struct *vma)  in mlock_vma_folio()  argument
990 	if (unlikely((vma->vm_flags & (VM_LOCKED|VM_SPECIAL)) == VM_LOCKED))  in mlock_vma_folio()
996 					struct vm_area_struct *vma)  in munlock_vma_folio()  argument
1007 	if (unlikely(vma->vm_flags & VM_LOCKED))  in munlock_vma_folio()
1016 extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma);
1018 static inline unsigned long __vma_address(const struct vm_area_struct *vma,  in __vma_address()  argument
1024 		address = vma->vm_start +  in __vma_address()
1027 		if (address < vma->vm_start || address >= vma->vm_end)  in __vma_address()
1031 		address = vma->vm_start;  in __vma_address()
1048 static inline unsigned long vma_filebacked_address(const struct vm_area_struct *vma,  in vma_filebacked_address()  argument
1051 	VM_WARN_ON_ONCE(vma_is_anonymous(vma));  in vma_filebacked_address()
1053 	return __vma_address(vma, pgoff, vma_start_pgoff(vma), nr_pages);  in vma_filebacked_address()
1068 static inline unsigned long vma_anon_address(const struct vm_area_struct *vma,  in vma_anon_address()  argument
1071 	VM_WARN_ON_ONCE(!vma_is_cow_mapping(vma));  in vma_anon_address()
1073 	return __vma_address(vma, pgoff_anon, vma_start_anon_pgoff(vma), nr_pages);  in vma_anon_address()
1083 	const struct vm_area_struct *vma = pvmw->vma;  in vma_address_end()  local
1092 		pgoff_vma_start = vma_start_anon_pgoff(vma);  in vma_address_end()
1094 		pgoff_vma_start = vma_start_pgoff(vma);  in vma_address_end()
1096 	address = vma->vm_start +  in vma_address_end()
1099 	if (address < vma->vm_start || address > vma->vm_end)  in vma_address_end()
1100 		address = vma->vm_end;  in vma_address_end()
1119 		fpin = get_file(vmf->vma->vm_file);  in maybe_unlock_mmap_for_io()
1125 static inline bool vma_supports_mlock(const struct vm_area_struct *vma)  in vma_supports_mlock()  argument
1127 	if (vma_test_any_mask(vma, VMA_SPECIAL_FLAGS))  in vma_supports_mlock()
1129 	if (vma_test_single_mask(vma, VMA_DROPPABLE))  in vma_supports_mlock()
1131 	if (vma_is_dax(vma) || is_vm_hugetlb_page(vma))  in vma_supports_mlock()
1133 	return vma != get_gate_vma(current->mm);  in vma_supports_mlock()
1187 		     struct vm_area_struct *vma, struct list_head *to_kill,
1190 		struct vm_area_struct *vma);
1252 static inline bool vma_is_single_threaded_private(struct vm_area_struct *vma)  in vma_is_single_threaded_private()  argument
1254 	if (vma->vm_flags & VM_SHARED)  in vma_is_single_threaded_private()
1257 	return atomic_read(&vma->vm_mm->mm_users) == 1;  in vma_is_single_threaded_private()
1261 bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma,
1266 		struct vm_area_struct *vma, bool is_private_single_threaded)  in folio_can_map_prot_numa()  argument
1288 void touch_pud(struct vm_area_struct *vma, unsigned long addr,
1290 bool touch_pmd(struct vm_area_struct *vma, unsigned long addr,
1356 static inline bool gup_must_unshare(struct vm_area_struct *vma,  in gup_must_unshare()  argument
1380 		if (!vma)  in gup_must_unshare()
1387 		return vma_is_cow_mapping(vma);  in gup_must_unshare()
1402 static inline bool vma_soft_dirty_enabled(struct vm_area_struct *vma)  in vma_soft_dirty_enabled()  argument
1417 	return !(vma->vm_flags & VM_SOFTDIRTY);  in vma_soft_dirty_enabled()
1420 static inline bool pmd_needs_soft_dirty_wp(struct vm_area_struct *vma, pmd_t pmd)  in pmd_needs_soft_dirty_wp()  argument
1422 	return vma_soft_dirty_enabled(vma) && !pmd_soft_dirty(pmd);  in pmd_needs_soft_dirty_wp()
1425 static inline bool pte_needs_soft_dirty_wp(struct vm_area_struct *vma, pte_t pte)  in pte_needs_soft_dirty_wp()  argument
1427 	return vma_soft_dirty_enabled(vma) && !pte_soft_dirty(pte);  in pte_needs_soft_dirty_wp()
1510 int walk_page_range_vma_unsafe(struct vm_area_struct *vma, unsigned long start,
1521 int remap_pfn_range_complete(struct vm_area_struct *vma,
1550 static inline void maybe_rmap_unlock_action(struct vm_area_struct *vma,  in maybe_rmap_unlock_action()  argument
1558 	VM_WARN_ON_ONCE(vma_is_anonymous(vma));  in maybe_rmap_unlock_action()
1559 	file = vma->vm_file;  in maybe_rmap_unlock_action()
1565 static inline bool clear_flush_young_ptes_notify(struct vm_area_struct *vma,  in clear_flush_young_ptes_notify()  argument
1570 	young = clear_flush_young_ptes(vma, addr, ptep, nr);  in clear_flush_young_ptes_notify()
1571 	young |= mmu_notifier_clear_flush_young(vma->vm_mm, addr,  in clear_flush_young_ptes_notify()
1576 static inline bool pmdp_clear_flush_young_notify(struct vm_area_struct *vma,  in pmdp_clear_flush_young_notify()  argument
1581 	young = pmdp_clear_flush_young(vma, addr, pmdp);  in pmdp_clear_flush_young_notify()
1582 	young |= mmu_notifier_clear_flush_young(vma->vm_mm, addr, addr + PMD_SIZE);  in pmdp_clear_flush_young_notify()
1586 static inline bool test_and_clear_young_ptes_notify(struct vm_area_struct *vma,  in test_and_clear_young_ptes_notify()  argument
1591 	young = test_and_clear_young_ptes(vma, addr, ptep, nr);  in test_and_clear_young_ptes_notify()
1592 	young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + nr * PAGE_SIZE);  in test_and_clear_young_ptes_notify()
1596 static inline bool pmdp_test_and_clear_young_notify(struct vm_area_struct *vma,  in pmdp_test_and_clear_young_notify()  argument
1601 	young = pmdp_test_and_clear_young(vma, addr, pmdp);  in pmdp_test_and_clear_young_notify()
1602 	young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + PMD_SIZE);  in pmdp_test_and_clear_young_notify()