31 static inline void sanity_check_pinned_pages(struct page **pages,  in sanity_check_pinned_pages()
50 		struct page *page = *pages;  in sanity_check_pinned_pages()  local
53 		if (!page)  in sanity_check_pinned_pages()
56 		folio = page_folio(page);  in sanity_check_pinned_pages()
58 		if (is_zero_page(page) ||  in sanity_check_pinned_pages()
62 			VM_WARN_ON_ONCE_FOLIO(!PageAnonExclusive(&folio->page), folio);  in sanity_check_pinned_pages()
65 			VM_WARN_ON_ONCE_PAGE(!PageAnonExclusive(&folio->page) &&  in sanity_check_pinned_pages()
66 					     !PageAnonExclusive(page), page);  in sanity_check_pinned_pages()
74 static inline struct folio *try_get_folio(struct page *page, int refs)  in try_get_folio()  argument
79 	folio = page_folio(page);  in try_get_folio()
94 	if (unlikely(page_folio(page) != folio)) {  in try_get_folio()
185 void unpin_user_page(struct page *page)  in unpin_user_page()  argument
187 	sanity_check_pinned_pages(&page, 1);  in unpin_user_page()
188 	gup_put_folio(page_folio(page), 1, FOLL_PIN);  in unpin_user_page()
232 static inline struct folio *gup_folio_range_next(struct page *start,  in gup_folio_range_next()
235 	struct page *next = start + i;  in gup_folio_range_next()
247 static inline struct folio *gup_folio_next(struct page **list,  in gup_folio_next()
284 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,  in unpin_user_pages_dirty_lock()
354 void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages,  in unpin_user_page_range_dirty_lock()  argument
361 	VM_WARN_ON_ONCE(!page_range_contiguous(page, npages));  in unpin_user_page_range_dirty_lock()
364 		folio = gup_folio_range_next(page, npages, i, &nr);  in unpin_user_page_range_dirty_lock()
375 static void gup_fast_unpin_user_pages(struct page **pages, unsigned long npages)  in gup_fast_unpin_user_pages()
401 void unpin_user_pages(struct page **pages, unsigned long npages)  in unpin_user_pages()
517 static struct folio *try_grab_folio_fast(struct page *page, int refs,  in try_grab_folio_fast()  argument
528 	if (unlikely(!(flags & FOLL_PCI_P2PDMA) && is_pci_p2pdma_page(page)))  in try_grab_folio_fast()
532 		return try_get_folio(page, refs);  in try_grab_folio_fast()
540 	if (is_zero_page(page))  in try_grab_folio_fast()
541 		return page_folio(page);  in try_grab_folio_fast()
543 	folio = try_get_folio(page, refs);  in try_grab_folio_fast()
585 static inline bool can_follow_write_common(struct page *page,  in can_follow_write_common()  argument
608 	return page && PageAnon(page) && PageAnonExclusive(page);  in can_follow_write_common()
611 static struct page *no_page_table(struct vm_area_struct *vma,  in no_page_table()
638 static inline bool can_follow_write_pud(pud_t pud, struct page *page,  in can_follow_write_pud()  argument
646 	return can_follow_write_common(page, vma, flags);  in can_follow_write_pud()
649 static struct page *follow_huge_pud(struct vm_area_struct *vma,  in follow_huge_pud()
654 	struct page *page;  in follow_huge_pud()  local
669 	page = pfn_to_page(pfn);  in follow_huge_pud()
671 	if (!pud_write(pud) && gup_must_unshare(vma, flags, page))  in follow_huge_pud()
674 	ret = try_grab_folio(page_folio(page), 1, flags);  in follow_huge_pud()
676 		page = ERR_PTR(ret);  in follow_huge_pud()
680 	return page;  in follow_huge_pud()
684 static inline bool can_follow_write_pmd(pmd_t pmd, struct page *page,  in can_follow_write_pmd()  argument
692 	if (!can_follow_write_common(page, vma, flags))  in can_follow_write_pmd()
701 static struct page *follow_huge_pmd(struct vm_area_struct *vma,  in follow_huge_pmd()
708 	struct page *page;  in follow_huge_pmd()  local
713 	page = pmd_page(pmdval);  in follow_huge_pmd()
715 	    !can_follow_write_pmd(pmdval, page, vma, flags))  in follow_huge_pmd()
725 	if (!pmd_write(pmdval) && gup_must_unshare(vma, flags, page))  in follow_huge_pmd()
728 	VM_WARN_ON_ONCE_PAGE((flags & FOLL_PIN) && PageAnon(page) &&  in follow_huge_pmd()
729 			     !PageAnonExclusive(page), page);  in follow_huge_pmd()
731 	ret = try_grab_folio(page_folio(page), 1, flags);  in follow_huge_pmd()
740 	page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;  in follow_huge_pmd()
743 	return page;  in follow_huge_pmd()
747 static struct page *follow_huge_pud(struct vm_area_struct *vma,  in follow_huge_pud()
754 static struct page *follow_huge_pmd(struct vm_area_struct *vma,  in follow_huge_pmd()
785 static inline bool can_follow_write_pte(pte_t pte, struct page *page,  in can_follow_write_pte()  argument
793 	if (!can_follow_write_common(page, vma, flags))  in can_follow_write_pte()
802 static struct page *follow_page_pte(struct vm_area_struct *vma,  in follow_page_pte()
807 	struct page *page;  in follow_page_pte()  local
821 	page = vm_normal_page(vma, address, pte);  in follow_page_pte()
827 	    !can_follow_write_pte(pte, page, vma, flags)) {  in follow_page_pte()
828 		page = NULL;  in follow_page_pte()
832 	if (unlikely(!page)) {  in follow_page_pte()
835 			page = ERR_PTR(-EFAULT);  in follow_page_pte()
840 			page = pte_page(pte);  in follow_page_pte()
843 			page = ERR_PTR(ret);  in follow_page_pte()
847 	folio = page_folio(page);  in follow_page_pte()
849 	if (!pte_write(pte) && gup_must_unshare(vma, flags, page)) {  in follow_page_pte()
850 		page = ERR_PTR(-EMLINK);  in follow_page_pte()
854 	VM_WARN_ON_ONCE_PAGE((flags & FOLL_PIN) && PageAnon(page) &&  in follow_page_pte()
855 			     !PageAnonExclusive(page), page);  in follow_page_pte()
860 		page = ERR_PTR(ret);  in follow_page_pte()
872 			unpin_user_page(page);  in follow_page_pte()
873 			page = ERR_PTR(ret);  in follow_page_pte()
890 	return page;  in follow_page_pte()
898 static struct page *follow_pmd_mask(struct vm_area_struct *vma,  in follow_pmd_mask()
905 	struct page *page;  in follow_pmd_mask()  local
937 	page = follow_huge_pmd(vma, address, pmd, flags, page_mask);  in follow_pmd_mask()
939 	return page;  in follow_pmd_mask()
942 static struct page *follow_pud_mask(struct vm_area_struct *vma,  in follow_pud_mask()
949 	struct page *page;  in follow_pud_mask()  local
958 		page = follow_huge_pud(vma, address, pudp, flags, page_mask);  in follow_pud_mask()
960 		if (page)  in follow_pud_mask()
961 			return page;  in follow_pud_mask()
970 static struct page *follow_p4d_mask(struct vm_area_struct *vma,  in follow_p4d_mask()
1007 static struct page *follow_page_mask(struct vm_area_struct *vma,  in follow_page_mask()
1013 	struct page *page;  in follow_page_mask()  local
1021 		page = no_page_table(vma, flags, address);  in follow_page_mask()
1023 		page = follow_p4d_mask(vma, address, pgd, flags, page_mask);  in follow_page_mask()
1027 	return page;  in follow_page_mask()
1032 		struct page **page)  in get_gate_page()  argument
1064 	if (!page)  in get_gate_page()
1066 	*page = vm_normal_page(*vma, address, entry);  in get_gate_page()
1067 	if (!*page) {  in get_gate_page()
1070 		*page = pte_page(entry);  in get_gate_page()
1072 	ret = try_grab_folio(page_folio(*page), 1, gup_flags);  in get_gate_page()
1356 		unsigned int gup_flags, struct page **pages,  in __get_user_pages()
1375 		struct page *page;  in __get_user_pages()  local
1400 						pages ? &page : NULL);  in __get_user_pages()
1426 		page = follow_page_mask(vma, start, gup_flags, &page_mask);  in __get_user_pages()
1427 		if (!page || PTR_ERR(page) == -EMLINK) {  in __get_user_pages()
1429 					   PTR_ERR(page) == -EMLINK, locked);  in __get_user_pages()
1443 		} else if (PTR_ERR(page) == -EEXIST) {  in __get_user_pages()
1451 				ret = PTR_ERR(page);  in __get_user_pages()
1454 		} else if (IS_ERR(page)) {  in __get_user_pages()
1455 			ret = PTR_ERR(page);  in __get_user_pages()
1464 			struct page *subpage;  in __get_user_pages()
1478 				struct folio *folio = page_folio(page);  in __get_user_pages()
1497 				subpage = page + j;  in __get_user_pages()
1652 						struct page **pages,  in __get_user_pages_locked()
1980 		unsigned long nr_pages, struct page **pages,  in __get_user_pages_locked()
2187 struct page *get_dump_page(unsigned long addr, int *locked)  in get_dump_page()
2189 	struct page *page;  in get_dump_page()  local
2192 	ret = __get_user_pages_locked(current->mm, addr, 1, &page, locked,  in get_dump_page()
2194 	return (ret == 1) ? page : NULL;  in get_dump_page()
2209 		struct page **pages;
2432 					    struct page **pages)  in check_and_migrate_movable_pages()
2444 					    struct page **pages)  in check_and_migrate_movable_pages()
2463 				  struct page **pages,  in __gup_longterm_locked()
2495 static bool is_valid_gup_args(struct page **pages, int *locked,  in is_valid_gup_args()
2600 		unsigned int gup_flags, struct page **pages,  in get_user_pages_remote()
2618 			   unsigned int gup_flags, struct page **pages,  in get_user_pages_remote()
2640 		    unsigned int gup_flags, struct page **pages)  in get_user_pages()
2668 			     struct page **pages, unsigned int gup_flags)  in get_user_pages_unlocked()
2830 		unsigned long end, unsigned int flags, struct page **pages,  in gup_fast_pte_range()
2841 		struct page *page;  in gup_fast_pte_range()  local
2862 		page = pte_page(pte);  in gup_fast_pte_range()
2864 		folio = try_grab_folio_fast(page, 1, flags);  in gup_fast_pte_range()
2879 		if (!pte_write(pte) && gup_must_unshare(NULL, flags, page)) {  in gup_fast_pte_range()
2895 		pages[*nr] = page;  in gup_fast_pte_range()
2917 		unsigned long end, unsigned int flags, struct page **pages,  in gup_fast_pte_range()
2925 		unsigned long end, unsigned int flags, struct page **pages,  in gup_fast_pmd_leaf()
2928 	struct page *page;  in gup_fast_pmd_leaf()  local
2939 	page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);  in gup_fast_pmd_leaf()
2941 	folio = try_grab_folio_fast(page, refs, flags);  in gup_fast_pmd_leaf()
2954 	if (!pmd_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) {  in gup_fast_pmd_leaf()
2962 		*(pages++) = page++;  in gup_fast_pmd_leaf()
2968 		unsigned long end, unsigned int flags, struct page **pages,  in gup_fast_pud_leaf()
2971 	struct page *page;  in gup_fast_pud_leaf()  local
2982 	page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);  in gup_fast_pud_leaf()
2984 	folio = try_grab_folio_fast(page, refs, flags);  in gup_fast_pud_leaf()
2998 	if (!pud_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) {  in gup_fast_pud_leaf()
3006 		*(pages++) = page++;  in gup_fast_pud_leaf()
3012 		unsigned long end, unsigned int flags, struct page **pages,  in gup_fast_pmd_range()
3044 		unsigned long end, unsigned int flags, struct page **pages,  in gup_fast_pud_range()
3070 		unsigned long end, unsigned int flags, struct page **pages,  in gup_fast_p4d_range()
3093 		unsigned int flags, struct page **pages, int *nr)  in gup_fast_pgd_range()
3113 		unsigned int flags, struct page **pages, int *nr)  in gup_fast_pgd_range()
3130 		unsigned int gup_flags, struct page **pages)  in gup_fast()
3176 		unsigned int gup_flags, struct page **pages)  in gup_fast_fallback()
3243 			     unsigned int gup_flags, struct page **pages)  in get_user_pages_fast_only()
3277 			unsigned int gup_flags, struct page **pages)  in get_user_pages_fast()
3311 			unsigned int gup_flags, struct page **pages)  in pin_user_pages_fast()
3344 			   unsigned int gup_flags, struct page **pages,  in pin_user_pages_remote()
3377 		    unsigned int gup_flags, struct page **pages)  in pin_user_pages()
3397 			     struct page **pages, unsigned int gup_flags)  in pin_user_pages_unlocked()