74 static inline struct folio *try_get_folio(struct page *page, int refs)  in try_get_folio()  argument
82 	if (unlikely(!folio_ref_try_add(folio, refs)))  in try_get_folio()
95 		folio_put_refs(folio, refs);  in try_get_folio()
102 static void gup_put_folio(struct folio *folio, int refs, unsigned int flags)  in gup_put_folio()  argument
107 		node_stat_mod_folio(folio, NR_FOLL_PIN_RELEASED, refs);  in gup_put_folio()
109 			atomic_sub(refs, &folio->_pincount);  in gup_put_folio()
111 			refs *= GUP_PIN_COUNTING_BIAS;  in gup_put_folio()
114 	folio_put_refs(folio, refs);  in gup_put_folio()
140 int __must_check try_grab_folio(struct folio *folio, int refs,  in try_grab_folio()  argument
150 		folio_ref_add(folio, refs);  in try_grab_folio()
164 			folio_ref_add(folio, refs);  in try_grab_folio()
165 			atomic_add(refs, &folio->_pincount);  in try_grab_folio()
167 			folio_ref_add(folio, refs * GUP_PIN_COUNTING_BIAS);  in try_grab_folio()
170 		node_stat_mod_folio(folio, NR_FOLL_PIN_ACQUIRED, refs);  in try_grab_folio()
517 static struct folio *try_grab_folio_fast(struct page *page, int refs,  in try_grab_folio_fast()  argument
532 		return try_get_folio(page, refs);  in try_grab_folio_fast()
543 	folio = try_get_folio(page, refs);  in try_grab_folio_fast()
554 		folio_put_refs(folio, refs);  in try_grab_folio_fast()
567 		atomic_add(refs, &folio->_pincount);  in try_grab_folio_fast()
570 				refs * (GUP_PIN_COUNTING_BIAS - 1));  in try_grab_folio_fast()
578 	node_stat_mod_folio(folio, NR_FOLL_PIN_ACQUIRED, refs);  in try_grab_folio_fast()
2941 	int refs;  in gup_fast_pmd_leaf()  local
2949 	refs = (end - addr) >> PAGE_SHIFT;  in gup_fast_pmd_leaf()
2952 	folio = try_grab_folio_fast(page, refs, flags);  in gup_fast_pmd_leaf()
2957 		gup_put_folio(folio, refs, flags);  in gup_fast_pmd_leaf()
2962 		gup_put_folio(folio, refs, flags);  in gup_fast_pmd_leaf()
2966 		gup_put_folio(folio, refs, flags);  in gup_fast_pmd_leaf()
2971 	*nr += refs;  in gup_fast_pmd_leaf()
2972 	for (; refs; refs--)  in gup_fast_pmd_leaf()
2984 	int refs;  in gup_fast_pud_leaf()  local
2992 	refs = (end - addr) >> PAGE_SHIFT;  in gup_fast_pud_leaf()
2995 	folio = try_grab_folio_fast(page, refs, flags);  in gup_fast_pud_leaf()
3000 		gup_put_folio(folio, refs, flags);  in gup_fast_pud_leaf()
3005 		gup_put_folio(folio, refs, flags);  in gup_fast_pud_leaf()
3010 		gup_put_folio(folio, refs, flags);  in gup_fast_pud_leaf()
3015 	*nr += refs;  in gup_fast_pud_leaf()
3016 	for (; refs; refs--)  in gup_fast_pud_leaf()