204 static __always_inline const struct page *page_fixed_fake_head(const struct page *page)  in page_fixed_fake_head()  argument
207 		return page;  in page_fixed_fake_head()
215 	if (IS_ALIGNED((unsigned long)page, PAGE_SIZE) &&  in page_fixed_fake_head()
216 	    test_bit(PG_head, &page->flags)) {  in page_fixed_fake_head()
222 		unsigned long head = READ_ONCE(page[1].compound_head);  in page_fixed_fake_head()
225 			return (const struct page *)(head - 1);  in page_fixed_fake_head()
227 	return page;  in page_fixed_fake_head()
230 static inline const struct page *page_fixed_fake_head(const struct page *page)  in page_fixed_fake_head()  argument
232 	return page;  in page_fixed_fake_head()
236 static __always_inline int page_is_fake_head(const struct page *page)  in page_is_fake_head()  argument
238 	return page_fixed_fake_head(page) != page;  in page_is_fake_head()
241 static __always_inline unsigned long _compound_head(const struct page *page)  in _compound_head()  argument
243 	unsigned long head = READ_ONCE(page->compound_head);  in _compound_head()
247 	return (unsigned long)page_fixed_fake_head(page);  in _compound_head()
250 #define compound_head(page)	((typeof(page))_compound_head(page))  argument
266 	const struct page *:	(const struct folio *)_compound_head(p), \
267 	struct page *:		(struct folio *)_compound_head(p)))
278 #define folio_page(folio, n)	nth_page(&(folio)->page, n)
280 static __always_inline int PageTail(const struct page *page)  in PageTail()  argument
282 	return READ_ONCE(page->compound_head) & 1 || page_is_fake_head(page);  in PageTail()
285 static __always_inline int PageCompound(const struct page *page)  in PageCompound()  argument
287 	return test_bit(PG_head, &page->flags) ||  in PageCompound()
288 	       READ_ONCE(page->compound_head) & 1;  in PageCompound()
292 static inline int PagePoisoned(const struct page *page)  in PagePoisoned()  argument
294 	return READ_ONCE(page->flags) == PAGE_POISON_PATTERN;  in PagePoisoned()
298 void page_init_poison(struct page *page, size_t size);
300 static inline void page_init_poison(struct page *page, size_t size)  in page_init_poison()  argument
308 	const struct page *page = &folio->page;  in const_folio_flags()  local
310 	VM_BUG_ON_PGFLAGS(page->compound_head & 1, page);  in const_folio_flags()
311 	VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page);  in const_folio_flags()
312 	return &page[n].flags;  in const_folio_flags()
317 	struct page *page = &folio->page;  in folio_flags()  local
319 	VM_BUG_ON_PGFLAGS(page->compound_head & 1, page);  in folio_flags()
320 	VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page);  in folio_flags()
321 	return &page[n].flags;  in folio_flags()
347 #define PF_POISONED_CHECK(page) ({					\  argument
348 		VM_BUG_ON_PGFLAGS(PagePoisoned(page), page);		\
349 		page; })
350 #define PF_ANY(page, enforce)	PF_POISONED_CHECK(page)  argument
351 #define PF_HEAD(page, enforce)	PF_POISONED_CHECK(compound_head(page))  argument
352 #define PF_NO_TAIL(page, enforce) ({					\  argument
353 		VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page);	\
354 		PF_POISONED_CHECK(compound_head(page)); })
355 #define PF_NO_COMPOUND(page, enforce) ({				\  argument
356 		VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page);	\
357 		PF_POISONED_CHECK(page); })
358 #define PF_SECOND(page, enforce) ({					\  argument
359 		VM_BUG_ON_PGFLAGS(!PageHead(page), page);		\
360 		PF_POISONED_CHECK(&page[1]); })
375 #define FOLIO_TEST_FLAG(name, page)					\  argument
377 { return test_bit(PG_##name, const_folio_flags(folio, page)); }
379 #define FOLIO_SET_FLAG(name, page)					\  argument
381 { set_bit(PG_##name, folio_flags(folio, page)); }
383 #define FOLIO_CLEAR_FLAG(name, page)					\  argument
385 { clear_bit(PG_##name, folio_flags(folio, page)); }
387 #define __FOLIO_SET_FLAG(name, page)					\  argument
389 { __set_bit(PG_##name, folio_flags(folio, page)); }
391 #define __FOLIO_CLEAR_FLAG(name, page)					\  argument
393 { __clear_bit(PG_##name, folio_flags(folio, page)); }
395 #define FOLIO_TEST_SET_FLAG(name, page)					\  argument
397 { return test_and_set_bit(PG_##name, folio_flags(folio, page)); }
399 #define FOLIO_TEST_CLEAR_FLAG(name, page)				\  argument
401 { return test_and_clear_bit(PG_##name, folio_flags(folio, page)); }
403 #define FOLIO_FLAG(name, page)						\  argument
404 FOLIO_TEST_FLAG(name, page)						\
405 FOLIO_SET_FLAG(name, page)						\
406 FOLIO_CLEAR_FLAG(name, page)
410 static __always_inline int Page##uname(const struct page *page)		\
411 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
415 static __always_inline void SetPage##uname(struct page *page)		\
416 { set_bit(PG_##lname, &policy(page, 1)->flags); }
420 static __always_inline void ClearPage##uname(struct page *page)		\
421 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
425 static __always_inline void __SetPage##uname(struct page *page)		\
426 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
430 static __always_inline void __ClearPage##uname(struct page *page)	\
431 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
435 static __always_inline int TestSetPage##uname(struct page *page)	\
436 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
440 static __always_inline int TestClearPage##uname(struct page *page)	\
441 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
482 static inline int Page##uname(const struct page *page) { return 0; }
486 static inline void SetPage##uname(struct page *page) {  }
490 static inline void ClearPage##uname(struct page *page) {  }
494 static inline void __ClearPage##uname(struct page *page) {  }
498 static inline int TestSetPage##uname(struct page *page) { return 0; }
502 static inline int TestClearPage##uname(struct page *page) { return 0; }
687 static __always_inline bool PageMappingFlags(const struct page *page)  in PageMappingFlags()  argument
689 	return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0;  in PageMappingFlags()
697 static __always_inline bool PageAnonNotKsm(const struct page *page)  in PageAnonNotKsm()  argument
699 	unsigned long flags = (unsigned long)page_folio(page)->mapping;  in PageAnonNotKsm()
704 static __always_inline bool PageAnon(const struct page *page)  in PageAnon()  argument
706 	return folio_test_anon(page_folio(page));  in PageAnon()
715 static __always_inline bool __PageMovable(const struct page *page)  in __PageMovable()  argument
717 	return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==  in __PageMovable()
737 u64 stable_page_flags(const struct page *page);
785 static inline bool PageUptodate(const struct page *page)  in PageUptodate()  argument
787 	return folio_test_uptodate(page_folio(page));  in PageUptodate()
807 static __always_inline void __SetPageUptodate(struct page *page)  in __SetPageUptodate()  argument
809 	__folio_mark_uptodate((struct folio *)page);  in __SetPageUptodate()
812 static __always_inline void SetPageUptodate(struct page *page)  in SetPageUptodate()  argument
814 	folio_mark_uptodate((struct folio *)page);  in SetPageUptodate()
820 void set_page_writeback(struct page *page);
832 static __always_inline int PageHead(const struct page *page)  in PageHead()  argument
834 	PF_POISONED_CHECK(page);  in PageHead()
835 	return test_bit(PG_head, &page->flags) && !page_is_fake_head(page);  in PageHead()
853 static __always_inline void set_compound_head(struct page *page, struct page *head)  in set_compound_head()  argument
855 	WRITE_ONCE(page->compound_head, (unsigned long)head + 1);  in set_compound_head()
858 static __always_inline void clear_compound_head(struct page *page)  in clear_compound_head()  argument
860 	WRITE_ONCE(page->compound_head, 0);  in clear_compound_head()
864 static inline void ClearPageCompound(struct page *page)  in ClearPageCompound()  argument
866 	BUG_ON(!PageHead(page));  in ClearPageCompound()
867 	ClearPageHead(page);  in ClearPageCompound()
887 static inline int PageTransHuge(const struct page *page)  in FOLIO_FLAG()
889 	VM_BUG_ON_PAGE(PageTail(page), page);  in FOLIO_FLAG()
890 	return PageHead(page);  in FOLIO_FLAG()
898 static inline int PageTransCompound(const struct page *page)  in PageTransCompound()  argument
900 	return PageCompound(page);  in PageTransCompound()
951 static inline bool page_has_type(const struct page *page)  in page_has_type()  argument
953 	return page_mapcount_is_type(data_race(page->page_type));  in page_has_type()
959 	return data_race(folio->page.page_type >> 24) == PGTY_##lname;	\
965 	VM_BUG_ON_FOLIO(data_race(folio->page.page_type) != UINT_MAX,	\
967 	folio->page.page_type = (unsigned int)PGTY_##lname << 24;	\
971 	if (folio->page.page_type == UINT_MAX)				\
974 	folio->page.page_type = UINT_MAX;				\
979 static __always_inline int Page##uname(const struct page *page)		\
981 	return data_race(page->page_type >> 24) == PGTY_##lname;	\
983 static __always_inline void __SetPage##uname(struct page *page)		\
985 	if (Page##uname(page))						\
987 	VM_BUG_ON_PAGE(data_race(page->page_type) != UINT_MAX, page);	\
988 	page->page_type = (unsigned int)PGTY_##lname << 24;		\
990 static __always_inline void __ClearPage##uname(struct page *page)	\
992 	if (page->page_type == UINT_MAX)				\
994 	VM_BUG_ON_PAGE(!Page##uname(page), page);			\
995 	page->page_type = UINT_MAX;					\
1059 static inline bool PageSlab(const struct page *page)  in PAGE_TYPE_OPS()
1061 	return folio_test_slab(page_folio(page));  in PAGE_TYPE_OPS()
1087 static inline bool PageHuge(const struct page *page)  in FOLIO_TYPE_OPS()
1089 	return folio_test_hugetlb(page_folio(page));  in FOLIO_TYPE_OPS()
1097 static inline bool is_page_hwpoison(const struct page *page)  in is_page_hwpoison()  argument
1101 	if (PageHWPoison(page))  in is_page_hwpoison()
1103 	folio = page_folio(page);  in is_page_hwpoison()
1104 	return folio_test_hugetlb(folio) && PageHWPoison(&folio->page);  in is_page_hwpoison()
1107 bool is_free_buddy_page(const struct page *page);
1111 static __always_inline int PageAnonExclusive(const struct page *page)  in PageAnonExclusive()  argument
1113 	VM_BUG_ON_PGFLAGS(!PageAnon(page), page);  in PageAnonExclusive()
1118 	if (PageHuge(page))  in PageAnonExclusive()
1119 		page = compound_head(page);  in PageAnonExclusive()
1120 	return test_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in PageAnonExclusive()
1123 static __always_inline void SetPageAnonExclusive(struct page *page)  in SetPageAnonExclusive()  argument
1125 	VM_BUG_ON_PGFLAGS(!PageAnonNotKsm(page), page);  in SetPageAnonExclusive()
1126 	VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);  in SetPageAnonExclusive()
1127 	set_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in SetPageAnonExclusive()
1130 static __always_inline void ClearPageAnonExclusive(struct page *page)  in ClearPageAnonExclusive()  argument
1132 	VM_BUG_ON_PGFLAGS(!PageAnonNotKsm(page), page);  in ClearPageAnonExclusive()
1133 	VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);  in ClearPageAnonExclusive()
1134 	clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in ClearPageAnonExclusive()
1137 static __always_inline void __ClearPageAnonExclusive(struct page *page)  in __ClearPageAnonExclusive()  argument
1139 	VM_BUG_ON_PGFLAGS(!PageAnon(page), page);  in __ClearPageAnonExclusive()
1140 	VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);  in __ClearPageAnonExclusive()
1141 	__clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in __ClearPageAnonExclusive()