92 			 struct folio *folio, enum btrfs_subpage_type type)  in btrfs_attach_subpage()  argument
100 	if (folio->mapping)  in btrfs_attach_subpage()
101 		ASSERT(folio_test_locked(folio));  in btrfs_attach_subpage()
104 	if (!btrfs_is_subpage(fs_info, folio->mapping) || folio_test_private(folio))  in btrfs_attach_subpage()
111 	folio_attach_private(folio, subpage);  in btrfs_attach_subpage()
115 void btrfs_detach_subpage(const struct btrfs_fs_info *fs_info, struct folio *folio)  in btrfs_detach_subpage()  argument
120 	if (!btrfs_is_subpage(fs_info, folio->mapping) || !folio_test_private(folio))  in btrfs_detach_subpage()
123 	subpage = folio_detach_private(folio);  in btrfs_detach_subpage()
164 void btrfs_folio_inc_eb_refs(const struct btrfs_fs_info *fs_info, struct folio *folio)  in btrfs_folio_inc_eb_refs()  argument
168 	if (!btrfs_is_subpage(fs_info, folio->mapping))  in btrfs_folio_inc_eb_refs()
171 	ASSERT(folio_test_private(folio) && folio->mapping);  in btrfs_folio_inc_eb_refs()
172 	lockdep_assert_held(&folio->mapping->i_private_lock);  in btrfs_folio_inc_eb_refs()
174 	subpage = folio_get_private(folio);  in btrfs_folio_inc_eb_refs()
178 void btrfs_folio_dec_eb_refs(const struct btrfs_fs_info *fs_info, struct folio *folio)  in btrfs_folio_dec_eb_refs()  argument
182 	if (!btrfs_is_subpage(fs_info, folio->mapping))  in btrfs_folio_dec_eb_refs()
185 	ASSERT(folio_test_private(folio) && folio->mapping);  in btrfs_folio_dec_eb_refs()
186 	lockdep_assert_held(&folio->mapping->i_private_lock);  in btrfs_folio_dec_eb_refs()
188 	subpage = folio_get_private(folio);  in btrfs_folio_dec_eb_refs()
194 				 struct folio *folio, u64 start, u32 len)  in btrfs_subpage_assert()  argument
197 	ASSERT(folio_order(folio) == 0);  in btrfs_subpage_assert()
200 	ASSERT(folio_test_private(folio) && folio_get_private(folio));  in btrfs_subpage_assert()
207 	if (folio->mapping)  in btrfs_subpage_assert()
208 		ASSERT(folio_pos(folio) <= start &&  in btrfs_subpage_assert()
209 		       start + len <= folio_pos(folio) + PAGE_SIZE);  in btrfs_subpage_assert()
212 #define subpage_calc_start_bit(fs_info, folio, name, start, len)	\  argument
216 	btrfs_subpage_assert(fs_info, folio, start, len);		\
222 static void btrfs_subpage_clamp_range(struct folio *folio, u64 *start, u32 *len)  in btrfs_subpage_clamp_range()  argument
227 	*start = max_t(u64, folio_pos(folio), orig_start);  in btrfs_subpage_clamp_range()
233 	if (folio_pos(folio) >= orig_start + orig_len)  in btrfs_subpage_clamp_range()
236 		*len = min_t(u64, folio_pos(folio) + PAGE_SIZE,  in btrfs_subpage_clamp_range()
241 					    struct folio *folio, u64 start, u32 len)  in btrfs_subpage_end_and_test_lock()  argument
243 	struct btrfs_subpage *subpage = folio_get_private(folio);  in btrfs_subpage_end_and_test_lock()
244 	const int start_bit = subpage_calc_start_bit(fs_info, folio, locked, start, len);  in btrfs_subpage_end_and_test_lock()
251 	btrfs_subpage_assert(fs_info, folio, start, len);  in btrfs_subpage_end_and_test_lock()
293 			  struct folio *folio, u64 start, u32 len)  in btrfs_folio_end_lock()  argument
295 	struct btrfs_subpage *subpage = folio_get_private(folio);  in btrfs_folio_end_lock()
297 	ASSERT(folio_test_locked(folio));  in btrfs_folio_end_lock()
299 	if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, folio->mapping)) {  in btrfs_folio_end_lock()
300 		folio_unlock(folio);  in btrfs_folio_end_lock()
313 		folio_unlock(folio);  in btrfs_folio_end_lock()
317 	btrfs_subpage_clamp_range(folio, &start, &len);  in btrfs_folio_end_lock()
318 	if (btrfs_subpage_end_and_test_lock(fs_info, folio, start, len))  in btrfs_folio_end_lock()
319 		folio_unlock(folio);  in btrfs_folio_end_lock()
323 				 struct folio *folio, unsigned long bitmap)  in btrfs_folio_end_lock_bitmap()  argument
325 	struct btrfs_subpage *subpage = folio_get_private(folio);  in btrfs_folio_end_lock_bitmap()
332 	if (!btrfs_is_subpage(fs_info, folio->mapping)) {  in btrfs_folio_end_lock_bitmap()
333 		folio_unlock(folio);  in btrfs_folio_end_lock_bitmap()
339 		folio_unlock(folio);  in btrfs_folio_end_lock_bitmap()
352 		folio_unlock(folio);  in btrfs_folio_end_lock_bitmap()
366 				struct folio *folio, u64 start, u32 len)  in btrfs_subpage_set_uptodate()  argument
368 	struct btrfs_subpage *subpage = folio_get_private(folio);  in btrfs_subpage_set_uptodate()
369 	unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,  in btrfs_subpage_set_uptodate()
376 		folio_mark_uptodate(folio);  in btrfs_subpage_set_uptodate()
381 				  struct folio *folio, u64 start, u32 len)  in btrfs_subpage_clear_uptodate()  argument
383 	struct btrfs_subpage *subpage = folio_get_private(folio);  in btrfs_subpage_clear_uptodate()
384 	unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,  in btrfs_subpage_clear_uptodate()
390 	folio_clear_uptodate(folio);  in btrfs_subpage_clear_uptodate()
395 			     struct folio *folio, u64 start, u32 len)  in btrfs_subpage_set_dirty()  argument
397 	struct btrfs_subpage *subpage = folio_get_private(folio);  in btrfs_subpage_set_dirty()
398 	unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,  in btrfs_subpage_set_dirty()
405 	folio_mark_dirty(folio);  in btrfs_subpage_set_dirty()
419 					struct folio *folio, u64 start, u32 len)  in btrfs_subpage_clear_and_test_dirty()  argument
421 	struct btrfs_subpage *subpage = folio_get_private(folio);  in btrfs_subpage_clear_and_test_dirty()
422 	unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,  in btrfs_subpage_clear_and_test_dirty()
436 			       struct folio *folio, u64 start, u32 len)  in btrfs_subpage_clear_dirty()  argument
440 	last = btrfs_subpage_clear_and_test_dirty(fs_info, folio, start, len);  in btrfs_subpage_clear_dirty()
442 		folio_clear_dirty_for_io(folio);  in btrfs_subpage_clear_dirty()
446 				 struct folio *folio, u64 start, u32 len)  in btrfs_subpage_set_writeback()  argument
448 	struct btrfs_subpage *subpage = folio_get_private(folio);  in btrfs_subpage_set_writeback()
449 	unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,  in btrfs_subpage_set_writeback()
455 	if (!folio_test_writeback(folio))  in btrfs_subpage_set_writeback()
456 		folio_start_writeback(folio);  in btrfs_subpage_set_writeback()
461 				   struct folio *folio, u64 start, u32 len)  in btrfs_subpage_clear_writeback()  argument
463 	struct btrfs_subpage *subpage = folio_get_private(folio);  in btrfs_subpage_clear_writeback()
464 	unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,  in btrfs_subpage_clear_writeback()
471 		ASSERT(folio_test_writeback(folio));  in btrfs_subpage_clear_writeback()
472 		folio_end_writeback(folio);  in btrfs_subpage_clear_writeback()
478 			       struct folio *folio, u64 start, u32 len)  in btrfs_subpage_set_ordered()  argument
480 	struct btrfs_subpage *subpage = folio_get_private(folio);  in btrfs_subpage_set_ordered()
481 	unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,  in btrfs_subpage_set_ordered()
487 	folio_set_ordered(folio);  in btrfs_subpage_set_ordered()
492 				 struct folio *folio, u64 start, u32 len)  in btrfs_subpage_clear_ordered()  argument
494 	struct btrfs_subpage *subpage = folio_get_private(folio);  in btrfs_subpage_clear_ordered()
495 	unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,  in btrfs_subpage_clear_ordered()
502 		folio_clear_ordered(folio);  in btrfs_subpage_clear_ordered()
507 			       struct folio *folio, u64 start, u32 len)  in btrfs_subpage_set_checked()  argument
509 	struct btrfs_subpage *subpage = folio_get_private(folio);  in btrfs_subpage_set_checked()
510 	unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,  in btrfs_subpage_set_checked()
517 		folio_set_checked(folio);  in btrfs_subpage_set_checked()
522 				 struct folio *folio, u64 start, u32 len)  in btrfs_subpage_clear_checked()  argument
524 	struct btrfs_subpage *subpage = folio_get_private(folio);  in btrfs_subpage_clear_checked()
525 	unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,  in btrfs_subpage_clear_checked()
531 	folio_clear_checked(folio);  in btrfs_subpage_clear_checked()
541 			       struct folio *folio, u64 start, u32 len)	\
543 	struct btrfs_subpage *subpage = folio_get_private(folio);	\
544 	unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,	\
569 			    struct folio *folio, u64 start, u32 len)	\
572 	    !btrfs_is_subpage(fs_info, folio->mapping)) {		\
573 		folio_set_func(folio);					\
576 	btrfs_subpage_set_##name(fs_info, folio, start, len);		\
579 			      struct folio *folio, u64 start, u32 len)	\
582 	    !btrfs_is_subpage(fs_info, folio->mapping)) {		\
583 		folio_clear_func(folio);				\
586 	btrfs_subpage_clear_##name(fs_info, folio, start, len);		\
589 			     struct folio *folio, u64 start, u32 len)	\
592 	    !btrfs_is_subpage(fs_info, folio->mapping))			\
593 		return folio_test_func(folio);				\
594 	return btrfs_subpage_test_##name(fs_info, folio, start, len);	\
597 				  struct folio *folio, u64 start, u32 len) \
600 	    !btrfs_is_subpage(fs_info, folio->mapping)) {		\
601 		folio_set_func(folio);					\
604 	btrfs_subpage_clamp_range(folio, &start, &len);			\
605 	btrfs_subpage_set_##name(fs_info, folio, start, len);		\
608 				    struct folio *folio, u64 start, u32 len) \
611 	    !btrfs_is_subpage(fs_info, folio->mapping)) {		\
612 		folio_clear_func(folio);				\
615 	btrfs_subpage_clamp_range(folio, &start, &len);			\
616 	btrfs_subpage_clear_##name(fs_info, folio, start, len);		\
619 				   struct folio *folio, u64 start, u32 len) \
622 	    !btrfs_is_subpage(fs_info, folio->mapping))			\
623 		return folio_test_func(folio);				\
624 	btrfs_subpage_clamp_range(folio, &start, &len);			\
625 	return btrfs_subpage_test_##name(fs_info, folio, start, len);	\
648 #define SUBPAGE_DUMP_BITMAP(fs_info, folio, name, start, len)		\  argument
650 	const struct btrfs_subpage *subpage = folio_get_private(folio);	\
656 		   start, len, folio_pos(folio),			\
665 				  struct folio *folio, u64 start, u32 len)  in btrfs_folio_assert_not_dirty()  argument
675 	if (!btrfs_is_subpage(fs_info, folio->mapping)) {  in btrfs_folio_assert_not_dirty()
676 		ASSERT(!folio_test_dirty(folio));  in btrfs_folio_assert_not_dirty()
680 	start_bit = subpage_calc_start_bit(fs_info, folio, dirty, start, len);  in btrfs_folio_assert_not_dirty()
682 	subpage = folio_get_private(folio);  in btrfs_folio_assert_not_dirty()
686 		SUBPAGE_DUMP_BITMAP(fs_info, folio, dirty, start, len);  in btrfs_folio_assert_not_dirty()
701 			  struct folio *folio, u64 start, u32 len)  in btrfs_folio_set_lock()  argument
709 	ASSERT(folio_test_locked(folio));  in btrfs_folio_set_lock()
710 	if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, folio->mapping))  in btrfs_folio_set_lock()
713 	subpage = folio_get_private(folio);  in btrfs_folio_set_lock()
714 	start_bit = subpage_calc_start_bit(fs_info, folio, locked, start, len);  in btrfs_folio_set_lock()
719 		SUBPAGE_DUMP_BITMAP(fs_info, folio, locked, start, len);  in btrfs_folio_set_lock()
729 				      struct folio *folio, u64 start, u32 len)  in btrfs_subpage_dump_bitmap()  argument
741 	ASSERT(folio_test_private(folio) && folio_get_private(folio));  in btrfs_subpage_dump_bitmap()
743 	subpage = folio_get_private(folio);  in btrfs_subpage_dump_bitmap()
754 	dump_page(folio_page(folio, 0), "btrfs subpage dump");  in btrfs_subpage_dump_bitmap()
757 		    start, len, folio_pos(folio),  in btrfs_subpage_dump_bitmap()
767 				    struct folio *folio,  in btrfs_get_subpage_dirty_bitmap()  argument
773 	ASSERT(folio_test_private(folio) && folio_get_private(folio));  in btrfs_get_subpage_dirty_bitmap()
775 	subpage = folio_get_private(folio);  in btrfs_get_subpage_dirty_bitmap()