1cac06d84SQu Wenruo // SPDX-License-Identifier: GPL-2.0
2cac06d84SQu Wenruo
3cac06d84SQu Wenruo #include <linux/slab.h>
49b569ea0SJosef Bacik #include "messages.h"
5cac06d84SQu Wenruo #include "ctree.h"
6cac06d84SQu Wenruo #include "subpage.h"
73d078efaSQu Wenruo #include "btrfs_inode.h"
8cac06d84SQu Wenruo
9894d1378SQu Wenruo /*
10894d1378SQu Wenruo * Subpage (sectorsize < PAGE_SIZE) support overview:
11894d1378SQu Wenruo *
12894d1378SQu Wenruo * Limitations:
13894d1378SQu Wenruo *
14894d1378SQu Wenruo * - Only support 64K page size for now
15894d1378SQu Wenruo * This is to make metadata handling easier, as 64K page would ensure
16894d1378SQu Wenruo * all nodesize would fit inside one page, thus we don't need to handle
17894d1378SQu Wenruo * cases where a tree block crosses several pages.
18894d1378SQu Wenruo *
19894d1378SQu Wenruo * - Only metadata read-write for now
20894d1378SQu Wenruo * The data read-write part is in development.
21894d1378SQu Wenruo *
22894d1378SQu Wenruo * - Metadata can't cross 64K page boundary
23894d1378SQu Wenruo * btrfs-progs and kernel have done that for a while, thus only ancient
24894d1378SQu Wenruo * filesystems could have such problem. For such case, do a graceful
25894d1378SQu Wenruo * rejection.
26894d1378SQu Wenruo *
27894d1378SQu Wenruo * Special behavior:
28894d1378SQu Wenruo *
29894d1378SQu Wenruo * - Metadata
30894d1378SQu Wenruo * Metadata read is fully supported.
31894d1378SQu Wenruo * Meaning when reading one tree block will only trigger the read for the
32894d1378SQu Wenruo * needed range, other unrelated range in the same page will not be touched.
33894d1378SQu Wenruo *
34894d1378SQu Wenruo * Metadata write support is partial.
35894d1378SQu Wenruo * The writeback is still for the full page, but we will only submit
36894d1378SQu Wenruo * the dirty extent buffers in the page.
37894d1378SQu Wenruo *
38894d1378SQu Wenruo * This means, if we have a metadata page like this:
39894d1378SQu Wenruo *
40894d1378SQu Wenruo * Page offset
41894d1378SQu Wenruo * 0 16K 32K 48K 64K
42894d1378SQu Wenruo * |/////////| |///////////|
43894d1378SQu Wenruo * \- Tree block A \- Tree block B
44894d1378SQu Wenruo *
45894d1378SQu Wenruo * Even if we just want to writeback tree block A, we will also writeback
46894d1378SQu Wenruo * tree block B if it's also dirty.
47894d1378SQu Wenruo *
48894d1378SQu Wenruo * This may cause extra metadata writeback which results more COW.
49894d1378SQu Wenruo *
50894d1378SQu Wenruo * Implementation:
51894d1378SQu Wenruo *
52894d1378SQu Wenruo * - Common
53894d1378SQu Wenruo * Both metadata and data will use a new structure, btrfs_subpage, to
54894d1378SQu Wenruo * record the status of each sector inside a page. This provides the extra
55894d1378SQu Wenruo * granularity needed.
56894d1378SQu Wenruo *
57894d1378SQu Wenruo * - Metadata
58894d1378SQu Wenruo * Since we have multiple tree blocks inside one page, we can't rely on page
59894d1378SQu Wenruo * locking anymore, or we will have greatly reduced concurrency or even
60894d1378SQu Wenruo * deadlocks (hold one tree lock while trying to lock another tree lock in
61894d1378SQu Wenruo * the same page).
62894d1378SQu Wenruo *
63894d1378SQu Wenruo * Thus for metadata locking, subpage support relies on io_tree locking only.
64894d1378SQu Wenruo * This means a slightly higher tree locking latency.
65894d1378SQu Wenruo */
66894d1378SQu Wenruo
67efffb803SQu Wenruo #if PAGE_SIZE > SZ_4K
btrfs_is_subpage(const struct btrfs_fs_info * fs_info,struct address_space * mapping)6813df3775SQu Wenruo bool btrfs_is_subpage(const struct btrfs_fs_info *fs_info, struct address_space *mapping)
69fbca46ebSQu Wenruo {
70fbca46ebSQu Wenruo if (fs_info->sectorsize >= PAGE_SIZE)
71fbca46ebSQu Wenruo return false;
72fbca46ebSQu Wenruo
73fbca46ebSQu Wenruo /*
74fbca46ebSQu Wenruo * Only data pages (either through DIO or compression) can have no
75fbca46ebSQu Wenruo * mapping. And if page->mapping->host is data inode, it's subpage.
76fbca46ebSQu Wenruo * As we have ruled our sectorsize >= PAGE_SIZE case already.
77fbca46ebSQu Wenruo */
788610ba7eSDavid Sterba if (!mapping || !mapping->host || is_data_inode(BTRFS_I(mapping->host)))
79fbca46ebSQu Wenruo return true;
80fbca46ebSQu Wenruo
81fbca46ebSQu Wenruo /*
82fbca46ebSQu Wenruo * Now the only remaining case is metadata, which we only go subpage
83fbca46ebSQu Wenruo * routine if nodesize < PAGE_SIZE.
84fbca46ebSQu Wenruo */
85fbca46ebSQu Wenruo if (fs_info->nodesize < PAGE_SIZE)
86fbca46ebSQu Wenruo return true;
87fbca46ebSQu Wenruo return false;
88fbca46ebSQu Wenruo }
89efffb803SQu Wenruo #endif
90fbca46ebSQu Wenruo
btrfs_attach_subpage(const struct btrfs_fs_info * fs_info,struct folio * folio,enum btrfs_subpage_type type)91cac06d84SQu Wenruo int btrfs_attach_subpage(const struct btrfs_fs_info *fs_info,
9255151ea9SQu Wenruo struct folio *folio, enum btrfs_subpage_type type)
93cac06d84SQu Wenruo {
94651fb419SQu Wenruo struct btrfs_subpage *subpage;
95cac06d84SQu Wenruo
96cac06d84SQu Wenruo /*
97143823cfSDavid Sterba * We have cases like a dummy extent buffer page, which is not mapped
98cac06d84SQu Wenruo * and doesn't need to be locked.
99cac06d84SQu Wenruo */
10055151ea9SQu Wenruo if (folio->mapping)
10155151ea9SQu Wenruo ASSERT(folio_test_locked(folio));
102651fb419SQu Wenruo
103cfbf07e2SQu Wenruo /* Either not subpage, or the folio already has private attached. */
10455151ea9SQu Wenruo if (!btrfs_is_subpage(fs_info, folio->mapping) || folio_test_private(folio))
105cac06d84SQu Wenruo return 0;
106cac06d84SQu Wenruo
107651fb419SQu Wenruo subpage = btrfs_alloc_subpage(fs_info, type);
108651fb419SQu Wenruo if (IS_ERR(subpage))
109651fb419SQu Wenruo return PTR_ERR(subpage);
110651fb419SQu Wenruo
111cfbf07e2SQu Wenruo folio_attach_private(folio, subpage);
112cac06d84SQu Wenruo return 0;
113cac06d84SQu Wenruo }
114cac06d84SQu Wenruo
btrfs_detach_subpage(const struct btrfs_fs_info * fs_info,struct folio * folio)11555151ea9SQu Wenruo void btrfs_detach_subpage(const struct btrfs_fs_info *fs_info, struct folio *folio)
116cac06d84SQu Wenruo {
117cac06d84SQu Wenruo struct btrfs_subpage *subpage;
118cac06d84SQu Wenruo
119cfbf07e2SQu Wenruo /* Either not subpage, or the folio already has private attached. */
12055151ea9SQu Wenruo if (!btrfs_is_subpage(fs_info, folio->mapping) || !folio_test_private(folio))
121cac06d84SQu Wenruo return;
122cac06d84SQu Wenruo
123cfbf07e2SQu Wenruo subpage = folio_detach_private(folio);
124cac06d84SQu Wenruo ASSERT(subpage);
125760f991fSQu Wenruo btrfs_free_subpage(subpage);
126760f991fSQu Wenruo }
127760f991fSQu Wenruo
btrfs_alloc_subpage(const struct btrfs_fs_info * fs_info,enum btrfs_subpage_type type)128651fb419SQu Wenruo struct btrfs_subpage *btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info,
129760f991fSQu Wenruo enum btrfs_subpage_type type)
130760f991fSQu Wenruo {
131651fb419SQu Wenruo struct btrfs_subpage *ret;
13272a69cd0SQu Wenruo unsigned int real_size;
133651fb419SQu Wenruo
134fdf250dbSQu Wenruo ASSERT(fs_info->sectorsize < PAGE_SIZE);
135760f991fSQu Wenruo
13672a69cd0SQu Wenruo real_size = struct_size(ret, bitmaps,
137ce4a71eeSQu Wenruo BITS_TO_LONGS(btrfs_bitmap_nr_max * fs_info->sectors_per_page));
13872a69cd0SQu Wenruo ret = kzalloc(real_size, GFP_NOFS);
139651fb419SQu Wenruo if (!ret)
140651fb419SQu Wenruo return ERR_PTR(-ENOMEM);
141651fb419SQu Wenruo
142651fb419SQu Wenruo spin_lock_init(&ret->lock);
1431e1de387SQu Wenruo if (type == BTRFS_SUBPAGE_METADATA) {
144651fb419SQu Wenruo atomic_set(&ret->eb_refs, 0);
1451e1de387SQu Wenruo } else {
146651fb419SQu Wenruo atomic_set(&ret->readers, 0);
147651fb419SQu Wenruo atomic_set(&ret->writers, 0);
1481e1de387SQu Wenruo }
149651fb419SQu Wenruo return ret;
150760f991fSQu Wenruo }
151760f991fSQu Wenruo
btrfs_free_subpage(struct btrfs_subpage * subpage)152760f991fSQu Wenruo void btrfs_free_subpage(struct btrfs_subpage *subpage)
153760f991fSQu Wenruo {
154cac06d84SQu Wenruo kfree(subpage);
155cac06d84SQu Wenruo }
1568ff8466dSQu Wenruo
1578ff8466dSQu Wenruo /*
1588ff8466dSQu Wenruo * Increase the eb_refs of current subpage.
1598ff8466dSQu Wenruo *
1608ff8466dSQu Wenruo * This is important for eb allocation, to prevent race with last eb freeing
1618ff8466dSQu Wenruo * of the same page.
1628ff8466dSQu Wenruo * With the eb_refs increased before the eb inserted into radix tree,
163cfbf07e2SQu Wenruo * detach_extent_buffer_page() won't detach the folio private while we're still
1648ff8466dSQu Wenruo * allocating the extent buffer.
1658ff8466dSQu Wenruo */
btrfs_folio_inc_eb_refs(const struct btrfs_fs_info * fs_info,struct folio * folio)16613df3775SQu Wenruo void btrfs_folio_inc_eb_refs(const struct btrfs_fs_info *fs_info, struct folio *folio)
1678ff8466dSQu Wenruo {
1688ff8466dSQu Wenruo struct btrfs_subpage *subpage;
1698ff8466dSQu Wenruo
17013df3775SQu Wenruo if (!btrfs_is_subpage(fs_info, folio->mapping))
1718ff8466dSQu Wenruo return;
1728ff8466dSQu Wenruo
17313df3775SQu Wenruo ASSERT(folio_test_private(folio) && folio->mapping);
174affc5af3SLinus Torvalds lockdep_assert_held(&folio->mapping->i_private_lock);
1758ff8466dSQu Wenruo
176cfbf07e2SQu Wenruo subpage = folio_get_private(folio);
1778ff8466dSQu Wenruo atomic_inc(&subpage->eb_refs);
1788ff8466dSQu Wenruo }
1798ff8466dSQu Wenruo
btrfs_folio_dec_eb_refs(const struct btrfs_fs_info * fs_info,struct folio * folio)18013df3775SQu Wenruo void btrfs_folio_dec_eb_refs(const struct btrfs_fs_info *fs_info, struct folio *folio)
1818ff8466dSQu Wenruo {
1828ff8466dSQu Wenruo struct btrfs_subpage *subpage;
1838ff8466dSQu Wenruo
18413df3775SQu Wenruo if (!btrfs_is_subpage(fs_info, folio->mapping))
1858ff8466dSQu Wenruo return;
1868ff8466dSQu Wenruo
18713df3775SQu Wenruo ASSERT(folio_test_private(folio) && folio->mapping);
188affc5af3SLinus Torvalds lockdep_assert_held(&folio->mapping->i_private_lock);
1898ff8466dSQu Wenruo
190cfbf07e2SQu Wenruo subpage = folio_get_private(folio);
1918ff8466dSQu Wenruo ASSERT(atomic_read(&subpage->eb_refs));
1928ff8466dSQu Wenruo atomic_dec(&subpage->eb_refs);
1938ff8466dSQu Wenruo }
194a1d767c1SQu Wenruo
btrfs_subpage_assert(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)19592082d40SQu Wenruo static void btrfs_subpage_assert(const struct btrfs_fs_info *fs_info,
19655151ea9SQu Wenruo struct folio *folio, u64 start, u32 len)
19792082d40SQu Wenruo {
19855151ea9SQu Wenruo /* For subpage support, the folio must be single page. */
19955151ea9SQu Wenruo ASSERT(folio_order(folio) == 0);
200cfbf07e2SQu Wenruo
20192082d40SQu Wenruo /* Basic checks */
202cfbf07e2SQu Wenruo ASSERT(folio_test_private(folio) && folio_get_private(folio));
20392082d40SQu Wenruo ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&
20492082d40SQu Wenruo IS_ALIGNED(len, fs_info->sectorsize));
20592082d40SQu Wenruo /*
20692082d40SQu Wenruo * The range check only works for mapped page, we can still have
20792082d40SQu Wenruo * unmapped page like dummy extent buffer pages.
20892082d40SQu Wenruo */
20955151ea9SQu Wenruo if (folio->mapping)
21055151ea9SQu Wenruo ASSERT(folio_pos(folio) <= start &&
21155151ea9SQu Wenruo start + len <= folio_pos(folio) + PAGE_SIZE);
21292082d40SQu Wenruo }
21392082d40SQu Wenruo
2148e7e9c67SQu Wenruo #define subpage_calc_start_bit(fs_info, folio, name, start, len) \
2158e7e9c67SQu Wenruo ({ \
21691629e6dSDavid Sterba unsigned int __start_bit; \
2178e7e9c67SQu Wenruo \
2188e7e9c67SQu Wenruo btrfs_subpage_assert(fs_info, folio, start, len); \
21991629e6dSDavid Sterba __start_bit = offset_in_page(start) >> fs_info->sectorsize_bits; \
220ce4a71eeSQu Wenruo __start_bit += fs_info->sectors_per_page * btrfs_bitmap_nr_##name; \
22191629e6dSDavid Sterba __start_bit; \
2228e7e9c67SQu Wenruo })
2238e7e9c67SQu Wenruo
btrfs_subpage_start_reader(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)22492082d40SQu Wenruo void btrfs_subpage_start_reader(const struct btrfs_fs_info *fs_info,
22555151ea9SQu Wenruo struct folio *folio, u64 start, u32 len)
22692082d40SQu Wenruo {
227cfbf07e2SQu Wenruo struct btrfs_subpage *subpage = folio_get_private(folio);
2288e7e9c67SQu Wenruo const int start_bit = subpage_calc_start_bit(fs_info, folio, locked, start, len);
22992082d40SQu Wenruo const int nbits = len >> fs_info->sectorsize_bits;
2308e7e9c67SQu Wenruo unsigned long flags;
2318e7e9c67SQu Wenruo
23292082d40SQu Wenruo
23355151ea9SQu Wenruo btrfs_subpage_assert(fs_info, folio, start, len);
23492082d40SQu Wenruo
2358e7e9c67SQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
2368e7e9c67SQu Wenruo /*
2378e7e9c67SQu Wenruo * Even though it's just for reading the page, no one should have
2388e7e9c67SQu Wenruo * locked the subpage range.
2398e7e9c67SQu Wenruo */
2408e7e9c67SQu Wenruo ASSERT(bitmap_test_range_all_zero(subpage->bitmaps, start_bit, nbits));
2418e7e9c67SQu Wenruo bitmap_set(subpage->bitmaps, start_bit, nbits);
2423d078efaSQu Wenruo atomic_add(nbits, &subpage->readers);
2438e7e9c67SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
24492082d40SQu Wenruo }
24592082d40SQu Wenruo
btrfs_subpage_end_reader(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)24692082d40SQu Wenruo void btrfs_subpage_end_reader(const struct btrfs_fs_info *fs_info,
24755151ea9SQu Wenruo struct folio *folio, u64 start, u32 len)
24892082d40SQu Wenruo {
249cfbf07e2SQu Wenruo struct btrfs_subpage *subpage = folio_get_private(folio);
2508e7e9c67SQu Wenruo const int start_bit = subpage_calc_start_bit(fs_info, folio, locked, start, len);
25192082d40SQu Wenruo const int nbits = len >> fs_info->sectorsize_bits;
2528e7e9c67SQu Wenruo unsigned long flags;
2533d078efaSQu Wenruo bool is_data;
2543d078efaSQu Wenruo bool last;
25592082d40SQu Wenruo
25655151ea9SQu Wenruo btrfs_subpage_assert(fs_info, folio, start, len);
2578610ba7eSDavid Sterba is_data = is_data_inode(BTRFS_I(folio->mapping->host));
2588e7e9c67SQu Wenruo
2598e7e9c67SQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
2608e7e9c67SQu Wenruo
2618e7e9c67SQu Wenruo /* The range should have already been locked. */
2628e7e9c67SQu Wenruo ASSERT(bitmap_test_range_all_set(subpage->bitmaps, start_bit, nbits));
26392082d40SQu Wenruo ASSERT(atomic_read(&subpage->readers) >= nbits);
2648e7e9c67SQu Wenruo
2658e7e9c67SQu Wenruo bitmap_clear(subpage->bitmaps, start_bit, nbits);
2663d078efaSQu Wenruo last = atomic_sub_and_test(nbits, &subpage->readers);
2673d078efaSQu Wenruo
2683d078efaSQu Wenruo /*
2693d078efaSQu Wenruo * For data we need to unlock the page if the last read has finished.
2703d078efaSQu Wenruo *
2713d078efaSQu Wenruo * And please don't replace @last with atomic_sub_and_test() call
2723d078efaSQu Wenruo * inside if () condition.
2733d078efaSQu Wenruo * As we want the atomic_sub_and_test() to be always executed.
2743d078efaSQu Wenruo */
2753d078efaSQu Wenruo if (is_data && last)
27655151ea9SQu Wenruo folio_unlock(folio);
2778e7e9c67SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
27892082d40SQu Wenruo }
27992082d40SQu Wenruo
btrfs_subpage_clamp_range(struct folio * folio,u64 * start,u32 * len)28055151ea9SQu Wenruo static void btrfs_subpage_clamp_range(struct folio *folio, u64 *start, u32 *len)
2811e1de387SQu Wenruo {
2821e1de387SQu Wenruo u64 orig_start = *start;
2831e1de387SQu Wenruo u32 orig_len = *len;
2841e1de387SQu Wenruo
28555151ea9SQu Wenruo *start = max_t(u64, folio_pos(folio), orig_start);
286e4f94347SQu Wenruo /*
287e4f94347SQu Wenruo * For certain call sites like btrfs_drop_pages(), we may have pages
288e4f94347SQu Wenruo * beyond the target range. In that case, just set @len to 0, subpage
289e4f94347SQu Wenruo * helpers can handle @len == 0 without any problem.
290e4f94347SQu Wenruo */
29155151ea9SQu Wenruo if (folio_pos(folio) >= orig_start + orig_len)
292e4f94347SQu Wenruo *len = 0;
293e4f94347SQu Wenruo else
29455151ea9SQu Wenruo *len = min_t(u64, folio_pos(folio) + PAGE_SIZE,
2951e1de387SQu Wenruo orig_start + orig_len) - *start;
2961e1de387SQu Wenruo }
2971e1de387SQu Wenruo
btrfs_subpage_start_writer(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)298621b9ff1SQu Wenruo static void btrfs_subpage_start_writer(const struct btrfs_fs_info *fs_info,
29955151ea9SQu Wenruo struct folio *folio, u64 start, u32 len)
3001e1de387SQu Wenruo {
301cfbf07e2SQu Wenruo struct btrfs_subpage *subpage = folio_get_private(folio);
302b086c5bdSQu Wenruo const int start_bit = subpage_calc_start_bit(fs_info, folio, locked, start, len);
3031e1de387SQu Wenruo const int nbits = (len >> fs_info->sectorsize_bits);
304b086c5bdSQu Wenruo unsigned long flags;
3051e1de387SQu Wenruo int ret;
3061e1de387SQu Wenruo
30755151ea9SQu Wenruo btrfs_subpage_assert(fs_info, folio, start, len);
3081e1de387SQu Wenruo
309b086c5bdSQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
3101e1de387SQu Wenruo ASSERT(atomic_read(&subpage->readers) == 0);
311b086c5bdSQu Wenruo ASSERT(bitmap_test_range_all_zero(subpage->bitmaps, start_bit, nbits));
312b086c5bdSQu Wenruo bitmap_set(subpage->bitmaps, start_bit, nbits);
3131e1de387SQu Wenruo ret = atomic_add_return(nbits, &subpage->writers);
3141e1de387SQu Wenruo ASSERT(ret == nbits);
315b086c5bdSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
3161e1de387SQu Wenruo }
3171e1de387SQu Wenruo
btrfs_subpage_end_and_test_writer(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)318621b9ff1SQu Wenruo static bool btrfs_subpage_end_and_test_writer(const struct btrfs_fs_info *fs_info,
31955151ea9SQu Wenruo struct folio *folio, u64 start, u32 len)
3201e1de387SQu Wenruo {
321cfbf07e2SQu Wenruo struct btrfs_subpage *subpage = folio_get_private(folio);
322b086c5bdSQu Wenruo const int start_bit = subpage_calc_start_bit(fs_info, folio, locked, start, len);
3231e1de387SQu Wenruo const int nbits = (len >> fs_info->sectorsize_bits);
324b086c5bdSQu Wenruo unsigned long flags;
325ab6eac7cSQu Wenruo unsigned int cleared = 0;
326ab6eac7cSQu Wenruo int bit = start_bit;
327b086c5bdSQu Wenruo bool last;
3281e1de387SQu Wenruo
32955151ea9SQu Wenruo btrfs_subpage_assert(fs_info, folio, start, len);
3301e1de387SQu Wenruo
331b086c5bdSQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
332164674a7SQu Wenruo /*
333164674a7SQu Wenruo * We have call sites passing @lock_page into
334164674a7SQu Wenruo * extent_clear_unlock_delalloc() for compression path.
335164674a7SQu Wenruo *
336164674a7SQu Wenruo * This @locked_page is locked by plain lock_page(), thus its
337164674a7SQu Wenruo * subpage::writers is 0. Handle them in a special way.
338164674a7SQu Wenruo */
339b086c5bdSQu Wenruo if (atomic_read(&subpage->writers) == 0) {
340b086c5bdSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
341164674a7SQu Wenruo return true;
342b086c5bdSQu Wenruo }
343164674a7SQu Wenruo
344ab6eac7cSQu Wenruo for_each_set_bit_from(bit, subpage->bitmaps, start_bit + nbits) {
345ab6eac7cSQu Wenruo clear_bit(bit, subpage->bitmaps);
346ab6eac7cSQu Wenruo cleared++;
347ab6eac7cSQu Wenruo }
348ab6eac7cSQu Wenruo ASSERT(atomic_read(&subpage->writers) >= cleared);
349ab6eac7cSQu Wenruo last = atomic_sub_and_test(cleared, &subpage->writers);
350b086c5bdSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
351b086c5bdSQu Wenruo return last;
3521e1de387SQu Wenruo }
3531e1de387SQu Wenruo
3541e1de387SQu Wenruo /*
35555151ea9SQu Wenruo * Lock a folio for delalloc page writeback.
3561e1de387SQu Wenruo *
3571e1de387SQu Wenruo * Return -EAGAIN if the page is not properly initialized.
3581e1de387SQu Wenruo * Return 0 with the page locked, and writer counter updated.
3591e1de387SQu Wenruo *
3601e1de387SQu Wenruo * Even with 0 returned, the page still need extra check to make sure
3611e1de387SQu Wenruo * it's really the correct page, as the caller is using
36247d55419SVishal Moola (Oracle) * filemap_get_folios_contig(), which can race with page invalidating.
3631e1de387SQu Wenruo */
btrfs_folio_start_writer_lock(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)36455151ea9SQu Wenruo int btrfs_folio_start_writer_lock(const struct btrfs_fs_info *fs_info,
36555151ea9SQu Wenruo struct folio *folio, u64 start, u32 len)
3661e1de387SQu Wenruo {
36755151ea9SQu Wenruo if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, folio->mapping)) {
36855151ea9SQu Wenruo folio_lock(folio);
3691e1de387SQu Wenruo return 0;
3701e1de387SQu Wenruo }
37155151ea9SQu Wenruo folio_lock(folio);
372cfbf07e2SQu Wenruo if (!folio_test_private(folio) || !folio_get_private(folio)) {
37355151ea9SQu Wenruo folio_unlock(folio);
3741e1de387SQu Wenruo return -EAGAIN;
3751e1de387SQu Wenruo }
37655151ea9SQu Wenruo btrfs_subpage_clamp_range(folio, &start, &len);
37755151ea9SQu Wenruo btrfs_subpage_start_writer(fs_info, folio, start, len);
3781e1de387SQu Wenruo return 0;
3791e1de387SQu Wenruo }
3801e1de387SQu Wenruo
38149a99073SQu Wenruo /*
38249a99073SQu Wenruo * Handle different locked folios:
38349a99073SQu Wenruo *
38449a99073SQu Wenruo * - Non-subpage folio
38549a99073SQu Wenruo * Just unlock it.
38649a99073SQu Wenruo *
38749a99073SQu Wenruo * - folio locked but without any subpage locked
38849a99073SQu Wenruo * This happens either before writepage_delalloc() or the delalloc range is
38949a99073SQu Wenruo * already handled by previous folio.
39049a99073SQu Wenruo * We can simple unlock it.
39149a99073SQu Wenruo *
39249a99073SQu Wenruo * - folio locked with subpage range locked.
39349a99073SQu Wenruo * We go through the locked sectors inside the range and clear their locked
39449a99073SQu Wenruo * bitmap, reduce the writer lock number, and unlock the page if that's
39549a99073SQu Wenruo * the last locked range.
39649a99073SQu Wenruo */
btrfs_folio_end_writer_lock(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)39755151ea9SQu Wenruo void btrfs_folio_end_writer_lock(const struct btrfs_fs_info *fs_info,
39855151ea9SQu Wenruo struct folio *folio, u64 start, u32 len)
3991e1de387SQu Wenruo {
40049a99073SQu Wenruo struct btrfs_subpage *subpage = folio_get_private(folio);
40149a99073SQu Wenruo
40249a99073SQu Wenruo ASSERT(folio_test_locked(folio));
40349a99073SQu Wenruo
40455151ea9SQu Wenruo if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, folio->mapping)) {
40555151ea9SQu Wenruo folio_unlock(folio);
40655151ea9SQu Wenruo return;
40755151ea9SQu Wenruo }
40849a99073SQu Wenruo
40949a99073SQu Wenruo /*
41049a99073SQu Wenruo * For subpage case, there are two types of locked page. With or
41149a99073SQu Wenruo * without writers number.
41249a99073SQu Wenruo *
41349a99073SQu Wenruo * Since we own the page lock, no one else could touch subpage::writers
41449a99073SQu Wenruo * and we are safe to do several atomic operations without spinlock.
41549a99073SQu Wenruo */
41649a99073SQu Wenruo if (atomic_read(&subpage->writers) == 0) {
41749a99073SQu Wenruo /* No writers, locked by plain lock_page(). */
41849a99073SQu Wenruo folio_unlock(folio);
41949a99073SQu Wenruo return;
42049a99073SQu Wenruo }
42149a99073SQu Wenruo
42255151ea9SQu Wenruo btrfs_subpage_clamp_range(folio, &start, &len);
42355151ea9SQu Wenruo if (btrfs_subpage_end_and_test_writer(fs_info, folio, start, len))
42455151ea9SQu Wenruo folio_unlock(folio);
4251e1de387SQu Wenruo }
4261e1de387SQu Wenruo
btrfs_folio_end_writer_lock_bitmap(const struct btrfs_fs_info * fs_info,struct folio * folio,unsigned long bitmap)427*bd610c09SQu Wenruo void btrfs_folio_end_writer_lock_bitmap(const struct btrfs_fs_info *fs_info,
428*bd610c09SQu Wenruo struct folio *folio, unsigned long bitmap)
429*bd610c09SQu Wenruo {
430*bd610c09SQu Wenruo struct btrfs_subpage *subpage = folio_get_private(folio);
431*bd610c09SQu Wenruo const int start_bit = fs_info->sectors_per_page * btrfs_bitmap_nr_locked;
432*bd610c09SQu Wenruo unsigned long flags;
433*bd610c09SQu Wenruo bool last = false;
434*bd610c09SQu Wenruo int cleared = 0;
435*bd610c09SQu Wenruo int bit;
436*bd610c09SQu Wenruo
437*bd610c09SQu Wenruo if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, folio->mapping)) {
438*bd610c09SQu Wenruo folio_unlock(folio);
439*bd610c09SQu Wenruo return;
440*bd610c09SQu Wenruo }
441*bd610c09SQu Wenruo
442*bd610c09SQu Wenruo if (atomic_read(&subpage->writers) == 0) {
443*bd610c09SQu Wenruo /* No writers, locked by plain lock_page(). */
444*bd610c09SQu Wenruo folio_unlock(folio);
445*bd610c09SQu Wenruo return;
446*bd610c09SQu Wenruo }
447*bd610c09SQu Wenruo
448*bd610c09SQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
449*bd610c09SQu Wenruo for_each_set_bit(bit, &bitmap, fs_info->sectors_per_page) {
450*bd610c09SQu Wenruo if (test_and_clear_bit(bit + start_bit, subpage->bitmaps))
451*bd610c09SQu Wenruo cleared++;
452*bd610c09SQu Wenruo }
453*bd610c09SQu Wenruo ASSERT(atomic_read(&subpage->writers) >= cleared);
454*bd610c09SQu Wenruo last = atomic_sub_and_test(cleared, &subpage->writers);
455*bd610c09SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
456*bd610c09SQu Wenruo if (last)
457*bd610c09SQu Wenruo folio_unlock(folio);
458*bd610c09SQu Wenruo }
459*bd610c09SQu Wenruo
46072a69cd0SQu Wenruo #define subpage_test_bitmap_all_set(fs_info, subpage, name) \
46172a69cd0SQu Wenruo bitmap_test_range_all_set(subpage->bitmaps, \
462ce4a71eeSQu Wenruo fs_info->sectors_per_page * btrfs_bitmap_nr_##name, \
463ce4a71eeSQu Wenruo fs_info->sectors_per_page)
46472a69cd0SQu Wenruo
46572a69cd0SQu Wenruo #define subpage_test_bitmap_all_zero(fs_info, subpage, name) \
46672a69cd0SQu Wenruo bitmap_test_range_all_zero(subpage->bitmaps, \
467ce4a71eeSQu Wenruo fs_info->sectors_per_page * btrfs_bitmap_nr_##name, \
468ce4a71eeSQu Wenruo fs_info->sectors_per_page)
46972a69cd0SQu Wenruo
btrfs_subpage_set_uptodate(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)470a1d767c1SQu Wenruo void btrfs_subpage_set_uptodate(const struct btrfs_fs_info *fs_info,
47155151ea9SQu Wenruo struct folio *folio, u64 start, u32 len)
472a1d767c1SQu Wenruo {
473cfbf07e2SQu Wenruo struct btrfs_subpage *subpage = folio_get_private(folio);
47455151ea9SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,
47572a69cd0SQu Wenruo uptodate, start, len);
476a1d767c1SQu Wenruo unsigned long flags;
477a1d767c1SQu Wenruo
478a1d767c1SQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
47972a69cd0SQu Wenruo bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
48072a69cd0SQu Wenruo if (subpage_test_bitmap_all_set(fs_info, subpage, uptodate))
48155151ea9SQu Wenruo folio_mark_uptodate(folio);
482a1d767c1SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
483a1d767c1SQu Wenruo }
484a1d767c1SQu Wenruo
btrfs_subpage_clear_uptodate(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)485a1d767c1SQu Wenruo void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info,
48655151ea9SQu Wenruo struct folio *folio, u64 start, u32 len)
487a1d767c1SQu Wenruo {
488cfbf07e2SQu Wenruo struct btrfs_subpage *subpage = folio_get_private(folio);
48955151ea9SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,
49072a69cd0SQu Wenruo uptodate, start, len);
491a1d767c1SQu Wenruo unsigned long flags;
492a1d767c1SQu Wenruo
493a1d767c1SQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
49472a69cd0SQu Wenruo bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
49555151ea9SQu Wenruo folio_clear_uptodate(folio);
496a1d767c1SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
497a1d767c1SQu Wenruo }
498a1d767c1SQu Wenruo
btrfs_subpage_set_dirty(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)499d8a5713eSQu Wenruo void btrfs_subpage_set_dirty(const struct btrfs_fs_info *fs_info,
50055151ea9SQu Wenruo struct folio *folio, u64 start, u32 len)
501d8a5713eSQu Wenruo {
502cfbf07e2SQu Wenruo struct btrfs_subpage *subpage = folio_get_private(folio);
50355151ea9SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,
50472a69cd0SQu Wenruo dirty, start, len);
505d8a5713eSQu Wenruo unsigned long flags;
506d8a5713eSQu Wenruo
507d8a5713eSQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
50872a69cd0SQu Wenruo bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
509d8a5713eSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
51055151ea9SQu Wenruo folio_mark_dirty(folio);
511d8a5713eSQu Wenruo }
512d8a5713eSQu Wenruo
513d8a5713eSQu Wenruo /*
514d8a5713eSQu Wenruo * Extra clear_and_test function for subpage dirty bitmap.
515d8a5713eSQu Wenruo *
516d8a5713eSQu Wenruo * Return true if we're the last bits in the dirty_bitmap and clear the
517d8a5713eSQu Wenruo * dirty_bitmap.
518d8a5713eSQu Wenruo * Return false otherwise.
519d8a5713eSQu Wenruo *
520d8a5713eSQu Wenruo * NOTE: Callers should manually clear page dirty for true case, as we have
521d8a5713eSQu Wenruo * extra handling for tree blocks.
522d8a5713eSQu Wenruo */
btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)523d8a5713eSQu Wenruo bool btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info *fs_info,
52455151ea9SQu Wenruo struct folio *folio, u64 start, u32 len)
525d8a5713eSQu Wenruo {
526cfbf07e2SQu Wenruo struct btrfs_subpage *subpage = folio_get_private(folio);
52755151ea9SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,
52872a69cd0SQu Wenruo dirty, start, len);
529d8a5713eSQu Wenruo unsigned long flags;
530d8a5713eSQu Wenruo bool last = false;
531d8a5713eSQu Wenruo
532d8a5713eSQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
53372a69cd0SQu Wenruo bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
53472a69cd0SQu Wenruo if (subpage_test_bitmap_all_zero(fs_info, subpage, dirty))
535d8a5713eSQu Wenruo last = true;
536d8a5713eSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
537d8a5713eSQu Wenruo return last;
538d8a5713eSQu Wenruo }
539d8a5713eSQu Wenruo
btrfs_subpage_clear_dirty(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)540d8a5713eSQu Wenruo void btrfs_subpage_clear_dirty(const struct btrfs_fs_info *fs_info,
54155151ea9SQu Wenruo struct folio *folio, u64 start, u32 len)
542d8a5713eSQu Wenruo {
543d8a5713eSQu Wenruo bool last;
544d8a5713eSQu Wenruo
54555151ea9SQu Wenruo last = btrfs_subpage_clear_and_test_dirty(fs_info, folio, start, len);
546d8a5713eSQu Wenruo if (last)
54755151ea9SQu Wenruo folio_clear_dirty_for_io(folio);
548d8a5713eSQu Wenruo }
549d8a5713eSQu Wenruo
btrfs_subpage_set_writeback(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)5503470da3bSQu Wenruo void btrfs_subpage_set_writeback(const struct btrfs_fs_info *fs_info,
55155151ea9SQu Wenruo struct folio *folio, u64 start, u32 len)
5523470da3bSQu Wenruo {
553cfbf07e2SQu Wenruo struct btrfs_subpage *subpage = folio_get_private(folio);
55455151ea9SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,
55572a69cd0SQu Wenruo writeback, start, len);
5563470da3bSQu Wenruo unsigned long flags;
5573470da3bSQu Wenruo
5583470da3bSQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
55972a69cd0SQu Wenruo bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
5601e61b8c6SJosef Bacik if (!folio_test_writeback(folio))
56155151ea9SQu Wenruo folio_start_writeback(folio);
5623470da3bSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
5633470da3bSQu Wenruo }
5643470da3bSQu Wenruo
btrfs_subpage_clear_writeback(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)5653470da3bSQu Wenruo void btrfs_subpage_clear_writeback(const struct btrfs_fs_info *fs_info,
56655151ea9SQu Wenruo struct folio *folio, u64 start, u32 len)
5673470da3bSQu Wenruo {
568cfbf07e2SQu Wenruo struct btrfs_subpage *subpage = folio_get_private(folio);
56955151ea9SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,
57072a69cd0SQu Wenruo writeback, start, len);
5713470da3bSQu Wenruo unsigned long flags;
5723470da3bSQu Wenruo
5733470da3bSQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
57472a69cd0SQu Wenruo bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
57572a69cd0SQu Wenruo if (subpage_test_bitmap_all_zero(fs_info, subpage, writeback)) {
57655151ea9SQu Wenruo ASSERT(folio_test_writeback(folio));
57755151ea9SQu Wenruo folio_end_writeback(folio);
5787c11d0aeSQu Wenruo }
5793470da3bSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
5803470da3bSQu Wenruo }
5813470da3bSQu Wenruo
btrfs_subpage_set_ordered(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)5826f17400bSQu Wenruo void btrfs_subpage_set_ordered(const struct btrfs_fs_info *fs_info,
58355151ea9SQu Wenruo struct folio *folio, u64 start, u32 len)
5846f17400bSQu Wenruo {
585cfbf07e2SQu Wenruo struct btrfs_subpage *subpage = folio_get_private(folio);
58655151ea9SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,
58772a69cd0SQu Wenruo ordered, start, len);
5886f17400bSQu Wenruo unsigned long flags;
5896f17400bSQu Wenruo
5906f17400bSQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
59172a69cd0SQu Wenruo bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
59255151ea9SQu Wenruo folio_set_ordered(folio);
5936f17400bSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
5946f17400bSQu Wenruo }
5956f17400bSQu Wenruo
btrfs_subpage_clear_ordered(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)5966f17400bSQu Wenruo void btrfs_subpage_clear_ordered(const struct btrfs_fs_info *fs_info,
59755151ea9SQu Wenruo struct folio *folio, u64 start, u32 len)
5986f17400bSQu Wenruo {
599cfbf07e2SQu Wenruo struct btrfs_subpage *subpage = folio_get_private(folio);
60055151ea9SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,
60172a69cd0SQu Wenruo ordered, start, len);
6026f17400bSQu Wenruo unsigned long flags;
6036f17400bSQu Wenruo
6046f17400bSQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
60572a69cd0SQu Wenruo bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
60672a69cd0SQu Wenruo if (subpage_test_bitmap_all_zero(fs_info, subpage, ordered))
60755151ea9SQu Wenruo folio_clear_ordered(folio);
6086f17400bSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
6096f17400bSQu Wenruo }
610e4f94347SQu Wenruo
btrfs_subpage_set_checked(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)611e4f94347SQu Wenruo void btrfs_subpage_set_checked(const struct btrfs_fs_info *fs_info,
61255151ea9SQu Wenruo struct folio *folio, u64 start, u32 len)
613e4f94347SQu Wenruo {
614cfbf07e2SQu Wenruo struct btrfs_subpage *subpage = folio_get_private(folio);
61555151ea9SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,
616e4f94347SQu Wenruo checked, start, len);
617e4f94347SQu Wenruo unsigned long flags;
618e4f94347SQu Wenruo
619e4f94347SQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
620e4f94347SQu Wenruo bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
621e4f94347SQu Wenruo if (subpage_test_bitmap_all_set(fs_info, subpage, checked))
62255151ea9SQu Wenruo folio_set_checked(folio);
623e4f94347SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
624e4f94347SQu Wenruo }
625e4f94347SQu Wenruo
btrfs_subpage_clear_checked(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)626e4f94347SQu Wenruo void btrfs_subpage_clear_checked(const struct btrfs_fs_info *fs_info,
62755151ea9SQu Wenruo struct folio *folio, u64 start, u32 len)
628e4f94347SQu Wenruo {
629cfbf07e2SQu Wenruo struct btrfs_subpage *subpage = folio_get_private(folio);
63055151ea9SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,
631e4f94347SQu Wenruo checked, start, len);
632e4f94347SQu Wenruo unsigned long flags;
633e4f94347SQu Wenruo
634e4f94347SQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
635e4f94347SQu Wenruo bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
63655151ea9SQu Wenruo folio_clear_checked(folio);
637e4f94347SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
638e4f94347SQu Wenruo }
639e4f94347SQu Wenruo
640a1d767c1SQu Wenruo /*
641a1d767c1SQu Wenruo * Unlike set/clear which is dependent on each page status, for test all bits
642a1d767c1SQu Wenruo * are tested in the same way.
643a1d767c1SQu Wenruo */
644a1d767c1SQu Wenruo #define IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(name) \
645a1d767c1SQu Wenruo bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \
64655151ea9SQu Wenruo struct folio *folio, u64 start, u32 len) \
647a1d767c1SQu Wenruo { \
648cfbf07e2SQu Wenruo struct btrfs_subpage *subpage = folio_get_private(folio); \
64955151ea9SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, \
65072a69cd0SQu Wenruo name, start, len); \
651a1d767c1SQu Wenruo unsigned long flags; \
652a1d767c1SQu Wenruo bool ret; \
653a1d767c1SQu Wenruo \
654a1d767c1SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); \
65572a69cd0SQu Wenruo ret = bitmap_test_range_all_set(subpage->bitmaps, start_bit, \
65672a69cd0SQu Wenruo len >> fs_info->sectorsize_bits); \
657a1d767c1SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); \
658a1d767c1SQu Wenruo return ret; \
659a1d767c1SQu Wenruo }
660a1d767c1SQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate);
661d8a5713eSQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(dirty);
6623470da3bSQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(writeback);
6636f17400bSQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(ordered);
664e4f94347SQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(checked);
665a1d767c1SQu Wenruo
666a1d767c1SQu Wenruo /*
667a1d767c1SQu Wenruo * Note that, in selftests (extent-io-tests), we can have empty fs_info passed
668a1d767c1SQu Wenruo * in. We only test sectorsize == PAGE_SIZE cases so far, thus we can fall
669a1d767c1SQu Wenruo * back to regular sectorsize branch.
670a1d767c1SQu Wenruo */
67155151ea9SQu Wenruo #define IMPLEMENT_BTRFS_PAGE_OPS(name, folio_set_func, \
67255151ea9SQu Wenruo folio_clear_func, folio_test_func) \
67355151ea9SQu Wenruo void btrfs_folio_set_##name(const struct btrfs_fs_info *fs_info, \
67455151ea9SQu Wenruo struct folio *folio, u64 start, u32 len) \
675a1d767c1SQu Wenruo { \
67613df3775SQu Wenruo if (unlikely(!fs_info) || \
67755151ea9SQu Wenruo !btrfs_is_subpage(fs_info, folio->mapping)) { \
67855151ea9SQu Wenruo folio_set_func(folio); \
679a1d767c1SQu Wenruo return; \
680a1d767c1SQu Wenruo } \
68155151ea9SQu Wenruo btrfs_subpage_set_##name(fs_info, folio, start, len); \
682a1d767c1SQu Wenruo } \
68355151ea9SQu Wenruo void btrfs_folio_clear_##name(const struct btrfs_fs_info *fs_info, \
68455151ea9SQu Wenruo struct folio *folio, u64 start, u32 len) \
685a1d767c1SQu Wenruo { \
68613df3775SQu Wenruo if (unlikely(!fs_info) || \
68755151ea9SQu Wenruo !btrfs_is_subpage(fs_info, folio->mapping)) { \
68855151ea9SQu Wenruo folio_clear_func(folio); \
689a1d767c1SQu Wenruo return; \
690a1d767c1SQu Wenruo } \
69155151ea9SQu Wenruo btrfs_subpage_clear_##name(fs_info, folio, start, len); \
692a1d767c1SQu Wenruo } \
69355151ea9SQu Wenruo bool btrfs_folio_test_##name(const struct btrfs_fs_info *fs_info, \
69455151ea9SQu Wenruo struct folio *folio, u64 start, u32 len) \
695a1d767c1SQu Wenruo { \
69613df3775SQu Wenruo if (unlikely(!fs_info) || \
69755151ea9SQu Wenruo !btrfs_is_subpage(fs_info, folio->mapping)) \
69855151ea9SQu Wenruo return folio_test_func(folio); \
69955151ea9SQu Wenruo return btrfs_subpage_test_##name(fs_info, folio, start, len); \
70060e2d255SQu Wenruo } \
70155151ea9SQu Wenruo void btrfs_folio_clamp_set_##name(const struct btrfs_fs_info *fs_info, \
70255151ea9SQu Wenruo struct folio *folio, u64 start, u32 len) \
70360e2d255SQu Wenruo { \
70413df3775SQu Wenruo if (unlikely(!fs_info) || \
70555151ea9SQu Wenruo !btrfs_is_subpage(fs_info, folio->mapping)) { \
70655151ea9SQu Wenruo folio_set_func(folio); \
70760e2d255SQu Wenruo return; \
70860e2d255SQu Wenruo } \
70955151ea9SQu Wenruo btrfs_subpage_clamp_range(folio, &start, &len); \
71055151ea9SQu Wenruo btrfs_subpage_set_##name(fs_info, folio, start, len); \
71160e2d255SQu Wenruo } \
71255151ea9SQu Wenruo void btrfs_folio_clamp_clear_##name(const struct btrfs_fs_info *fs_info, \
71355151ea9SQu Wenruo struct folio *folio, u64 start, u32 len) \
71460e2d255SQu Wenruo { \
71513df3775SQu Wenruo if (unlikely(!fs_info) || \
71655151ea9SQu Wenruo !btrfs_is_subpage(fs_info, folio->mapping)) { \
71755151ea9SQu Wenruo folio_clear_func(folio); \
71860e2d255SQu Wenruo return; \
71960e2d255SQu Wenruo } \
72055151ea9SQu Wenruo btrfs_subpage_clamp_range(folio, &start, &len); \
72155151ea9SQu Wenruo btrfs_subpage_clear_##name(fs_info, folio, start, len); \
72260e2d255SQu Wenruo } \
72355151ea9SQu Wenruo bool btrfs_folio_clamp_test_##name(const struct btrfs_fs_info *fs_info, \
72455151ea9SQu Wenruo struct folio *folio, u64 start, u32 len) \
72560e2d255SQu Wenruo { \
72613df3775SQu Wenruo if (unlikely(!fs_info) || \
72755151ea9SQu Wenruo !btrfs_is_subpage(fs_info, folio->mapping)) \
72855151ea9SQu Wenruo return folio_test_func(folio); \
72955151ea9SQu Wenruo btrfs_subpage_clamp_range(folio, &start, &len); \
73055151ea9SQu Wenruo return btrfs_subpage_test_##name(fs_info, folio, start, len); \
731a1d767c1SQu Wenruo }
73255151ea9SQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(uptodate, folio_mark_uptodate, folio_clear_uptodate,
73355151ea9SQu Wenruo folio_test_uptodate);
73455151ea9SQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(dirty, folio_mark_dirty, folio_clear_dirty_for_io,
73555151ea9SQu Wenruo folio_test_dirty);
73655151ea9SQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(writeback, folio_start_writeback, folio_end_writeback,
73755151ea9SQu Wenruo folio_test_writeback);
73855151ea9SQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(ordered, folio_set_ordered, folio_clear_ordered,
73955151ea9SQu Wenruo folio_test_ordered);
74055151ea9SQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(checked, folio_set_checked, folio_clear_checked,
74155151ea9SQu Wenruo folio_test_checked);
742cc1d0d93SQu Wenruo
743cc1d0d93SQu Wenruo /*
744cc1d0d93SQu Wenruo * Make sure not only the page dirty bit is cleared, but also subpage dirty bit
745cc1d0d93SQu Wenruo * is cleared.
746cc1d0d93SQu Wenruo */
btrfs_folio_assert_not_dirty(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)74721b5bef2SQu Wenruo void btrfs_folio_assert_not_dirty(const struct btrfs_fs_info *fs_info,
74821b5bef2SQu Wenruo struct folio *folio, u64 start, u32 len)
749cc1d0d93SQu Wenruo {
75021b5bef2SQu Wenruo struct btrfs_subpage *subpage;
75121b5bef2SQu Wenruo unsigned int start_bit;
75221b5bef2SQu Wenruo unsigned int nbits;
75321b5bef2SQu Wenruo unsigned long flags;
754cc1d0d93SQu Wenruo
755cc1d0d93SQu Wenruo if (!IS_ENABLED(CONFIG_BTRFS_ASSERT))
756cc1d0d93SQu Wenruo return;
757cc1d0d93SQu Wenruo
75821b5bef2SQu Wenruo if (!btrfs_is_subpage(fs_info, folio->mapping)) {
75955151ea9SQu Wenruo ASSERT(!folio_test_dirty(folio));
760cc1d0d93SQu Wenruo return;
76121b5bef2SQu Wenruo }
762cc1d0d93SQu Wenruo
76321b5bef2SQu Wenruo start_bit = subpage_calc_start_bit(fs_info, folio, dirty, start, len);
76421b5bef2SQu Wenruo nbits = len >> fs_info->sectorsize_bits;
76521b5bef2SQu Wenruo subpage = folio_get_private(folio);
76621b5bef2SQu Wenruo ASSERT(subpage);
76721b5bef2SQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
76821b5bef2SQu Wenruo ASSERT(bitmap_test_range_all_zero(subpage->bitmaps, start_bit, nbits));
76921b5bef2SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
770cc1d0d93SQu Wenruo }
771e55a0de1SQu Wenruo
772e55a0de1SQu Wenruo /*
773bca707e5SQu Wenruo * This is for folio already locked by plain lock_page()/folio_lock(), which
774bca707e5SQu Wenruo * doesn't have any subpage awareness.
775bca707e5SQu Wenruo *
776bca707e5SQu Wenruo * This populates the involved subpage ranges so that subpage helpers can
777bca707e5SQu Wenruo * properly unlock them.
778bca707e5SQu Wenruo */
btrfs_folio_set_writer_lock(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)779bca707e5SQu Wenruo void btrfs_folio_set_writer_lock(const struct btrfs_fs_info *fs_info,
780bca707e5SQu Wenruo struct folio *folio, u64 start, u32 len)
781bca707e5SQu Wenruo {
782bca707e5SQu Wenruo struct btrfs_subpage *subpage;
783bca707e5SQu Wenruo unsigned long flags;
784bca707e5SQu Wenruo unsigned int start_bit;
785bca707e5SQu Wenruo unsigned int nbits;
786bca707e5SQu Wenruo int ret;
787bca707e5SQu Wenruo
788bca707e5SQu Wenruo ASSERT(folio_test_locked(folio));
789bca707e5SQu Wenruo if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, folio->mapping))
790bca707e5SQu Wenruo return;
791bca707e5SQu Wenruo
792bca707e5SQu Wenruo subpage = folio_get_private(folio);
793bca707e5SQu Wenruo start_bit = subpage_calc_start_bit(fs_info, folio, locked, start, len);
794bca707e5SQu Wenruo nbits = len >> fs_info->sectorsize_bits;
795bca707e5SQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
796bca707e5SQu Wenruo /* Target range should not yet be locked. */
797bca707e5SQu Wenruo ASSERT(bitmap_test_range_all_zero(subpage->bitmaps, start_bit, nbits));
798bca707e5SQu Wenruo bitmap_set(subpage->bitmaps, start_bit, nbits);
799bca707e5SQu Wenruo ret = atomic_add_return(nbits, &subpage->writers);
800ce4a71eeSQu Wenruo ASSERT(ret <= fs_info->sectors_per_page);
801bca707e5SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
802bca707e5SQu Wenruo }
803bca707e5SQu Wenruo
804bca707e5SQu Wenruo /*
805bca707e5SQu Wenruo * Find any subpage writer locked range inside @folio, starting at file offset
806bca707e5SQu Wenruo * @search_start. The caller should ensure the folio is locked.
807bca707e5SQu Wenruo *
808bca707e5SQu Wenruo * Return true and update @found_start_ret and @found_len_ret to the first
809bca707e5SQu Wenruo * writer locked range.
810bca707e5SQu Wenruo * Return false if there is no writer locked range.
811bca707e5SQu Wenruo */
btrfs_subpage_find_writer_locked(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 search_start,u64 * found_start_ret,u32 * found_len_ret)812bca707e5SQu Wenruo bool btrfs_subpage_find_writer_locked(const struct btrfs_fs_info *fs_info,
813bca707e5SQu Wenruo struct folio *folio, u64 search_start,
814bca707e5SQu Wenruo u64 *found_start_ret, u32 *found_len_ret)
815bca707e5SQu Wenruo {
816bca707e5SQu Wenruo struct btrfs_subpage *subpage = folio_get_private(folio);
817ce4a71eeSQu Wenruo const u32 sectors_per_page = fs_info->sectors_per_page;
818bca707e5SQu Wenruo const unsigned int len = PAGE_SIZE - offset_in_page(search_start);
819bca707e5SQu Wenruo const unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,
820bca707e5SQu Wenruo locked, search_start, len);
821ce4a71eeSQu Wenruo const unsigned int locked_bitmap_start = sectors_per_page * btrfs_bitmap_nr_locked;
822ce4a71eeSQu Wenruo const unsigned int locked_bitmap_end = locked_bitmap_start + sectors_per_page;
823bca707e5SQu Wenruo unsigned long flags;
824bca707e5SQu Wenruo int first_zero;
825bca707e5SQu Wenruo int first_set;
826bca707e5SQu Wenruo bool found = false;
827bca707e5SQu Wenruo
828bca707e5SQu Wenruo ASSERT(folio_test_locked(folio));
829bca707e5SQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
830bca707e5SQu Wenruo first_set = find_next_bit(subpage->bitmaps, locked_bitmap_end, start_bit);
831bca707e5SQu Wenruo if (first_set >= locked_bitmap_end)
832bca707e5SQu Wenruo goto out;
833bca707e5SQu Wenruo
834bca707e5SQu Wenruo found = true;
835bca707e5SQu Wenruo
836bca707e5SQu Wenruo *found_start_ret = folio_pos(folio) +
837bca707e5SQu Wenruo ((first_set - locked_bitmap_start) << fs_info->sectorsize_bits);
838bca707e5SQu Wenruo /*
839bca707e5SQu Wenruo * Since @first_set is ensured to be smaller than locked_bitmap_end
840bca707e5SQu Wenruo * here, @found_start_ret should be inside the folio.
841bca707e5SQu Wenruo */
842bca707e5SQu Wenruo ASSERT(*found_start_ret < folio_pos(folio) + PAGE_SIZE);
843bca707e5SQu Wenruo
844bca707e5SQu Wenruo first_zero = find_next_zero_bit(subpage->bitmaps, locked_bitmap_end, first_set);
845bca707e5SQu Wenruo *found_len_ret = (first_zero - first_set) << fs_info->sectorsize_bits;
846bca707e5SQu Wenruo out:
847bca707e5SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
848bca707e5SQu Wenruo return found;
849bca707e5SQu Wenruo }
850bca707e5SQu Wenruo
851ce4a71eeSQu Wenruo #define GET_SUBPAGE_BITMAP(subpage, fs_info, name, dst) \
85277b0b98bSQu Wenruo { \
853ce4a71eeSQu Wenruo const int sectors_per_page = fs_info->sectors_per_page; \
85477b0b98bSQu Wenruo \
855ce4a71eeSQu Wenruo ASSERT(sectors_per_page < BITS_PER_LONG); \
85677b0b98bSQu Wenruo *dst = bitmap_read(subpage->bitmaps, \
857ce4a71eeSQu Wenruo sectors_per_page * btrfs_bitmap_nr_##name, \
858ce4a71eeSQu Wenruo sectors_per_page); \
85977b0b98bSQu Wenruo }
86075258f20SQu Wenruo
btrfs_subpage_dump_bitmap(const struct btrfs_fs_info * fs_info,struct folio * folio,u64 start,u32 len)86175258f20SQu Wenruo void __cold btrfs_subpage_dump_bitmap(const struct btrfs_fs_info *fs_info,
86255151ea9SQu Wenruo struct folio *folio, u64 start, u32 len)
86375258f20SQu Wenruo {
86475258f20SQu Wenruo struct btrfs_subpage *subpage;
865ce4a71eeSQu Wenruo const u32 sectors_per_page = fs_info->sectors_per_page;
86675258f20SQu Wenruo unsigned long uptodate_bitmap;
86775258f20SQu Wenruo unsigned long dirty_bitmap;
86875258f20SQu Wenruo unsigned long writeback_bitmap;
86975258f20SQu Wenruo unsigned long ordered_bitmap;
87075258f20SQu Wenruo unsigned long checked_bitmap;
87175258f20SQu Wenruo unsigned long flags;
87275258f20SQu Wenruo
873cfbf07e2SQu Wenruo ASSERT(folio_test_private(folio) && folio_get_private(folio));
874ce4a71eeSQu Wenruo ASSERT(sectors_per_page > 1);
875cfbf07e2SQu Wenruo subpage = folio_get_private(folio);
87675258f20SQu Wenruo
87775258f20SQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
878ce4a71eeSQu Wenruo GET_SUBPAGE_BITMAP(subpage, fs_info, uptodate, &uptodate_bitmap);
879ce4a71eeSQu Wenruo GET_SUBPAGE_BITMAP(subpage, fs_info, dirty, &dirty_bitmap);
880ce4a71eeSQu Wenruo GET_SUBPAGE_BITMAP(subpage, fs_info, writeback, &writeback_bitmap);
881ce4a71eeSQu Wenruo GET_SUBPAGE_BITMAP(subpage, fs_info, ordered, &ordered_bitmap);
882ce4a71eeSQu Wenruo GET_SUBPAGE_BITMAP(subpage, fs_info, checked, &checked_bitmap);
883ce4a71eeSQu Wenruo GET_SUBPAGE_BITMAP(subpage, fs_info, locked, &checked_bitmap);
88475258f20SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
88575258f20SQu Wenruo
88655151ea9SQu Wenruo dump_page(folio_page(folio, 0), "btrfs subpage dump");
88775258f20SQu Wenruo btrfs_warn(fs_info,
88890df2c10SQu Wenruo "start=%llu len=%u page=%llu, bitmaps uptodate=%*pbl dirty=%*pbl writeback=%*pbl ordered=%*pbl checked=%*pbl",
88955151ea9SQu Wenruo start, len, folio_pos(folio),
890ce4a71eeSQu Wenruo sectors_per_page, &uptodate_bitmap,
891ce4a71eeSQu Wenruo sectors_per_page, &dirty_bitmap,
892ce4a71eeSQu Wenruo sectors_per_page, &writeback_bitmap,
893ce4a71eeSQu Wenruo sectors_per_page, &ordered_bitmap,
894ce4a71eeSQu Wenruo sectors_per_page, &checked_bitmap);
89575258f20SQu Wenruo }
89681891974SQu Wenruo
btrfs_get_subpage_dirty_bitmap(struct btrfs_fs_info * fs_info,struct folio * folio,unsigned long * ret_bitmap)89781891974SQu Wenruo void btrfs_get_subpage_dirty_bitmap(struct btrfs_fs_info *fs_info,
89881891974SQu Wenruo struct folio *folio,
89981891974SQu Wenruo unsigned long *ret_bitmap)
90081891974SQu Wenruo {
90181891974SQu Wenruo struct btrfs_subpage *subpage;
90281891974SQu Wenruo unsigned long flags;
90381891974SQu Wenruo
90481891974SQu Wenruo ASSERT(folio_test_private(folio) && folio_get_private(folio));
905ce4a71eeSQu Wenruo ASSERT(fs_info->sectors_per_page > 1);
90681891974SQu Wenruo subpage = folio_get_private(folio);
90781891974SQu Wenruo
90881891974SQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
909ce4a71eeSQu Wenruo GET_SUBPAGE_BITMAP(subpage, fs_info, dirty, ret_bitmap);
91081891974SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
91181891974SQu Wenruo }
912