1 /* SPDX-License-Identifier: GPL-2.0 */ 2 3 #ifndef BTRFS_SUBPAGE_H 4 #define BTRFS_SUBPAGE_H 5 6 #include <linux/spinlock.h> 7 #include <linux/atomic.h> 8 #include <linux/sizes.h> 9 #include "btrfs_inode.h" 10 11 struct address_space; 12 struct folio; 13 14 /* 15 * Extra info for subpage bitmap. 16 * 17 * For subpage we pack all uptodate/dirty/writeback/fixup bitmaps into 18 * one larger bitmap. 19 * 20 * This structure records how they are organized in the bitmap: 21 * 22 * /- uptodate /- dirty /- writeback /- fixup 23 * | | | | 24 * v v v v 25 * |u|u|u|u|........|u|u|d|d|.......|d|d|w|w|.....|w|w|f|f|.....|f|f| 26 * |< sectors_per_page >| 27 * 28 * Unlike regular macro-like enums, here we do not go upper-case names, as 29 * these names will be utilized in various macros to define function names. 30 */ 31 enum { 32 btrfs_bitmap_nr_uptodate = 0, 33 btrfs_bitmap_nr_dirty, 34 35 /* 36 * This can be changed to atomic eventually. But this change will rely 37 * on the async delalloc range rework for locked bitmap. As async 38 * delalloc can unlock its range and mark blocks writeback at random 39 * timing. 40 */ 41 btrfs_bitmap_nr_writeback, 42 43 /* 44 * Blocks dirtied by the dirty_folio callback instead of a reserving 45 * write path (e.g. set_page_dirty_lock() on a GUP pin). They have 46 * no space reservation and need the writepage fixup before they can 47 * be submitted. 48 */ 49 btrfs_bitmap_nr_fixup, 50 51 btrfs_bitmap_nr_max 52 }; 53 54 /* 55 * Structure to trace status of each sector inside a page, attached to 56 * page::private for both data and metadata inodes. 57 */ 58 struct btrfs_folio_state { 59 /* Common members for both data and metadata pages */ 60 spinlock_t lock; 61 union { 62 /* 63 * Structures only used by metadata 64 * 65 * @eb_refs should only be operated under private_lock, as it 66 * manages whether the btrfs_folio_state can be detached. 67 */ 68 atomic_t eb_refs; 69 70 /* 71 * Structures only used by data, 72 * 73 * How many sectors inside the page is locked. 74 */ 75 atomic_t nr_locked; 76 }; 77 unsigned long bitmaps[]; 78 }; 79 80 enum btrfs_folio_type { 81 BTRFS_SUBPAGE_METADATA, 82 BTRFS_SUBPAGE_DATA, 83 }; 84 85 /* 86 * Subpage support for metadata is more complex, as we can have dummy extent 87 * buffers, where folios have no mapping to determine the owning inode. 88 * 89 * Thankfully we only need to check if node size is smaller than page size. 90 * Even with larger folio support, we will only allocate a folio as large as 91 * node size. 92 * Thus if nodesize < PAGE_SIZE, we know metadata needs need to subpage routine. 93 */ 94 static inline bool btrfs_meta_is_subpage(const struct btrfs_fs_info *fs_info) 95 { 96 return fs_info->nodesize < PAGE_SIZE; 97 } 98 static inline bool btrfs_is_subpage(const struct btrfs_fs_info *fs_info, 99 struct folio *folio) 100 { 101 if (folio->mapping && folio->mapping->host) 102 ASSERT(is_data_inode(BTRFS_I(folio->mapping->host))); 103 return fs_info->sectorsize < folio_size(folio); 104 } 105 106 int btrfs_attach_folio_state(const struct btrfs_fs_info *fs_info, 107 struct folio *folio, enum btrfs_folio_type type); 108 void btrfs_detach_folio_state(const struct btrfs_fs_info *fs_info, struct folio *folio, 109 enum btrfs_folio_type type); 110 111 /* Allocate additional data where page represents more than one sector */ 112 struct btrfs_folio_state *btrfs_alloc_folio_state(const struct btrfs_fs_info *fs_info, 113 size_t fsize, enum btrfs_folio_type type); 114 static inline void btrfs_free_folio_state(struct btrfs_folio_state *bfs) 115 { 116 kfree(bfs); 117 } 118 119 void btrfs_folio_inc_eb_refs(const struct btrfs_fs_info *fs_info, struct folio *folio); 120 void btrfs_folio_dec_eb_refs(const struct btrfs_fs_info *fs_info, struct folio *folio); 121 122 void btrfs_folio_end_lock(const struct btrfs_fs_info *fs_info, 123 struct folio *folio, u64 start, u32 len); 124 void btrfs_folio_set_lock(const struct btrfs_fs_info *fs_info, 125 struct folio *folio, u64 start, u32 len); 126 void btrfs_folio_end_lock_bitmap(const struct btrfs_fs_info *fs_info, 127 struct folio *folio, unsigned long *bitmap); 128 /* 129 * Template for subpage related operations. 130 * 131 * btrfs_subpage_*() are for call sites where the folio has subpage attached and 132 * the range is ensured to be inside the folio's single page. 133 * 134 * btrfs_folio_*() are for call sites where the page can either be subpage 135 * specific or regular folios. The function will handle both cases. 136 * But the range still needs to be inside one single page. 137 * 138 * btrfs_folio_clamp_*() are similar to btrfs_folio_*(), except the range doesn't 139 * need to be inside the page. Those functions will truncate the range 140 * automatically. 141 * 142 * Both btrfs_folio_*() and btrfs_folio_clamp_*() are for data folios. 143 * 144 * For metadata, one should use btrfs_meta_folio_*() helpers instead, and there 145 * is no clamp version for metadata helpers, as we either go subpage 146 * (nodesize < PAGE_SIZE) or go regular folio helpers (nodesize >= PAGE_SIZE, 147 * and our folio is never larger than nodesize). 148 */ 149 #define DECLARE_BTRFS_SUBPAGE_OPS(name) \ 150 void btrfs_subpage_set_##name(const struct btrfs_fs_info *fs_info, \ 151 struct folio *folio, u64 start, u32 len); \ 152 void btrfs_subpage_clear_##name(const struct btrfs_fs_info *fs_info, \ 153 struct folio *folio, u64 start, u32 len); \ 154 bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \ 155 struct folio *folio, u64 start, u32 len); \ 156 void btrfs_folio_set_##name(const struct btrfs_fs_info *fs_info, \ 157 struct folio *folio, u64 start, u32 len); \ 158 void btrfs_folio_clear_##name(const struct btrfs_fs_info *fs_info, \ 159 struct folio *folio, u64 start, u32 len); \ 160 bool btrfs_folio_test_##name(const struct btrfs_fs_info *fs_info, \ 161 struct folio *folio, u64 start, u32 len); \ 162 void btrfs_folio_clamp_set_##name(const struct btrfs_fs_info *fs_info, \ 163 struct folio *folio, u64 start, u32 len); \ 164 void btrfs_folio_clamp_clear_##name(const struct btrfs_fs_info *fs_info, \ 165 struct folio *folio, u64 start, u32 len); \ 166 bool btrfs_folio_clamp_test_##name(const struct btrfs_fs_info *fs_info, \ 167 struct folio *folio, u64 start, u32 len); \ 168 void btrfs_meta_folio_set_##name(struct folio *folio, const struct extent_buffer *eb); \ 169 void btrfs_meta_folio_clear_##name(struct folio *folio, const struct extent_buffer *eb); \ 170 bool btrfs_meta_folio_test_##name(struct folio *folio, const struct extent_buffer *eb); 171 172 DECLARE_BTRFS_SUBPAGE_OPS(uptodate); 173 DECLARE_BTRFS_SUBPAGE_OPS(dirty); 174 DECLARE_BTRFS_SUBPAGE_OPS(writeback); 175 176 /* 177 * Fixup bit helpers. 178 * 179 * The fixup bit is data-only and has no plain set helper (setting happens 180 * together with dirtying in btrfs_subpage_set_fixup_dirty()), so it does not 181 * go through DECLARE_BTRFS_SUBPAGE_OPS(). For single-block folios the 182 * folio_*_fixup_pending() flag takes the place of the bitmap. 183 */ 184 void btrfs_subpage_clear_fixup(const struct btrfs_fs_info *fs_info, 185 struct folio *folio, u64 start, u32 len); 186 bool btrfs_subpage_test_fixup(const struct btrfs_fs_info *fs_info, 187 struct folio *folio, u64 start, u32 len); 188 bool btrfs_folio_test_fixup(const struct btrfs_fs_info *fs_info, 189 struct folio *folio, u64 start, u32 len); 190 void btrfs_folio_set_fixup_dirty(const struct btrfs_fs_info *fs_info, 191 struct folio *folio, u64 start, u32 len); 192 /* For a block that just got its space reserved; it stays dirty. */ 193 void btrfs_folio_clear_fixup(const struct btrfs_fs_info *fs_info, 194 struct folio *folio, u64 start, u32 len); 195 /* For callers discarding the data; clears the dirty bits too. */ 196 void btrfs_folio_clear_fixup_dirty(const struct btrfs_fs_info *fs_info, 197 struct folio *folio, u64 start, u32 len); 198 199 /* 200 * Helper for error cleanup, where a folio will have its dirty flag cleared, 201 * with writeback started and finished. 202 */ 203 static inline void btrfs_folio_clamp_finish_io(struct btrfs_fs_info *fs_info, 204 struct folio *locked_folio, 205 u64 start, u32 len) 206 { 207 btrfs_folio_clamp_clear_dirty(fs_info, locked_folio, start, len); 208 btrfs_folio_clamp_set_writeback(fs_info, locked_folio, start, len); 209 btrfs_folio_clamp_clear_writeback(fs_info, locked_folio, start, len); 210 } 211 212 bool btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info *fs_info, 213 struct folio *folio, u64 start, u32 len); 214 215 void btrfs_folio_assert_not_dirty(const struct btrfs_fs_info *fs_info, 216 struct folio *folio, u64 start, u32 len); 217 bool btrfs_meta_folio_clear_and_test_dirty(struct folio *folio, const struct extent_buffer *eb); 218 void btrfs_copy_subpage_dirty_bitmap(struct btrfs_fs_info *fs_info, 219 struct folio *folio, 220 unsigned long *dst); 221 void __cold btrfs_subpage_dump_bitmap(const struct btrfs_fs_info *fs_info, 222 struct folio *folio, u64 start, u32 len); 223 224 #endif 225