1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/slab.h> 4 #include "messages.h" 5 #include "subpage.h" 6 #include "btrfs_inode.h" 7 8 /* 9 * Subpage (block size < folio size) support overview: 10 * 11 * Limitations: 12 * 13 * - Metadata must be fully aligned to node size 14 * So when nodesize <= page size, the metadata can never cross folio boundaries. 15 * 16 * - Only support blocks per folio <= min(BTRFS_MAX_FOLIO_SIZE / fs block size, 17 * BTRFS_MAX_BLOCKS_PER_FOLIO) 18 * This is to ensure we can afford an on-stack bitmap, without the need to allocate 19 * bitmap memory at runtime. 20 * 21 * Implementation: 22 * 23 * - Common 24 * Both metadata and data will use a new structure, btrfs_folio_state, to 25 * record the status of each sector inside a page. This provides the extra 26 * granularity needed. 27 * 28 * - Metadata 29 * Since we have multiple tree blocks inside one page, we can't rely on page 30 * locking anymore, or we will have greatly reduced concurrency or even 31 * deadlocks (hold one tree lock while trying to lock another tree lock in 32 * the same page). 33 * 34 * Thus for metadata locking, subpage support relies on io_tree locking only. 35 * This means a slightly higher tree locking latency. 36 */ 37 38 int btrfs_attach_folio_state(const struct btrfs_fs_info *fs_info, 39 struct folio *folio, enum btrfs_folio_type type) 40 { 41 struct btrfs_folio_state *bfs; 42 43 /* For metadata we don't support large folio yet. */ 44 if (type == BTRFS_SUBPAGE_METADATA) 45 ASSERT(!folio_test_large(folio)); 46 47 /* 48 * We have cases like a dummy extent buffer page, which is not mapped 49 * and doesn't need to be locked. 50 */ 51 if (folio->mapping) 52 ASSERT(folio_test_locked(folio)); 53 54 /* Either not subpage, or the folio already has private attached. */ 55 if (folio_test_private(folio)) 56 return 0; 57 if (type == BTRFS_SUBPAGE_METADATA && !btrfs_meta_is_subpage(fs_info)) 58 return 0; 59 if (type == BTRFS_SUBPAGE_DATA && !btrfs_is_subpage(fs_info, folio)) 60 return 0; 61 62 bfs = btrfs_alloc_folio_state(fs_info, folio_size(folio), type); 63 if (IS_ERR(bfs)) 64 return PTR_ERR(bfs); 65 66 folio_attach_private(folio, bfs); 67 return 0; 68 } 69 70 void btrfs_detach_folio_state(const struct btrfs_fs_info *fs_info, struct folio *folio, 71 enum btrfs_folio_type type) 72 { 73 struct btrfs_folio_state *bfs; 74 75 /* Either not subpage, or the folio already has private attached. */ 76 if (!folio_test_private(folio)) 77 return; 78 if (type == BTRFS_SUBPAGE_METADATA && !btrfs_meta_is_subpage(fs_info)) 79 return; 80 if (type == BTRFS_SUBPAGE_DATA && !btrfs_is_subpage(fs_info, folio)) 81 return; 82 83 bfs = folio_detach_private(folio); 84 ASSERT(bfs); 85 btrfs_free_folio_state(bfs); 86 } 87 88 struct btrfs_folio_state *btrfs_alloc_folio_state(const struct btrfs_fs_info *fs_info, 89 size_t fsize, enum btrfs_folio_type type) 90 { 91 struct btrfs_folio_state *ret; 92 unsigned int real_size; 93 94 ASSERT(fs_info->sectorsize < fsize); 95 96 real_size = struct_size(ret, bitmaps, 97 BITS_TO_LONGS(btrfs_bitmap_nr_max * 98 (fsize >> fs_info->sectorsize_bits))); 99 ret = kzalloc(real_size, GFP_NOFS); 100 if (!ret) 101 return ERR_PTR(-ENOMEM); 102 103 spin_lock_init(&ret->lock); 104 if (type == BTRFS_SUBPAGE_METADATA) 105 atomic_set(&ret->eb_refs, 0); 106 else 107 atomic_set(&ret->nr_locked, 0); 108 return ret; 109 } 110 111 /* 112 * Increase the eb_refs of current subpage. 113 * 114 * This is important for eb allocation, to prevent race with last eb freeing 115 * of the same page. 116 * With the eb_refs increased before the eb inserted into radix tree, 117 * detach_extent_buffer_page() won't detach the folio private while we're still 118 * allocating the extent buffer. 119 */ 120 void btrfs_folio_inc_eb_refs(const struct btrfs_fs_info *fs_info, struct folio *folio) 121 { 122 struct btrfs_folio_state *bfs; 123 124 if (!btrfs_meta_is_subpage(fs_info)) 125 return; 126 127 ASSERT(folio_test_private(folio) && folio->mapping); 128 lockdep_assert_held(&folio->mapping->i_private_lock); 129 130 bfs = folio_get_private(folio); 131 atomic_inc(&bfs->eb_refs); 132 } 133 134 void btrfs_folio_dec_eb_refs(const struct btrfs_fs_info *fs_info, struct folio *folio) 135 { 136 struct btrfs_folio_state *bfs; 137 138 if (!btrfs_meta_is_subpage(fs_info)) 139 return; 140 141 ASSERT(folio_test_private(folio) && folio->mapping); 142 lockdep_assert_held(&folio->mapping->i_private_lock); 143 144 bfs = folio_get_private(folio); 145 ASSERT(atomic_read(&bfs->eb_refs)); 146 atomic_dec(&bfs->eb_refs); 147 } 148 149 static void btrfs_subpage_assert(const struct btrfs_fs_info *fs_info, 150 struct folio *folio, u64 start, u32 len) 151 { 152 /* Basic checks */ 153 ASSERT(folio_test_private(folio) && folio_get_private(folio)); 154 ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && 155 IS_ALIGNED(len, fs_info->sectorsize), "start=%llu len=%u", start, len); 156 /* 157 * The range check only works for mapped page, we can still have 158 * unmapped page like dummy extent buffer pages. 159 */ 160 if (folio->mapping) 161 ASSERT(folio_pos(folio) <= start && 162 start + len <= folio_next_pos(folio), 163 "start=%llu len=%u folio_pos=%llu folio_size=%zu", 164 start, len, folio_pos(folio), folio_size(folio)); 165 } 166 167 #define subpage_calc_start_bit(fs_info, folio, name, start, len) \ 168 ({ \ 169 unsigned int __start_bit; \ 170 const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ 171 \ 172 btrfs_subpage_assert(fs_info, folio, start, len); \ 173 __start_bit = offset_in_folio(folio, start) >> fs_info->sectorsize_bits; \ 174 __start_bit += __bpf * btrfs_bitmap_nr_##name; \ 175 __start_bit; \ 176 }) 177 178 static void btrfs_subpage_clamp_range(struct folio *folio, u64 *start, u32 *len) 179 { 180 u64 orig_start = *start; 181 u32 orig_len = *len; 182 183 *start = max_t(u64, folio_pos(folio), orig_start); 184 /* 185 * For certain call sites like btrfs_drop_pages(), we may have pages 186 * beyond the target range. In that case, just set @len to 0, subpage 187 * helpers can handle @len == 0 without any problem. 188 */ 189 if (folio_pos(folio) >= orig_start + orig_len) 190 *len = 0; 191 else 192 *len = min_t(u64, folio_next_pos(folio), orig_start + orig_len) - *start; 193 } 194 195 static bool btrfs_subpage_end_and_test_lock(const struct btrfs_fs_info *fs_info, 196 struct folio *folio, u64 start, u32 len) 197 { 198 struct btrfs_folio_state *bfs = folio_get_private(folio); 199 const int nbits = (len >> fs_info->sectorsize_bits); 200 unsigned long flags; 201 bool last; 202 203 btrfs_subpage_assert(fs_info, folio, start, len); 204 205 spin_lock_irqsave(&bfs->lock, flags); 206 /* 207 * We have call sites passing @lock_page into 208 * extent_clear_unlock_delalloc() for compression path. 209 * 210 * This @locked_page is locked by plain lock_page(), thus its 211 * subpage::locked is 0. Handle them in a special way. 212 */ 213 if (atomic_read(&bfs->nr_locked) == 0) { 214 spin_unlock_irqrestore(&bfs->lock, flags); 215 return true; 216 } 217 ASSERT(atomic_read(&bfs->nr_locked) >= nbits, 218 "atomic_read(&bfs->nr_locked)=%d nbits=%d", 219 atomic_read(&bfs->nr_locked), nbits); 220 last = atomic_sub_and_test(nbits, &bfs->nr_locked); 221 spin_unlock_irqrestore(&bfs->lock, flags); 222 return last; 223 } 224 225 /* 226 * Handle different locked folios: 227 * 228 * - Non-subpage folio 229 * Just unlock it. 230 * 231 * - folio locked but without any subpage locked 232 * This happens either before writepage_delalloc() or the delalloc range is 233 * already handled by previous folio. 234 * We can simple unlock it. 235 * 236 * - folio locked with subpage range locked. 237 * We go through the locked sectors inside the range and clear their locked 238 * bitmap, reduce the writer lock number, and unlock the page if that's 239 * the last locked range. 240 */ 241 void btrfs_folio_end_lock(const struct btrfs_fs_info *fs_info, 242 struct folio *folio, u64 start, u32 len) 243 { 244 struct btrfs_folio_state *bfs = folio_get_private(folio); 245 246 ASSERT(folio_test_locked(folio)); 247 248 if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, folio)) { 249 folio_unlock(folio); 250 return; 251 } 252 253 /* 254 * For subpage case, there are two types of locked page. With or 255 * without locked number. 256 * 257 * Since we own the page lock, no one else could touch subpage::locked 258 * and we are safe to do several atomic operations without spinlock. 259 */ 260 if (atomic_read(&bfs->nr_locked) == 0) { 261 /* No subpage lock, locked by plain lock_page(). */ 262 folio_unlock(folio); 263 return; 264 } 265 266 btrfs_subpage_clamp_range(folio, &start, &len); 267 if (btrfs_subpage_end_and_test_lock(fs_info, folio, start, len)) 268 folio_unlock(folio); 269 } 270 271 void btrfs_folio_end_lock_bitmap(const struct btrfs_fs_info *fs_info, 272 struct folio *folio, unsigned long *bitmap) 273 { 274 struct btrfs_folio_state *bfs = folio_get_private(folio); 275 const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); 276 const unsigned int nbits = bitmap_weight(bitmap, blocks_per_folio); 277 unsigned long flags; 278 bool last = false; 279 280 if (!btrfs_is_subpage(fs_info, folio)) { 281 folio_unlock(folio); 282 return; 283 } 284 285 if (atomic_read(&bfs->nr_locked) == 0) { 286 /* No subpage lock, locked by plain lock_page(). */ 287 folio_unlock(folio); 288 return; 289 } 290 291 spin_lock_irqsave(&bfs->lock, flags); 292 ASSERT(atomic_read(&bfs->nr_locked) >= nbits, 293 "atomic_read(&bfs->nr_locked)=%d nbits=%d", 294 atomic_read(&bfs->nr_locked), nbits); 295 last = atomic_sub_and_test(nbits, &bfs->nr_locked); 296 spin_unlock_irqrestore(&bfs->lock, flags); 297 if (last) 298 folio_unlock(folio); 299 } 300 301 #define subpage_test_bitmap_all_set(fs_info, folio, name) \ 302 ({ \ 303 struct btrfs_folio_state *__bfs = folio_get_private(folio); \ 304 const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ 305 \ 306 bitmap_test_range_all_set(__bfs->bitmaps, \ 307 __bpf * btrfs_bitmap_nr_##name, __bpf); \ 308 }) 309 310 #define subpage_test_bitmap_all_zero(fs_info, folio, name) \ 311 ({ \ 312 struct btrfs_folio_state *__bfs = folio_get_private(folio); \ 313 const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ 314 \ 315 bitmap_test_range_all_zero(__bfs->bitmaps, \ 316 __bpf * btrfs_bitmap_nr_##name, __bpf); \ 317 }) 318 319 void btrfs_subpage_set_uptodate(const struct btrfs_fs_info *fs_info, 320 struct folio *folio, u64 start, u32 len) 321 { 322 struct btrfs_folio_state *bfs = folio_get_private(folio); 323 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, 324 uptodate, start, len); 325 unsigned long flags; 326 327 spin_lock_irqsave(&bfs->lock, flags); 328 bitmap_set(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 329 if (subpage_test_bitmap_all_set(fs_info, folio, uptodate)) 330 folio_mark_uptodate(folio); 331 spin_unlock_irqrestore(&bfs->lock, flags); 332 } 333 334 void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info, 335 struct folio *folio, u64 start, u32 len) 336 { 337 struct btrfs_folio_state *bfs = folio_get_private(folio); 338 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, 339 uptodate, start, len); 340 unsigned long flags; 341 342 spin_lock_irqsave(&bfs->lock, flags); 343 bitmap_clear(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 344 folio_clear_uptodate(folio); 345 spin_unlock_irqrestore(&bfs->lock, flags); 346 } 347 348 void btrfs_subpage_set_dirty(const struct btrfs_fs_info *fs_info, 349 struct folio *folio, u64 start, u32 len) 350 { 351 struct btrfs_folio_state *bfs = folio_get_private(folio); 352 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, 353 dirty, start, len); 354 unsigned long flags; 355 356 spin_lock_irqsave(&bfs->lock, flags); 357 bitmap_set(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 358 spin_unlock_irqrestore(&bfs->lock, flags); 359 folio_mark_dirty(folio); 360 } 361 362 static void folio_clear_tags(struct folio *folio) 363 { 364 struct address_space *mapping = folio_mapping(folio); 365 XA_STATE(xas, &mapping->i_pages, folio->index); 366 unsigned long flags; 367 368 ASSERT(folio_test_locked(folio)); 369 ASSERT(mapping); 370 ASSERT(mapping_use_writeback_tags(mapping)); 371 372 xas_lock_irqsave(&xas, flags); 373 xas_load(&xas); 374 xas_clear_mark(&xas, PAGECACHE_TAG_DIRTY); 375 xas_clear_mark(&xas, PAGECACHE_TAG_TOWRITE); 376 xas_unlock_irqrestore(&xas, flags); 377 } 378 379 /* 380 * Extra clear_and_test function for subpage dirty bitmap. 381 * 382 * Return true if we're the last bits in the dirty_bitmap and clear the 383 * dirty_bitmap. 384 * Return false otherwise. 385 * 386 * NOTE: Callers should manually clear page dirty for true case, as we have 387 * extra handling for tree blocks. 388 */ 389 bool btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info *fs_info, 390 struct folio *folio, u64 start, u32 len) 391 { 392 struct btrfs_folio_state *bfs = folio_get_private(folio); 393 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, 394 dirty, start, len); 395 unsigned long flags; 396 bool last = false; 397 398 spin_lock_irqsave(&bfs->lock, flags); 399 bitmap_clear(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 400 if (subpage_test_bitmap_all_zero(fs_info, folio, dirty)) 401 last = true; 402 spin_unlock_irqrestore(&bfs->lock, flags); 403 return last; 404 } 405 406 void btrfs_subpage_clear_dirty(const struct btrfs_fs_info *fs_info, 407 struct folio *folio, u64 start, u32 len) 408 { 409 bool last; 410 411 last = btrfs_subpage_clear_and_test_dirty(fs_info, folio, start, len); 412 if (last) 413 folio_clear_dirty_for_io(folio); 414 } 415 416 void btrfs_subpage_set_writeback(const struct btrfs_fs_info *fs_info, 417 struct folio *folio, u64 start, u32 len) 418 { 419 struct btrfs_folio_state *bfs = folio_get_private(folio); 420 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, 421 writeback, start, len); 422 unsigned long flags; 423 424 spin_lock_irqsave(&bfs->lock, flags); 425 bitmap_set(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 426 427 /* 428 * Don't clear the TOWRITE tag when starting writeback on a still-dirty 429 * folio. Doing so can cause WB_SYNC_ALL writepages() to overlook it, 430 * assume writeback is complete, and exit too early — violating sync 431 * ordering guarantees. 432 * 433 * Instead we manually clear the DIRTY and TOWRITE tags after the folio 434 * is no longer dirty. 435 */ 436 if (!folio_test_writeback(folio)) 437 __folio_start_writeback(folio, true); 438 if (!folio_test_dirty(folio)) 439 folio_clear_tags(folio); 440 spin_unlock_irqrestore(&bfs->lock, flags); 441 } 442 443 void btrfs_subpage_clear_writeback(const struct btrfs_fs_info *fs_info, 444 struct folio *folio, u64 start, u32 len) 445 { 446 struct btrfs_folio_state *bfs = folio_get_private(folio); 447 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, 448 writeback, start, len); 449 unsigned long flags; 450 451 spin_lock_irqsave(&bfs->lock, flags); 452 bitmap_clear(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 453 if (subpage_test_bitmap_all_zero(fs_info, folio, writeback)) { 454 ASSERT(folio_test_writeback(folio)); 455 folio_end_writeback(folio); 456 } 457 spin_unlock_irqrestore(&bfs->lock, flags); 458 } 459 460 /* 461 * Unlike set/clear which is dependent on each page status, for test all bits 462 * are tested in the same way. 463 */ 464 #define IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(name) \ 465 bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \ 466 struct folio *folio, u64 start, u32 len) \ 467 { \ 468 struct btrfs_folio_state *bfs = folio_get_private(folio); \ 469 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, \ 470 name, start, len); \ 471 unsigned long flags; \ 472 bool ret; \ 473 \ 474 spin_lock_irqsave(&bfs->lock, flags); \ 475 ret = bitmap_test_range_all_set(bfs->bitmaps, start_bit, \ 476 len >> fs_info->sectorsize_bits); \ 477 spin_unlock_irqrestore(&bfs->lock, flags); \ 478 return ret; \ 479 } 480 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate); 481 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(dirty); 482 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(writeback); 483 484 /* 485 * Note that, in selftests (extent-io-tests), we can have empty fs_info passed 486 * in. We only test sectorsize == PAGE_SIZE cases so far, thus we can fall 487 * back to regular sectorsize branch. 488 */ 489 #define IMPLEMENT_BTRFS_PAGE_OPS(name, folio_set_func, \ 490 folio_clear_func, folio_test_func) \ 491 void btrfs_folio_set_##name(const struct btrfs_fs_info *fs_info, \ 492 struct folio *folio, u64 start, u32 len) \ 493 { \ 494 if (unlikely(!fs_info) || \ 495 !btrfs_is_subpage(fs_info, folio)) { \ 496 folio_set_func(folio); \ 497 return; \ 498 } \ 499 btrfs_subpage_set_##name(fs_info, folio, start, len); \ 500 } \ 501 void btrfs_folio_clear_##name(const struct btrfs_fs_info *fs_info, \ 502 struct folio *folio, u64 start, u32 len) \ 503 { \ 504 if (unlikely(!fs_info) || \ 505 !btrfs_is_subpage(fs_info, folio)) { \ 506 folio_clear_func(folio); \ 507 return; \ 508 } \ 509 btrfs_subpage_clear_##name(fs_info, folio, start, len); \ 510 } \ 511 bool btrfs_folio_test_##name(const struct btrfs_fs_info *fs_info, \ 512 struct folio *folio, u64 start, u32 len) \ 513 { \ 514 if (unlikely(!fs_info) || \ 515 !btrfs_is_subpage(fs_info, folio)) \ 516 return folio_test_func(folio); \ 517 return btrfs_subpage_test_##name(fs_info, folio, start, len); \ 518 } \ 519 void btrfs_folio_clamp_set_##name(const struct btrfs_fs_info *fs_info, \ 520 struct folio *folio, u64 start, u32 len) \ 521 { \ 522 if (unlikely(!fs_info) || \ 523 !btrfs_is_subpage(fs_info, folio)) { \ 524 folio_set_func(folio); \ 525 return; \ 526 } \ 527 btrfs_subpage_clamp_range(folio, &start, &len); \ 528 btrfs_subpage_set_##name(fs_info, folio, start, len); \ 529 } \ 530 void btrfs_folio_clamp_clear_##name(const struct btrfs_fs_info *fs_info, \ 531 struct folio *folio, u64 start, u32 len) \ 532 { \ 533 if (unlikely(!fs_info) || \ 534 !btrfs_is_subpage(fs_info, folio)) { \ 535 folio_clear_func(folio); \ 536 return; \ 537 } \ 538 btrfs_subpage_clamp_range(folio, &start, &len); \ 539 btrfs_subpage_clear_##name(fs_info, folio, start, len); \ 540 } \ 541 bool btrfs_folio_clamp_test_##name(const struct btrfs_fs_info *fs_info, \ 542 struct folio *folio, u64 start, u32 len) \ 543 { \ 544 if (unlikely(!fs_info) || \ 545 !btrfs_is_subpage(fs_info, folio)) \ 546 return folio_test_func(folio); \ 547 btrfs_subpage_clamp_range(folio, &start, &len); \ 548 return btrfs_subpage_test_##name(fs_info, folio, start, len); \ 549 } \ 550 void btrfs_meta_folio_set_##name(struct folio *folio, const struct extent_buffer *eb) \ 551 { \ 552 if (!btrfs_meta_is_subpage(eb->fs_info)) { \ 553 folio_set_func(folio); \ 554 return; \ 555 } \ 556 btrfs_subpage_set_##name(eb->fs_info, folio, eb->start, eb->len); \ 557 } \ 558 void btrfs_meta_folio_clear_##name(struct folio *folio, const struct extent_buffer *eb) \ 559 { \ 560 if (!btrfs_meta_is_subpage(eb->fs_info)) { \ 561 folio_clear_func(folio); \ 562 return; \ 563 } \ 564 btrfs_subpage_clear_##name(eb->fs_info, folio, eb->start, eb->len); \ 565 } \ 566 bool btrfs_meta_folio_test_##name(struct folio *folio, const struct extent_buffer *eb) \ 567 { \ 568 if (!btrfs_meta_is_subpage(eb->fs_info)) \ 569 return folio_test_func(folio); \ 570 return btrfs_subpage_test_##name(eb->fs_info, folio, eb->start, eb->len); \ 571 } 572 IMPLEMENT_BTRFS_PAGE_OPS(uptodate, folio_mark_uptodate, folio_clear_uptodate, 573 folio_test_uptodate); 574 IMPLEMENT_BTRFS_PAGE_OPS(dirty, folio_mark_dirty, folio_clear_dirty_for_io, 575 folio_test_dirty); 576 IMPLEMENT_BTRFS_PAGE_OPS(writeback, folio_start_writeback, folio_end_writeback, 577 folio_test_writeback); 578 579 #define DEFINE_GET_SUBPAGE_BITMAP(name) \ 580 static inline unsigned long get_bitmap_value_##name( \ 581 const struct btrfs_fs_info *fs_info, \ 582 struct folio *folio) \ 583 { \ 584 const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ 585 const struct btrfs_folio_state *__bfs = folio_get_private(folio); \ 586 unsigned long value; \ 587 \ 588 ASSERT(__bpf <= BITS_PER_LONG); \ 589 value = bitmap_read(__bfs->bitmaps, __bpf * btrfs_bitmap_nr_##name, \ 590 __bpf); \ 591 return value; \ 592 } \ 593 static inline const unsigned long *get_bitmap_pointer_##name( \ 594 const struct btrfs_fs_info *fs_info, \ 595 struct folio *folio) \ 596 { \ 597 const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ 598 struct btrfs_folio_state *__bfs = folio_get_private(folio); \ 599 unsigned long *pointer; \ 600 \ 601 ASSERT(__bpf >= BITS_PER_LONG); \ 602 ASSERT(IS_ALIGNED(__bpf, BITS_PER_LONG)); \ 603 pointer = __bfs->bitmaps + (BIT_WORD(__bpf) * btrfs_bitmap_nr_##name); \ 604 return pointer; \ 605 } 606 607 DEFINE_GET_SUBPAGE_BITMAP(uptodate); 608 DEFINE_GET_SUBPAGE_BITMAP(dirty); 609 DEFINE_GET_SUBPAGE_BITMAP(writeback); 610 611 #define SUBPAGE_DUMP_BITMAP(fs_info, folio, name, start, len) \ 612 { \ 613 const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ 614 \ 615 if (__bpf <= BITS_PER_LONG) { \ 616 unsigned long bitmap = get_bitmap_value_##name(fs_info, folio); \ 617 \ 618 btrfs_warn(fs_info, \ 619 "dumping bitmap start=%llu len=%u folio=%llu " #name "_bitmap=%*pbl", \ 620 start, len, folio_pos(folio), __bpf, &bitmap); \ 621 } else { \ 622 btrfs_warn(fs_info, \ 623 "dumping bitmap start=%llu len=%u folio=%llu " #name "_bitmap=%*pbl", \ 624 start, len, folio_pos(folio), __bpf, \ 625 get_bitmap_pointer_##name(fs_info, folio)); \ 626 } \ 627 } 628 629 /* 630 * Make sure not only the page dirty bit is cleared, but also subpage dirty bit 631 * is cleared. 632 */ 633 void btrfs_folio_assert_not_dirty(const struct btrfs_fs_info *fs_info, 634 struct folio *folio, u64 start, u32 len) 635 { 636 struct btrfs_folio_state *bfs; 637 unsigned int start_bit; 638 unsigned int nbits; 639 unsigned long flags; 640 641 if (!IS_ENABLED(CONFIG_BTRFS_ASSERT)) 642 return; 643 644 if (!btrfs_is_subpage(fs_info, folio)) { 645 ASSERT(!folio_test_dirty(folio)); 646 return; 647 } 648 649 start_bit = subpage_calc_start_bit(fs_info, folio, dirty, start, len); 650 nbits = len >> fs_info->sectorsize_bits; 651 bfs = folio_get_private(folio); 652 ASSERT(bfs); 653 spin_lock_irqsave(&bfs->lock, flags); 654 if (unlikely(!bitmap_test_range_all_zero(bfs->bitmaps, start_bit, nbits))) { 655 SUBPAGE_DUMP_BITMAP(fs_info, folio, dirty, start, len); 656 ASSERT(bitmap_test_range_all_zero(bfs->bitmaps, start_bit, nbits)); 657 } 658 ASSERT(bitmap_test_range_all_zero(bfs->bitmaps, start_bit, nbits)); 659 spin_unlock_irqrestore(&bfs->lock, flags); 660 } 661 662 /* 663 * This is for folio already locked by plain lock_page()/folio_lock(), which 664 * doesn't have any subpage awareness. 665 * 666 * This populates the involved subpage ranges so that subpage helpers can 667 * properly unlock them. 668 */ 669 void btrfs_folio_set_lock(const struct btrfs_fs_info *fs_info, 670 struct folio *folio, u64 start, u32 len) 671 { 672 struct btrfs_folio_state *bfs; 673 unsigned long flags; 674 unsigned int nbits; 675 int ret; 676 677 ASSERT(folio_test_locked(folio)); 678 if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, folio)) 679 return; 680 681 bfs = folio_get_private(folio); 682 nbits = len >> fs_info->sectorsize_bits; 683 spin_lock_irqsave(&bfs->lock, flags); 684 ret = atomic_add_return(nbits, &bfs->nr_locked); 685 ASSERT(ret <= btrfs_blocks_per_folio(fs_info, folio)); 686 spin_unlock_irqrestore(&bfs->lock, flags); 687 } 688 689 /* 690 * Clear the dirty flag for the folio. 691 * 692 * If the affected folio is no longer dirty, return true. Otherwise return false. 693 */ 694 bool btrfs_meta_folio_clear_and_test_dirty(struct folio *folio, const struct extent_buffer *eb) 695 { 696 bool last; 697 698 if (!btrfs_meta_is_subpage(eb->fs_info)) { 699 folio_clear_dirty_for_io(folio); 700 return true; 701 } 702 703 last = btrfs_subpage_clear_and_test_dirty(eb->fs_info, folio, eb->start, eb->len); 704 if (last) { 705 folio_clear_dirty_for_io(folio); 706 return true; 707 } 708 return false; 709 } 710 711 void __cold btrfs_subpage_dump_bitmap(const struct btrfs_fs_info *fs_info, 712 struct folio *folio, u64 start, u32 len) 713 { 714 struct btrfs_folio_state *bfs; 715 const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); 716 unsigned long flags; 717 718 ASSERT(folio_test_private(folio) && folio_get_private(folio)); 719 ASSERT(blocks_per_folio > 1); 720 bfs = folio_get_private(folio); 721 722 dump_page(folio_page(folio, 0), "btrfs folio state dump"); 723 724 if (blocks_per_folio <= BITS_PER_LONG) { 725 unsigned long uptodate; 726 unsigned long dirty; 727 unsigned long writeback; 728 729 spin_lock_irqsave(&bfs->lock, flags); 730 uptodate = get_bitmap_value_uptodate(fs_info, folio); 731 dirty = get_bitmap_value_dirty(fs_info, folio); 732 writeback = get_bitmap_value_writeback(fs_info, folio); 733 734 spin_unlock_irqrestore(&bfs->lock, flags); 735 736 btrfs_warn(fs_info, 737 "start=%llu len=%u page=%llu, bitmaps uptodate=%*pbl dirty=%*pbl writeback=%*pbl", 738 start, len, folio_pos(folio), 739 blocks_per_folio, &uptodate, 740 blocks_per_folio, &dirty, 741 blocks_per_folio, &writeback); 742 return; 743 } 744 745 spin_lock_irqsave(&bfs->lock, flags); 746 btrfs_warn(fs_info, 747 "start=%llu len=%u page=%llu, bitmaps uptodate=%*pbl dirty=%*pbl writeback=%*pbl", 748 start, len, folio_pos(folio), 749 blocks_per_folio, get_bitmap_pointer_uptodate(fs_info, folio), 750 blocks_per_folio, get_bitmap_pointer_dirty(fs_info, folio), 751 blocks_per_folio, get_bitmap_pointer_writeback(fs_info, folio)); 752 spin_unlock_irqrestore(&bfs->lock, flags); 753 } 754 755 void btrfs_copy_subpage_dirty_bitmap(struct btrfs_fs_info *fs_info, 756 struct folio *folio, 757 unsigned long *dst) 758 { 759 struct btrfs_folio_state *bfs; 760 const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); 761 unsigned long flags; 762 unsigned long value; 763 764 if (blocks_per_folio == 1) { 765 value = 1; 766 bitmap_copy(dst, &value, 1); 767 return; 768 } 769 770 ASSERT(folio_test_private(folio) && folio_get_private(folio)); 771 ASSERT(blocks_per_folio > 1); 772 bfs = folio_get_private(folio); 773 774 if (blocks_per_folio <= BITS_PER_LONG) { 775 spin_lock_irqsave(&bfs->lock, flags); 776 value = bitmap_read(bfs->bitmaps, btrfs_bitmap_nr_dirty * blocks_per_folio, 777 blocks_per_folio); 778 spin_unlock_irqrestore(&bfs->lock, flags); 779 bitmap_copy(dst, &value, blocks_per_folio); 780 return; 781 } 782 spin_lock_irqsave(&bfs->lock, flags); 783 bitmap_copy(dst, get_bitmap_pointer_dirty(fs_info, folio), 784 blocks_per_folio); 785 spin_unlock_irqrestore(&bfs->lock, flags); 786 } 787