1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/slab.h> 4 #include "messages.h" 5 #include "subpage.h" 6 #include "btrfs_inode.h" 7 8 /* 9 * Subpage (block size < folio size) support overview: 10 * 11 * Limitations: 12 * 13 * - Metadata must be fully aligned to node size 14 * So when nodesize <= page size, the metadata can never cross folio boundaries. 15 * 16 * - Only support blocks per folio <= min(BTRFS_MAX_FOLIO_SIZE / fs block size, 17 * BTRFS_MAX_BLOCKS_PER_FOLIO) 18 * This is to ensure we can afford an on-stack bitmap, without the need to allocate 19 * bitmap memory at runtime. 20 * 21 * Implementation: 22 * 23 * - Common 24 * Both metadata and data will use a new structure, btrfs_folio_state, to 25 * record the status of each sector inside a page. This provides the extra 26 * granularity needed. 27 * 28 * - Metadata 29 * Since we have multiple tree blocks inside one page, we can't rely on page 30 * locking anymore, or we will have greatly reduced concurrency or even 31 * deadlocks (hold one tree lock while trying to lock another tree lock in 32 * the same page). 33 * 34 * Thus for metadata locking, subpage support relies on io_tree locking only. 35 * This means a slightly higher tree locking latency. 36 */ 37 38 int btrfs_attach_folio_state(const struct btrfs_fs_info *fs_info, 39 struct folio *folio, enum btrfs_folio_type type) 40 { 41 struct btrfs_folio_state *bfs; 42 43 /* For metadata we don't support large folio yet. */ 44 if (type == BTRFS_SUBPAGE_METADATA) 45 ASSERT(!folio_test_large(folio)); 46 47 /* 48 * We have cases like a dummy extent buffer page, which is not mapped 49 * and doesn't need to be locked. 50 */ 51 if (folio->mapping) 52 ASSERT(folio_test_locked(folio)); 53 54 /* Either not subpage, or the folio already has private attached. */ 55 if (folio_test_private(folio)) 56 return 0; 57 if (type == BTRFS_SUBPAGE_METADATA && !btrfs_meta_is_subpage(fs_info)) 58 return 0; 59 if (type == BTRFS_SUBPAGE_DATA && !btrfs_is_subpage(fs_info, folio)) 60 return 0; 61 62 bfs = btrfs_alloc_folio_state(fs_info, folio_size(folio), type); 63 if (IS_ERR(bfs)) 64 return PTR_ERR(bfs); 65 66 folio_attach_private(folio, bfs); 67 return 0; 68 } 69 70 void btrfs_detach_folio_state(const struct btrfs_fs_info *fs_info, struct folio *folio, 71 enum btrfs_folio_type type) 72 { 73 struct btrfs_folio_state *bfs; 74 75 /* Either not subpage, or the folio already has private attached. */ 76 if (!folio_test_private(folio)) 77 return; 78 if (type == BTRFS_SUBPAGE_METADATA && !btrfs_meta_is_subpage(fs_info)) 79 return; 80 if (type == BTRFS_SUBPAGE_DATA && !btrfs_is_subpage(fs_info, folio)) 81 return; 82 83 bfs = folio_detach_private(folio); 84 ASSERT(bfs); 85 btrfs_free_folio_state(bfs); 86 } 87 88 struct btrfs_folio_state *btrfs_alloc_folio_state(const struct btrfs_fs_info *fs_info, 89 size_t fsize, enum btrfs_folio_type type) 90 { 91 struct btrfs_folio_state *ret; 92 unsigned int real_size; 93 94 ASSERT(fs_info->sectorsize < fsize); 95 96 real_size = struct_size(ret, bitmaps, 97 BITS_TO_LONGS(btrfs_bitmap_nr_max * 98 (fsize >> fs_info->sectorsize_bits))); 99 ret = kzalloc(real_size, GFP_NOFS); 100 if (!ret) 101 return ERR_PTR(-ENOMEM); 102 103 spin_lock_init(&ret->lock); 104 if (type == BTRFS_SUBPAGE_METADATA) 105 atomic_set(&ret->eb_refs, 0); 106 else 107 atomic_set(&ret->nr_locked, 0); 108 return ret; 109 } 110 111 /* 112 * Increase the eb_refs of current subpage. 113 * 114 * This is important for eb allocation, to prevent race with last eb freeing 115 * of the same page. 116 * With the eb_refs increased before the eb inserted into radix tree, 117 * detach_extent_buffer_page() won't detach the folio private while we're still 118 * allocating the extent buffer. 119 */ 120 void btrfs_folio_inc_eb_refs(const struct btrfs_fs_info *fs_info, struct folio *folio) 121 { 122 struct btrfs_folio_state *bfs; 123 124 if (!btrfs_meta_is_subpage(fs_info)) 125 return; 126 127 ASSERT(folio_test_private(folio) && folio->mapping); 128 lockdep_assert_held(&folio->mapping->i_private_lock); 129 130 bfs = folio_get_private(folio); 131 atomic_inc(&bfs->eb_refs); 132 } 133 134 void btrfs_folio_dec_eb_refs(const struct btrfs_fs_info *fs_info, struct folio *folio) 135 { 136 struct btrfs_folio_state *bfs; 137 138 if (!btrfs_meta_is_subpage(fs_info)) 139 return; 140 141 ASSERT(folio_test_private(folio) && folio->mapping); 142 lockdep_assert_held(&folio->mapping->i_private_lock); 143 144 bfs = folio_get_private(folio); 145 ASSERT(atomic_read(&bfs->eb_refs)); 146 atomic_dec(&bfs->eb_refs); 147 } 148 149 static void btrfs_subpage_assert(const struct btrfs_fs_info *fs_info, 150 struct folio *folio, u64 start, u32 len) 151 { 152 /* Basic checks */ 153 ASSERT(folio_test_private(folio) && folio_get_private(folio)); 154 ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && 155 IS_ALIGNED(len, fs_info->sectorsize), "start=%llu len=%u", start, len); 156 /* 157 * The range check only works for mapped page, we can still have 158 * unmapped page like dummy extent buffer pages. 159 */ 160 if (folio->mapping) 161 ASSERT(folio_pos(folio) <= start && 162 start + len <= folio_next_pos(folio), 163 "start=%llu len=%u folio_pos=%llu folio_size=%zu", 164 start, len, folio_pos(folio), folio_size(folio)); 165 } 166 167 #define subpage_calc_start_bit(fs_info, folio, name, start, len) \ 168 ({ \ 169 unsigned int __start_bit; \ 170 const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ 171 \ 172 btrfs_subpage_assert(fs_info, folio, start, len); \ 173 __start_bit = offset_in_folio(folio, start) >> fs_info->sectorsize_bits; \ 174 __start_bit += __bpf * btrfs_bitmap_nr_##name; \ 175 __start_bit; \ 176 }) 177 178 static void btrfs_subpage_clamp_range(struct folio *folio, u64 *start, u32 *len) 179 { 180 u64 orig_start = *start; 181 u32 orig_len = *len; 182 183 *start = max_t(u64, folio_pos(folio), orig_start); 184 /* 185 * For certain call sites like btrfs_drop_pages(), we may have pages 186 * beyond the target range. In that case, just set @len to 0, subpage 187 * helpers can handle @len == 0 without any problem. 188 */ 189 if (folio_pos(folio) >= orig_start + orig_len) 190 *len = 0; 191 else 192 *len = min_t(u64, folio_next_pos(folio), orig_start + orig_len) - *start; 193 } 194 195 static bool btrfs_subpage_end_and_test_lock(const struct btrfs_fs_info *fs_info, 196 struct folio *folio, u64 start, u32 len) 197 { 198 struct btrfs_folio_state *bfs = folio_get_private(folio); 199 const int nbits = (len >> fs_info->sectorsize_bits); 200 unsigned long flags; 201 bool last; 202 203 btrfs_subpage_assert(fs_info, folio, start, len); 204 205 spin_lock_irqsave(&bfs->lock, flags); 206 /* 207 * We have call sites passing @lock_page into 208 * extent_clear_unlock_delalloc() for compression path. 209 * 210 * This @locked_page is locked by plain lock_page(), thus its 211 * subpage::locked is 0. Handle them in a special way. 212 */ 213 if (atomic_read(&bfs->nr_locked) == 0) { 214 spin_unlock_irqrestore(&bfs->lock, flags); 215 return true; 216 } 217 ASSERT(atomic_read(&bfs->nr_locked) >= nbits, 218 "atomic_read(&bfs->nr_locked)=%d nbits=%d", 219 atomic_read(&bfs->nr_locked), nbits); 220 last = atomic_sub_and_test(nbits, &bfs->nr_locked); 221 spin_unlock_irqrestore(&bfs->lock, flags); 222 return last; 223 } 224 225 /* 226 * Handle different locked folios: 227 * 228 * - Non-subpage folio 229 * Just unlock it. 230 * 231 * - folio locked but without any subpage locked 232 * This happens either before writepage_delalloc() or the delalloc range is 233 * already handled by previous folio. 234 * We can simple unlock it. 235 * 236 * - folio locked with subpage range locked. 237 * We go through the locked sectors inside the range and clear their locked 238 * bitmap, reduce the writer lock number, and unlock the page if that's 239 * the last locked range. 240 */ 241 void btrfs_folio_end_lock(const struct btrfs_fs_info *fs_info, 242 struct folio *folio, u64 start, u32 len) 243 { 244 struct btrfs_folio_state *bfs = folio_get_private(folio); 245 246 ASSERT(folio_test_locked(folio)); 247 248 if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, folio)) { 249 folio_unlock(folio); 250 return; 251 } 252 253 /* 254 * For subpage case, there are two types of locked page. With or 255 * without locked number. 256 * 257 * Since we own the page lock, no one else could touch subpage::locked 258 * and we are safe to do several atomic operations without spinlock. 259 */ 260 if (atomic_read(&bfs->nr_locked) == 0) { 261 /* No subpage lock, locked by plain lock_page(). */ 262 folio_unlock(folio); 263 return; 264 } 265 266 btrfs_subpage_clamp_range(folio, &start, &len); 267 if (btrfs_subpage_end_and_test_lock(fs_info, folio, start, len)) 268 folio_unlock(folio); 269 } 270 271 void btrfs_folio_end_lock_bitmap(const struct btrfs_fs_info *fs_info, 272 struct folio *folio, unsigned long *bitmap) 273 { 274 struct btrfs_folio_state *bfs = folio_get_private(folio); 275 const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); 276 const unsigned int nbits = bitmap_weight(bitmap, blocks_per_folio); 277 unsigned long flags; 278 bool last = false; 279 280 if (!btrfs_is_subpage(fs_info, folio)) { 281 folio_unlock(folio); 282 return; 283 } 284 285 if (atomic_read(&bfs->nr_locked) == 0) { 286 /* No subpage lock, locked by plain lock_page(). */ 287 folio_unlock(folio); 288 return; 289 } 290 291 spin_lock_irqsave(&bfs->lock, flags); 292 ASSERT(atomic_read(&bfs->nr_locked) >= nbits, 293 "atomic_read(&bfs->nr_locked)=%d nbits=%d", 294 atomic_read(&bfs->nr_locked), nbits); 295 last = atomic_sub_and_test(nbits, &bfs->nr_locked); 296 spin_unlock_irqrestore(&bfs->lock, flags); 297 if (last) 298 folio_unlock(folio); 299 } 300 301 #define subpage_test_bitmap_all_set(fs_info, folio, name) \ 302 ({ \ 303 struct btrfs_folio_state *__bfs = folio_get_private(folio); \ 304 const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ 305 \ 306 bitmap_test_range_all_set(__bfs->bitmaps, \ 307 __bpf * btrfs_bitmap_nr_##name, __bpf); \ 308 }) 309 310 #define subpage_test_bitmap_all_zero(fs_info, folio, name) \ 311 ({ \ 312 struct btrfs_folio_state *__bfs = folio_get_private(folio); \ 313 const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ 314 \ 315 bitmap_test_range_all_zero(__bfs->bitmaps, \ 316 __bpf * btrfs_bitmap_nr_##name, __bpf); \ 317 }) 318 319 void btrfs_subpage_set_uptodate(const struct btrfs_fs_info *fs_info, 320 struct folio *folio, u64 start, u32 len) 321 { 322 struct btrfs_folio_state *bfs = folio_get_private(folio); 323 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, 324 uptodate, start, len); 325 unsigned long flags; 326 327 spin_lock_irqsave(&bfs->lock, flags); 328 bitmap_set(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 329 if (subpage_test_bitmap_all_set(fs_info, folio, uptodate)) 330 folio_mark_uptodate(folio); 331 spin_unlock_irqrestore(&bfs->lock, flags); 332 } 333 334 void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info, 335 struct folio *folio, u64 start, u32 len) 336 { 337 struct btrfs_folio_state *bfs = folio_get_private(folio); 338 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, 339 uptodate, start, len); 340 unsigned long flags; 341 342 spin_lock_irqsave(&bfs->lock, flags); 343 bitmap_clear(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 344 folio_clear_uptodate(folio); 345 spin_unlock_irqrestore(&bfs->lock, flags); 346 } 347 348 /* 349 * folio_mark_dirty() for a folio we are dirtying with a space reservation. 350 * 351 * Dirtiers without a reservation use btrfs_data_dirty_folio(). 352 */ 353 static void btrfs_folio_mark_dirty(struct folio *folio) 354 { 355 struct address_space *mapping = folio_mapping(folio); 356 357 if (!mapping || !mapping->host || !is_data_inode(BTRFS_I(mapping->host))) { 358 folio_mark_dirty(folio); 359 return; 360 } 361 if (folio_test_reclaim(folio)) 362 folio_clear_reclaim(folio); 363 filemap_dirty_folio(mapping, folio); 364 } 365 366 /* 367 * The set helper of the dirty ops, so it only runs for folios without a 368 * fixup bitmap: for those the folio flag is the whole fixup state, and this 369 * reserving write covers the block, so retire it. Metadata never has the 370 * flag set and only pays the test. 371 */ 372 static void btrfs_folio_mark_dirty_reserved(struct folio *folio) 373 { 374 if (folio_test_fixup_pending(folio)) 375 folio_clear_fixup_pending(folio); 376 btrfs_folio_mark_dirty(folio); 377 } 378 379 void btrfs_subpage_set_dirty(const struct btrfs_fs_info *fs_info, 380 struct folio *folio, u64 start, u32 len) 381 { 382 struct btrfs_folio_state *bfs = folio_get_private(folio); 383 unsigned int dirty_bit = subpage_calc_start_bit(fs_info, folio, 384 dirty, start, len); 385 unsigned int fixup_bit = subpage_calc_start_bit(fs_info, folio, 386 fixup, start, len); 387 const unsigned int nbits = len >> fs_info->sectorsize_bits; 388 unsigned long flags; 389 390 spin_lock_irqsave(&bfs->lock, flags); 391 bitmap_set(bfs->bitmaps, dirty_bit, nbits); 392 /* Proper dirtying obviates the need for fixup. */ 393 bitmap_clear(bfs->bitmaps, fixup_bit, nbits); 394 if (folio_test_fixup_pending(folio) && 395 subpage_test_bitmap_all_zero(fs_info, folio, fixup)) 396 folio_clear_fixup_pending(folio); 397 spin_unlock_irqrestore(&bfs->lock, flags); 398 btrfs_folio_mark_dirty(folio); 399 } 400 401 static void folio_clear_tags(struct folio *folio) 402 { 403 struct address_space *mapping = folio_mapping(folio); 404 XA_STATE(xas, &mapping->i_pages, folio->index); 405 unsigned long flags; 406 407 ASSERT(folio_test_locked(folio)); 408 ASSERT(mapping); 409 ASSERT(mapping_use_writeback_tags(mapping)); 410 411 xas_lock_irqsave(&xas, flags); 412 xas_load(&xas); 413 xas_clear_mark(&xas, PAGECACHE_TAG_DIRTY); 414 xas_clear_mark(&xas, PAGECACHE_TAG_TOWRITE); 415 xas_unlock_irqrestore(&xas, flags); 416 } 417 418 /* 419 * Extra clear_and_test function for subpage dirty bitmap. 420 * 421 * Return true if we're the last bits in the dirty_bitmap and clear the 422 * dirty_bitmap. 423 * Return false otherwise. 424 * 425 * NOTE: Callers should manually clear page dirty for true case, as we have 426 * extra handling for tree blocks. 427 */ 428 bool btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info *fs_info, 429 struct folio *folio, u64 start, u32 len) 430 { 431 struct btrfs_folio_state *bfs = folio_get_private(folio); 432 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, 433 dirty, start, len); 434 unsigned long flags; 435 bool last = false; 436 437 spin_lock_irqsave(&bfs->lock, flags); 438 bitmap_clear(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 439 if (subpage_test_bitmap_all_zero(fs_info, folio, dirty)) 440 last = true; 441 spin_unlock_irqrestore(&bfs->lock, flags); 442 return last; 443 } 444 445 void btrfs_subpage_clear_dirty(const struct btrfs_fs_info *fs_info, 446 struct folio *folio, u64 start, u32 len) 447 { 448 bool last; 449 450 last = btrfs_subpage_clear_and_test_dirty(fs_info, folio, start, len); 451 if (last) 452 folio_clear_dirty_for_io(folio); 453 } 454 455 void btrfs_subpage_set_writeback(const struct btrfs_fs_info *fs_info, 456 struct folio *folio, u64 start, u32 len) 457 { 458 struct btrfs_folio_state *bfs = folio_get_private(folio); 459 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, 460 writeback, start, len); 461 unsigned long flags; 462 463 spin_lock_irqsave(&bfs->lock, flags); 464 bitmap_set(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 465 466 /* 467 * Don't clear the TOWRITE tag when starting writeback on a still-dirty 468 * folio. Doing so can cause WB_SYNC_ALL writepages() to overlook it, 469 * assume writeback is complete, and exit too early — violating sync 470 * ordering guarantees. 471 * 472 * Instead we manually clear the DIRTY and TOWRITE tags after the folio 473 * is no longer dirty. 474 */ 475 if (!folio_test_writeback(folio)) 476 __folio_start_writeback(folio, true); 477 if (!folio_test_dirty(folio)) 478 folio_clear_tags(folio); 479 spin_unlock_irqrestore(&bfs->lock, flags); 480 } 481 482 void btrfs_subpage_clear_writeback(const struct btrfs_fs_info *fs_info, 483 struct folio *folio, u64 start, u32 len) 484 { 485 struct btrfs_folio_state *bfs = folio_get_private(folio); 486 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, 487 writeback, start, len); 488 unsigned long flags; 489 490 spin_lock_irqsave(&bfs->lock, flags); 491 bitmap_clear(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 492 if (subpage_test_bitmap_all_zero(fs_info, folio, writeback)) { 493 ASSERT(folio_test_writeback(folio)); 494 folio_end_writeback(folio); 495 } 496 spin_unlock_irqrestore(&bfs->lock, flags); 497 } 498 499 void btrfs_subpage_clear_fixup(const struct btrfs_fs_info *fs_info, 500 struct folio *folio, u64 start, u32 len) 501 { 502 struct btrfs_folio_state *bfs = folio_get_private(folio); 503 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, 504 fixup, start, len); 505 unsigned long flags; 506 507 spin_lock_irqsave(&bfs->lock, flags); 508 bitmap_clear(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 509 if (subpage_test_bitmap_all_zero(fs_info, folio, fixup)) 510 folio_clear_fixup_pending(folio); 511 spin_unlock_irqrestore(&bfs->lock, flags); 512 } 513 514 /* 515 * In one pass under bfs->lock, mark every block with a clear dirty bit in the 516 * range both dirty and needing fixup. 517 * 518 * Only called from the dirty_folio callback, which owns the folio-level 519 * dirty flag; calling folio_mark_dirty() here would recurse. 520 * 521 * The folio fixup flag and bits are both set under bfs->lock so that a 522 * writeback pass observing the new bits also observes the flag. 523 */ 524 static void btrfs_subpage_set_fixup_dirty(const struct btrfs_fs_info *fs_info, 525 struct folio *folio, u64 start, u32 len) 526 { 527 struct btrfs_folio_state *bfs = folio_get_private(folio); 528 unsigned int dirty_bit = subpage_calc_start_bit(fs_info, folio, 529 dirty, start, len); 530 unsigned int fixup_bit = subpage_calc_start_bit(fs_info, folio, 531 fixup, start, len); 532 const unsigned int nbits = len >> fs_info->sectorsize_bits; 533 unsigned long flags; 534 bool marked = false; 535 536 spin_lock_irqsave(&bfs->lock, flags); 537 for (unsigned int i = 0; i < nbits; i++) { 538 if (test_bit(dirty_bit + i, bfs->bitmaps)) 539 continue; 540 set_bit(dirty_bit + i, bfs->bitmaps); 541 set_bit(fixup_bit + i, bfs->bitmaps); 542 marked = true; 543 } 544 if (marked) 545 folio_set_fixup_pending(folio); 546 spin_unlock_irqrestore(&bfs->lock, flags); 547 } 548 549 /* 550 * Mark the still-clean blocks of a folio dirty and needing fixup, for 551 * btrfs_data_dirty_folio(). 552 * 553 * A subpage block size folio that is not uptodate is left alone: its clean 554 * blocks may hold content that was never read in, which must not be marked 555 * dirty. 556 */ 557 void btrfs_folio_set_fixup_dirty(const struct btrfs_fs_info *fs_info, 558 struct folio *folio, u64 start, u32 len) 559 { 560 if (!btrfs_is_subpage(fs_info, folio)) { 561 if (!folio_test_dirty(folio)) 562 folio_set_fixup_pending(folio); 563 return; 564 } 565 if (!folio_test_uptodate(folio)) 566 return; 567 btrfs_subpage_set_fixup_dirty(fs_info, folio, start, len); 568 } 569 570 /* 571 * Drop the fixup blocks inside the range: clear both their fixup and dirty 572 * bits. 573 * 574 * Fixup blocks carry no space reservation, so their fixup and dirty bits 575 * must be dropped together. Clearing only the fixup bit would leave a 576 * dirty block without a reservation which is not a valid state. 577 * 578 * Returns true if the folio has no dirty blocks left. 579 */ 580 static bool btrfs_subpage_clear_fixup_dirty(const struct btrfs_fs_info *fs_info, 581 struct folio *folio, u64 start, u32 len) 582 { 583 struct btrfs_folio_state *bfs = folio_get_private(folio); 584 unsigned int dirty_bit = subpage_calc_start_bit(fs_info, folio, 585 dirty, start, len); 586 unsigned int fixup_bit = subpage_calc_start_bit(fs_info, folio, 587 fixup, start, len); 588 const unsigned int nbits = len >> fs_info->sectorsize_bits; 589 unsigned long flags; 590 bool last; 591 592 spin_lock_irqsave(&bfs->lock, flags); 593 for (unsigned int i = 0; i < nbits; i++) { 594 if (!test_bit(fixup_bit + i, bfs->bitmaps)) 595 continue; 596 clear_bit(fixup_bit + i, bfs->bitmaps); 597 clear_bit(dirty_bit + i, bfs->bitmaps); 598 } 599 if (subpage_test_bitmap_all_zero(fs_info, folio, fixup)) 600 folio_clear_fixup_pending(folio); 601 last = subpage_test_bitmap_all_zero(fs_info, folio, dirty); 602 spin_unlock_irqrestore(&bfs->lock, flags); 603 return last; 604 } 605 606 /* 607 * Drop the fixup blocks inside the range, for callers discarding their data: 608 * btrfs_invalidate_folio() and the writepage fixup worker's error path. 609 * 610 * Callers that have just reserved space for a block want 611 * btrfs_folio_clear_fixup() instead - there the block stays dirty and gets 612 * written. 613 * 614 * The range can be byte-granular (an unaligned truncate through 615 * btrfs_invalidate_folio()); only blocks fully inside it are dropped, as a 616 * partially covered block still holds live data outside the range. For 617 * single-block folios the folio flag is the fixup state, so it is dropped 618 * only when the range covers the whole folio. 619 */ 620 void btrfs_folio_clear_fixup_dirty(const struct btrfs_fs_info *fs_info, 621 struct folio *folio, u64 start, u32 len) 622 { 623 u64 aligned_start; 624 u64 aligned_end; 625 626 /* The folio flag is set whenever any fixup bitmap bit is. */ 627 if (!folio_test_fixup_pending(folio)) 628 return; 629 if (!btrfs_is_subpage(fs_info, folio)) { 630 if (start <= folio_pos(folio) && 631 start + len >= folio_next_pos(folio)) { 632 folio_clear_fixup_pending(folio); 633 folio_clear_dirty_for_io(folio); 634 } 635 return; 636 } 637 btrfs_subpage_clamp_range(folio, &start, &len); 638 aligned_start = round_up(start, fs_info->sectorsize); 639 aligned_end = round_down(start + len, fs_info->sectorsize); 640 if (aligned_end <= aligned_start) 641 return; 642 if (btrfs_subpage_clear_fixup_dirty(fs_info, folio, aligned_start, 643 aligned_end - aligned_start)) 644 folio_clear_dirty_for_io(folio); 645 } 646 647 bool btrfs_folio_test_fixup(const struct btrfs_fs_info *fs_info, 648 struct folio *folio, u64 start, u32 len) 649 { 650 if (!btrfs_is_subpage(fs_info, folio)) 651 return folio_test_fixup_pending(folio); 652 return btrfs_subpage_test_fixup(fs_info, folio, start, len); 653 } 654 655 void btrfs_folio_clear_fixup(const struct btrfs_fs_info *fs_info, 656 struct folio *folio, u64 start, u32 len) 657 { 658 if (!btrfs_is_subpage(fs_info, folio)) { 659 folio_clear_fixup_pending(folio); 660 return; 661 } 662 btrfs_subpage_clear_fixup(fs_info, folio, start, len); 663 } 664 665 /* 666 * Unlike set/clear which is dependent on each page status, for test all bits 667 * are tested in the same way. 668 */ 669 #define IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(name) \ 670 bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \ 671 struct folio *folio, u64 start, u32 len) \ 672 { \ 673 struct btrfs_folio_state *bfs = folio_get_private(folio); \ 674 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, \ 675 name, start, len); \ 676 unsigned long flags; \ 677 bool ret; \ 678 \ 679 spin_lock_irqsave(&bfs->lock, flags); \ 680 ret = bitmap_test_range_all_set(bfs->bitmaps, start_bit, \ 681 len >> fs_info->sectorsize_bits); \ 682 spin_unlock_irqrestore(&bfs->lock, flags); \ 683 return ret; \ 684 } 685 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate); 686 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(dirty); 687 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(writeback); 688 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(fixup); 689 690 /* 691 * Note that, in selftests (extent-io-tests), we can have empty fs_info passed 692 * in. We only test sectorsize == PAGE_SIZE cases so far, thus we can fall 693 * back to regular sectorsize branch. 694 */ 695 #define IMPLEMENT_BTRFS_PAGE_OPS(name, folio_set_func, \ 696 folio_clear_func, folio_test_func) \ 697 void btrfs_folio_set_##name(const struct btrfs_fs_info *fs_info, \ 698 struct folio *folio, u64 start, u32 len) \ 699 { \ 700 if (unlikely(!fs_info) || \ 701 !btrfs_is_subpage(fs_info, folio)) { \ 702 folio_set_func(folio); \ 703 return; \ 704 } \ 705 btrfs_subpage_set_##name(fs_info, folio, start, len); \ 706 } \ 707 void btrfs_folio_clear_##name(const struct btrfs_fs_info *fs_info, \ 708 struct folio *folio, u64 start, u32 len) \ 709 { \ 710 if (unlikely(!fs_info) || \ 711 !btrfs_is_subpage(fs_info, folio)) { \ 712 folio_clear_func(folio); \ 713 return; \ 714 } \ 715 btrfs_subpage_clear_##name(fs_info, folio, start, len); \ 716 } \ 717 bool btrfs_folio_test_##name(const struct btrfs_fs_info *fs_info, \ 718 struct folio *folio, u64 start, u32 len) \ 719 { \ 720 if (unlikely(!fs_info) || \ 721 !btrfs_is_subpage(fs_info, folio)) \ 722 return folio_test_func(folio); \ 723 return btrfs_subpage_test_##name(fs_info, folio, start, len); \ 724 } \ 725 void btrfs_folio_clamp_set_##name(const struct btrfs_fs_info *fs_info, \ 726 struct folio *folio, u64 start, u32 len) \ 727 { \ 728 if (unlikely(!fs_info) || \ 729 !btrfs_is_subpage(fs_info, folio)) { \ 730 folio_set_func(folio); \ 731 return; \ 732 } \ 733 btrfs_subpage_clamp_range(folio, &start, &len); \ 734 btrfs_subpage_set_##name(fs_info, folio, start, len); \ 735 } \ 736 void btrfs_folio_clamp_clear_##name(const struct btrfs_fs_info *fs_info, \ 737 struct folio *folio, u64 start, u32 len) \ 738 { \ 739 if (unlikely(!fs_info) || \ 740 !btrfs_is_subpage(fs_info, folio)) { \ 741 folio_clear_func(folio); \ 742 return; \ 743 } \ 744 btrfs_subpage_clamp_range(folio, &start, &len); \ 745 btrfs_subpage_clear_##name(fs_info, folio, start, len); \ 746 } \ 747 bool btrfs_folio_clamp_test_##name(const struct btrfs_fs_info *fs_info, \ 748 struct folio *folio, u64 start, u32 len) \ 749 { \ 750 if (unlikely(!fs_info) || \ 751 !btrfs_is_subpage(fs_info, folio)) \ 752 return folio_test_func(folio); \ 753 btrfs_subpage_clamp_range(folio, &start, &len); \ 754 return btrfs_subpage_test_##name(fs_info, folio, start, len); \ 755 } \ 756 void btrfs_meta_folio_set_##name(struct folio *folio, const struct extent_buffer *eb) \ 757 { \ 758 if (!btrfs_meta_is_subpage(eb->fs_info)) { \ 759 folio_set_func(folio); \ 760 return; \ 761 } \ 762 btrfs_subpage_set_##name(eb->fs_info, folio, eb->start, eb->len); \ 763 } \ 764 void btrfs_meta_folio_clear_##name(struct folio *folio, const struct extent_buffer *eb) \ 765 { \ 766 if (!btrfs_meta_is_subpage(eb->fs_info)) { \ 767 folio_clear_func(folio); \ 768 return; \ 769 } \ 770 btrfs_subpage_clear_##name(eb->fs_info, folio, eb->start, eb->len); \ 771 } \ 772 bool btrfs_meta_folio_test_##name(struct folio *folio, const struct extent_buffer *eb) \ 773 { \ 774 if (!btrfs_meta_is_subpage(eb->fs_info)) \ 775 return folio_test_func(folio); \ 776 return btrfs_subpage_test_##name(eb->fs_info, folio, eb->start, eb->len); \ 777 } 778 IMPLEMENT_BTRFS_PAGE_OPS(uptodate, folio_mark_uptodate, folio_clear_uptodate, 779 folio_test_uptodate); 780 IMPLEMENT_BTRFS_PAGE_OPS(dirty, btrfs_folio_mark_dirty_reserved, 781 folio_clear_dirty_for_io, folio_test_dirty); 782 IMPLEMENT_BTRFS_PAGE_OPS(writeback, folio_start_writeback, folio_end_writeback, 783 folio_test_writeback); 784 785 #define DEFINE_GET_SUBPAGE_BITMAP(name) \ 786 static inline unsigned long get_bitmap_value_##name( \ 787 const struct btrfs_fs_info *fs_info, \ 788 struct folio *folio) \ 789 { \ 790 const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ 791 const struct btrfs_folio_state *__bfs = folio_get_private(folio); \ 792 unsigned long value; \ 793 \ 794 ASSERT(__bpf <= BITS_PER_LONG); \ 795 value = bitmap_read(__bfs->bitmaps, __bpf * btrfs_bitmap_nr_##name, \ 796 __bpf); \ 797 return value; \ 798 } \ 799 static inline const unsigned long *get_bitmap_pointer_##name( \ 800 const struct btrfs_fs_info *fs_info, \ 801 struct folio *folio) \ 802 { \ 803 const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ 804 struct btrfs_folio_state *__bfs = folio_get_private(folio); \ 805 unsigned long *pointer; \ 806 \ 807 ASSERT(__bpf >= BITS_PER_LONG); \ 808 ASSERT(IS_ALIGNED(__bpf, BITS_PER_LONG)); \ 809 pointer = __bfs->bitmaps + (BIT_WORD(__bpf) * btrfs_bitmap_nr_##name); \ 810 return pointer; \ 811 } 812 813 DEFINE_GET_SUBPAGE_BITMAP(uptodate); 814 DEFINE_GET_SUBPAGE_BITMAP(dirty); 815 DEFINE_GET_SUBPAGE_BITMAP(writeback); 816 817 #define SUBPAGE_DUMP_BITMAP(fs_info, folio, name, start, len) \ 818 { \ 819 const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ 820 \ 821 if (__bpf <= BITS_PER_LONG) { \ 822 unsigned long bitmap = get_bitmap_value_##name(fs_info, folio); \ 823 \ 824 btrfs_warn(fs_info, \ 825 "dumping bitmap start=%llu len=%u folio=%llu " #name "_bitmap=%*pbl", \ 826 start, len, folio_pos(folio), __bpf, &bitmap); \ 827 } else { \ 828 btrfs_warn(fs_info, \ 829 "dumping bitmap start=%llu len=%u folio=%llu " #name "_bitmap=%*pbl", \ 830 start, len, folio_pos(folio), __bpf, \ 831 get_bitmap_pointer_##name(fs_info, folio)); \ 832 } \ 833 } 834 835 /* 836 * Make sure not only the page dirty bit is cleared, but also subpage dirty bit 837 * is cleared. 838 */ 839 void btrfs_folio_assert_not_dirty(const struct btrfs_fs_info *fs_info, 840 struct folio *folio, u64 start, u32 len) 841 { 842 struct btrfs_folio_state *bfs; 843 unsigned int start_bit; 844 unsigned int nbits; 845 unsigned long flags; 846 847 if (!IS_ENABLED(CONFIG_BTRFS_ASSERT)) 848 return; 849 850 if (!btrfs_is_subpage(fs_info, folio)) { 851 ASSERT(!folio_test_dirty(folio)); 852 return; 853 } 854 855 start_bit = subpage_calc_start_bit(fs_info, folio, dirty, start, len); 856 nbits = len >> fs_info->sectorsize_bits; 857 bfs = folio_get_private(folio); 858 ASSERT(bfs); 859 spin_lock_irqsave(&bfs->lock, flags); 860 if (unlikely(!bitmap_test_range_all_zero(bfs->bitmaps, start_bit, nbits))) { 861 SUBPAGE_DUMP_BITMAP(fs_info, folio, dirty, start, len); 862 ASSERT(bitmap_test_range_all_zero(bfs->bitmaps, start_bit, nbits)); 863 } 864 ASSERT(bitmap_test_range_all_zero(bfs->bitmaps, start_bit, nbits)); 865 spin_unlock_irqrestore(&bfs->lock, flags); 866 } 867 868 /* 869 * This is for folio already locked by plain lock_page()/folio_lock(), which 870 * doesn't have any subpage awareness. 871 * 872 * This populates the involved subpage ranges so that subpage helpers can 873 * properly unlock them. 874 */ 875 void btrfs_folio_set_lock(const struct btrfs_fs_info *fs_info, 876 struct folio *folio, u64 start, u32 len) 877 { 878 struct btrfs_folio_state *bfs; 879 unsigned long flags; 880 unsigned int nbits; 881 int ret; 882 883 ASSERT(folio_test_locked(folio)); 884 if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, folio)) 885 return; 886 887 bfs = folio_get_private(folio); 888 nbits = len >> fs_info->sectorsize_bits; 889 spin_lock_irqsave(&bfs->lock, flags); 890 ret = atomic_add_return(nbits, &bfs->nr_locked); 891 ASSERT(ret <= btrfs_blocks_per_folio(fs_info, folio)); 892 spin_unlock_irqrestore(&bfs->lock, flags); 893 } 894 895 /* 896 * Clear the dirty flag for the folio. 897 * 898 * If the affected folio is no longer dirty, return true. Otherwise return false. 899 */ 900 bool btrfs_meta_folio_clear_and_test_dirty(struct folio *folio, const struct extent_buffer *eb) 901 { 902 bool last; 903 904 if (!btrfs_meta_is_subpage(eb->fs_info)) { 905 folio_clear_dirty_for_io(folio); 906 return true; 907 } 908 909 last = btrfs_subpage_clear_and_test_dirty(eb->fs_info, folio, eb->start, eb->len); 910 if (last) { 911 folio_clear_dirty_for_io(folio); 912 return true; 913 } 914 return false; 915 } 916 917 void __cold btrfs_subpage_dump_bitmap(const struct btrfs_fs_info *fs_info, 918 struct folio *folio, u64 start, u32 len) 919 { 920 struct btrfs_folio_state *bfs; 921 const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); 922 unsigned long flags; 923 924 ASSERT(folio_test_private(folio) && folio_get_private(folio)); 925 ASSERT(blocks_per_folio > 1); 926 bfs = folio_get_private(folio); 927 928 dump_page(folio_page(folio, 0), "btrfs folio state dump"); 929 930 if (blocks_per_folio <= BITS_PER_LONG) { 931 unsigned long uptodate; 932 unsigned long dirty; 933 unsigned long writeback; 934 935 spin_lock_irqsave(&bfs->lock, flags); 936 uptodate = get_bitmap_value_uptodate(fs_info, folio); 937 dirty = get_bitmap_value_dirty(fs_info, folio); 938 writeback = get_bitmap_value_writeback(fs_info, folio); 939 940 spin_unlock_irqrestore(&bfs->lock, flags); 941 942 btrfs_warn(fs_info, 943 "start=%llu len=%u page=%llu, bitmaps uptodate=%*pbl dirty=%*pbl writeback=%*pbl", 944 start, len, folio_pos(folio), 945 blocks_per_folio, &uptodate, 946 blocks_per_folio, &dirty, 947 blocks_per_folio, &writeback); 948 return; 949 } 950 951 spin_lock_irqsave(&bfs->lock, flags); 952 btrfs_warn(fs_info, 953 "start=%llu len=%u page=%llu, bitmaps uptodate=%*pbl dirty=%*pbl writeback=%*pbl", 954 start, len, folio_pos(folio), 955 blocks_per_folio, get_bitmap_pointer_uptodate(fs_info, folio), 956 blocks_per_folio, get_bitmap_pointer_dirty(fs_info, folio), 957 blocks_per_folio, get_bitmap_pointer_writeback(fs_info, folio)); 958 spin_unlock_irqrestore(&bfs->lock, flags); 959 } 960 961 void btrfs_copy_subpage_dirty_bitmap(struct btrfs_fs_info *fs_info, 962 struct folio *folio, 963 unsigned long *dst) 964 { 965 struct btrfs_folio_state *bfs; 966 const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); 967 unsigned long flags; 968 unsigned long value; 969 970 if (blocks_per_folio == 1) { 971 value = 1; 972 bitmap_copy(dst, &value, 1); 973 return; 974 } 975 976 ASSERT(folio_test_private(folio) && folio_get_private(folio)); 977 ASSERT(blocks_per_folio > 1); 978 bfs = folio_get_private(folio); 979 980 if (blocks_per_folio <= BITS_PER_LONG) { 981 spin_lock_irqsave(&bfs->lock, flags); 982 value = bitmap_read(bfs->bitmaps, btrfs_bitmap_nr_dirty * blocks_per_folio, 983 blocks_per_folio); 984 spin_unlock_irqrestore(&bfs->lock, flags); 985 bitmap_copy(dst, &value, blocks_per_folio); 986 return; 987 } 988 spin_lock_irqsave(&bfs->lock, flags); 989 bitmap_copy(dst, get_bitmap_pointer_dirty(fs_info, folio), 990 blocks_per_folio); 991 spin_unlock_irqrestore(&bfs->lock, flags); 992 } 993