1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/bitops.h> 4 #include <linux/slab.h> 5 #include <linux/bio.h> 6 #include <linux/mm.h> 7 #include <linux/pagemap.h> 8 #include <linux/page-flags.h> 9 #include <linux/spinlock.h> 10 #include <linux/blkdev.h> 11 #include <linux/swap.h> 12 #include <linux/writeback.h> 13 #include <linux/pagevec.h> 14 #include <linux/prefetch.h> 15 #include <linux/cleancache.h> 16 #include "misc.h" 17 #include "extent_io.h" 18 #include "extent-io-tree.h" 19 #include "extent_map.h" 20 #include "ctree.h" 21 #include "btrfs_inode.h" 22 #include "volumes.h" 23 #include "check-integrity.h" 24 #include "locking.h" 25 #include "rcu-string.h" 26 #include "backref.h" 27 #include "disk-io.h" 28 #include "subpage.h" 29 #include "zoned.h" 30 #include "block-group.h" 31 32 static struct kmem_cache *extent_state_cache; 33 static struct kmem_cache *extent_buffer_cache; 34 static struct bio_set btrfs_bioset; 35 36 static inline bool extent_state_in_tree(const struct extent_state *state) 37 { 38 return !RB_EMPTY_NODE(&state->rb_node); 39 } 40 41 #ifdef CONFIG_BTRFS_DEBUG 42 static LIST_HEAD(states); 43 static DEFINE_SPINLOCK(leak_lock); 44 45 static inline void btrfs_leak_debug_add(spinlock_t *lock, 46 struct list_head *new, 47 struct list_head *head) 48 { 49 unsigned long flags; 50 51 spin_lock_irqsave(lock, flags); 52 list_add(new, head); 53 spin_unlock_irqrestore(lock, flags); 54 } 55 56 static inline void btrfs_leak_debug_del(spinlock_t *lock, 57 struct list_head *entry) 58 { 59 unsigned long flags; 60 61 spin_lock_irqsave(lock, flags); 62 list_del(entry); 63 spin_unlock_irqrestore(lock, flags); 64 } 65 66 void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info) 67 { 68 struct extent_buffer *eb; 69 unsigned long flags; 70 71 /* 72 * If we didn't get into open_ctree our allocated_ebs will not be 73 * initialized, so just skip this. 74 */ 75 if (!fs_info->allocated_ebs.next) 76 return; 77 78 spin_lock_irqsave(&fs_info->eb_leak_lock, flags); 79 while (!list_empty(&fs_info->allocated_ebs)) { 80 eb = list_first_entry(&fs_info->allocated_ebs, 81 struct extent_buffer, leak_list); 82 pr_err( 83 "BTRFS: buffer leak start %llu len %lu refs %d bflags %lu owner %llu\n", 84 eb->start, eb->len, atomic_read(&eb->refs), eb->bflags, 85 btrfs_header_owner(eb)); 86 list_del(&eb->leak_list); 87 kmem_cache_free(extent_buffer_cache, eb); 88 } 89 spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags); 90 } 91 92 static inline void btrfs_extent_state_leak_debug_check(void) 93 { 94 struct extent_state *state; 95 96 while (!list_empty(&states)) { 97 state = list_entry(states.next, struct extent_state, leak_list); 98 pr_err("BTRFS: state leak: start %llu end %llu state %u in tree %d refs %d\n", 99 state->start, state->end, state->state, 100 extent_state_in_tree(state), 101 refcount_read(&state->refs)); 102 list_del(&state->leak_list); 103 kmem_cache_free(extent_state_cache, state); 104 } 105 } 106 107 #define btrfs_debug_check_extent_io_range(tree, start, end) \ 108 __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end)) 109 static inline void __btrfs_debug_check_extent_io_range(const char *caller, 110 struct extent_io_tree *tree, u64 start, u64 end) 111 { 112 struct inode *inode = tree->private_data; 113 u64 isize; 114 115 if (!inode || !is_data_inode(inode)) 116 return; 117 118 isize = i_size_read(inode); 119 if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) { 120 btrfs_debug_rl(BTRFS_I(inode)->root->fs_info, 121 "%s: ino %llu isize %llu odd range [%llu,%llu]", 122 caller, btrfs_ino(BTRFS_I(inode)), isize, start, end); 123 } 124 } 125 #else 126 #define btrfs_leak_debug_add(lock, new, head) do {} while (0) 127 #define btrfs_leak_debug_del(lock, entry) do {} while (0) 128 #define btrfs_extent_state_leak_debug_check() do {} while (0) 129 #define btrfs_debug_check_extent_io_range(c, s, e) do {} while (0) 130 #endif 131 132 struct tree_entry { 133 u64 start; 134 u64 end; 135 struct rb_node rb_node; 136 }; 137 138 struct extent_page_data { 139 struct bio *bio; 140 /* tells writepage not to lock the state bits for this range 141 * it still does the unlocking 142 */ 143 unsigned int extent_locked:1; 144 145 /* tells the submit_bio code to use REQ_SYNC */ 146 unsigned int sync_io:1; 147 }; 148 149 static int add_extent_changeset(struct extent_state *state, u32 bits, 150 struct extent_changeset *changeset, 151 int set) 152 { 153 int ret; 154 155 if (!changeset) 156 return 0; 157 if (set && (state->state & bits) == bits) 158 return 0; 159 if (!set && (state->state & bits) == 0) 160 return 0; 161 changeset->bytes_changed += state->end - state->start + 1; 162 ret = ulist_add(&changeset->range_changed, state->start, state->end, 163 GFP_ATOMIC); 164 return ret; 165 } 166 167 int __must_check submit_one_bio(struct bio *bio, int mirror_num, 168 unsigned long bio_flags) 169 { 170 blk_status_t ret = 0; 171 struct extent_io_tree *tree = bio->bi_private; 172 173 bio->bi_private = NULL; 174 175 if (is_data_inode(tree->private_data)) 176 ret = btrfs_submit_data_bio(tree->private_data, bio, mirror_num, 177 bio_flags); 178 else 179 ret = btrfs_submit_metadata_bio(tree->private_data, bio, 180 mirror_num, bio_flags); 181 182 return blk_status_to_errno(ret); 183 } 184 185 /* Cleanup unsubmitted bios */ 186 static void end_write_bio(struct extent_page_data *epd, int ret) 187 { 188 if (epd->bio) { 189 epd->bio->bi_status = errno_to_blk_status(ret); 190 bio_endio(epd->bio); 191 epd->bio = NULL; 192 } 193 } 194 195 /* 196 * Submit bio from extent page data via submit_one_bio 197 * 198 * Return 0 if everything is OK. 199 * Return <0 for error. 200 */ 201 static int __must_check flush_write_bio(struct extent_page_data *epd) 202 { 203 int ret = 0; 204 205 if (epd->bio) { 206 ret = submit_one_bio(epd->bio, 0, 0); 207 /* 208 * Clean up of epd->bio is handled by its endio function. 209 * And endio is either triggered by successful bio execution 210 * or the error handler of submit bio hook. 211 * So at this point, no matter what happened, we don't need 212 * to clean up epd->bio. 213 */ 214 epd->bio = NULL; 215 } 216 return ret; 217 } 218 219 int __init extent_state_cache_init(void) 220 { 221 extent_state_cache = kmem_cache_create("btrfs_extent_state", 222 sizeof(struct extent_state), 0, 223 SLAB_MEM_SPREAD, NULL); 224 if (!extent_state_cache) 225 return -ENOMEM; 226 return 0; 227 } 228 229 int __init extent_io_init(void) 230 { 231 extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer", 232 sizeof(struct extent_buffer), 0, 233 SLAB_MEM_SPREAD, NULL); 234 if (!extent_buffer_cache) 235 return -ENOMEM; 236 237 if (bioset_init(&btrfs_bioset, BIO_POOL_SIZE, 238 offsetof(struct btrfs_io_bio, bio), 239 BIOSET_NEED_BVECS)) 240 goto free_buffer_cache; 241 242 if (bioset_integrity_create(&btrfs_bioset, BIO_POOL_SIZE)) 243 goto free_bioset; 244 245 return 0; 246 247 free_bioset: 248 bioset_exit(&btrfs_bioset); 249 250 free_buffer_cache: 251 kmem_cache_destroy(extent_buffer_cache); 252 extent_buffer_cache = NULL; 253 return -ENOMEM; 254 } 255 256 void __cold extent_state_cache_exit(void) 257 { 258 btrfs_extent_state_leak_debug_check(); 259 kmem_cache_destroy(extent_state_cache); 260 } 261 262 void __cold extent_io_exit(void) 263 { 264 /* 265 * Make sure all delayed rcu free are flushed before we 266 * destroy caches. 267 */ 268 rcu_barrier(); 269 kmem_cache_destroy(extent_buffer_cache); 270 bioset_exit(&btrfs_bioset); 271 } 272 273 /* 274 * For the file_extent_tree, we want to hold the inode lock when we lookup and 275 * update the disk_i_size, but lockdep will complain because our io_tree we hold 276 * the tree lock and get the inode lock when setting delalloc. These two things 277 * are unrelated, so make a class for the file_extent_tree so we don't get the 278 * two locking patterns mixed up. 279 */ 280 static struct lock_class_key file_extent_tree_class; 281 282 void extent_io_tree_init(struct btrfs_fs_info *fs_info, 283 struct extent_io_tree *tree, unsigned int owner, 284 void *private_data) 285 { 286 tree->fs_info = fs_info; 287 tree->state = RB_ROOT; 288 tree->dirty_bytes = 0; 289 spin_lock_init(&tree->lock); 290 tree->private_data = private_data; 291 tree->owner = owner; 292 if (owner == IO_TREE_INODE_FILE_EXTENT) 293 lockdep_set_class(&tree->lock, &file_extent_tree_class); 294 } 295 296 void extent_io_tree_release(struct extent_io_tree *tree) 297 { 298 spin_lock(&tree->lock); 299 /* 300 * Do a single barrier for the waitqueue_active check here, the state 301 * of the waitqueue should not change once extent_io_tree_release is 302 * called. 303 */ 304 smp_mb(); 305 while (!RB_EMPTY_ROOT(&tree->state)) { 306 struct rb_node *node; 307 struct extent_state *state; 308 309 node = rb_first(&tree->state); 310 state = rb_entry(node, struct extent_state, rb_node); 311 rb_erase(&state->rb_node, &tree->state); 312 RB_CLEAR_NODE(&state->rb_node); 313 /* 314 * btree io trees aren't supposed to have tasks waiting for 315 * changes in the flags of extent states ever. 316 */ 317 ASSERT(!waitqueue_active(&state->wq)); 318 free_extent_state(state); 319 320 cond_resched_lock(&tree->lock); 321 } 322 spin_unlock(&tree->lock); 323 } 324 325 static struct extent_state *alloc_extent_state(gfp_t mask) 326 { 327 struct extent_state *state; 328 329 /* 330 * The given mask might be not appropriate for the slab allocator, 331 * drop the unsupported bits 332 */ 333 mask &= ~(__GFP_DMA32|__GFP_HIGHMEM); 334 state = kmem_cache_alloc(extent_state_cache, mask); 335 if (!state) 336 return state; 337 state->state = 0; 338 state->failrec = NULL; 339 RB_CLEAR_NODE(&state->rb_node); 340 btrfs_leak_debug_add(&leak_lock, &state->leak_list, &states); 341 refcount_set(&state->refs, 1); 342 init_waitqueue_head(&state->wq); 343 trace_alloc_extent_state(state, mask, _RET_IP_); 344 return state; 345 } 346 347 void free_extent_state(struct extent_state *state) 348 { 349 if (!state) 350 return; 351 if (refcount_dec_and_test(&state->refs)) { 352 WARN_ON(extent_state_in_tree(state)); 353 btrfs_leak_debug_del(&leak_lock, &state->leak_list); 354 trace_free_extent_state(state, _RET_IP_); 355 kmem_cache_free(extent_state_cache, state); 356 } 357 } 358 359 static struct rb_node *tree_insert(struct rb_root *root, 360 struct rb_node *search_start, 361 u64 offset, 362 struct rb_node *node, 363 struct rb_node ***p_in, 364 struct rb_node **parent_in) 365 { 366 struct rb_node **p; 367 struct rb_node *parent = NULL; 368 struct tree_entry *entry; 369 370 if (p_in && parent_in) { 371 p = *p_in; 372 parent = *parent_in; 373 goto do_insert; 374 } 375 376 p = search_start ? &search_start : &root->rb_node; 377 while (*p) { 378 parent = *p; 379 entry = rb_entry(parent, struct tree_entry, rb_node); 380 381 if (offset < entry->start) 382 p = &(*p)->rb_left; 383 else if (offset > entry->end) 384 p = &(*p)->rb_right; 385 else 386 return parent; 387 } 388 389 do_insert: 390 rb_link_node(node, parent, p); 391 rb_insert_color(node, root); 392 return NULL; 393 } 394 395 /** 396 * Search @tree for an entry that contains @offset. Such entry would have 397 * entry->start <= offset && entry->end >= offset. 398 * 399 * @tree: the tree to search 400 * @offset: offset that should fall within an entry in @tree 401 * @next_ret: pointer to the first entry whose range ends after @offset 402 * @prev_ret: pointer to the first entry whose range begins before @offset 403 * @p_ret: pointer where new node should be anchored (used when inserting an 404 * entry in the tree) 405 * @parent_ret: points to entry which would have been the parent of the entry, 406 * containing @offset 407 * 408 * This function returns a pointer to the entry that contains @offset byte 409 * address. If no such entry exists, then NULL is returned and the other 410 * pointer arguments to the function are filled, otherwise the found entry is 411 * returned and other pointers are left untouched. 412 */ 413 static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset, 414 struct rb_node **next_ret, 415 struct rb_node **prev_ret, 416 struct rb_node ***p_ret, 417 struct rb_node **parent_ret) 418 { 419 struct rb_root *root = &tree->state; 420 struct rb_node **n = &root->rb_node; 421 struct rb_node *prev = NULL; 422 struct rb_node *orig_prev = NULL; 423 struct tree_entry *entry; 424 struct tree_entry *prev_entry = NULL; 425 426 while (*n) { 427 prev = *n; 428 entry = rb_entry(prev, struct tree_entry, rb_node); 429 prev_entry = entry; 430 431 if (offset < entry->start) 432 n = &(*n)->rb_left; 433 else if (offset > entry->end) 434 n = &(*n)->rb_right; 435 else 436 return *n; 437 } 438 439 if (p_ret) 440 *p_ret = n; 441 if (parent_ret) 442 *parent_ret = prev; 443 444 if (next_ret) { 445 orig_prev = prev; 446 while (prev && offset > prev_entry->end) { 447 prev = rb_next(prev); 448 prev_entry = rb_entry(prev, struct tree_entry, rb_node); 449 } 450 *next_ret = prev; 451 prev = orig_prev; 452 } 453 454 if (prev_ret) { 455 prev_entry = rb_entry(prev, struct tree_entry, rb_node); 456 while (prev && offset < prev_entry->start) { 457 prev = rb_prev(prev); 458 prev_entry = rb_entry(prev, struct tree_entry, rb_node); 459 } 460 *prev_ret = prev; 461 } 462 return NULL; 463 } 464 465 static inline struct rb_node * 466 tree_search_for_insert(struct extent_io_tree *tree, 467 u64 offset, 468 struct rb_node ***p_ret, 469 struct rb_node **parent_ret) 470 { 471 struct rb_node *next= NULL; 472 struct rb_node *ret; 473 474 ret = __etree_search(tree, offset, &next, NULL, p_ret, parent_ret); 475 if (!ret) 476 return next; 477 return ret; 478 } 479 480 static inline struct rb_node *tree_search(struct extent_io_tree *tree, 481 u64 offset) 482 { 483 return tree_search_for_insert(tree, offset, NULL, NULL); 484 } 485 486 /* 487 * utility function to look for merge candidates inside a given range. 488 * Any extents with matching state are merged together into a single 489 * extent in the tree. Extents with EXTENT_IO in their state field 490 * are not merged because the end_io handlers need to be able to do 491 * operations on them without sleeping (or doing allocations/splits). 492 * 493 * This should be called with the tree lock held. 494 */ 495 static void merge_state(struct extent_io_tree *tree, 496 struct extent_state *state) 497 { 498 struct extent_state *other; 499 struct rb_node *other_node; 500 501 if (state->state & (EXTENT_LOCKED | EXTENT_BOUNDARY)) 502 return; 503 504 other_node = rb_prev(&state->rb_node); 505 if (other_node) { 506 other = rb_entry(other_node, struct extent_state, rb_node); 507 if (other->end == state->start - 1 && 508 other->state == state->state) { 509 if (tree->private_data && 510 is_data_inode(tree->private_data)) 511 btrfs_merge_delalloc_extent(tree->private_data, 512 state, other); 513 state->start = other->start; 514 rb_erase(&other->rb_node, &tree->state); 515 RB_CLEAR_NODE(&other->rb_node); 516 free_extent_state(other); 517 } 518 } 519 other_node = rb_next(&state->rb_node); 520 if (other_node) { 521 other = rb_entry(other_node, struct extent_state, rb_node); 522 if (other->start == state->end + 1 && 523 other->state == state->state) { 524 if (tree->private_data && 525 is_data_inode(tree->private_data)) 526 btrfs_merge_delalloc_extent(tree->private_data, 527 state, other); 528 state->end = other->end; 529 rb_erase(&other->rb_node, &tree->state); 530 RB_CLEAR_NODE(&other->rb_node); 531 free_extent_state(other); 532 } 533 } 534 } 535 536 static void set_state_bits(struct extent_io_tree *tree, 537 struct extent_state *state, u32 *bits, 538 struct extent_changeset *changeset); 539 540 /* 541 * insert an extent_state struct into the tree. 'bits' are set on the 542 * struct before it is inserted. 543 * 544 * This may return -EEXIST if the extent is already there, in which case the 545 * state struct is freed. 546 * 547 * The tree lock is not taken internally. This is a utility function and 548 * probably isn't what you want to call (see set/clear_extent_bit). 549 */ 550 static int insert_state(struct extent_io_tree *tree, 551 struct extent_state *state, u64 start, u64 end, 552 struct rb_node ***p, 553 struct rb_node **parent, 554 u32 *bits, struct extent_changeset *changeset) 555 { 556 struct rb_node *node; 557 558 if (end < start) { 559 btrfs_err(tree->fs_info, 560 "insert state: end < start %llu %llu", end, start); 561 WARN_ON(1); 562 } 563 state->start = start; 564 state->end = end; 565 566 set_state_bits(tree, state, bits, changeset); 567 568 node = tree_insert(&tree->state, NULL, end, &state->rb_node, p, parent); 569 if (node) { 570 struct extent_state *found; 571 found = rb_entry(node, struct extent_state, rb_node); 572 btrfs_err(tree->fs_info, 573 "found node %llu %llu on insert of %llu %llu", 574 found->start, found->end, start, end); 575 return -EEXIST; 576 } 577 merge_state(tree, state); 578 return 0; 579 } 580 581 /* 582 * split a given extent state struct in two, inserting the preallocated 583 * struct 'prealloc' as the newly created second half. 'split' indicates an 584 * offset inside 'orig' where it should be split. 585 * 586 * Before calling, 587 * the tree has 'orig' at [orig->start, orig->end]. After calling, there 588 * are two extent state structs in the tree: 589 * prealloc: [orig->start, split - 1] 590 * orig: [ split, orig->end ] 591 * 592 * The tree locks are not taken by this function. They need to be held 593 * by the caller. 594 */ 595 static int split_state(struct extent_io_tree *tree, struct extent_state *orig, 596 struct extent_state *prealloc, u64 split) 597 { 598 struct rb_node *node; 599 600 if (tree->private_data && is_data_inode(tree->private_data)) 601 btrfs_split_delalloc_extent(tree->private_data, orig, split); 602 603 prealloc->start = orig->start; 604 prealloc->end = split - 1; 605 prealloc->state = orig->state; 606 orig->start = split; 607 608 node = tree_insert(&tree->state, &orig->rb_node, prealloc->end, 609 &prealloc->rb_node, NULL, NULL); 610 if (node) { 611 free_extent_state(prealloc); 612 return -EEXIST; 613 } 614 return 0; 615 } 616 617 static struct extent_state *next_state(struct extent_state *state) 618 { 619 struct rb_node *next = rb_next(&state->rb_node); 620 if (next) 621 return rb_entry(next, struct extent_state, rb_node); 622 else 623 return NULL; 624 } 625 626 /* 627 * utility function to clear some bits in an extent state struct. 628 * it will optionally wake up anyone waiting on this state (wake == 1). 629 * 630 * If no bits are set on the state struct after clearing things, the 631 * struct is freed and removed from the tree 632 */ 633 static struct extent_state *clear_state_bit(struct extent_io_tree *tree, 634 struct extent_state *state, 635 u32 *bits, int wake, 636 struct extent_changeset *changeset) 637 { 638 struct extent_state *next; 639 u32 bits_to_clear = *bits & ~EXTENT_CTLBITS; 640 int ret; 641 642 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) { 643 u64 range = state->end - state->start + 1; 644 WARN_ON(range > tree->dirty_bytes); 645 tree->dirty_bytes -= range; 646 } 647 648 if (tree->private_data && is_data_inode(tree->private_data)) 649 btrfs_clear_delalloc_extent(tree->private_data, state, bits); 650 651 ret = add_extent_changeset(state, bits_to_clear, changeset, 0); 652 BUG_ON(ret < 0); 653 state->state &= ~bits_to_clear; 654 if (wake) 655 wake_up(&state->wq); 656 if (state->state == 0) { 657 next = next_state(state); 658 if (extent_state_in_tree(state)) { 659 rb_erase(&state->rb_node, &tree->state); 660 RB_CLEAR_NODE(&state->rb_node); 661 free_extent_state(state); 662 } else { 663 WARN_ON(1); 664 } 665 } else { 666 merge_state(tree, state); 667 next = next_state(state); 668 } 669 return next; 670 } 671 672 static struct extent_state * 673 alloc_extent_state_atomic(struct extent_state *prealloc) 674 { 675 if (!prealloc) 676 prealloc = alloc_extent_state(GFP_ATOMIC); 677 678 return prealloc; 679 } 680 681 static void extent_io_tree_panic(struct extent_io_tree *tree, int err) 682 { 683 btrfs_panic(tree->fs_info, err, 684 "locking error: extent tree was modified by another thread while locked"); 685 } 686 687 /* 688 * clear some bits on a range in the tree. This may require splitting 689 * or inserting elements in the tree, so the gfp mask is used to 690 * indicate which allocations or sleeping are allowed. 691 * 692 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove 693 * the given range from the tree regardless of state (ie for truncate). 694 * 695 * the range [start, end] is inclusive. 696 * 697 * This takes the tree lock, and returns 0 on success and < 0 on error. 698 */ 699 int __clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, 700 u32 bits, int wake, int delete, 701 struct extent_state **cached_state, 702 gfp_t mask, struct extent_changeset *changeset) 703 { 704 struct extent_state *state; 705 struct extent_state *cached; 706 struct extent_state *prealloc = NULL; 707 struct rb_node *node; 708 u64 last_end; 709 int err; 710 int clear = 0; 711 712 btrfs_debug_check_extent_io_range(tree, start, end); 713 trace_btrfs_clear_extent_bit(tree, start, end - start + 1, bits); 714 715 if (bits & EXTENT_DELALLOC) 716 bits |= EXTENT_NORESERVE; 717 718 if (delete) 719 bits |= ~EXTENT_CTLBITS; 720 721 if (bits & (EXTENT_LOCKED | EXTENT_BOUNDARY)) 722 clear = 1; 723 again: 724 if (!prealloc && gfpflags_allow_blocking(mask)) { 725 /* 726 * Don't care for allocation failure here because we might end 727 * up not needing the pre-allocated extent state at all, which 728 * is the case if we only have in the tree extent states that 729 * cover our input range and don't cover too any other range. 730 * If we end up needing a new extent state we allocate it later. 731 */ 732 prealloc = alloc_extent_state(mask); 733 } 734 735 spin_lock(&tree->lock); 736 if (cached_state) { 737 cached = *cached_state; 738 739 if (clear) { 740 *cached_state = NULL; 741 cached_state = NULL; 742 } 743 744 if (cached && extent_state_in_tree(cached) && 745 cached->start <= start && cached->end > start) { 746 if (clear) 747 refcount_dec(&cached->refs); 748 state = cached; 749 goto hit_next; 750 } 751 if (clear) 752 free_extent_state(cached); 753 } 754 /* 755 * this search will find the extents that end after 756 * our range starts 757 */ 758 node = tree_search(tree, start); 759 if (!node) 760 goto out; 761 state = rb_entry(node, struct extent_state, rb_node); 762 hit_next: 763 if (state->start > end) 764 goto out; 765 WARN_ON(state->end < start); 766 last_end = state->end; 767 768 /* the state doesn't have the wanted bits, go ahead */ 769 if (!(state->state & bits)) { 770 state = next_state(state); 771 goto next; 772 } 773 774 /* 775 * | ---- desired range ---- | 776 * | state | or 777 * | ------------- state -------------- | 778 * 779 * We need to split the extent we found, and may flip 780 * bits on second half. 781 * 782 * If the extent we found extends past our range, we 783 * just split and search again. It'll get split again 784 * the next time though. 785 * 786 * If the extent we found is inside our range, we clear 787 * the desired bit on it. 788 */ 789 790 if (state->start < start) { 791 prealloc = alloc_extent_state_atomic(prealloc); 792 BUG_ON(!prealloc); 793 err = split_state(tree, state, prealloc, start); 794 if (err) 795 extent_io_tree_panic(tree, err); 796 797 prealloc = NULL; 798 if (err) 799 goto out; 800 if (state->end <= end) { 801 state = clear_state_bit(tree, state, &bits, wake, 802 changeset); 803 goto next; 804 } 805 goto search_again; 806 } 807 /* 808 * | ---- desired range ---- | 809 * | state | 810 * We need to split the extent, and clear the bit 811 * on the first half 812 */ 813 if (state->start <= end && state->end > end) { 814 prealloc = alloc_extent_state_atomic(prealloc); 815 BUG_ON(!prealloc); 816 err = split_state(tree, state, prealloc, end + 1); 817 if (err) 818 extent_io_tree_panic(tree, err); 819 820 if (wake) 821 wake_up(&state->wq); 822 823 clear_state_bit(tree, prealloc, &bits, wake, changeset); 824 825 prealloc = NULL; 826 goto out; 827 } 828 829 state = clear_state_bit(tree, state, &bits, wake, changeset); 830 next: 831 if (last_end == (u64)-1) 832 goto out; 833 start = last_end + 1; 834 if (start <= end && state && !need_resched()) 835 goto hit_next; 836 837 search_again: 838 if (start > end) 839 goto out; 840 spin_unlock(&tree->lock); 841 if (gfpflags_allow_blocking(mask)) 842 cond_resched(); 843 goto again; 844 845 out: 846 spin_unlock(&tree->lock); 847 if (prealloc) 848 free_extent_state(prealloc); 849 850 return 0; 851 852 } 853 854 static void wait_on_state(struct extent_io_tree *tree, 855 struct extent_state *state) 856 __releases(tree->lock) 857 __acquires(tree->lock) 858 { 859 DEFINE_WAIT(wait); 860 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE); 861 spin_unlock(&tree->lock); 862 schedule(); 863 spin_lock(&tree->lock); 864 finish_wait(&state->wq, &wait); 865 } 866 867 /* 868 * waits for one or more bits to clear on a range in the state tree. 869 * The range [start, end] is inclusive. 870 * The tree lock is taken by this function 871 */ 872 static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, 873 u32 bits) 874 { 875 struct extent_state *state; 876 struct rb_node *node; 877 878 btrfs_debug_check_extent_io_range(tree, start, end); 879 880 spin_lock(&tree->lock); 881 again: 882 while (1) { 883 /* 884 * this search will find all the extents that end after 885 * our range starts 886 */ 887 node = tree_search(tree, start); 888 process_node: 889 if (!node) 890 break; 891 892 state = rb_entry(node, struct extent_state, rb_node); 893 894 if (state->start > end) 895 goto out; 896 897 if (state->state & bits) { 898 start = state->start; 899 refcount_inc(&state->refs); 900 wait_on_state(tree, state); 901 free_extent_state(state); 902 goto again; 903 } 904 start = state->end + 1; 905 906 if (start > end) 907 break; 908 909 if (!cond_resched_lock(&tree->lock)) { 910 node = rb_next(node); 911 goto process_node; 912 } 913 } 914 out: 915 spin_unlock(&tree->lock); 916 } 917 918 static void set_state_bits(struct extent_io_tree *tree, 919 struct extent_state *state, 920 u32 *bits, struct extent_changeset *changeset) 921 { 922 u32 bits_to_set = *bits & ~EXTENT_CTLBITS; 923 int ret; 924 925 if (tree->private_data && is_data_inode(tree->private_data)) 926 btrfs_set_delalloc_extent(tree->private_data, state, bits); 927 928 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) { 929 u64 range = state->end - state->start + 1; 930 tree->dirty_bytes += range; 931 } 932 ret = add_extent_changeset(state, bits_to_set, changeset, 1); 933 BUG_ON(ret < 0); 934 state->state |= bits_to_set; 935 } 936 937 static void cache_state_if_flags(struct extent_state *state, 938 struct extent_state **cached_ptr, 939 unsigned flags) 940 { 941 if (cached_ptr && !(*cached_ptr)) { 942 if (!flags || (state->state & flags)) { 943 *cached_ptr = state; 944 refcount_inc(&state->refs); 945 } 946 } 947 } 948 949 static void cache_state(struct extent_state *state, 950 struct extent_state **cached_ptr) 951 { 952 return cache_state_if_flags(state, cached_ptr, 953 EXTENT_LOCKED | EXTENT_BOUNDARY); 954 } 955 956 /* 957 * set some bits on a range in the tree. This may require allocations or 958 * sleeping, so the gfp mask is used to indicate what is allowed. 959 * 960 * If any of the exclusive bits are set, this will fail with -EEXIST if some 961 * part of the range already has the desired bits set. The start of the 962 * existing range is returned in failed_start in this case. 963 * 964 * [start, end] is inclusive This takes the tree lock. 965 */ 966 int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, u32 bits, 967 u32 exclusive_bits, u64 *failed_start, 968 struct extent_state **cached_state, gfp_t mask, 969 struct extent_changeset *changeset) 970 { 971 struct extent_state *state; 972 struct extent_state *prealloc = NULL; 973 struct rb_node *node; 974 struct rb_node **p; 975 struct rb_node *parent; 976 int err = 0; 977 u64 last_start; 978 u64 last_end; 979 980 btrfs_debug_check_extent_io_range(tree, start, end); 981 trace_btrfs_set_extent_bit(tree, start, end - start + 1, bits); 982 983 if (exclusive_bits) 984 ASSERT(failed_start); 985 else 986 ASSERT(failed_start == NULL); 987 again: 988 if (!prealloc && gfpflags_allow_blocking(mask)) { 989 /* 990 * Don't care for allocation failure here because we might end 991 * up not needing the pre-allocated extent state at all, which 992 * is the case if we only have in the tree extent states that 993 * cover our input range and don't cover too any other range. 994 * If we end up needing a new extent state we allocate it later. 995 */ 996 prealloc = alloc_extent_state(mask); 997 } 998 999 spin_lock(&tree->lock); 1000 if (cached_state && *cached_state) { 1001 state = *cached_state; 1002 if (state->start <= start && state->end > start && 1003 extent_state_in_tree(state)) { 1004 node = &state->rb_node; 1005 goto hit_next; 1006 } 1007 } 1008 /* 1009 * this search will find all the extents that end after 1010 * our range starts. 1011 */ 1012 node = tree_search_for_insert(tree, start, &p, &parent); 1013 if (!node) { 1014 prealloc = alloc_extent_state_atomic(prealloc); 1015 BUG_ON(!prealloc); 1016 err = insert_state(tree, prealloc, start, end, 1017 &p, &parent, &bits, changeset); 1018 if (err) 1019 extent_io_tree_panic(tree, err); 1020 1021 cache_state(prealloc, cached_state); 1022 prealloc = NULL; 1023 goto out; 1024 } 1025 state = rb_entry(node, struct extent_state, rb_node); 1026 hit_next: 1027 last_start = state->start; 1028 last_end = state->end; 1029 1030 /* 1031 * | ---- desired range ---- | 1032 * | state | 1033 * 1034 * Just lock what we found and keep going 1035 */ 1036 if (state->start == start && state->end <= end) { 1037 if (state->state & exclusive_bits) { 1038 *failed_start = state->start; 1039 err = -EEXIST; 1040 goto out; 1041 } 1042 1043 set_state_bits(tree, state, &bits, changeset); 1044 cache_state(state, cached_state); 1045 merge_state(tree, state); 1046 if (last_end == (u64)-1) 1047 goto out; 1048 start = last_end + 1; 1049 state = next_state(state); 1050 if (start < end && state && state->start == start && 1051 !need_resched()) 1052 goto hit_next; 1053 goto search_again; 1054 } 1055 1056 /* 1057 * | ---- desired range ---- | 1058 * | state | 1059 * or 1060 * | ------------- state -------------- | 1061 * 1062 * We need to split the extent we found, and may flip bits on 1063 * second half. 1064 * 1065 * If the extent we found extends past our 1066 * range, we just split and search again. It'll get split 1067 * again the next time though. 1068 * 1069 * If the extent we found is inside our range, we set the 1070 * desired bit on it. 1071 */ 1072 if (state->start < start) { 1073 if (state->state & exclusive_bits) { 1074 *failed_start = start; 1075 err = -EEXIST; 1076 goto out; 1077 } 1078 1079 /* 1080 * If this extent already has all the bits we want set, then 1081 * skip it, not necessary to split it or do anything with it. 1082 */ 1083 if ((state->state & bits) == bits) { 1084 start = state->end + 1; 1085 cache_state(state, cached_state); 1086 goto search_again; 1087 } 1088 1089 prealloc = alloc_extent_state_atomic(prealloc); 1090 BUG_ON(!prealloc); 1091 err = split_state(tree, state, prealloc, start); 1092 if (err) 1093 extent_io_tree_panic(tree, err); 1094 1095 prealloc = NULL; 1096 if (err) 1097 goto out; 1098 if (state->end <= end) { 1099 set_state_bits(tree, state, &bits, changeset); 1100 cache_state(state, cached_state); 1101 merge_state(tree, state); 1102 if (last_end == (u64)-1) 1103 goto out; 1104 start = last_end + 1; 1105 state = next_state(state); 1106 if (start < end && state && state->start == start && 1107 !need_resched()) 1108 goto hit_next; 1109 } 1110 goto search_again; 1111 } 1112 /* 1113 * | ---- desired range ---- | 1114 * | state | or | state | 1115 * 1116 * There's a hole, we need to insert something in it and 1117 * ignore the extent we found. 1118 */ 1119 if (state->start > start) { 1120 u64 this_end; 1121 if (end < last_start) 1122 this_end = end; 1123 else 1124 this_end = last_start - 1; 1125 1126 prealloc = alloc_extent_state_atomic(prealloc); 1127 BUG_ON(!prealloc); 1128 1129 /* 1130 * Avoid to free 'prealloc' if it can be merged with 1131 * the later extent. 1132 */ 1133 err = insert_state(tree, prealloc, start, this_end, 1134 NULL, NULL, &bits, changeset); 1135 if (err) 1136 extent_io_tree_panic(tree, err); 1137 1138 cache_state(prealloc, cached_state); 1139 prealloc = NULL; 1140 start = this_end + 1; 1141 goto search_again; 1142 } 1143 /* 1144 * | ---- desired range ---- | 1145 * | state | 1146 * We need to split the extent, and set the bit 1147 * on the first half 1148 */ 1149 if (state->start <= end && state->end > end) { 1150 if (state->state & exclusive_bits) { 1151 *failed_start = start; 1152 err = -EEXIST; 1153 goto out; 1154 } 1155 1156 prealloc = alloc_extent_state_atomic(prealloc); 1157 BUG_ON(!prealloc); 1158 err = split_state(tree, state, prealloc, end + 1); 1159 if (err) 1160 extent_io_tree_panic(tree, err); 1161 1162 set_state_bits(tree, prealloc, &bits, changeset); 1163 cache_state(prealloc, cached_state); 1164 merge_state(tree, prealloc); 1165 prealloc = NULL; 1166 goto out; 1167 } 1168 1169 search_again: 1170 if (start > end) 1171 goto out; 1172 spin_unlock(&tree->lock); 1173 if (gfpflags_allow_blocking(mask)) 1174 cond_resched(); 1175 goto again; 1176 1177 out: 1178 spin_unlock(&tree->lock); 1179 if (prealloc) 1180 free_extent_state(prealloc); 1181 1182 return err; 1183 1184 } 1185 1186 /** 1187 * convert_extent_bit - convert all bits in a given range from one bit to 1188 * another 1189 * @tree: the io tree to search 1190 * @start: the start offset in bytes 1191 * @end: the end offset in bytes (inclusive) 1192 * @bits: the bits to set in this range 1193 * @clear_bits: the bits to clear in this range 1194 * @cached_state: state that we're going to cache 1195 * 1196 * This will go through and set bits for the given range. If any states exist 1197 * already in this range they are set with the given bit and cleared of the 1198 * clear_bits. This is only meant to be used by things that are mergeable, ie 1199 * converting from say DELALLOC to DIRTY. This is not meant to be used with 1200 * boundary bits like LOCK. 1201 * 1202 * All allocations are done with GFP_NOFS. 1203 */ 1204 int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, 1205 u32 bits, u32 clear_bits, 1206 struct extent_state **cached_state) 1207 { 1208 struct extent_state *state; 1209 struct extent_state *prealloc = NULL; 1210 struct rb_node *node; 1211 struct rb_node **p; 1212 struct rb_node *parent; 1213 int err = 0; 1214 u64 last_start; 1215 u64 last_end; 1216 bool first_iteration = true; 1217 1218 btrfs_debug_check_extent_io_range(tree, start, end); 1219 trace_btrfs_convert_extent_bit(tree, start, end - start + 1, bits, 1220 clear_bits); 1221 1222 again: 1223 if (!prealloc) { 1224 /* 1225 * Best effort, don't worry if extent state allocation fails 1226 * here for the first iteration. We might have a cached state 1227 * that matches exactly the target range, in which case no 1228 * extent state allocations are needed. We'll only know this 1229 * after locking the tree. 1230 */ 1231 prealloc = alloc_extent_state(GFP_NOFS); 1232 if (!prealloc && !first_iteration) 1233 return -ENOMEM; 1234 } 1235 1236 spin_lock(&tree->lock); 1237 if (cached_state && *cached_state) { 1238 state = *cached_state; 1239 if (state->start <= start && state->end > start && 1240 extent_state_in_tree(state)) { 1241 node = &state->rb_node; 1242 goto hit_next; 1243 } 1244 } 1245 1246 /* 1247 * this search will find all the extents that end after 1248 * our range starts. 1249 */ 1250 node = tree_search_for_insert(tree, start, &p, &parent); 1251 if (!node) { 1252 prealloc = alloc_extent_state_atomic(prealloc); 1253 if (!prealloc) { 1254 err = -ENOMEM; 1255 goto out; 1256 } 1257 err = insert_state(tree, prealloc, start, end, 1258 &p, &parent, &bits, NULL); 1259 if (err) 1260 extent_io_tree_panic(tree, err); 1261 cache_state(prealloc, cached_state); 1262 prealloc = NULL; 1263 goto out; 1264 } 1265 state = rb_entry(node, struct extent_state, rb_node); 1266 hit_next: 1267 last_start = state->start; 1268 last_end = state->end; 1269 1270 /* 1271 * | ---- desired range ---- | 1272 * | state | 1273 * 1274 * Just lock what we found and keep going 1275 */ 1276 if (state->start == start && state->end <= end) { 1277 set_state_bits(tree, state, &bits, NULL); 1278 cache_state(state, cached_state); 1279 state = clear_state_bit(tree, state, &clear_bits, 0, NULL); 1280 if (last_end == (u64)-1) 1281 goto out; 1282 start = last_end + 1; 1283 if (start < end && state && state->start == start && 1284 !need_resched()) 1285 goto hit_next; 1286 goto search_again; 1287 } 1288 1289 /* 1290 * | ---- desired range ---- | 1291 * | state | 1292 * or 1293 * | ------------- state -------------- | 1294 * 1295 * We need to split the extent we found, and may flip bits on 1296 * second half. 1297 * 1298 * If the extent we found extends past our 1299 * range, we just split and search again. It'll get split 1300 * again the next time though. 1301 * 1302 * If the extent we found is inside our range, we set the 1303 * desired bit on it. 1304 */ 1305 if (state->start < start) { 1306 prealloc = alloc_extent_state_atomic(prealloc); 1307 if (!prealloc) { 1308 err = -ENOMEM; 1309 goto out; 1310 } 1311 err = split_state(tree, state, prealloc, start); 1312 if (err) 1313 extent_io_tree_panic(tree, err); 1314 prealloc = NULL; 1315 if (err) 1316 goto out; 1317 if (state->end <= end) { 1318 set_state_bits(tree, state, &bits, NULL); 1319 cache_state(state, cached_state); 1320 state = clear_state_bit(tree, state, &clear_bits, 0, 1321 NULL); 1322 if (last_end == (u64)-1) 1323 goto out; 1324 start = last_end + 1; 1325 if (start < end && state && state->start == start && 1326 !need_resched()) 1327 goto hit_next; 1328 } 1329 goto search_again; 1330 } 1331 /* 1332 * | ---- desired range ---- | 1333 * | state | or | state | 1334 * 1335 * There's a hole, we need to insert something in it and 1336 * ignore the extent we found. 1337 */ 1338 if (state->start > start) { 1339 u64 this_end; 1340 if (end < last_start) 1341 this_end = end; 1342 else 1343 this_end = last_start - 1; 1344 1345 prealloc = alloc_extent_state_atomic(prealloc); 1346 if (!prealloc) { 1347 err = -ENOMEM; 1348 goto out; 1349 } 1350 1351 /* 1352 * Avoid to free 'prealloc' if it can be merged with 1353 * the later extent. 1354 */ 1355 err = insert_state(tree, prealloc, start, this_end, 1356 NULL, NULL, &bits, NULL); 1357 if (err) 1358 extent_io_tree_panic(tree, err); 1359 cache_state(prealloc, cached_state); 1360 prealloc = NULL; 1361 start = this_end + 1; 1362 goto search_again; 1363 } 1364 /* 1365 * | ---- desired range ---- | 1366 * | state | 1367 * We need to split the extent, and set the bit 1368 * on the first half 1369 */ 1370 if (state->start <= end && state->end > end) { 1371 prealloc = alloc_extent_state_atomic(prealloc); 1372 if (!prealloc) { 1373 err = -ENOMEM; 1374 goto out; 1375 } 1376 1377 err = split_state(tree, state, prealloc, end + 1); 1378 if (err) 1379 extent_io_tree_panic(tree, err); 1380 1381 set_state_bits(tree, prealloc, &bits, NULL); 1382 cache_state(prealloc, cached_state); 1383 clear_state_bit(tree, prealloc, &clear_bits, 0, NULL); 1384 prealloc = NULL; 1385 goto out; 1386 } 1387 1388 search_again: 1389 if (start > end) 1390 goto out; 1391 spin_unlock(&tree->lock); 1392 cond_resched(); 1393 first_iteration = false; 1394 goto again; 1395 1396 out: 1397 spin_unlock(&tree->lock); 1398 if (prealloc) 1399 free_extent_state(prealloc); 1400 1401 return err; 1402 } 1403 1404 /* wrappers around set/clear extent bit */ 1405 int set_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, 1406 u32 bits, struct extent_changeset *changeset) 1407 { 1408 /* 1409 * We don't support EXTENT_LOCKED yet, as current changeset will 1410 * record any bits changed, so for EXTENT_LOCKED case, it will 1411 * either fail with -EEXIST or changeset will record the whole 1412 * range. 1413 */ 1414 BUG_ON(bits & EXTENT_LOCKED); 1415 1416 return set_extent_bit(tree, start, end, bits, 0, NULL, NULL, GFP_NOFS, 1417 changeset); 1418 } 1419 1420 int set_extent_bits_nowait(struct extent_io_tree *tree, u64 start, u64 end, 1421 u32 bits) 1422 { 1423 return set_extent_bit(tree, start, end, bits, 0, NULL, NULL, 1424 GFP_NOWAIT, NULL); 1425 } 1426 1427 int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, 1428 u32 bits, int wake, int delete, 1429 struct extent_state **cached) 1430 { 1431 return __clear_extent_bit(tree, start, end, bits, wake, delete, 1432 cached, GFP_NOFS, NULL); 1433 } 1434 1435 int clear_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, 1436 u32 bits, struct extent_changeset *changeset) 1437 { 1438 /* 1439 * Don't support EXTENT_LOCKED case, same reason as 1440 * set_record_extent_bits(). 1441 */ 1442 BUG_ON(bits & EXTENT_LOCKED); 1443 1444 return __clear_extent_bit(tree, start, end, bits, 0, 0, NULL, GFP_NOFS, 1445 changeset); 1446 } 1447 1448 /* 1449 * either insert or lock state struct between start and end use mask to tell 1450 * us if waiting is desired. 1451 */ 1452 int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, 1453 struct extent_state **cached_state) 1454 { 1455 int err; 1456 u64 failed_start; 1457 1458 while (1) { 1459 err = set_extent_bit(tree, start, end, EXTENT_LOCKED, 1460 EXTENT_LOCKED, &failed_start, 1461 cached_state, GFP_NOFS, NULL); 1462 if (err == -EEXIST) { 1463 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED); 1464 start = failed_start; 1465 } else 1466 break; 1467 WARN_ON(start > end); 1468 } 1469 return err; 1470 } 1471 1472 int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end) 1473 { 1474 int err; 1475 u64 failed_start; 1476 1477 err = set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED, 1478 &failed_start, NULL, GFP_NOFS, NULL); 1479 if (err == -EEXIST) { 1480 if (failed_start > start) 1481 clear_extent_bit(tree, start, failed_start - 1, 1482 EXTENT_LOCKED, 1, 0, NULL); 1483 return 0; 1484 } 1485 return 1; 1486 } 1487 1488 void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end) 1489 { 1490 unsigned long index = start >> PAGE_SHIFT; 1491 unsigned long end_index = end >> PAGE_SHIFT; 1492 struct page *page; 1493 1494 while (index <= end_index) { 1495 page = find_get_page(inode->i_mapping, index); 1496 BUG_ON(!page); /* Pages should be in the extent_io_tree */ 1497 clear_page_dirty_for_io(page); 1498 put_page(page); 1499 index++; 1500 } 1501 } 1502 1503 void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end) 1504 { 1505 unsigned long index = start >> PAGE_SHIFT; 1506 unsigned long end_index = end >> PAGE_SHIFT; 1507 struct page *page; 1508 1509 while (index <= end_index) { 1510 page = find_get_page(inode->i_mapping, index); 1511 BUG_ON(!page); /* Pages should be in the extent_io_tree */ 1512 __set_page_dirty_nobuffers(page); 1513 account_page_redirty(page); 1514 put_page(page); 1515 index++; 1516 } 1517 } 1518 1519 /* find the first state struct with 'bits' set after 'start', and 1520 * return it. tree->lock must be held. NULL will returned if 1521 * nothing was found after 'start' 1522 */ 1523 static struct extent_state * 1524 find_first_extent_bit_state(struct extent_io_tree *tree, u64 start, u32 bits) 1525 { 1526 struct rb_node *node; 1527 struct extent_state *state; 1528 1529 /* 1530 * this search will find all the extents that end after 1531 * our range starts. 1532 */ 1533 node = tree_search(tree, start); 1534 if (!node) 1535 goto out; 1536 1537 while (1) { 1538 state = rb_entry(node, struct extent_state, rb_node); 1539 if (state->end >= start && (state->state & bits)) 1540 return state; 1541 1542 node = rb_next(node); 1543 if (!node) 1544 break; 1545 } 1546 out: 1547 return NULL; 1548 } 1549 1550 /* 1551 * Find the first offset in the io tree with one or more @bits set. 1552 * 1553 * Note: If there are multiple bits set in @bits, any of them will match. 1554 * 1555 * Return 0 if we find something, and update @start_ret and @end_ret. 1556 * Return 1 if we found nothing. 1557 */ 1558 int find_first_extent_bit(struct extent_io_tree *tree, u64 start, 1559 u64 *start_ret, u64 *end_ret, u32 bits, 1560 struct extent_state **cached_state) 1561 { 1562 struct extent_state *state; 1563 int ret = 1; 1564 1565 spin_lock(&tree->lock); 1566 if (cached_state && *cached_state) { 1567 state = *cached_state; 1568 if (state->end == start - 1 && extent_state_in_tree(state)) { 1569 while ((state = next_state(state)) != NULL) { 1570 if (state->state & bits) 1571 goto got_it; 1572 } 1573 free_extent_state(*cached_state); 1574 *cached_state = NULL; 1575 goto out; 1576 } 1577 free_extent_state(*cached_state); 1578 *cached_state = NULL; 1579 } 1580 1581 state = find_first_extent_bit_state(tree, start, bits); 1582 got_it: 1583 if (state) { 1584 cache_state_if_flags(state, cached_state, 0); 1585 *start_ret = state->start; 1586 *end_ret = state->end; 1587 ret = 0; 1588 } 1589 out: 1590 spin_unlock(&tree->lock); 1591 return ret; 1592 } 1593 1594 /** 1595 * Find a contiguous area of bits 1596 * 1597 * @tree: io tree to check 1598 * @start: offset to start the search from 1599 * @start_ret: the first offset we found with the bits set 1600 * @end_ret: the final contiguous range of the bits that were set 1601 * @bits: bits to look for 1602 * 1603 * set_extent_bit and clear_extent_bit can temporarily split contiguous ranges 1604 * to set bits appropriately, and then merge them again. During this time it 1605 * will drop the tree->lock, so use this helper if you want to find the actual 1606 * contiguous area for given bits. We will search to the first bit we find, and 1607 * then walk down the tree until we find a non-contiguous area. The area 1608 * returned will be the full contiguous area with the bits set. 1609 */ 1610 int find_contiguous_extent_bit(struct extent_io_tree *tree, u64 start, 1611 u64 *start_ret, u64 *end_ret, u32 bits) 1612 { 1613 struct extent_state *state; 1614 int ret = 1; 1615 1616 spin_lock(&tree->lock); 1617 state = find_first_extent_bit_state(tree, start, bits); 1618 if (state) { 1619 *start_ret = state->start; 1620 *end_ret = state->end; 1621 while ((state = next_state(state)) != NULL) { 1622 if (state->start > (*end_ret + 1)) 1623 break; 1624 *end_ret = state->end; 1625 } 1626 ret = 0; 1627 } 1628 spin_unlock(&tree->lock); 1629 return ret; 1630 } 1631 1632 /** 1633 * Find the first range that has @bits not set. This range could start before 1634 * @start. 1635 * 1636 * @tree: the tree to search 1637 * @start: offset at/after which the found extent should start 1638 * @start_ret: records the beginning of the range 1639 * @end_ret: records the end of the range (inclusive) 1640 * @bits: the set of bits which must be unset 1641 * 1642 * Since unallocated range is also considered one which doesn't have the bits 1643 * set it's possible that @end_ret contains -1, this happens in case the range 1644 * spans (last_range_end, end of device]. In this case it's up to the caller to 1645 * trim @end_ret to the appropriate size. 1646 */ 1647 void find_first_clear_extent_bit(struct extent_io_tree *tree, u64 start, 1648 u64 *start_ret, u64 *end_ret, u32 bits) 1649 { 1650 struct extent_state *state; 1651 struct rb_node *node, *prev = NULL, *next; 1652 1653 spin_lock(&tree->lock); 1654 1655 /* Find first extent with bits cleared */ 1656 while (1) { 1657 node = __etree_search(tree, start, &next, &prev, NULL, NULL); 1658 if (!node && !next && !prev) { 1659 /* 1660 * Tree is completely empty, send full range and let 1661 * caller deal with it 1662 */ 1663 *start_ret = 0; 1664 *end_ret = -1; 1665 goto out; 1666 } else if (!node && !next) { 1667 /* 1668 * We are past the last allocated chunk, set start at 1669 * the end of the last extent. 1670 */ 1671 state = rb_entry(prev, struct extent_state, rb_node); 1672 *start_ret = state->end + 1; 1673 *end_ret = -1; 1674 goto out; 1675 } else if (!node) { 1676 node = next; 1677 } 1678 /* 1679 * At this point 'node' either contains 'start' or start is 1680 * before 'node' 1681 */ 1682 state = rb_entry(node, struct extent_state, rb_node); 1683 1684 if (in_range(start, state->start, state->end - state->start + 1)) { 1685 if (state->state & bits) { 1686 /* 1687 * |--range with bits sets--| 1688 * | 1689 * start 1690 */ 1691 start = state->end + 1; 1692 } else { 1693 /* 1694 * 'start' falls within a range that doesn't 1695 * have the bits set, so take its start as 1696 * the beginning of the desired range 1697 * 1698 * |--range with bits cleared----| 1699 * | 1700 * start 1701 */ 1702 *start_ret = state->start; 1703 break; 1704 } 1705 } else { 1706 /* 1707 * |---prev range---|---hole/unset---|---node range---| 1708 * | 1709 * start 1710 * 1711 * or 1712 * 1713 * |---hole/unset--||--first node--| 1714 * 0 | 1715 * start 1716 */ 1717 if (prev) { 1718 state = rb_entry(prev, struct extent_state, 1719 rb_node); 1720 *start_ret = state->end + 1; 1721 } else { 1722 *start_ret = 0; 1723 } 1724 break; 1725 } 1726 } 1727 1728 /* 1729 * Find the longest stretch from start until an entry which has the 1730 * bits set 1731 */ 1732 while (1) { 1733 state = rb_entry(node, struct extent_state, rb_node); 1734 if (state->end >= start && !(state->state & bits)) { 1735 *end_ret = state->end; 1736 } else { 1737 *end_ret = state->start - 1; 1738 break; 1739 } 1740 1741 node = rb_next(node); 1742 if (!node) 1743 break; 1744 } 1745 out: 1746 spin_unlock(&tree->lock); 1747 } 1748 1749 /* 1750 * find a contiguous range of bytes in the file marked as delalloc, not 1751 * more than 'max_bytes'. start and end are used to return the range, 1752 * 1753 * true is returned if we find something, false if nothing was in the tree 1754 */ 1755 bool btrfs_find_delalloc_range(struct extent_io_tree *tree, u64 *start, 1756 u64 *end, u64 max_bytes, 1757 struct extent_state **cached_state) 1758 { 1759 struct rb_node *node; 1760 struct extent_state *state; 1761 u64 cur_start = *start; 1762 bool found = false; 1763 u64 total_bytes = 0; 1764 1765 spin_lock(&tree->lock); 1766 1767 /* 1768 * this search will find all the extents that end after 1769 * our range starts. 1770 */ 1771 node = tree_search(tree, cur_start); 1772 if (!node) { 1773 *end = (u64)-1; 1774 goto out; 1775 } 1776 1777 while (1) { 1778 state = rb_entry(node, struct extent_state, rb_node); 1779 if (found && (state->start != cur_start || 1780 (state->state & EXTENT_BOUNDARY))) { 1781 goto out; 1782 } 1783 if (!(state->state & EXTENT_DELALLOC)) { 1784 if (!found) 1785 *end = state->end; 1786 goto out; 1787 } 1788 if (!found) { 1789 *start = state->start; 1790 *cached_state = state; 1791 refcount_inc(&state->refs); 1792 } 1793 found = true; 1794 *end = state->end; 1795 cur_start = state->end + 1; 1796 node = rb_next(node); 1797 total_bytes += state->end - state->start + 1; 1798 if (total_bytes >= max_bytes) 1799 break; 1800 if (!node) 1801 break; 1802 } 1803 out: 1804 spin_unlock(&tree->lock); 1805 return found; 1806 } 1807 1808 static int __process_pages_contig(struct address_space *mapping, 1809 struct page *locked_page, 1810 pgoff_t start_index, pgoff_t end_index, 1811 unsigned long page_ops, pgoff_t *index_ret); 1812 1813 static noinline void __unlock_for_delalloc(struct inode *inode, 1814 struct page *locked_page, 1815 u64 start, u64 end) 1816 { 1817 unsigned long index = start >> PAGE_SHIFT; 1818 unsigned long end_index = end >> PAGE_SHIFT; 1819 1820 ASSERT(locked_page); 1821 if (index == locked_page->index && end_index == index) 1822 return; 1823 1824 __process_pages_contig(inode->i_mapping, locked_page, index, end_index, 1825 PAGE_UNLOCK, NULL); 1826 } 1827 1828 static noinline int lock_delalloc_pages(struct inode *inode, 1829 struct page *locked_page, 1830 u64 delalloc_start, 1831 u64 delalloc_end) 1832 { 1833 unsigned long index = delalloc_start >> PAGE_SHIFT; 1834 unsigned long index_ret = index; 1835 unsigned long end_index = delalloc_end >> PAGE_SHIFT; 1836 int ret; 1837 1838 ASSERT(locked_page); 1839 if (index == locked_page->index && index == end_index) 1840 return 0; 1841 1842 ret = __process_pages_contig(inode->i_mapping, locked_page, index, 1843 end_index, PAGE_LOCK, &index_ret); 1844 if (ret == -EAGAIN) 1845 __unlock_for_delalloc(inode, locked_page, delalloc_start, 1846 (u64)index_ret << PAGE_SHIFT); 1847 return ret; 1848 } 1849 1850 /* 1851 * Find and lock a contiguous range of bytes in the file marked as delalloc, no 1852 * more than @max_bytes. @Start and @end are used to return the range, 1853 * 1854 * Return: true if we find something 1855 * false if nothing was in the tree 1856 */ 1857 EXPORT_FOR_TESTS 1858 noinline_for_stack bool find_lock_delalloc_range(struct inode *inode, 1859 struct page *locked_page, u64 *start, 1860 u64 *end) 1861 { 1862 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree; 1863 u64 max_bytes = BTRFS_MAX_EXTENT_SIZE; 1864 u64 delalloc_start; 1865 u64 delalloc_end; 1866 bool found; 1867 struct extent_state *cached_state = NULL; 1868 int ret; 1869 int loops = 0; 1870 1871 again: 1872 /* step one, find a bunch of delalloc bytes starting at start */ 1873 delalloc_start = *start; 1874 delalloc_end = 0; 1875 found = btrfs_find_delalloc_range(tree, &delalloc_start, &delalloc_end, 1876 max_bytes, &cached_state); 1877 if (!found || delalloc_end <= *start) { 1878 *start = delalloc_start; 1879 *end = delalloc_end; 1880 free_extent_state(cached_state); 1881 return false; 1882 } 1883 1884 /* 1885 * start comes from the offset of locked_page. We have to lock 1886 * pages in order, so we can't process delalloc bytes before 1887 * locked_page 1888 */ 1889 if (delalloc_start < *start) 1890 delalloc_start = *start; 1891 1892 /* 1893 * make sure to limit the number of pages we try to lock down 1894 */ 1895 if (delalloc_end + 1 - delalloc_start > max_bytes) 1896 delalloc_end = delalloc_start + max_bytes - 1; 1897 1898 /* step two, lock all the pages after the page that has start */ 1899 ret = lock_delalloc_pages(inode, locked_page, 1900 delalloc_start, delalloc_end); 1901 ASSERT(!ret || ret == -EAGAIN); 1902 if (ret == -EAGAIN) { 1903 /* some of the pages are gone, lets avoid looping by 1904 * shortening the size of the delalloc range we're searching 1905 */ 1906 free_extent_state(cached_state); 1907 cached_state = NULL; 1908 if (!loops) { 1909 max_bytes = PAGE_SIZE; 1910 loops = 1; 1911 goto again; 1912 } else { 1913 found = false; 1914 goto out_failed; 1915 } 1916 } 1917 1918 /* step three, lock the state bits for the whole range */ 1919 lock_extent_bits(tree, delalloc_start, delalloc_end, &cached_state); 1920 1921 /* then test to make sure it is all still delalloc */ 1922 ret = test_range_bit(tree, delalloc_start, delalloc_end, 1923 EXTENT_DELALLOC, 1, cached_state); 1924 if (!ret) { 1925 unlock_extent_cached(tree, delalloc_start, delalloc_end, 1926 &cached_state); 1927 __unlock_for_delalloc(inode, locked_page, 1928 delalloc_start, delalloc_end); 1929 cond_resched(); 1930 goto again; 1931 } 1932 free_extent_state(cached_state); 1933 *start = delalloc_start; 1934 *end = delalloc_end; 1935 out_failed: 1936 return found; 1937 } 1938 1939 static int __process_pages_contig(struct address_space *mapping, 1940 struct page *locked_page, 1941 pgoff_t start_index, pgoff_t end_index, 1942 unsigned long page_ops, pgoff_t *index_ret) 1943 { 1944 unsigned long nr_pages = end_index - start_index + 1; 1945 unsigned long pages_processed = 0; 1946 pgoff_t index = start_index; 1947 struct page *pages[16]; 1948 unsigned ret; 1949 int err = 0; 1950 int i; 1951 1952 if (page_ops & PAGE_LOCK) { 1953 ASSERT(page_ops == PAGE_LOCK); 1954 ASSERT(index_ret && *index_ret == start_index); 1955 } 1956 1957 if ((page_ops & PAGE_SET_ERROR) && nr_pages > 0) 1958 mapping_set_error(mapping, -EIO); 1959 1960 while (nr_pages > 0) { 1961 ret = find_get_pages_contig(mapping, index, 1962 min_t(unsigned long, 1963 nr_pages, ARRAY_SIZE(pages)), pages); 1964 if (ret == 0) { 1965 /* 1966 * Only if we're going to lock these pages, 1967 * can we find nothing at @index. 1968 */ 1969 ASSERT(page_ops & PAGE_LOCK); 1970 err = -EAGAIN; 1971 goto out; 1972 } 1973 1974 for (i = 0; i < ret; i++) { 1975 if (page_ops & PAGE_SET_PRIVATE2) 1976 SetPagePrivate2(pages[i]); 1977 1978 if (locked_page && pages[i] == locked_page) { 1979 put_page(pages[i]); 1980 pages_processed++; 1981 continue; 1982 } 1983 if (page_ops & PAGE_START_WRITEBACK) { 1984 clear_page_dirty_for_io(pages[i]); 1985 set_page_writeback(pages[i]); 1986 } 1987 if (page_ops & PAGE_SET_ERROR) 1988 SetPageError(pages[i]); 1989 if (page_ops & PAGE_END_WRITEBACK) 1990 end_page_writeback(pages[i]); 1991 if (page_ops & PAGE_UNLOCK) 1992 unlock_page(pages[i]); 1993 if (page_ops & PAGE_LOCK) { 1994 lock_page(pages[i]); 1995 if (!PageDirty(pages[i]) || 1996 pages[i]->mapping != mapping) { 1997 unlock_page(pages[i]); 1998 for (; i < ret; i++) 1999 put_page(pages[i]); 2000 err = -EAGAIN; 2001 goto out; 2002 } 2003 } 2004 put_page(pages[i]); 2005 pages_processed++; 2006 } 2007 nr_pages -= ret; 2008 index += ret; 2009 cond_resched(); 2010 } 2011 out: 2012 if (err && index_ret) 2013 *index_ret = start_index + pages_processed - 1; 2014 return err; 2015 } 2016 2017 void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end, 2018 struct page *locked_page, 2019 u32 clear_bits, unsigned long page_ops) 2020 { 2021 clear_extent_bit(&inode->io_tree, start, end, clear_bits, 1, 0, NULL); 2022 2023 __process_pages_contig(inode->vfs_inode.i_mapping, locked_page, 2024 start >> PAGE_SHIFT, end >> PAGE_SHIFT, 2025 page_ops, NULL); 2026 } 2027 2028 /* 2029 * count the number of bytes in the tree that have a given bit(s) 2030 * set. This can be fairly slow, except for EXTENT_DIRTY which is 2031 * cached. The total number found is returned. 2032 */ 2033 u64 count_range_bits(struct extent_io_tree *tree, 2034 u64 *start, u64 search_end, u64 max_bytes, 2035 u32 bits, int contig) 2036 { 2037 struct rb_node *node; 2038 struct extent_state *state; 2039 u64 cur_start = *start; 2040 u64 total_bytes = 0; 2041 u64 last = 0; 2042 int found = 0; 2043 2044 if (WARN_ON(search_end <= cur_start)) 2045 return 0; 2046 2047 spin_lock(&tree->lock); 2048 if (cur_start == 0 && bits == EXTENT_DIRTY) { 2049 total_bytes = tree->dirty_bytes; 2050 goto out; 2051 } 2052 /* 2053 * this search will find all the extents that end after 2054 * our range starts. 2055 */ 2056 node = tree_search(tree, cur_start); 2057 if (!node) 2058 goto out; 2059 2060 while (1) { 2061 state = rb_entry(node, struct extent_state, rb_node); 2062 if (state->start > search_end) 2063 break; 2064 if (contig && found && state->start > last + 1) 2065 break; 2066 if (state->end >= cur_start && (state->state & bits) == bits) { 2067 total_bytes += min(search_end, state->end) + 1 - 2068 max(cur_start, state->start); 2069 if (total_bytes >= max_bytes) 2070 break; 2071 if (!found) { 2072 *start = max(cur_start, state->start); 2073 found = 1; 2074 } 2075 last = state->end; 2076 } else if (contig && found) { 2077 break; 2078 } 2079 node = rb_next(node); 2080 if (!node) 2081 break; 2082 } 2083 out: 2084 spin_unlock(&tree->lock); 2085 return total_bytes; 2086 } 2087 2088 /* 2089 * set the private field for a given byte offset in the tree. If there isn't 2090 * an extent_state there already, this does nothing. 2091 */ 2092 int set_state_failrec(struct extent_io_tree *tree, u64 start, 2093 struct io_failure_record *failrec) 2094 { 2095 struct rb_node *node; 2096 struct extent_state *state; 2097 int ret = 0; 2098 2099 spin_lock(&tree->lock); 2100 /* 2101 * this search will find all the extents that end after 2102 * our range starts. 2103 */ 2104 node = tree_search(tree, start); 2105 if (!node) { 2106 ret = -ENOENT; 2107 goto out; 2108 } 2109 state = rb_entry(node, struct extent_state, rb_node); 2110 if (state->start != start) { 2111 ret = -ENOENT; 2112 goto out; 2113 } 2114 state->failrec = failrec; 2115 out: 2116 spin_unlock(&tree->lock); 2117 return ret; 2118 } 2119 2120 struct io_failure_record *get_state_failrec(struct extent_io_tree *tree, u64 start) 2121 { 2122 struct rb_node *node; 2123 struct extent_state *state; 2124 struct io_failure_record *failrec; 2125 2126 spin_lock(&tree->lock); 2127 /* 2128 * this search will find all the extents that end after 2129 * our range starts. 2130 */ 2131 node = tree_search(tree, start); 2132 if (!node) { 2133 failrec = ERR_PTR(-ENOENT); 2134 goto out; 2135 } 2136 state = rb_entry(node, struct extent_state, rb_node); 2137 if (state->start != start) { 2138 failrec = ERR_PTR(-ENOENT); 2139 goto out; 2140 } 2141 2142 failrec = state->failrec; 2143 out: 2144 spin_unlock(&tree->lock); 2145 return failrec; 2146 } 2147 2148 /* 2149 * searches a range in the state tree for a given mask. 2150 * If 'filled' == 1, this returns 1 only if every extent in the tree 2151 * has the bits set. Otherwise, 1 is returned if any bit in the 2152 * range is found set. 2153 */ 2154 int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end, 2155 u32 bits, int filled, struct extent_state *cached) 2156 { 2157 struct extent_state *state = NULL; 2158 struct rb_node *node; 2159 int bitset = 0; 2160 2161 spin_lock(&tree->lock); 2162 if (cached && extent_state_in_tree(cached) && cached->start <= start && 2163 cached->end > start) 2164 node = &cached->rb_node; 2165 else 2166 node = tree_search(tree, start); 2167 while (node && start <= end) { 2168 state = rb_entry(node, struct extent_state, rb_node); 2169 2170 if (filled && state->start > start) { 2171 bitset = 0; 2172 break; 2173 } 2174 2175 if (state->start > end) 2176 break; 2177 2178 if (state->state & bits) { 2179 bitset = 1; 2180 if (!filled) 2181 break; 2182 } else if (filled) { 2183 bitset = 0; 2184 break; 2185 } 2186 2187 if (state->end == (u64)-1) 2188 break; 2189 2190 start = state->end + 1; 2191 if (start > end) 2192 break; 2193 node = rb_next(node); 2194 if (!node) { 2195 if (filled) 2196 bitset = 0; 2197 break; 2198 } 2199 } 2200 spin_unlock(&tree->lock); 2201 return bitset; 2202 } 2203 2204 /* 2205 * helper function to set a given page up to date if all the 2206 * extents in the tree for that page are up to date 2207 */ 2208 static void check_page_uptodate(struct extent_io_tree *tree, struct page *page) 2209 { 2210 u64 start = page_offset(page); 2211 u64 end = start + PAGE_SIZE - 1; 2212 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL)) 2213 SetPageUptodate(page); 2214 } 2215 2216 int free_io_failure(struct extent_io_tree *failure_tree, 2217 struct extent_io_tree *io_tree, 2218 struct io_failure_record *rec) 2219 { 2220 int ret; 2221 int err = 0; 2222 2223 set_state_failrec(failure_tree, rec->start, NULL); 2224 ret = clear_extent_bits(failure_tree, rec->start, 2225 rec->start + rec->len - 1, 2226 EXTENT_LOCKED | EXTENT_DIRTY); 2227 if (ret) 2228 err = ret; 2229 2230 ret = clear_extent_bits(io_tree, rec->start, 2231 rec->start + rec->len - 1, 2232 EXTENT_DAMAGED); 2233 if (ret && !err) 2234 err = ret; 2235 2236 kfree(rec); 2237 return err; 2238 } 2239 2240 /* 2241 * this bypasses the standard btrfs submit functions deliberately, as 2242 * the standard behavior is to write all copies in a raid setup. here we only 2243 * want to write the one bad copy. so we do the mapping for ourselves and issue 2244 * submit_bio directly. 2245 * to avoid any synchronization issues, wait for the data after writing, which 2246 * actually prevents the read that triggered the error from finishing. 2247 * currently, there can be no more than two copies of every data bit. thus, 2248 * exactly one rewrite is required. 2249 */ 2250 int repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start, 2251 u64 length, u64 logical, struct page *page, 2252 unsigned int pg_offset, int mirror_num) 2253 { 2254 struct bio *bio; 2255 struct btrfs_device *dev; 2256 u64 map_length = 0; 2257 u64 sector; 2258 struct btrfs_bio *bbio = NULL; 2259 int ret; 2260 2261 ASSERT(!(fs_info->sb->s_flags & SB_RDONLY)); 2262 BUG_ON(!mirror_num); 2263 2264 if (btrfs_is_zoned(fs_info)) 2265 return btrfs_repair_one_zone(fs_info, logical); 2266 2267 bio = btrfs_io_bio_alloc(1); 2268 bio->bi_iter.bi_size = 0; 2269 map_length = length; 2270 2271 /* 2272 * Avoid races with device replace and make sure our bbio has devices 2273 * associated to its stripes that don't go away while we are doing the 2274 * read repair operation. 2275 */ 2276 btrfs_bio_counter_inc_blocked(fs_info); 2277 if (btrfs_is_parity_mirror(fs_info, logical, length)) { 2278 /* 2279 * Note that we don't use BTRFS_MAP_WRITE because it's supposed 2280 * to update all raid stripes, but here we just want to correct 2281 * bad stripe, thus BTRFS_MAP_READ is abused to only get the bad 2282 * stripe's dev and sector. 2283 */ 2284 ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, logical, 2285 &map_length, &bbio, 0); 2286 if (ret) { 2287 btrfs_bio_counter_dec(fs_info); 2288 bio_put(bio); 2289 return -EIO; 2290 } 2291 ASSERT(bbio->mirror_num == 1); 2292 } else { 2293 ret = btrfs_map_block(fs_info, BTRFS_MAP_WRITE, logical, 2294 &map_length, &bbio, mirror_num); 2295 if (ret) { 2296 btrfs_bio_counter_dec(fs_info); 2297 bio_put(bio); 2298 return -EIO; 2299 } 2300 BUG_ON(mirror_num != bbio->mirror_num); 2301 } 2302 2303 sector = bbio->stripes[bbio->mirror_num - 1].physical >> 9; 2304 bio->bi_iter.bi_sector = sector; 2305 dev = bbio->stripes[bbio->mirror_num - 1].dev; 2306 btrfs_put_bbio(bbio); 2307 if (!dev || !dev->bdev || 2308 !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) { 2309 btrfs_bio_counter_dec(fs_info); 2310 bio_put(bio); 2311 return -EIO; 2312 } 2313 bio_set_dev(bio, dev->bdev); 2314 bio->bi_opf = REQ_OP_WRITE | REQ_SYNC; 2315 bio_add_page(bio, page, length, pg_offset); 2316 2317 if (btrfsic_submit_bio_wait(bio)) { 2318 /* try to remap that extent elsewhere? */ 2319 btrfs_bio_counter_dec(fs_info); 2320 bio_put(bio); 2321 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS); 2322 return -EIO; 2323 } 2324 2325 btrfs_info_rl_in_rcu(fs_info, 2326 "read error corrected: ino %llu off %llu (dev %s sector %llu)", 2327 ino, start, 2328 rcu_str_deref(dev->name), sector); 2329 btrfs_bio_counter_dec(fs_info); 2330 bio_put(bio); 2331 return 0; 2332 } 2333 2334 int btrfs_repair_eb_io_failure(const struct extent_buffer *eb, int mirror_num) 2335 { 2336 struct btrfs_fs_info *fs_info = eb->fs_info; 2337 u64 start = eb->start; 2338 int i, num_pages = num_extent_pages(eb); 2339 int ret = 0; 2340 2341 if (sb_rdonly(fs_info->sb)) 2342 return -EROFS; 2343 2344 for (i = 0; i < num_pages; i++) { 2345 struct page *p = eb->pages[i]; 2346 2347 ret = repair_io_failure(fs_info, 0, start, PAGE_SIZE, start, p, 2348 start - page_offset(p), mirror_num); 2349 if (ret) 2350 break; 2351 start += PAGE_SIZE; 2352 } 2353 2354 return ret; 2355 } 2356 2357 /* 2358 * each time an IO finishes, we do a fast check in the IO failure tree 2359 * to see if we need to process or clean up an io_failure_record 2360 */ 2361 int clean_io_failure(struct btrfs_fs_info *fs_info, 2362 struct extent_io_tree *failure_tree, 2363 struct extent_io_tree *io_tree, u64 start, 2364 struct page *page, u64 ino, unsigned int pg_offset) 2365 { 2366 u64 private; 2367 struct io_failure_record *failrec; 2368 struct extent_state *state; 2369 int num_copies; 2370 int ret; 2371 2372 private = 0; 2373 ret = count_range_bits(failure_tree, &private, (u64)-1, 1, 2374 EXTENT_DIRTY, 0); 2375 if (!ret) 2376 return 0; 2377 2378 failrec = get_state_failrec(failure_tree, start); 2379 if (IS_ERR(failrec)) 2380 return 0; 2381 2382 BUG_ON(!failrec->this_mirror); 2383 2384 if (failrec->in_validation) { 2385 /* there was no real error, just free the record */ 2386 btrfs_debug(fs_info, 2387 "clean_io_failure: freeing dummy error at %llu", 2388 failrec->start); 2389 goto out; 2390 } 2391 if (sb_rdonly(fs_info->sb)) 2392 goto out; 2393 2394 spin_lock(&io_tree->lock); 2395 state = find_first_extent_bit_state(io_tree, 2396 failrec->start, 2397 EXTENT_LOCKED); 2398 spin_unlock(&io_tree->lock); 2399 2400 if (state && state->start <= failrec->start && 2401 state->end >= failrec->start + failrec->len - 1) { 2402 num_copies = btrfs_num_copies(fs_info, failrec->logical, 2403 failrec->len); 2404 if (num_copies > 1) { 2405 repair_io_failure(fs_info, ino, start, failrec->len, 2406 failrec->logical, page, pg_offset, 2407 failrec->failed_mirror); 2408 } 2409 } 2410 2411 out: 2412 free_io_failure(failure_tree, io_tree, failrec); 2413 2414 return 0; 2415 } 2416 2417 /* 2418 * Can be called when 2419 * - hold extent lock 2420 * - under ordered extent 2421 * - the inode is freeing 2422 */ 2423 void btrfs_free_io_failure_record(struct btrfs_inode *inode, u64 start, u64 end) 2424 { 2425 struct extent_io_tree *failure_tree = &inode->io_failure_tree; 2426 struct io_failure_record *failrec; 2427 struct extent_state *state, *next; 2428 2429 if (RB_EMPTY_ROOT(&failure_tree->state)) 2430 return; 2431 2432 spin_lock(&failure_tree->lock); 2433 state = find_first_extent_bit_state(failure_tree, start, EXTENT_DIRTY); 2434 while (state) { 2435 if (state->start > end) 2436 break; 2437 2438 ASSERT(state->end <= end); 2439 2440 next = next_state(state); 2441 2442 failrec = state->failrec; 2443 free_extent_state(state); 2444 kfree(failrec); 2445 2446 state = next; 2447 } 2448 spin_unlock(&failure_tree->lock); 2449 } 2450 2451 static struct io_failure_record *btrfs_get_io_failure_record(struct inode *inode, 2452 u64 start, u64 end) 2453 { 2454 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2455 struct io_failure_record *failrec; 2456 struct extent_map *em; 2457 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree; 2458 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree; 2459 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 2460 int ret; 2461 u64 logical; 2462 2463 failrec = get_state_failrec(failure_tree, start); 2464 if (!IS_ERR(failrec)) { 2465 btrfs_debug(fs_info, 2466 "Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu, validation=%d", 2467 failrec->logical, failrec->start, failrec->len, 2468 failrec->in_validation); 2469 /* 2470 * when data can be on disk more than twice, add to failrec here 2471 * (e.g. with a list for failed_mirror) to make 2472 * clean_io_failure() clean all those errors at once. 2473 */ 2474 2475 return failrec; 2476 } 2477 2478 failrec = kzalloc(sizeof(*failrec), GFP_NOFS); 2479 if (!failrec) 2480 return ERR_PTR(-ENOMEM); 2481 2482 failrec->start = start; 2483 failrec->len = end - start + 1; 2484 failrec->this_mirror = 0; 2485 failrec->bio_flags = 0; 2486 failrec->in_validation = 0; 2487 2488 read_lock(&em_tree->lock); 2489 em = lookup_extent_mapping(em_tree, start, failrec->len); 2490 if (!em) { 2491 read_unlock(&em_tree->lock); 2492 kfree(failrec); 2493 return ERR_PTR(-EIO); 2494 } 2495 2496 if (em->start > start || em->start + em->len <= start) { 2497 free_extent_map(em); 2498 em = NULL; 2499 } 2500 read_unlock(&em_tree->lock); 2501 if (!em) { 2502 kfree(failrec); 2503 return ERR_PTR(-EIO); 2504 } 2505 2506 logical = start - em->start; 2507 logical = em->block_start + logical; 2508 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) { 2509 logical = em->block_start; 2510 failrec->bio_flags = EXTENT_BIO_COMPRESSED; 2511 extent_set_compress_type(&failrec->bio_flags, em->compress_type); 2512 } 2513 2514 btrfs_debug(fs_info, 2515 "Get IO Failure Record: (new) logical=%llu, start=%llu, len=%llu", 2516 logical, start, failrec->len); 2517 2518 failrec->logical = logical; 2519 free_extent_map(em); 2520 2521 /* Set the bits in the private failure tree */ 2522 ret = set_extent_bits(failure_tree, start, end, 2523 EXTENT_LOCKED | EXTENT_DIRTY); 2524 if (ret >= 0) { 2525 ret = set_state_failrec(failure_tree, start, failrec); 2526 /* Set the bits in the inode's tree */ 2527 ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED); 2528 } else if (ret < 0) { 2529 kfree(failrec); 2530 return ERR_PTR(ret); 2531 } 2532 2533 return failrec; 2534 } 2535 2536 static bool btrfs_check_repairable(struct inode *inode, bool needs_validation, 2537 struct io_failure_record *failrec, 2538 int failed_mirror) 2539 { 2540 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2541 int num_copies; 2542 2543 num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len); 2544 if (num_copies == 1) { 2545 /* 2546 * we only have a single copy of the data, so don't bother with 2547 * all the retry and error correction code that follows. no 2548 * matter what the error is, it is very likely to persist. 2549 */ 2550 btrfs_debug(fs_info, 2551 "Check Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d", 2552 num_copies, failrec->this_mirror, failed_mirror); 2553 return false; 2554 } 2555 2556 /* 2557 * there are two premises: 2558 * a) deliver good data to the caller 2559 * b) correct the bad sectors on disk 2560 */ 2561 if (needs_validation) { 2562 /* 2563 * to fulfill b), we need to know the exact failing sectors, as 2564 * we don't want to rewrite any more than the failed ones. thus, 2565 * we need separate read requests for the failed bio 2566 * 2567 * if the following BUG_ON triggers, our validation request got 2568 * merged. we need separate requests for our algorithm to work. 2569 */ 2570 BUG_ON(failrec->in_validation); 2571 failrec->in_validation = 1; 2572 failrec->this_mirror = failed_mirror; 2573 } else { 2574 /* 2575 * we're ready to fulfill a) and b) alongside. get a good copy 2576 * of the failed sector and if we succeed, we have setup 2577 * everything for repair_io_failure to do the rest for us. 2578 */ 2579 if (failrec->in_validation) { 2580 BUG_ON(failrec->this_mirror != failed_mirror); 2581 failrec->in_validation = 0; 2582 failrec->this_mirror = 0; 2583 } 2584 failrec->failed_mirror = failed_mirror; 2585 failrec->this_mirror++; 2586 if (failrec->this_mirror == failed_mirror) 2587 failrec->this_mirror++; 2588 } 2589 2590 if (failrec->this_mirror > num_copies) { 2591 btrfs_debug(fs_info, 2592 "Check Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d", 2593 num_copies, failrec->this_mirror, failed_mirror); 2594 return false; 2595 } 2596 2597 return true; 2598 } 2599 2600 static bool btrfs_io_needs_validation(struct inode *inode, struct bio *bio) 2601 { 2602 u64 len = 0; 2603 const u32 blocksize = inode->i_sb->s_blocksize; 2604 2605 /* 2606 * If bi_status is BLK_STS_OK, then this was a checksum error, not an 2607 * I/O error. In this case, we already know exactly which sector was 2608 * bad, so we don't need to validate. 2609 */ 2610 if (bio->bi_status == BLK_STS_OK) 2611 return false; 2612 2613 /* 2614 * We need to validate each sector individually if the failed I/O was 2615 * for multiple sectors. 2616 * 2617 * There are a few possible bios that can end up here: 2618 * 1. A buffered read bio, which is not cloned. 2619 * 2. A direct I/O read bio, which is cloned. 2620 * 3. A (buffered or direct) repair bio, which is not cloned. 2621 * 2622 * For cloned bios (case 2), we can get the size from 2623 * btrfs_io_bio->iter; for non-cloned bios (cases 1 and 3), we can get 2624 * it from the bvecs. 2625 */ 2626 if (bio_flagged(bio, BIO_CLONED)) { 2627 if (btrfs_io_bio(bio)->iter.bi_size > blocksize) 2628 return true; 2629 } else { 2630 struct bio_vec *bvec; 2631 int i; 2632 2633 bio_for_each_bvec_all(bvec, bio, i) { 2634 len += bvec->bv_len; 2635 if (len > blocksize) 2636 return true; 2637 } 2638 } 2639 return false; 2640 } 2641 2642 blk_status_t btrfs_submit_read_repair(struct inode *inode, 2643 struct bio *failed_bio, u32 bio_offset, 2644 struct page *page, unsigned int pgoff, 2645 u64 start, u64 end, int failed_mirror, 2646 submit_bio_hook_t *submit_bio_hook) 2647 { 2648 struct io_failure_record *failrec; 2649 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2650 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree; 2651 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree; 2652 struct btrfs_io_bio *failed_io_bio = btrfs_io_bio(failed_bio); 2653 const int icsum = bio_offset >> fs_info->sectorsize_bits; 2654 bool need_validation; 2655 struct bio *repair_bio; 2656 struct btrfs_io_bio *repair_io_bio; 2657 blk_status_t status; 2658 2659 btrfs_debug(fs_info, 2660 "repair read error: read error at %llu", start); 2661 2662 BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE); 2663 2664 failrec = btrfs_get_io_failure_record(inode, start, end); 2665 if (IS_ERR(failrec)) 2666 return errno_to_blk_status(PTR_ERR(failrec)); 2667 2668 need_validation = btrfs_io_needs_validation(inode, failed_bio); 2669 2670 if (!btrfs_check_repairable(inode, need_validation, failrec, 2671 failed_mirror)) { 2672 free_io_failure(failure_tree, tree, failrec); 2673 return BLK_STS_IOERR; 2674 } 2675 2676 repair_bio = btrfs_io_bio_alloc(1); 2677 repair_io_bio = btrfs_io_bio(repair_bio); 2678 repair_bio->bi_opf = REQ_OP_READ; 2679 if (need_validation) 2680 repair_bio->bi_opf |= REQ_FAILFAST_DEV; 2681 repair_bio->bi_end_io = failed_bio->bi_end_io; 2682 repair_bio->bi_iter.bi_sector = failrec->logical >> 9; 2683 repair_bio->bi_private = failed_bio->bi_private; 2684 2685 if (failed_io_bio->csum) { 2686 const u32 csum_size = fs_info->csum_size; 2687 2688 repair_io_bio->csum = repair_io_bio->csum_inline; 2689 memcpy(repair_io_bio->csum, 2690 failed_io_bio->csum + csum_size * icsum, csum_size); 2691 } 2692 2693 bio_add_page(repair_bio, page, failrec->len, pgoff); 2694 repair_io_bio->logical = failrec->start; 2695 repair_io_bio->iter = repair_bio->bi_iter; 2696 2697 btrfs_debug(btrfs_sb(inode->i_sb), 2698 "repair read error: submitting new read to mirror %d, in_validation=%d", 2699 failrec->this_mirror, failrec->in_validation); 2700 2701 status = submit_bio_hook(inode, repair_bio, failrec->this_mirror, 2702 failrec->bio_flags); 2703 if (status) { 2704 free_io_failure(failure_tree, tree, failrec); 2705 bio_put(repair_bio); 2706 } 2707 return status; 2708 } 2709 2710 /* lots and lots of room for performance fixes in the end_bio funcs */ 2711 2712 void end_extent_writepage(struct page *page, int err, u64 start, u64 end) 2713 { 2714 int uptodate = (err == 0); 2715 int ret = 0; 2716 2717 btrfs_writepage_endio_finish_ordered(page, start, end, uptodate); 2718 2719 if (!uptodate) { 2720 ClearPageUptodate(page); 2721 SetPageError(page); 2722 ret = err < 0 ? err : -EIO; 2723 mapping_set_error(page->mapping, ret); 2724 } 2725 } 2726 2727 /* 2728 * after a writepage IO is done, we need to: 2729 * clear the uptodate bits on error 2730 * clear the writeback bits in the extent tree for this IO 2731 * end_page_writeback if the page has no more pending IO 2732 * 2733 * Scheduling is not allowed, so the extent state tree is expected 2734 * to have one and only one object corresponding to this IO. 2735 */ 2736 static void end_bio_extent_writepage(struct bio *bio) 2737 { 2738 int error = blk_status_to_errno(bio->bi_status); 2739 struct bio_vec *bvec; 2740 u64 start; 2741 u64 end; 2742 struct bvec_iter_all iter_all; 2743 bool first_bvec = true; 2744 2745 ASSERT(!bio_flagged(bio, BIO_CLONED)); 2746 bio_for_each_segment_all(bvec, bio, iter_all) { 2747 struct page *page = bvec->bv_page; 2748 struct inode *inode = page->mapping->host; 2749 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2750 2751 /* We always issue full-page reads, but if some block 2752 * in a page fails to read, blk_update_request() will 2753 * advance bv_offset and adjust bv_len to compensate. 2754 * Print a warning for nonzero offsets, and an error 2755 * if they don't add up to a full page. */ 2756 if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) { 2757 if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE) 2758 btrfs_err(fs_info, 2759 "partial page write in btrfs with offset %u and length %u", 2760 bvec->bv_offset, bvec->bv_len); 2761 else 2762 btrfs_info(fs_info, 2763 "incomplete page write in btrfs with offset %u and length %u", 2764 bvec->bv_offset, bvec->bv_len); 2765 } 2766 2767 start = page_offset(page); 2768 end = start + bvec->bv_offset + bvec->bv_len - 1; 2769 2770 if (first_bvec) { 2771 btrfs_record_physical_zoned(inode, start, bio); 2772 first_bvec = false; 2773 } 2774 2775 end_extent_writepage(page, error, start, end); 2776 end_page_writeback(page); 2777 } 2778 2779 bio_put(bio); 2780 } 2781 2782 /* 2783 * Record previously processed extent range 2784 * 2785 * For endio_readpage_release_extent() to handle a full extent range, reducing 2786 * the extent io operations. 2787 */ 2788 struct processed_extent { 2789 struct btrfs_inode *inode; 2790 /* Start of the range in @inode */ 2791 u64 start; 2792 /* End of the range in @inode */ 2793 u64 end; 2794 bool uptodate; 2795 }; 2796 2797 /* 2798 * Try to release processed extent range 2799 * 2800 * May not release the extent range right now if the current range is 2801 * contiguous to processed extent. 2802 * 2803 * Will release processed extent when any of @inode, @uptodate, the range is 2804 * no longer contiguous to the processed range. 2805 * 2806 * Passing @inode == NULL will force processed extent to be released. 2807 */ 2808 static void endio_readpage_release_extent(struct processed_extent *processed, 2809 struct btrfs_inode *inode, u64 start, u64 end, 2810 bool uptodate) 2811 { 2812 struct extent_state *cached = NULL; 2813 struct extent_io_tree *tree; 2814 2815 /* The first extent, initialize @processed */ 2816 if (!processed->inode) 2817 goto update; 2818 2819 /* 2820 * Contiguous to processed extent, just uptodate the end. 2821 * 2822 * Several things to notice: 2823 * 2824 * - bio can be merged as long as on-disk bytenr is contiguous 2825 * This means we can have page belonging to other inodes, thus need to 2826 * check if the inode still matches. 2827 * - bvec can contain range beyond current page for multi-page bvec 2828 * Thus we need to do processed->end + 1 >= start check 2829 */ 2830 if (processed->inode == inode && processed->uptodate == uptodate && 2831 processed->end + 1 >= start && end >= processed->end) { 2832 processed->end = end; 2833 return; 2834 } 2835 2836 tree = &processed->inode->io_tree; 2837 /* 2838 * Now we don't have range contiguous to the processed range, release 2839 * the processed range now. 2840 */ 2841 if (processed->uptodate && tree->track_uptodate) 2842 set_extent_uptodate(tree, processed->start, processed->end, 2843 &cached, GFP_ATOMIC); 2844 unlock_extent_cached_atomic(tree, processed->start, processed->end, 2845 &cached); 2846 2847 update: 2848 /* Update processed to current range */ 2849 processed->inode = inode; 2850 processed->start = start; 2851 processed->end = end; 2852 processed->uptodate = uptodate; 2853 } 2854 2855 static void begin_page_read(struct btrfs_fs_info *fs_info, struct page *page) 2856 { 2857 ASSERT(PageLocked(page)); 2858 if (fs_info->sectorsize == PAGE_SIZE) 2859 return; 2860 2861 ASSERT(PagePrivate(page)); 2862 btrfs_subpage_start_reader(fs_info, page, page_offset(page), PAGE_SIZE); 2863 } 2864 2865 static void end_page_read(struct page *page, bool uptodate, u64 start, u32 len) 2866 { 2867 struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb); 2868 2869 ASSERT(page_offset(page) <= start && 2870 start + len <= page_offset(page) + PAGE_SIZE); 2871 2872 if (uptodate) { 2873 btrfs_page_set_uptodate(fs_info, page, start, len); 2874 } else { 2875 btrfs_page_clear_uptodate(fs_info, page, start, len); 2876 btrfs_page_set_error(fs_info, page, start, len); 2877 } 2878 2879 if (fs_info->sectorsize == PAGE_SIZE) 2880 unlock_page(page); 2881 else if (is_data_inode(page->mapping->host)) 2882 /* 2883 * For subpage data, unlock the page if we're the last reader. 2884 * For subpage metadata, page lock is not utilized for read. 2885 */ 2886 btrfs_subpage_end_reader(fs_info, page, start, len); 2887 } 2888 2889 /* 2890 * Find extent buffer for a givne bytenr. 2891 * 2892 * This is for end_bio_extent_readpage(), thus we can't do any unsafe locking 2893 * in endio context. 2894 */ 2895 static struct extent_buffer *find_extent_buffer_readpage( 2896 struct btrfs_fs_info *fs_info, struct page *page, u64 bytenr) 2897 { 2898 struct extent_buffer *eb; 2899 2900 /* 2901 * For regular sectorsize, we can use page->private to grab extent 2902 * buffer 2903 */ 2904 if (fs_info->sectorsize == PAGE_SIZE) { 2905 ASSERT(PagePrivate(page) && page->private); 2906 return (struct extent_buffer *)page->private; 2907 } 2908 2909 /* For subpage case, we need to lookup buffer radix tree */ 2910 rcu_read_lock(); 2911 eb = radix_tree_lookup(&fs_info->buffer_radix, 2912 bytenr >> fs_info->sectorsize_bits); 2913 rcu_read_unlock(); 2914 ASSERT(eb); 2915 return eb; 2916 } 2917 2918 /* 2919 * after a readpage IO is done, we need to: 2920 * clear the uptodate bits on error 2921 * set the uptodate bits if things worked 2922 * set the page up to date if all extents in the tree are uptodate 2923 * clear the lock bit in the extent tree 2924 * unlock the page if there are no other extents locked for it 2925 * 2926 * Scheduling is not allowed, so the extent state tree is expected 2927 * to have one and only one object corresponding to this IO. 2928 */ 2929 static void end_bio_extent_readpage(struct bio *bio) 2930 { 2931 struct bio_vec *bvec; 2932 int uptodate = !bio->bi_status; 2933 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 2934 struct extent_io_tree *tree, *failure_tree; 2935 struct processed_extent processed = { 0 }; 2936 /* 2937 * The offset to the beginning of a bio, since one bio can never be 2938 * larger than UINT_MAX, u32 here is enough. 2939 */ 2940 u32 bio_offset = 0; 2941 int mirror; 2942 int ret; 2943 struct bvec_iter_all iter_all; 2944 2945 ASSERT(!bio_flagged(bio, BIO_CLONED)); 2946 bio_for_each_segment_all(bvec, bio, iter_all) { 2947 struct page *page = bvec->bv_page; 2948 struct inode *inode = page->mapping->host; 2949 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2950 const u32 sectorsize = fs_info->sectorsize; 2951 u64 start; 2952 u64 end; 2953 u32 len; 2954 2955 btrfs_debug(fs_info, 2956 "end_bio_extent_readpage: bi_sector=%llu, err=%d, mirror=%u", 2957 bio->bi_iter.bi_sector, bio->bi_status, 2958 io_bio->mirror_num); 2959 tree = &BTRFS_I(inode)->io_tree; 2960 failure_tree = &BTRFS_I(inode)->io_failure_tree; 2961 2962 /* 2963 * We always issue full-sector reads, but if some block in a 2964 * page fails to read, blk_update_request() will advance 2965 * bv_offset and adjust bv_len to compensate. Print a warning 2966 * for unaligned offsets, and an error if they don't add up to 2967 * a full sector. 2968 */ 2969 if (!IS_ALIGNED(bvec->bv_offset, sectorsize)) 2970 btrfs_err(fs_info, 2971 "partial page read in btrfs with offset %u and length %u", 2972 bvec->bv_offset, bvec->bv_len); 2973 else if (!IS_ALIGNED(bvec->bv_offset + bvec->bv_len, 2974 sectorsize)) 2975 btrfs_info(fs_info, 2976 "incomplete page read with offset %u and length %u", 2977 bvec->bv_offset, bvec->bv_len); 2978 2979 start = page_offset(page) + bvec->bv_offset; 2980 end = start + bvec->bv_len - 1; 2981 len = bvec->bv_len; 2982 2983 mirror = io_bio->mirror_num; 2984 if (likely(uptodate)) { 2985 if (is_data_inode(inode)) 2986 ret = btrfs_verify_data_csum(io_bio, 2987 bio_offset, page, start, end); 2988 else 2989 ret = btrfs_validate_metadata_buffer(io_bio, 2990 page, start, end, mirror); 2991 if (ret) 2992 uptodate = 0; 2993 else 2994 clean_io_failure(BTRFS_I(inode)->root->fs_info, 2995 failure_tree, tree, start, 2996 page, 2997 btrfs_ino(BTRFS_I(inode)), 0); 2998 } 2999 3000 if (likely(uptodate)) 3001 goto readpage_ok; 3002 3003 if (is_data_inode(inode)) { 3004 3005 /* 3006 * The generic bio_readpage_error handles errors the 3007 * following way: If possible, new read requests are 3008 * created and submitted and will end up in 3009 * end_bio_extent_readpage as well (if we're lucky, 3010 * not in the !uptodate case). In that case it returns 3011 * 0 and we just go on with the next page in our bio. 3012 * If it can't handle the error it will return -EIO and 3013 * we remain responsible for that page. 3014 */ 3015 if (!btrfs_submit_read_repair(inode, bio, bio_offset, 3016 page, 3017 start - page_offset(page), 3018 start, end, mirror, 3019 btrfs_submit_data_bio)) { 3020 uptodate = !bio->bi_status; 3021 ASSERT(bio_offset + len > bio_offset); 3022 bio_offset += len; 3023 continue; 3024 } 3025 } else { 3026 struct extent_buffer *eb; 3027 3028 eb = find_extent_buffer_readpage(fs_info, page, start); 3029 set_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags); 3030 eb->read_mirror = mirror; 3031 atomic_dec(&eb->io_pages); 3032 if (test_and_clear_bit(EXTENT_BUFFER_READAHEAD, 3033 &eb->bflags)) 3034 btree_readahead_hook(eb, -EIO); 3035 } 3036 readpage_ok: 3037 if (likely(uptodate)) { 3038 loff_t i_size = i_size_read(inode); 3039 pgoff_t end_index = i_size >> PAGE_SHIFT; 3040 3041 /* 3042 * Zero out the remaining part if this range straddles 3043 * i_size. 3044 * 3045 * Here we should only zero the range inside the bvec, 3046 * not touch anything else. 3047 * 3048 * NOTE: i_size is exclusive while end is inclusive. 3049 */ 3050 if (page->index == end_index && i_size <= end) { 3051 u32 zero_start = max(offset_in_page(i_size), 3052 offset_in_page(start)); 3053 3054 zero_user_segment(page, zero_start, 3055 offset_in_page(end) + 1); 3056 } 3057 } 3058 ASSERT(bio_offset + len > bio_offset); 3059 bio_offset += len; 3060 3061 /* Update page status and unlock */ 3062 end_page_read(page, uptodate, start, len); 3063 endio_readpage_release_extent(&processed, BTRFS_I(inode), 3064 start, end, uptodate); 3065 } 3066 /* Release the last extent */ 3067 endio_readpage_release_extent(&processed, NULL, 0, 0, false); 3068 btrfs_io_bio_free_csum(io_bio); 3069 bio_put(bio); 3070 } 3071 3072 /* 3073 * Initialize the members up to but not including 'bio'. Use after allocating a 3074 * new bio by bio_alloc_bioset as it does not initialize the bytes outside of 3075 * 'bio' because use of __GFP_ZERO is not supported. 3076 */ 3077 static inline void btrfs_io_bio_init(struct btrfs_io_bio *btrfs_bio) 3078 { 3079 memset(btrfs_bio, 0, offsetof(struct btrfs_io_bio, bio)); 3080 } 3081 3082 /* 3083 * The following helpers allocate a bio. As it's backed by a bioset, it'll 3084 * never fail. We're returning a bio right now but you can call btrfs_io_bio 3085 * for the appropriate container_of magic 3086 */ 3087 struct bio *btrfs_bio_alloc(u64 first_byte) 3088 { 3089 struct bio *bio; 3090 3091 bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_VECS, &btrfs_bioset); 3092 bio->bi_iter.bi_sector = first_byte >> 9; 3093 btrfs_io_bio_init(btrfs_io_bio(bio)); 3094 return bio; 3095 } 3096 3097 struct bio *btrfs_bio_clone(struct bio *bio) 3098 { 3099 struct btrfs_io_bio *btrfs_bio; 3100 struct bio *new; 3101 3102 /* Bio allocation backed by a bioset does not fail */ 3103 new = bio_clone_fast(bio, GFP_NOFS, &btrfs_bioset); 3104 btrfs_bio = btrfs_io_bio(new); 3105 btrfs_io_bio_init(btrfs_bio); 3106 btrfs_bio->iter = bio->bi_iter; 3107 return new; 3108 } 3109 3110 struct bio *btrfs_io_bio_alloc(unsigned int nr_iovecs) 3111 { 3112 struct bio *bio; 3113 3114 /* Bio allocation backed by a bioset does not fail */ 3115 bio = bio_alloc_bioset(GFP_NOFS, nr_iovecs, &btrfs_bioset); 3116 btrfs_io_bio_init(btrfs_io_bio(bio)); 3117 return bio; 3118 } 3119 3120 struct bio *btrfs_bio_clone_partial(struct bio *orig, int offset, int size) 3121 { 3122 struct bio *bio; 3123 struct btrfs_io_bio *btrfs_bio; 3124 3125 /* this will never fail when it's backed by a bioset */ 3126 bio = bio_clone_fast(orig, GFP_NOFS, &btrfs_bioset); 3127 ASSERT(bio); 3128 3129 btrfs_bio = btrfs_io_bio(bio); 3130 btrfs_io_bio_init(btrfs_bio); 3131 3132 bio_trim(bio, offset >> 9, size >> 9); 3133 btrfs_bio->iter = bio->bi_iter; 3134 return bio; 3135 } 3136 3137 /** 3138 * Attempt to add a page to bio 3139 * 3140 * @bio: destination bio 3141 * @page: page to add to the bio 3142 * @disk_bytenr: offset of the new bio or to check whether we are adding 3143 * a contiguous page to the previous one 3144 * @pg_offset: starting offset in the page 3145 * @size: portion of page that we want to write 3146 * @prev_bio_flags: flags of previous bio to see if we can merge the current one 3147 * @bio_flags: flags of the current bio to see if we can merge them 3148 * @return: true if page was added, false otherwise 3149 * 3150 * Attempt to add a page to bio considering stripe alignment etc. 3151 * 3152 * Return true if successfully page added. Otherwise, return false. 3153 */ 3154 static bool btrfs_bio_add_page(struct bio *bio, struct page *page, 3155 u64 disk_bytenr, unsigned int size, 3156 unsigned int pg_offset, 3157 unsigned long prev_bio_flags, 3158 unsigned long bio_flags) 3159 { 3160 const sector_t sector = disk_bytenr >> SECTOR_SHIFT; 3161 bool contig; 3162 int ret; 3163 3164 if (prev_bio_flags != bio_flags) 3165 return false; 3166 3167 if (prev_bio_flags & EXTENT_BIO_COMPRESSED) 3168 contig = bio->bi_iter.bi_sector == sector; 3169 else 3170 contig = bio_end_sector(bio) == sector; 3171 if (!contig) 3172 return false; 3173 3174 if (btrfs_bio_fits_in_stripe(page, size, bio, bio_flags)) 3175 return false; 3176 3177 if (bio_op(bio) == REQ_OP_ZONE_APPEND) { 3178 struct page *first_page = bio_first_bvec_all(bio)->bv_page; 3179 3180 if (!btrfs_bio_fits_in_ordered_extent(first_page, bio, size)) 3181 return false; 3182 ret = bio_add_zone_append_page(bio, page, size, pg_offset); 3183 } else { 3184 ret = bio_add_page(bio, page, size, pg_offset); 3185 } 3186 3187 return ret == size; 3188 } 3189 3190 /* 3191 * @opf: bio REQ_OP_* and REQ_* flags as one value 3192 * @wbc: optional writeback control for io accounting 3193 * @page: page to add to the bio 3194 * @disk_bytenr: logical bytenr where the write will be 3195 * @size: portion of page that we want to write to 3196 * @pg_offset: offset of the new bio or to check whether we are adding 3197 * a contiguous page to the previous one 3198 * @bio_ret: must be valid pointer, newly allocated bio will be stored there 3199 * @end_io_func: end_io callback for new bio 3200 * @mirror_num: desired mirror to read/write 3201 * @prev_bio_flags: flags of previous bio to see if we can merge the current one 3202 * @bio_flags: flags of the current bio to see if we can merge them 3203 */ 3204 static int submit_extent_page(unsigned int opf, 3205 struct writeback_control *wbc, 3206 struct page *page, u64 disk_bytenr, 3207 size_t size, unsigned long pg_offset, 3208 struct bio **bio_ret, 3209 bio_end_io_t end_io_func, 3210 int mirror_num, 3211 unsigned long prev_bio_flags, 3212 unsigned long bio_flags, 3213 bool force_bio_submit) 3214 { 3215 int ret = 0; 3216 struct bio *bio; 3217 size_t io_size = min_t(size_t, size, PAGE_SIZE); 3218 struct btrfs_inode *inode = BTRFS_I(page->mapping->host); 3219 struct extent_io_tree *tree = &inode->io_tree; 3220 struct btrfs_fs_info *fs_info = inode->root->fs_info; 3221 3222 ASSERT(bio_ret); 3223 3224 if (*bio_ret) { 3225 bio = *bio_ret; 3226 if (force_bio_submit || 3227 !btrfs_bio_add_page(bio, page, disk_bytenr, io_size, 3228 pg_offset, prev_bio_flags, bio_flags)) { 3229 ret = submit_one_bio(bio, mirror_num, prev_bio_flags); 3230 if (ret < 0) { 3231 *bio_ret = NULL; 3232 return ret; 3233 } 3234 bio = NULL; 3235 } else { 3236 if (wbc) 3237 wbc_account_cgroup_owner(wbc, page, io_size); 3238 return 0; 3239 } 3240 } 3241 3242 bio = btrfs_bio_alloc(disk_bytenr); 3243 bio_add_page(bio, page, io_size, pg_offset); 3244 bio->bi_end_io = end_io_func; 3245 bio->bi_private = tree; 3246 bio->bi_write_hint = page->mapping->host->i_write_hint; 3247 bio->bi_opf = opf; 3248 if (wbc) { 3249 struct block_device *bdev; 3250 3251 bdev = fs_info->fs_devices->latest_bdev; 3252 bio_set_dev(bio, bdev); 3253 wbc_init_bio(wbc, bio); 3254 wbc_account_cgroup_owner(wbc, page, io_size); 3255 } 3256 if (btrfs_is_zoned(fs_info) && bio_op(bio) == REQ_OP_ZONE_APPEND) { 3257 struct extent_map *em; 3258 struct map_lookup *map; 3259 3260 em = btrfs_get_chunk_map(fs_info, disk_bytenr, io_size); 3261 if (IS_ERR(em)) 3262 return PTR_ERR(em); 3263 3264 map = em->map_lookup; 3265 /* We only support single profile for now */ 3266 ASSERT(map->num_stripes == 1); 3267 btrfs_io_bio(bio)->device = map->stripes[0].dev; 3268 3269 free_extent_map(em); 3270 } 3271 3272 *bio_ret = bio; 3273 3274 return ret; 3275 } 3276 3277 static int attach_extent_buffer_page(struct extent_buffer *eb, 3278 struct page *page, 3279 struct btrfs_subpage *prealloc) 3280 { 3281 struct btrfs_fs_info *fs_info = eb->fs_info; 3282 int ret = 0; 3283 3284 /* 3285 * If the page is mapped to btree inode, we should hold the private 3286 * lock to prevent race. 3287 * For cloned or dummy extent buffers, their pages are not mapped and 3288 * will not race with any other ebs. 3289 */ 3290 if (page->mapping) 3291 lockdep_assert_held(&page->mapping->private_lock); 3292 3293 if (fs_info->sectorsize == PAGE_SIZE) { 3294 if (!PagePrivate(page)) 3295 attach_page_private(page, eb); 3296 else 3297 WARN_ON(page->private != (unsigned long)eb); 3298 return 0; 3299 } 3300 3301 /* Already mapped, just free prealloc */ 3302 if (PagePrivate(page)) { 3303 btrfs_free_subpage(prealloc); 3304 return 0; 3305 } 3306 3307 if (prealloc) 3308 /* Has preallocated memory for subpage */ 3309 attach_page_private(page, prealloc); 3310 else 3311 /* Do new allocation to attach subpage */ 3312 ret = btrfs_attach_subpage(fs_info, page, 3313 BTRFS_SUBPAGE_METADATA); 3314 return ret; 3315 } 3316 3317 int set_page_extent_mapped(struct page *page) 3318 { 3319 struct btrfs_fs_info *fs_info; 3320 3321 ASSERT(page->mapping); 3322 3323 if (PagePrivate(page)) 3324 return 0; 3325 3326 fs_info = btrfs_sb(page->mapping->host->i_sb); 3327 3328 if (fs_info->sectorsize < PAGE_SIZE) 3329 return btrfs_attach_subpage(fs_info, page, BTRFS_SUBPAGE_DATA); 3330 3331 attach_page_private(page, (void *)EXTENT_PAGE_PRIVATE); 3332 return 0; 3333 } 3334 3335 void clear_page_extent_mapped(struct page *page) 3336 { 3337 struct btrfs_fs_info *fs_info; 3338 3339 ASSERT(page->mapping); 3340 3341 if (!PagePrivate(page)) 3342 return; 3343 3344 fs_info = btrfs_sb(page->mapping->host->i_sb); 3345 if (fs_info->sectorsize < PAGE_SIZE) 3346 return btrfs_detach_subpage(fs_info, page); 3347 3348 detach_page_private(page); 3349 } 3350 3351 static struct extent_map * 3352 __get_extent_map(struct inode *inode, struct page *page, size_t pg_offset, 3353 u64 start, u64 len, struct extent_map **em_cached) 3354 { 3355 struct extent_map *em; 3356 3357 if (em_cached && *em_cached) { 3358 em = *em_cached; 3359 if (extent_map_in_tree(em) && start >= em->start && 3360 start < extent_map_end(em)) { 3361 refcount_inc(&em->refs); 3362 return em; 3363 } 3364 3365 free_extent_map(em); 3366 *em_cached = NULL; 3367 } 3368 3369 em = btrfs_get_extent(BTRFS_I(inode), page, pg_offset, start, len); 3370 if (em_cached && !IS_ERR_OR_NULL(em)) { 3371 BUG_ON(*em_cached); 3372 refcount_inc(&em->refs); 3373 *em_cached = em; 3374 } 3375 return em; 3376 } 3377 /* 3378 * basic readpage implementation. Locked extent state structs are inserted 3379 * into the tree that are removed when the IO is done (by the end_io 3380 * handlers) 3381 * XXX JDM: This needs looking at to ensure proper page locking 3382 * return 0 on success, otherwise return error 3383 */ 3384 int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, 3385 struct bio **bio, unsigned long *bio_flags, 3386 unsigned int read_flags, u64 *prev_em_start) 3387 { 3388 struct inode *inode = page->mapping->host; 3389 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 3390 u64 start = page_offset(page); 3391 const u64 end = start + PAGE_SIZE - 1; 3392 u64 cur = start; 3393 u64 extent_offset; 3394 u64 last_byte = i_size_read(inode); 3395 u64 block_start; 3396 u64 cur_end; 3397 struct extent_map *em; 3398 int ret = 0; 3399 int nr = 0; 3400 size_t pg_offset = 0; 3401 size_t iosize; 3402 size_t blocksize = inode->i_sb->s_blocksize; 3403 unsigned long this_bio_flag = 0; 3404 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree; 3405 3406 ret = set_page_extent_mapped(page); 3407 if (ret < 0) { 3408 unlock_extent(tree, start, end); 3409 btrfs_page_set_error(fs_info, page, start, PAGE_SIZE); 3410 unlock_page(page); 3411 goto out; 3412 } 3413 3414 if (!PageUptodate(page)) { 3415 if (cleancache_get_page(page) == 0) { 3416 BUG_ON(blocksize != PAGE_SIZE); 3417 unlock_extent(tree, start, end); 3418 unlock_page(page); 3419 goto out; 3420 } 3421 } 3422 3423 if (page->index == last_byte >> PAGE_SHIFT) { 3424 size_t zero_offset = offset_in_page(last_byte); 3425 3426 if (zero_offset) { 3427 iosize = PAGE_SIZE - zero_offset; 3428 memzero_page(page, zero_offset, iosize); 3429 flush_dcache_page(page); 3430 } 3431 } 3432 begin_page_read(fs_info, page); 3433 while (cur <= end) { 3434 bool force_bio_submit = false; 3435 u64 disk_bytenr; 3436 3437 if (cur >= last_byte) { 3438 struct extent_state *cached = NULL; 3439 3440 iosize = PAGE_SIZE - pg_offset; 3441 memzero_page(page, pg_offset, iosize); 3442 flush_dcache_page(page); 3443 set_extent_uptodate(tree, cur, cur + iosize - 1, 3444 &cached, GFP_NOFS); 3445 unlock_extent_cached(tree, cur, 3446 cur + iosize - 1, &cached); 3447 end_page_read(page, true, cur, iosize); 3448 break; 3449 } 3450 em = __get_extent_map(inode, page, pg_offset, cur, 3451 end - cur + 1, em_cached); 3452 if (IS_ERR_OR_NULL(em)) { 3453 unlock_extent(tree, cur, end); 3454 end_page_read(page, false, cur, end + 1 - cur); 3455 break; 3456 } 3457 extent_offset = cur - em->start; 3458 BUG_ON(extent_map_end(em) <= cur); 3459 BUG_ON(end < cur); 3460 3461 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) { 3462 this_bio_flag |= EXTENT_BIO_COMPRESSED; 3463 extent_set_compress_type(&this_bio_flag, 3464 em->compress_type); 3465 } 3466 3467 iosize = min(extent_map_end(em) - cur, end - cur + 1); 3468 cur_end = min(extent_map_end(em) - 1, end); 3469 iosize = ALIGN(iosize, blocksize); 3470 if (this_bio_flag & EXTENT_BIO_COMPRESSED) 3471 disk_bytenr = em->block_start; 3472 else 3473 disk_bytenr = em->block_start + extent_offset; 3474 block_start = em->block_start; 3475 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 3476 block_start = EXTENT_MAP_HOLE; 3477 3478 /* 3479 * If we have a file range that points to a compressed extent 3480 * and it's followed by a consecutive file range that points 3481 * to the same compressed extent (possibly with a different 3482 * offset and/or length, so it either points to the whole extent 3483 * or only part of it), we must make sure we do not submit a 3484 * single bio to populate the pages for the 2 ranges because 3485 * this makes the compressed extent read zero out the pages 3486 * belonging to the 2nd range. Imagine the following scenario: 3487 * 3488 * File layout 3489 * [0 - 8K] [8K - 24K] 3490 * | | 3491 * | | 3492 * points to extent X, points to extent X, 3493 * offset 4K, length of 8K offset 0, length 16K 3494 * 3495 * [extent X, compressed length = 4K uncompressed length = 16K] 3496 * 3497 * If the bio to read the compressed extent covers both ranges, 3498 * it will decompress extent X into the pages belonging to the 3499 * first range and then it will stop, zeroing out the remaining 3500 * pages that belong to the other range that points to extent X. 3501 * So here we make sure we submit 2 bios, one for the first 3502 * range and another one for the third range. Both will target 3503 * the same physical extent from disk, but we can't currently 3504 * make the compressed bio endio callback populate the pages 3505 * for both ranges because each compressed bio is tightly 3506 * coupled with a single extent map, and each range can have 3507 * an extent map with a different offset value relative to the 3508 * uncompressed data of our extent and different lengths. This 3509 * is a corner case so we prioritize correctness over 3510 * non-optimal behavior (submitting 2 bios for the same extent). 3511 */ 3512 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) && 3513 prev_em_start && *prev_em_start != (u64)-1 && 3514 *prev_em_start != em->start) 3515 force_bio_submit = true; 3516 3517 if (prev_em_start) 3518 *prev_em_start = em->start; 3519 3520 free_extent_map(em); 3521 em = NULL; 3522 3523 /* we've found a hole, just zero and go on */ 3524 if (block_start == EXTENT_MAP_HOLE) { 3525 struct extent_state *cached = NULL; 3526 3527 memzero_page(page, pg_offset, iosize); 3528 flush_dcache_page(page); 3529 3530 set_extent_uptodate(tree, cur, cur + iosize - 1, 3531 &cached, GFP_NOFS); 3532 unlock_extent_cached(tree, cur, 3533 cur + iosize - 1, &cached); 3534 end_page_read(page, true, cur, iosize); 3535 cur = cur + iosize; 3536 pg_offset += iosize; 3537 continue; 3538 } 3539 /* the get_extent function already copied into the page */ 3540 if (test_range_bit(tree, cur, cur_end, 3541 EXTENT_UPTODATE, 1, NULL)) { 3542 check_page_uptodate(tree, page); 3543 unlock_extent(tree, cur, cur + iosize - 1); 3544 end_page_read(page, true, cur, iosize); 3545 cur = cur + iosize; 3546 pg_offset += iosize; 3547 continue; 3548 } 3549 /* we have an inline extent but it didn't get marked up 3550 * to date. Error out 3551 */ 3552 if (block_start == EXTENT_MAP_INLINE) { 3553 unlock_extent(tree, cur, cur + iosize - 1); 3554 end_page_read(page, false, cur, iosize); 3555 cur = cur + iosize; 3556 pg_offset += iosize; 3557 continue; 3558 } 3559 3560 ret = submit_extent_page(REQ_OP_READ | read_flags, NULL, 3561 page, disk_bytenr, iosize, 3562 pg_offset, bio, 3563 end_bio_extent_readpage, 0, 3564 *bio_flags, 3565 this_bio_flag, 3566 force_bio_submit); 3567 if (!ret) { 3568 nr++; 3569 *bio_flags = this_bio_flag; 3570 } else { 3571 unlock_extent(tree, cur, cur + iosize - 1); 3572 end_page_read(page, false, cur, iosize); 3573 goto out; 3574 } 3575 cur = cur + iosize; 3576 pg_offset += iosize; 3577 } 3578 out: 3579 return ret; 3580 } 3581 3582 static inline void contiguous_readpages(struct page *pages[], int nr_pages, 3583 u64 start, u64 end, 3584 struct extent_map **em_cached, 3585 struct bio **bio, 3586 unsigned long *bio_flags, 3587 u64 *prev_em_start) 3588 { 3589 struct btrfs_inode *inode = BTRFS_I(pages[0]->mapping->host); 3590 int index; 3591 3592 btrfs_lock_and_flush_ordered_range(inode, start, end, NULL); 3593 3594 for (index = 0; index < nr_pages; index++) { 3595 btrfs_do_readpage(pages[index], em_cached, bio, bio_flags, 3596 REQ_RAHEAD, prev_em_start); 3597 put_page(pages[index]); 3598 } 3599 } 3600 3601 static void update_nr_written(struct writeback_control *wbc, 3602 unsigned long nr_written) 3603 { 3604 wbc->nr_to_write -= nr_written; 3605 } 3606 3607 /* 3608 * helper for __extent_writepage, doing all of the delayed allocation setup. 3609 * 3610 * This returns 1 if btrfs_run_delalloc_range function did all the work required 3611 * to write the page (copy into inline extent). In this case the IO has 3612 * been started and the page is already unlocked. 3613 * 3614 * This returns 0 if all went well (page still locked) 3615 * This returns < 0 if there were errors (page still locked) 3616 */ 3617 static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode, 3618 struct page *page, struct writeback_control *wbc, 3619 u64 delalloc_start, unsigned long *nr_written) 3620 { 3621 u64 page_end = delalloc_start + PAGE_SIZE - 1; 3622 bool found; 3623 u64 delalloc_to_write = 0; 3624 u64 delalloc_end = 0; 3625 int ret; 3626 int page_started = 0; 3627 3628 3629 while (delalloc_end < page_end) { 3630 found = find_lock_delalloc_range(&inode->vfs_inode, page, 3631 &delalloc_start, 3632 &delalloc_end); 3633 if (!found) { 3634 delalloc_start = delalloc_end + 1; 3635 continue; 3636 } 3637 ret = btrfs_run_delalloc_range(inode, page, delalloc_start, 3638 delalloc_end, &page_started, nr_written, wbc); 3639 if (ret) { 3640 SetPageError(page); 3641 /* 3642 * btrfs_run_delalloc_range should return < 0 for error 3643 * but just in case, we use > 0 here meaning the IO is 3644 * started, so we don't want to return > 0 unless 3645 * things are going well. 3646 */ 3647 return ret < 0 ? ret : -EIO; 3648 } 3649 /* 3650 * delalloc_end is already one less than the total length, so 3651 * we don't subtract one from PAGE_SIZE 3652 */ 3653 delalloc_to_write += (delalloc_end - delalloc_start + 3654 PAGE_SIZE) >> PAGE_SHIFT; 3655 delalloc_start = delalloc_end + 1; 3656 } 3657 if (wbc->nr_to_write < delalloc_to_write) { 3658 int thresh = 8192; 3659 3660 if (delalloc_to_write < thresh * 2) 3661 thresh = delalloc_to_write; 3662 wbc->nr_to_write = min_t(u64, delalloc_to_write, 3663 thresh); 3664 } 3665 3666 /* did the fill delalloc function already unlock and start 3667 * the IO? 3668 */ 3669 if (page_started) { 3670 /* 3671 * we've unlocked the page, so we can't update 3672 * the mapping's writeback index, just update 3673 * nr_to_write. 3674 */ 3675 wbc->nr_to_write -= *nr_written; 3676 return 1; 3677 } 3678 3679 return 0; 3680 } 3681 3682 /* 3683 * helper for __extent_writepage. This calls the writepage start hooks, 3684 * and does the loop to map the page into extents and bios. 3685 * 3686 * We return 1 if the IO is started and the page is unlocked, 3687 * 0 if all went well (page still locked) 3688 * < 0 if there were errors (page still locked) 3689 */ 3690 static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, 3691 struct page *page, 3692 struct writeback_control *wbc, 3693 struct extent_page_data *epd, 3694 loff_t i_size, 3695 unsigned long nr_written, 3696 int *nr_ret) 3697 { 3698 struct btrfs_fs_info *fs_info = inode->root->fs_info; 3699 struct extent_io_tree *tree = &inode->io_tree; 3700 u64 start = page_offset(page); 3701 u64 end = start + PAGE_SIZE - 1; 3702 u64 cur = start; 3703 u64 extent_offset; 3704 u64 block_start; 3705 struct extent_map *em; 3706 int ret = 0; 3707 int nr = 0; 3708 u32 opf = REQ_OP_WRITE; 3709 const unsigned int write_flags = wbc_to_write_flags(wbc); 3710 bool compressed; 3711 3712 ret = btrfs_writepage_cow_fixup(page, start, end); 3713 if (ret) { 3714 /* Fixup worker will requeue */ 3715 redirty_page_for_writepage(wbc, page); 3716 update_nr_written(wbc, nr_written); 3717 unlock_page(page); 3718 return 1; 3719 } 3720 3721 /* 3722 * we don't want to touch the inode after unlocking the page, 3723 * so we update the mapping writeback index now 3724 */ 3725 update_nr_written(wbc, nr_written + 1); 3726 3727 while (cur <= end) { 3728 u64 disk_bytenr; 3729 u64 em_end; 3730 u32 iosize; 3731 3732 if (cur >= i_size) { 3733 btrfs_writepage_endio_finish_ordered(page, cur, end, 1); 3734 break; 3735 } 3736 em = btrfs_get_extent(inode, NULL, 0, cur, end - cur + 1); 3737 if (IS_ERR_OR_NULL(em)) { 3738 SetPageError(page); 3739 ret = PTR_ERR_OR_ZERO(em); 3740 break; 3741 } 3742 3743 extent_offset = cur - em->start; 3744 em_end = extent_map_end(em); 3745 ASSERT(cur <= em_end); 3746 ASSERT(cur < end); 3747 ASSERT(IS_ALIGNED(em->start, fs_info->sectorsize)); 3748 ASSERT(IS_ALIGNED(em->len, fs_info->sectorsize)); 3749 block_start = em->block_start; 3750 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags); 3751 disk_bytenr = em->block_start + extent_offset; 3752 3753 /* Note that em_end from extent_map_end() is exclusive */ 3754 iosize = min(em_end, end + 1) - cur; 3755 3756 if (btrfs_use_zone_append(inode, em)) 3757 opf = REQ_OP_ZONE_APPEND; 3758 3759 free_extent_map(em); 3760 em = NULL; 3761 3762 /* 3763 * compressed and inline extents are written through other 3764 * paths in the FS 3765 */ 3766 if (compressed || block_start == EXTENT_MAP_HOLE || 3767 block_start == EXTENT_MAP_INLINE) { 3768 if (compressed) 3769 nr++; 3770 else 3771 btrfs_writepage_endio_finish_ordered(page, cur, 3772 cur + iosize - 1, 1); 3773 cur += iosize; 3774 continue; 3775 } 3776 3777 btrfs_set_range_writeback(tree, cur, cur + iosize - 1); 3778 if (!PageWriteback(page)) { 3779 btrfs_err(inode->root->fs_info, 3780 "page %lu not writeback, cur %llu end %llu", 3781 page->index, cur, end); 3782 } 3783 3784 ret = submit_extent_page(opf | write_flags, wbc, page, 3785 disk_bytenr, iosize, 3786 cur - page_offset(page), &epd->bio, 3787 end_bio_extent_writepage, 3788 0, 0, 0, false); 3789 if (ret) { 3790 SetPageError(page); 3791 if (PageWriteback(page)) 3792 end_page_writeback(page); 3793 } 3794 3795 cur += iosize; 3796 nr++; 3797 } 3798 *nr_ret = nr; 3799 return ret; 3800 } 3801 3802 /* 3803 * the writepage semantics are similar to regular writepage. extent 3804 * records are inserted to lock ranges in the tree, and as dirty areas 3805 * are found, they are marked writeback. Then the lock bits are removed 3806 * and the end_io handler clears the writeback ranges 3807 * 3808 * Return 0 if everything goes well. 3809 * Return <0 for error. 3810 */ 3811 static int __extent_writepage(struct page *page, struct writeback_control *wbc, 3812 struct extent_page_data *epd) 3813 { 3814 struct inode *inode = page->mapping->host; 3815 u64 start = page_offset(page); 3816 u64 page_end = start + PAGE_SIZE - 1; 3817 int ret; 3818 int nr = 0; 3819 size_t pg_offset; 3820 loff_t i_size = i_size_read(inode); 3821 unsigned long end_index = i_size >> PAGE_SHIFT; 3822 unsigned long nr_written = 0; 3823 3824 trace___extent_writepage(page, inode, wbc); 3825 3826 WARN_ON(!PageLocked(page)); 3827 3828 ClearPageError(page); 3829 3830 pg_offset = offset_in_page(i_size); 3831 if (page->index > end_index || 3832 (page->index == end_index && !pg_offset)) { 3833 page->mapping->a_ops->invalidatepage(page, 0, PAGE_SIZE); 3834 unlock_page(page); 3835 return 0; 3836 } 3837 3838 if (page->index == end_index) { 3839 memzero_page(page, pg_offset, PAGE_SIZE - pg_offset); 3840 flush_dcache_page(page); 3841 } 3842 3843 ret = set_page_extent_mapped(page); 3844 if (ret < 0) { 3845 SetPageError(page); 3846 goto done; 3847 } 3848 3849 if (!epd->extent_locked) { 3850 ret = writepage_delalloc(BTRFS_I(inode), page, wbc, start, 3851 &nr_written); 3852 if (ret == 1) 3853 return 0; 3854 if (ret) 3855 goto done; 3856 } 3857 3858 ret = __extent_writepage_io(BTRFS_I(inode), page, wbc, epd, i_size, 3859 nr_written, &nr); 3860 if (ret == 1) 3861 return 0; 3862 3863 done: 3864 if (nr == 0) { 3865 /* make sure the mapping tag for page dirty gets cleared */ 3866 set_page_writeback(page); 3867 end_page_writeback(page); 3868 } 3869 if (PageError(page)) { 3870 ret = ret < 0 ? ret : -EIO; 3871 end_extent_writepage(page, ret, start, page_end); 3872 } 3873 unlock_page(page); 3874 ASSERT(ret <= 0); 3875 return ret; 3876 } 3877 3878 void wait_on_extent_buffer_writeback(struct extent_buffer *eb) 3879 { 3880 wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_WRITEBACK, 3881 TASK_UNINTERRUPTIBLE); 3882 } 3883 3884 static void end_extent_buffer_writeback(struct extent_buffer *eb) 3885 { 3886 clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags); 3887 smp_mb__after_atomic(); 3888 wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK); 3889 } 3890 3891 /* 3892 * Lock extent buffer status and pages for writeback. 3893 * 3894 * May try to flush write bio if we can't get the lock. 3895 * 3896 * Return 0 if the extent buffer doesn't need to be submitted. 3897 * (E.g. the extent buffer is not dirty) 3898 * Return >0 is the extent buffer is submitted to bio. 3899 * Return <0 if something went wrong, no page is locked. 3900 */ 3901 static noinline_for_stack int lock_extent_buffer_for_io(struct extent_buffer *eb, 3902 struct extent_page_data *epd) 3903 { 3904 struct btrfs_fs_info *fs_info = eb->fs_info; 3905 int i, num_pages, failed_page_nr; 3906 int flush = 0; 3907 int ret = 0; 3908 3909 if (!btrfs_try_tree_write_lock(eb)) { 3910 ret = flush_write_bio(epd); 3911 if (ret < 0) 3912 return ret; 3913 flush = 1; 3914 btrfs_tree_lock(eb); 3915 } 3916 3917 if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) { 3918 btrfs_tree_unlock(eb); 3919 if (!epd->sync_io) 3920 return 0; 3921 if (!flush) { 3922 ret = flush_write_bio(epd); 3923 if (ret < 0) 3924 return ret; 3925 flush = 1; 3926 } 3927 while (1) { 3928 wait_on_extent_buffer_writeback(eb); 3929 btrfs_tree_lock(eb); 3930 if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) 3931 break; 3932 btrfs_tree_unlock(eb); 3933 } 3934 } 3935 3936 /* 3937 * We need to do this to prevent races in people who check if the eb is 3938 * under IO since we can end up having no IO bits set for a short period 3939 * of time. 3940 */ 3941 spin_lock(&eb->refs_lock); 3942 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) { 3943 set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags); 3944 spin_unlock(&eb->refs_lock); 3945 btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN); 3946 percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, 3947 -eb->len, 3948 fs_info->dirty_metadata_batch); 3949 ret = 1; 3950 } else { 3951 spin_unlock(&eb->refs_lock); 3952 } 3953 3954 btrfs_tree_unlock(eb); 3955 3956 /* 3957 * Either we don't need to submit any tree block, or we're submitting 3958 * subpage eb. 3959 * Subpage metadata doesn't use page locking at all, so we can skip 3960 * the page locking. 3961 */ 3962 if (!ret || fs_info->sectorsize < PAGE_SIZE) 3963 return ret; 3964 3965 num_pages = num_extent_pages(eb); 3966 for (i = 0; i < num_pages; i++) { 3967 struct page *p = eb->pages[i]; 3968 3969 if (!trylock_page(p)) { 3970 if (!flush) { 3971 int err; 3972 3973 err = flush_write_bio(epd); 3974 if (err < 0) { 3975 ret = err; 3976 failed_page_nr = i; 3977 goto err_unlock; 3978 } 3979 flush = 1; 3980 } 3981 lock_page(p); 3982 } 3983 } 3984 3985 return ret; 3986 err_unlock: 3987 /* Unlock already locked pages */ 3988 for (i = 0; i < failed_page_nr; i++) 3989 unlock_page(eb->pages[i]); 3990 /* 3991 * Clear EXTENT_BUFFER_WRITEBACK and wake up anyone waiting on it. 3992 * Also set back EXTENT_BUFFER_DIRTY so future attempts to this eb can 3993 * be made and undo everything done before. 3994 */ 3995 btrfs_tree_lock(eb); 3996 spin_lock(&eb->refs_lock); 3997 set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags); 3998 end_extent_buffer_writeback(eb); 3999 spin_unlock(&eb->refs_lock); 4000 percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, eb->len, 4001 fs_info->dirty_metadata_batch); 4002 btrfs_clear_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN); 4003 btrfs_tree_unlock(eb); 4004 return ret; 4005 } 4006 4007 static void set_btree_ioerr(struct page *page, struct extent_buffer *eb) 4008 { 4009 struct btrfs_fs_info *fs_info = eb->fs_info; 4010 4011 btrfs_page_set_error(fs_info, page, eb->start, eb->len); 4012 if (test_and_set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) 4013 return; 4014 4015 /* 4016 * If we error out, we should add back the dirty_metadata_bytes 4017 * to make it consistent. 4018 */ 4019 percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, 4020 eb->len, fs_info->dirty_metadata_batch); 4021 4022 /* 4023 * If writeback for a btree extent that doesn't belong to a log tree 4024 * failed, increment the counter transaction->eb_write_errors. 4025 * We do this because while the transaction is running and before it's 4026 * committing (when we call filemap_fdata[write|wait]_range against 4027 * the btree inode), we might have 4028 * btree_inode->i_mapping->a_ops->writepages() called by the VM - if it 4029 * returns an error or an error happens during writeback, when we're 4030 * committing the transaction we wouldn't know about it, since the pages 4031 * can be no longer dirty nor marked anymore for writeback (if a 4032 * subsequent modification to the extent buffer didn't happen before the 4033 * transaction commit), which makes filemap_fdata[write|wait]_range not 4034 * able to find the pages tagged with SetPageError at transaction 4035 * commit time. So if this happens we must abort the transaction, 4036 * otherwise we commit a super block with btree roots that point to 4037 * btree nodes/leafs whose content on disk is invalid - either garbage 4038 * or the content of some node/leaf from a past generation that got 4039 * cowed or deleted and is no longer valid. 4040 * 4041 * Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would 4042 * not be enough - we need to distinguish between log tree extents vs 4043 * non-log tree extents, and the next filemap_fdatawait_range() call 4044 * will catch and clear such errors in the mapping - and that call might 4045 * be from a log sync and not from a transaction commit. Also, checking 4046 * for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is 4047 * not done and would not be reliable - the eb might have been released 4048 * from memory and reading it back again means that flag would not be 4049 * set (since it's a runtime flag, not persisted on disk). 4050 * 4051 * Using the flags below in the btree inode also makes us achieve the 4052 * goal of AS_EIO/AS_ENOSPC when writepages() returns success, started 4053 * writeback for all dirty pages and before filemap_fdatawait_range() 4054 * is called, the writeback for all dirty pages had already finished 4055 * with errors - because we were not using AS_EIO/AS_ENOSPC, 4056 * filemap_fdatawait_range() would return success, as it could not know 4057 * that writeback errors happened (the pages were no longer tagged for 4058 * writeback). 4059 */ 4060 switch (eb->log_index) { 4061 case -1: 4062 set_bit(BTRFS_FS_BTREE_ERR, &fs_info->flags); 4063 break; 4064 case 0: 4065 set_bit(BTRFS_FS_LOG1_ERR, &fs_info->flags); 4066 break; 4067 case 1: 4068 set_bit(BTRFS_FS_LOG2_ERR, &fs_info->flags); 4069 break; 4070 default: 4071 BUG(); /* unexpected, logic error */ 4072 } 4073 } 4074 4075 /* 4076 * The endio specific version which won't touch any unsafe spinlock in endio 4077 * context. 4078 */ 4079 static struct extent_buffer *find_extent_buffer_nolock( 4080 struct btrfs_fs_info *fs_info, u64 start) 4081 { 4082 struct extent_buffer *eb; 4083 4084 rcu_read_lock(); 4085 eb = radix_tree_lookup(&fs_info->buffer_radix, 4086 start >> fs_info->sectorsize_bits); 4087 if (eb && atomic_inc_not_zero(&eb->refs)) { 4088 rcu_read_unlock(); 4089 return eb; 4090 } 4091 rcu_read_unlock(); 4092 return NULL; 4093 } 4094 4095 /* 4096 * The endio function for subpage extent buffer write. 4097 * 4098 * Unlike end_bio_extent_buffer_writepage(), we only call end_page_writeback() 4099 * after all extent buffers in the page has finished their writeback. 4100 */ 4101 static void end_bio_subpage_eb_writepage(struct btrfs_fs_info *fs_info, 4102 struct bio *bio) 4103 { 4104 struct bio_vec *bvec; 4105 struct bvec_iter_all iter_all; 4106 4107 ASSERT(!bio_flagged(bio, BIO_CLONED)); 4108 bio_for_each_segment_all(bvec, bio, iter_all) { 4109 struct page *page = bvec->bv_page; 4110 u64 bvec_start = page_offset(page) + bvec->bv_offset; 4111 u64 bvec_end = bvec_start + bvec->bv_len - 1; 4112 u64 cur_bytenr = bvec_start; 4113 4114 ASSERT(IS_ALIGNED(bvec->bv_len, fs_info->nodesize)); 4115 4116 /* Iterate through all extent buffers in the range */ 4117 while (cur_bytenr <= bvec_end) { 4118 struct extent_buffer *eb; 4119 int done; 4120 4121 /* 4122 * Here we can't use find_extent_buffer(), as it may 4123 * try to lock eb->refs_lock, which is not safe in endio 4124 * context. 4125 */ 4126 eb = find_extent_buffer_nolock(fs_info, cur_bytenr); 4127 ASSERT(eb); 4128 4129 cur_bytenr = eb->start + eb->len; 4130 4131 ASSERT(test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)); 4132 done = atomic_dec_and_test(&eb->io_pages); 4133 ASSERT(done); 4134 4135 if (bio->bi_status || 4136 test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) { 4137 ClearPageUptodate(page); 4138 set_btree_ioerr(page, eb); 4139 } 4140 4141 btrfs_subpage_clear_writeback(fs_info, page, eb->start, 4142 eb->len); 4143 end_extent_buffer_writeback(eb); 4144 /* 4145 * free_extent_buffer() will grab spinlock which is not 4146 * safe in endio context. Thus here we manually dec 4147 * the ref. 4148 */ 4149 atomic_dec(&eb->refs); 4150 } 4151 } 4152 bio_put(bio); 4153 } 4154 4155 static void end_bio_extent_buffer_writepage(struct bio *bio) 4156 { 4157 struct btrfs_fs_info *fs_info; 4158 struct bio_vec *bvec; 4159 struct extent_buffer *eb; 4160 int done; 4161 struct bvec_iter_all iter_all; 4162 4163 fs_info = btrfs_sb(bio_first_page_all(bio)->mapping->host->i_sb); 4164 if (fs_info->sectorsize < PAGE_SIZE) 4165 return end_bio_subpage_eb_writepage(fs_info, bio); 4166 4167 ASSERT(!bio_flagged(bio, BIO_CLONED)); 4168 bio_for_each_segment_all(bvec, bio, iter_all) { 4169 struct page *page = bvec->bv_page; 4170 4171 eb = (struct extent_buffer *)page->private; 4172 BUG_ON(!eb); 4173 done = atomic_dec_and_test(&eb->io_pages); 4174 4175 if (bio->bi_status || 4176 test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) { 4177 ClearPageUptodate(page); 4178 set_btree_ioerr(page, eb); 4179 } 4180 4181 end_page_writeback(page); 4182 4183 if (!done) 4184 continue; 4185 4186 end_extent_buffer_writeback(eb); 4187 } 4188 4189 bio_put(bio); 4190 } 4191 4192 /* 4193 * Unlike the work in write_one_eb(), we rely completely on extent locking. 4194 * Page locking is only utilized at minimum to keep the VMM code happy. 4195 * 4196 * Caller should still call write_one_eb() other than this function directly. 4197 * As write_one_eb() has extra preparation before submitting the extent buffer. 4198 */ 4199 static int write_one_subpage_eb(struct extent_buffer *eb, 4200 struct writeback_control *wbc, 4201 struct extent_page_data *epd) 4202 { 4203 struct btrfs_fs_info *fs_info = eb->fs_info; 4204 struct page *page = eb->pages[0]; 4205 unsigned int write_flags = wbc_to_write_flags(wbc) | REQ_META; 4206 bool no_dirty_ebs = false; 4207 int ret; 4208 4209 /* clear_page_dirty_for_io() in subpage helper needs page locked */ 4210 lock_page(page); 4211 btrfs_subpage_set_writeback(fs_info, page, eb->start, eb->len); 4212 4213 /* Check if this is the last dirty bit to update nr_written */ 4214 no_dirty_ebs = btrfs_subpage_clear_and_test_dirty(fs_info, page, 4215 eb->start, eb->len); 4216 if (no_dirty_ebs) 4217 clear_page_dirty_for_io(page); 4218 4219 ret = submit_extent_page(REQ_OP_WRITE | write_flags, wbc, page, 4220 eb->start, eb->len, eb->start - page_offset(page), 4221 &epd->bio, end_bio_extent_buffer_writepage, 0, 0, 0, 4222 false); 4223 if (ret) { 4224 btrfs_subpage_clear_writeback(fs_info, page, eb->start, eb->len); 4225 set_btree_ioerr(page, eb); 4226 unlock_page(page); 4227 4228 if (atomic_dec_and_test(&eb->io_pages)) 4229 end_extent_buffer_writeback(eb); 4230 return -EIO; 4231 } 4232 unlock_page(page); 4233 /* 4234 * Submission finished without problem, if no range of the page is 4235 * dirty anymore, we have submitted a page. Update nr_written in wbc. 4236 */ 4237 if (no_dirty_ebs) 4238 update_nr_written(wbc, 1); 4239 return ret; 4240 } 4241 4242 static noinline_for_stack int write_one_eb(struct extent_buffer *eb, 4243 struct writeback_control *wbc, 4244 struct extent_page_data *epd) 4245 { 4246 u64 disk_bytenr = eb->start; 4247 u32 nritems; 4248 int i, num_pages; 4249 unsigned long start, end; 4250 unsigned int write_flags = wbc_to_write_flags(wbc) | REQ_META; 4251 int ret = 0; 4252 4253 clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags); 4254 num_pages = num_extent_pages(eb); 4255 atomic_set(&eb->io_pages, num_pages); 4256 4257 /* set btree blocks beyond nritems with 0 to avoid stale content. */ 4258 nritems = btrfs_header_nritems(eb); 4259 if (btrfs_header_level(eb) > 0) { 4260 end = btrfs_node_key_ptr_offset(nritems); 4261 4262 memzero_extent_buffer(eb, end, eb->len - end); 4263 } else { 4264 /* 4265 * leaf: 4266 * header 0 1 2 .. N ... data_N .. data_2 data_1 data_0 4267 */ 4268 start = btrfs_item_nr_offset(nritems); 4269 end = BTRFS_LEAF_DATA_OFFSET + leaf_data_end(eb); 4270 memzero_extent_buffer(eb, start, end - start); 4271 } 4272 4273 if (eb->fs_info->sectorsize < PAGE_SIZE) 4274 return write_one_subpage_eb(eb, wbc, epd); 4275 4276 for (i = 0; i < num_pages; i++) { 4277 struct page *p = eb->pages[i]; 4278 4279 clear_page_dirty_for_io(p); 4280 set_page_writeback(p); 4281 ret = submit_extent_page(REQ_OP_WRITE | write_flags, wbc, 4282 p, disk_bytenr, PAGE_SIZE, 0, 4283 &epd->bio, 4284 end_bio_extent_buffer_writepage, 4285 0, 0, 0, false); 4286 if (ret) { 4287 set_btree_ioerr(p, eb); 4288 if (PageWriteback(p)) 4289 end_page_writeback(p); 4290 if (atomic_sub_and_test(num_pages - i, &eb->io_pages)) 4291 end_extent_buffer_writeback(eb); 4292 ret = -EIO; 4293 break; 4294 } 4295 disk_bytenr += PAGE_SIZE; 4296 update_nr_written(wbc, 1); 4297 unlock_page(p); 4298 } 4299 4300 if (unlikely(ret)) { 4301 for (; i < num_pages; i++) { 4302 struct page *p = eb->pages[i]; 4303 clear_page_dirty_for_io(p); 4304 unlock_page(p); 4305 } 4306 } 4307 4308 return ret; 4309 } 4310 4311 /* 4312 * Submit one subpage btree page. 4313 * 4314 * The main difference to submit_eb_page() is: 4315 * - Page locking 4316 * For subpage, we don't rely on page locking at all. 4317 * 4318 * - Flush write bio 4319 * We only flush bio if we may be unable to fit current extent buffers into 4320 * current bio. 4321 * 4322 * Return >=0 for the number of submitted extent buffers. 4323 * Return <0 for fatal error. 4324 */ 4325 static int submit_eb_subpage(struct page *page, 4326 struct writeback_control *wbc, 4327 struct extent_page_data *epd) 4328 { 4329 struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb); 4330 int submitted = 0; 4331 u64 page_start = page_offset(page); 4332 int bit_start = 0; 4333 const int nbits = BTRFS_SUBPAGE_BITMAP_SIZE; 4334 int sectors_per_node = fs_info->nodesize >> fs_info->sectorsize_bits; 4335 int ret; 4336 4337 /* Lock and write each dirty extent buffers in the range */ 4338 while (bit_start < nbits) { 4339 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 4340 struct extent_buffer *eb; 4341 unsigned long flags; 4342 u64 start; 4343 4344 /* 4345 * Take private lock to ensure the subpage won't be detached 4346 * in the meantime. 4347 */ 4348 spin_lock(&page->mapping->private_lock); 4349 if (!PagePrivate(page)) { 4350 spin_unlock(&page->mapping->private_lock); 4351 break; 4352 } 4353 spin_lock_irqsave(&subpage->lock, flags); 4354 if (!((1 << bit_start) & subpage->dirty_bitmap)) { 4355 spin_unlock_irqrestore(&subpage->lock, flags); 4356 spin_unlock(&page->mapping->private_lock); 4357 bit_start++; 4358 continue; 4359 } 4360 4361 start = page_start + bit_start * fs_info->sectorsize; 4362 bit_start += sectors_per_node; 4363 4364 /* 4365 * Here we just want to grab the eb without touching extra 4366 * spin locks, so call find_extent_buffer_nolock(). 4367 */ 4368 eb = find_extent_buffer_nolock(fs_info, start); 4369 spin_unlock_irqrestore(&subpage->lock, flags); 4370 spin_unlock(&page->mapping->private_lock); 4371 4372 /* 4373 * The eb has already reached 0 refs thus find_extent_buffer() 4374 * doesn't return it. We don't need to write back such eb 4375 * anyway. 4376 */ 4377 if (!eb) 4378 continue; 4379 4380 ret = lock_extent_buffer_for_io(eb, epd); 4381 if (ret == 0) { 4382 free_extent_buffer(eb); 4383 continue; 4384 } 4385 if (ret < 0) { 4386 free_extent_buffer(eb); 4387 goto cleanup; 4388 } 4389 ret = write_one_eb(eb, wbc, epd); 4390 free_extent_buffer(eb); 4391 if (ret < 0) 4392 goto cleanup; 4393 submitted++; 4394 } 4395 return submitted; 4396 4397 cleanup: 4398 /* We hit error, end bio for the submitted extent buffers */ 4399 end_write_bio(epd, ret); 4400 return ret; 4401 } 4402 4403 /* 4404 * Submit all page(s) of one extent buffer. 4405 * 4406 * @page: the page of one extent buffer 4407 * @eb_context: to determine if we need to submit this page, if current page 4408 * belongs to this eb, we don't need to submit 4409 * 4410 * The caller should pass each page in their bytenr order, and here we use 4411 * @eb_context to determine if we have submitted pages of one extent buffer. 4412 * 4413 * If we have, we just skip until we hit a new page that doesn't belong to 4414 * current @eb_context. 4415 * 4416 * If not, we submit all the page(s) of the extent buffer. 4417 * 4418 * Return >0 if we have submitted the extent buffer successfully. 4419 * Return 0 if we don't need to submit the page, as it's already submitted by 4420 * previous call. 4421 * Return <0 for fatal error. 4422 */ 4423 static int submit_eb_page(struct page *page, struct writeback_control *wbc, 4424 struct extent_page_data *epd, 4425 struct extent_buffer **eb_context) 4426 { 4427 struct address_space *mapping = page->mapping; 4428 struct btrfs_block_group *cache = NULL; 4429 struct extent_buffer *eb; 4430 int ret; 4431 4432 if (!PagePrivate(page)) 4433 return 0; 4434 4435 if (btrfs_sb(page->mapping->host->i_sb)->sectorsize < PAGE_SIZE) 4436 return submit_eb_subpage(page, wbc, epd); 4437 4438 spin_lock(&mapping->private_lock); 4439 if (!PagePrivate(page)) { 4440 spin_unlock(&mapping->private_lock); 4441 return 0; 4442 } 4443 4444 eb = (struct extent_buffer *)page->private; 4445 4446 /* 4447 * Shouldn't happen and normally this would be a BUG_ON but no point 4448 * crashing the machine for something we can survive anyway. 4449 */ 4450 if (WARN_ON(!eb)) { 4451 spin_unlock(&mapping->private_lock); 4452 return 0; 4453 } 4454 4455 if (eb == *eb_context) { 4456 spin_unlock(&mapping->private_lock); 4457 return 0; 4458 } 4459 ret = atomic_inc_not_zero(&eb->refs); 4460 spin_unlock(&mapping->private_lock); 4461 if (!ret) 4462 return 0; 4463 4464 if (!btrfs_check_meta_write_pointer(eb->fs_info, eb, &cache)) { 4465 /* 4466 * If for_sync, this hole will be filled with 4467 * trasnsaction commit. 4468 */ 4469 if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync) 4470 ret = -EAGAIN; 4471 else 4472 ret = 0; 4473 free_extent_buffer(eb); 4474 return ret; 4475 } 4476 4477 *eb_context = eb; 4478 4479 ret = lock_extent_buffer_for_io(eb, epd); 4480 if (ret <= 0) { 4481 btrfs_revert_meta_write_pointer(cache, eb); 4482 if (cache) 4483 btrfs_put_block_group(cache); 4484 free_extent_buffer(eb); 4485 return ret; 4486 } 4487 if (cache) 4488 btrfs_put_block_group(cache); 4489 ret = write_one_eb(eb, wbc, epd); 4490 free_extent_buffer(eb); 4491 if (ret < 0) 4492 return ret; 4493 return 1; 4494 } 4495 4496 int btree_write_cache_pages(struct address_space *mapping, 4497 struct writeback_control *wbc) 4498 { 4499 struct extent_buffer *eb_context = NULL; 4500 struct extent_page_data epd = { 4501 .bio = NULL, 4502 .extent_locked = 0, 4503 .sync_io = wbc->sync_mode == WB_SYNC_ALL, 4504 }; 4505 struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info; 4506 int ret = 0; 4507 int done = 0; 4508 int nr_to_write_done = 0; 4509 struct pagevec pvec; 4510 int nr_pages; 4511 pgoff_t index; 4512 pgoff_t end; /* Inclusive */ 4513 int scanned = 0; 4514 xa_mark_t tag; 4515 4516 pagevec_init(&pvec); 4517 if (wbc->range_cyclic) { 4518 index = mapping->writeback_index; /* Start from prev offset */ 4519 end = -1; 4520 /* 4521 * Start from the beginning does not need to cycle over the 4522 * range, mark it as scanned. 4523 */ 4524 scanned = (index == 0); 4525 } else { 4526 index = wbc->range_start >> PAGE_SHIFT; 4527 end = wbc->range_end >> PAGE_SHIFT; 4528 scanned = 1; 4529 } 4530 if (wbc->sync_mode == WB_SYNC_ALL) 4531 tag = PAGECACHE_TAG_TOWRITE; 4532 else 4533 tag = PAGECACHE_TAG_DIRTY; 4534 btrfs_zoned_meta_io_lock(fs_info); 4535 retry: 4536 if (wbc->sync_mode == WB_SYNC_ALL) 4537 tag_pages_for_writeback(mapping, index, end); 4538 while (!done && !nr_to_write_done && (index <= end) && 4539 (nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, 4540 tag))) { 4541 unsigned i; 4542 4543 for (i = 0; i < nr_pages; i++) { 4544 struct page *page = pvec.pages[i]; 4545 4546 ret = submit_eb_page(page, wbc, &epd, &eb_context); 4547 if (ret == 0) 4548 continue; 4549 if (ret < 0) { 4550 done = 1; 4551 break; 4552 } 4553 4554 /* 4555 * the filesystem may choose to bump up nr_to_write. 4556 * We have to make sure to honor the new nr_to_write 4557 * at any time 4558 */ 4559 nr_to_write_done = wbc->nr_to_write <= 0; 4560 } 4561 pagevec_release(&pvec); 4562 cond_resched(); 4563 } 4564 if (!scanned && !done) { 4565 /* 4566 * We hit the last page and there is more work to be done: wrap 4567 * back to the start of the file 4568 */ 4569 scanned = 1; 4570 index = 0; 4571 goto retry; 4572 } 4573 if (ret < 0) { 4574 end_write_bio(&epd, ret); 4575 goto out; 4576 } 4577 /* 4578 * If something went wrong, don't allow any metadata write bio to be 4579 * submitted. 4580 * 4581 * This would prevent use-after-free if we had dirty pages not 4582 * cleaned up, which can still happen by fuzzed images. 4583 * 4584 * - Bad extent tree 4585 * Allowing existing tree block to be allocated for other trees. 4586 * 4587 * - Log tree operations 4588 * Exiting tree blocks get allocated to log tree, bumps its 4589 * generation, then get cleaned in tree re-balance. 4590 * Such tree block will not be written back, since it's clean, 4591 * thus no WRITTEN flag set. 4592 * And after log writes back, this tree block is not traced by 4593 * any dirty extent_io_tree. 4594 * 4595 * - Offending tree block gets re-dirtied from its original owner 4596 * Since it has bumped generation, no WRITTEN flag, it can be 4597 * reused without COWing. This tree block will not be traced 4598 * by btrfs_transaction::dirty_pages. 4599 * 4600 * Now such dirty tree block will not be cleaned by any dirty 4601 * extent io tree. Thus we don't want to submit such wild eb 4602 * if the fs already has error. 4603 */ 4604 if (!test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) { 4605 ret = flush_write_bio(&epd); 4606 } else { 4607 ret = -EROFS; 4608 end_write_bio(&epd, ret); 4609 } 4610 out: 4611 btrfs_zoned_meta_io_unlock(fs_info); 4612 return ret; 4613 } 4614 4615 /** 4616 * Walk the list of dirty pages of the given address space and write all of them. 4617 * 4618 * @mapping: address space structure to write 4619 * @wbc: subtract the number of written pages from *@wbc->nr_to_write 4620 * @epd: holds context for the write, namely the bio 4621 * 4622 * If a page is already under I/O, write_cache_pages() skips it, even 4623 * if it's dirty. This is desirable behaviour for memory-cleaning writeback, 4624 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync() 4625 * and msync() need to guarantee that all the data which was dirty at the time 4626 * the call was made get new I/O started against them. If wbc->sync_mode is 4627 * WB_SYNC_ALL then we were called for data integrity and we must wait for 4628 * existing IO to complete. 4629 */ 4630 static int extent_write_cache_pages(struct address_space *mapping, 4631 struct writeback_control *wbc, 4632 struct extent_page_data *epd) 4633 { 4634 struct inode *inode = mapping->host; 4635 int ret = 0; 4636 int done = 0; 4637 int nr_to_write_done = 0; 4638 struct pagevec pvec; 4639 int nr_pages; 4640 pgoff_t index; 4641 pgoff_t end; /* Inclusive */ 4642 pgoff_t done_index; 4643 int range_whole = 0; 4644 int scanned = 0; 4645 xa_mark_t tag; 4646 4647 /* 4648 * We have to hold onto the inode so that ordered extents can do their 4649 * work when the IO finishes. The alternative to this is failing to add 4650 * an ordered extent if the igrab() fails there and that is a huge pain 4651 * to deal with, so instead just hold onto the inode throughout the 4652 * writepages operation. If it fails here we are freeing up the inode 4653 * anyway and we'd rather not waste our time writing out stuff that is 4654 * going to be truncated anyway. 4655 */ 4656 if (!igrab(inode)) 4657 return 0; 4658 4659 pagevec_init(&pvec); 4660 if (wbc->range_cyclic) { 4661 index = mapping->writeback_index; /* Start from prev offset */ 4662 end = -1; 4663 /* 4664 * Start from the beginning does not need to cycle over the 4665 * range, mark it as scanned. 4666 */ 4667 scanned = (index == 0); 4668 } else { 4669 index = wbc->range_start >> PAGE_SHIFT; 4670 end = wbc->range_end >> PAGE_SHIFT; 4671 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 4672 range_whole = 1; 4673 scanned = 1; 4674 } 4675 4676 /* 4677 * We do the tagged writepage as long as the snapshot flush bit is set 4678 * and we are the first one who do the filemap_flush() on this inode. 4679 * 4680 * The nr_to_write == LONG_MAX is needed to make sure other flushers do 4681 * not race in and drop the bit. 4682 */ 4683 if (range_whole && wbc->nr_to_write == LONG_MAX && 4684 test_and_clear_bit(BTRFS_INODE_SNAPSHOT_FLUSH, 4685 &BTRFS_I(inode)->runtime_flags)) 4686 wbc->tagged_writepages = 1; 4687 4688 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 4689 tag = PAGECACHE_TAG_TOWRITE; 4690 else 4691 tag = PAGECACHE_TAG_DIRTY; 4692 retry: 4693 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 4694 tag_pages_for_writeback(mapping, index, end); 4695 done_index = index; 4696 while (!done && !nr_to_write_done && (index <= end) && 4697 (nr_pages = pagevec_lookup_range_tag(&pvec, mapping, 4698 &index, end, tag))) { 4699 unsigned i; 4700 4701 for (i = 0; i < nr_pages; i++) { 4702 struct page *page = pvec.pages[i]; 4703 4704 done_index = page->index + 1; 4705 /* 4706 * At this point we hold neither the i_pages lock nor 4707 * the page lock: the page may be truncated or 4708 * invalidated (changing page->mapping to NULL), 4709 * or even swizzled back from swapper_space to 4710 * tmpfs file mapping 4711 */ 4712 if (!trylock_page(page)) { 4713 ret = flush_write_bio(epd); 4714 BUG_ON(ret < 0); 4715 lock_page(page); 4716 } 4717 4718 if (unlikely(page->mapping != mapping)) { 4719 unlock_page(page); 4720 continue; 4721 } 4722 4723 if (wbc->sync_mode != WB_SYNC_NONE) { 4724 if (PageWriteback(page)) { 4725 ret = flush_write_bio(epd); 4726 BUG_ON(ret < 0); 4727 } 4728 wait_on_page_writeback(page); 4729 } 4730 4731 if (PageWriteback(page) || 4732 !clear_page_dirty_for_io(page)) { 4733 unlock_page(page); 4734 continue; 4735 } 4736 4737 ret = __extent_writepage(page, wbc, epd); 4738 if (ret < 0) { 4739 done = 1; 4740 break; 4741 } 4742 4743 /* 4744 * the filesystem may choose to bump up nr_to_write. 4745 * We have to make sure to honor the new nr_to_write 4746 * at any time 4747 */ 4748 nr_to_write_done = wbc->nr_to_write <= 0; 4749 } 4750 pagevec_release(&pvec); 4751 cond_resched(); 4752 } 4753 if (!scanned && !done) { 4754 /* 4755 * We hit the last page and there is more work to be done: wrap 4756 * back to the start of the file 4757 */ 4758 scanned = 1; 4759 index = 0; 4760 4761 /* 4762 * If we're looping we could run into a page that is locked by a 4763 * writer and that writer could be waiting on writeback for a 4764 * page in our current bio, and thus deadlock, so flush the 4765 * write bio here. 4766 */ 4767 ret = flush_write_bio(epd); 4768 if (!ret) 4769 goto retry; 4770 } 4771 4772 if (wbc->range_cyclic || (wbc->nr_to_write > 0 && range_whole)) 4773 mapping->writeback_index = done_index; 4774 4775 btrfs_add_delayed_iput(inode); 4776 return ret; 4777 } 4778 4779 int extent_write_full_page(struct page *page, struct writeback_control *wbc) 4780 { 4781 int ret; 4782 struct extent_page_data epd = { 4783 .bio = NULL, 4784 .extent_locked = 0, 4785 .sync_io = wbc->sync_mode == WB_SYNC_ALL, 4786 }; 4787 4788 ret = __extent_writepage(page, wbc, &epd); 4789 ASSERT(ret <= 0); 4790 if (ret < 0) { 4791 end_write_bio(&epd, ret); 4792 return ret; 4793 } 4794 4795 ret = flush_write_bio(&epd); 4796 ASSERT(ret <= 0); 4797 return ret; 4798 } 4799 4800 int extent_write_locked_range(struct inode *inode, u64 start, u64 end, 4801 int mode) 4802 { 4803 int ret = 0; 4804 struct address_space *mapping = inode->i_mapping; 4805 struct page *page; 4806 unsigned long nr_pages = (end - start + PAGE_SIZE) >> 4807 PAGE_SHIFT; 4808 4809 struct extent_page_data epd = { 4810 .bio = NULL, 4811 .extent_locked = 1, 4812 .sync_io = mode == WB_SYNC_ALL, 4813 }; 4814 struct writeback_control wbc_writepages = { 4815 .sync_mode = mode, 4816 .nr_to_write = nr_pages * 2, 4817 .range_start = start, 4818 .range_end = end + 1, 4819 /* We're called from an async helper function */ 4820 .punt_to_cgroup = 1, 4821 .no_cgroup_owner = 1, 4822 }; 4823 4824 wbc_attach_fdatawrite_inode(&wbc_writepages, inode); 4825 while (start <= end) { 4826 page = find_get_page(mapping, start >> PAGE_SHIFT); 4827 if (clear_page_dirty_for_io(page)) 4828 ret = __extent_writepage(page, &wbc_writepages, &epd); 4829 else { 4830 btrfs_writepage_endio_finish_ordered(page, start, 4831 start + PAGE_SIZE - 1, 1); 4832 unlock_page(page); 4833 } 4834 put_page(page); 4835 start += PAGE_SIZE; 4836 } 4837 4838 ASSERT(ret <= 0); 4839 if (ret == 0) 4840 ret = flush_write_bio(&epd); 4841 else 4842 end_write_bio(&epd, ret); 4843 4844 wbc_detach_inode(&wbc_writepages); 4845 return ret; 4846 } 4847 4848 int extent_writepages(struct address_space *mapping, 4849 struct writeback_control *wbc) 4850 { 4851 int ret = 0; 4852 struct extent_page_data epd = { 4853 .bio = NULL, 4854 .extent_locked = 0, 4855 .sync_io = wbc->sync_mode == WB_SYNC_ALL, 4856 }; 4857 4858 ret = extent_write_cache_pages(mapping, wbc, &epd); 4859 ASSERT(ret <= 0); 4860 if (ret < 0) { 4861 end_write_bio(&epd, ret); 4862 return ret; 4863 } 4864 ret = flush_write_bio(&epd); 4865 return ret; 4866 } 4867 4868 void extent_readahead(struct readahead_control *rac) 4869 { 4870 struct bio *bio = NULL; 4871 unsigned long bio_flags = 0; 4872 struct page *pagepool[16]; 4873 struct extent_map *em_cached = NULL; 4874 u64 prev_em_start = (u64)-1; 4875 int nr; 4876 4877 while ((nr = readahead_page_batch(rac, pagepool))) { 4878 u64 contig_start = readahead_pos(rac); 4879 u64 contig_end = contig_start + readahead_batch_length(rac) - 1; 4880 4881 contiguous_readpages(pagepool, nr, contig_start, contig_end, 4882 &em_cached, &bio, &bio_flags, &prev_em_start); 4883 } 4884 4885 if (em_cached) 4886 free_extent_map(em_cached); 4887 4888 if (bio) { 4889 if (submit_one_bio(bio, 0, bio_flags)) 4890 return; 4891 } 4892 } 4893 4894 /* 4895 * basic invalidatepage code, this waits on any locked or writeback 4896 * ranges corresponding to the page, and then deletes any extent state 4897 * records from the tree 4898 */ 4899 int extent_invalidatepage(struct extent_io_tree *tree, 4900 struct page *page, unsigned long offset) 4901 { 4902 struct extent_state *cached_state = NULL; 4903 u64 start = page_offset(page); 4904 u64 end = start + PAGE_SIZE - 1; 4905 size_t blocksize = page->mapping->host->i_sb->s_blocksize; 4906 4907 /* This function is only called for the btree inode */ 4908 ASSERT(tree->owner == IO_TREE_BTREE_INODE_IO); 4909 4910 start += ALIGN(offset, blocksize); 4911 if (start > end) 4912 return 0; 4913 4914 lock_extent_bits(tree, start, end, &cached_state); 4915 wait_on_page_writeback(page); 4916 4917 /* 4918 * Currently for btree io tree, only EXTENT_LOCKED is utilized, 4919 * so here we only need to unlock the extent range to free any 4920 * existing extent state. 4921 */ 4922 unlock_extent_cached(tree, start, end, &cached_state); 4923 return 0; 4924 } 4925 4926 /* 4927 * a helper for releasepage, this tests for areas of the page that 4928 * are locked or under IO and drops the related state bits if it is safe 4929 * to drop the page. 4930 */ 4931 static int try_release_extent_state(struct extent_io_tree *tree, 4932 struct page *page, gfp_t mask) 4933 { 4934 u64 start = page_offset(page); 4935 u64 end = start + PAGE_SIZE - 1; 4936 int ret = 1; 4937 4938 if (test_range_bit(tree, start, end, EXTENT_LOCKED, 0, NULL)) { 4939 ret = 0; 4940 } else { 4941 /* 4942 * At this point we can safely clear everything except the 4943 * locked bit, the nodatasum bit and the delalloc new bit. 4944 * The delalloc new bit will be cleared by ordered extent 4945 * completion. 4946 */ 4947 ret = __clear_extent_bit(tree, start, end, 4948 ~(EXTENT_LOCKED | EXTENT_NODATASUM | EXTENT_DELALLOC_NEW), 4949 0, 0, NULL, mask, NULL); 4950 4951 /* if clear_extent_bit failed for enomem reasons, 4952 * we can't allow the release to continue. 4953 */ 4954 if (ret < 0) 4955 ret = 0; 4956 else 4957 ret = 1; 4958 } 4959 return ret; 4960 } 4961 4962 /* 4963 * a helper for releasepage. As long as there are no locked extents 4964 * in the range corresponding to the page, both state records and extent 4965 * map records are removed 4966 */ 4967 int try_release_extent_mapping(struct page *page, gfp_t mask) 4968 { 4969 struct extent_map *em; 4970 u64 start = page_offset(page); 4971 u64 end = start + PAGE_SIZE - 1; 4972 struct btrfs_inode *btrfs_inode = BTRFS_I(page->mapping->host); 4973 struct extent_io_tree *tree = &btrfs_inode->io_tree; 4974 struct extent_map_tree *map = &btrfs_inode->extent_tree; 4975 4976 if (gfpflags_allow_blocking(mask) && 4977 page->mapping->host->i_size > SZ_16M) { 4978 u64 len; 4979 while (start <= end) { 4980 struct btrfs_fs_info *fs_info; 4981 u64 cur_gen; 4982 4983 len = end - start + 1; 4984 write_lock(&map->lock); 4985 em = lookup_extent_mapping(map, start, len); 4986 if (!em) { 4987 write_unlock(&map->lock); 4988 break; 4989 } 4990 if (test_bit(EXTENT_FLAG_PINNED, &em->flags) || 4991 em->start != start) { 4992 write_unlock(&map->lock); 4993 free_extent_map(em); 4994 break; 4995 } 4996 if (test_range_bit(tree, em->start, 4997 extent_map_end(em) - 1, 4998 EXTENT_LOCKED, 0, NULL)) 4999 goto next; 5000 /* 5001 * If it's not in the list of modified extents, used 5002 * by a fast fsync, we can remove it. If it's being 5003 * logged we can safely remove it since fsync took an 5004 * extra reference on the em. 5005 */ 5006 if (list_empty(&em->list) || 5007 test_bit(EXTENT_FLAG_LOGGING, &em->flags)) 5008 goto remove_em; 5009 /* 5010 * If it's in the list of modified extents, remove it 5011 * only if its generation is older then the current one, 5012 * in which case we don't need it for a fast fsync. 5013 * Otherwise don't remove it, we could be racing with an 5014 * ongoing fast fsync that could miss the new extent. 5015 */ 5016 fs_info = btrfs_inode->root->fs_info; 5017 spin_lock(&fs_info->trans_lock); 5018 cur_gen = fs_info->generation; 5019 spin_unlock(&fs_info->trans_lock); 5020 if (em->generation >= cur_gen) 5021 goto next; 5022 remove_em: 5023 /* 5024 * We only remove extent maps that are not in the list of 5025 * modified extents or that are in the list but with a 5026 * generation lower then the current generation, so there 5027 * is no need to set the full fsync flag on the inode (it 5028 * hurts the fsync performance for workloads with a data 5029 * size that exceeds or is close to the system's memory). 5030 */ 5031 remove_extent_mapping(map, em); 5032 /* once for the rb tree */ 5033 free_extent_map(em); 5034 next: 5035 start = extent_map_end(em); 5036 write_unlock(&map->lock); 5037 5038 /* once for us */ 5039 free_extent_map(em); 5040 5041 cond_resched(); /* Allow large-extent preemption. */ 5042 } 5043 } 5044 return try_release_extent_state(tree, page, mask); 5045 } 5046 5047 /* 5048 * helper function for fiemap, which doesn't want to see any holes. 5049 * This maps until we find something past 'last' 5050 */ 5051 static struct extent_map *get_extent_skip_holes(struct btrfs_inode *inode, 5052 u64 offset, u64 last) 5053 { 5054 u64 sectorsize = btrfs_inode_sectorsize(inode); 5055 struct extent_map *em; 5056 u64 len; 5057 5058 if (offset >= last) 5059 return NULL; 5060 5061 while (1) { 5062 len = last - offset; 5063 if (len == 0) 5064 break; 5065 len = ALIGN(len, sectorsize); 5066 em = btrfs_get_extent_fiemap(inode, offset, len); 5067 if (IS_ERR_OR_NULL(em)) 5068 return em; 5069 5070 /* if this isn't a hole return it */ 5071 if (em->block_start != EXTENT_MAP_HOLE) 5072 return em; 5073 5074 /* this is a hole, advance to the next extent */ 5075 offset = extent_map_end(em); 5076 free_extent_map(em); 5077 if (offset >= last) 5078 break; 5079 } 5080 return NULL; 5081 } 5082 5083 /* 5084 * To cache previous fiemap extent 5085 * 5086 * Will be used for merging fiemap extent 5087 */ 5088 struct fiemap_cache { 5089 u64 offset; 5090 u64 phys; 5091 u64 len; 5092 u32 flags; 5093 bool cached; 5094 }; 5095 5096 /* 5097 * Helper to submit fiemap extent. 5098 * 5099 * Will try to merge current fiemap extent specified by @offset, @phys, 5100 * @len and @flags with cached one. 5101 * And only when we fails to merge, cached one will be submitted as 5102 * fiemap extent. 5103 * 5104 * Return value is the same as fiemap_fill_next_extent(). 5105 */ 5106 static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo, 5107 struct fiemap_cache *cache, 5108 u64 offset, u64 phys, u64 len, u32 flags) 5109 { 5110 int ret = 0; 5111 5112 if (!cache->cached) 5113 goto assign; 5114 5115 /* 5116 * Sanity check, extent_fiemap() should have ensured that new 5117 * fiemap extent won't overlap with cached one. 5118 * Not recoverable. 5119 * 5120 * NOTE: Physical address can overlap, due to compression 5121 */ 5122 if (cache->offset + cache->len > offset) { 5123 WARN_ON(1); 5124 return -EINVAL; 5125 } 5126 5127 /* 5128 * Only merges fiemap extents if 5129 * 1) Their logical addresses are continuous 5130 * 5131 * 2) Their physical addresses are continuous 5132 * So truly compressed (physical size smaller than logical size) 5133 * extents won't get merged with each other 5134 * 5135 * 3) Share same flags except FIEMAP_EXTENT_LAST 5136 * So regular extent won't get merged with prealloc extent 5137 */ 5138 if (cache->offset + cache->len == offset && 5139 cache->phys + cache->len == phys && 5140 (cache->flags & ~FIEMAP_EXTENT_LAST) == 5141 (flags & ~FIEMAP_EXTENT_LAST)) { 5142 cache->len += len; 5143 cache->flags |= flags; 5144 goto try_submit_last; 5145 } 5146 5147 /* Not mergeable, need to submit cached one */ 5148 ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys, 5149 cache->len, cache->flags); 5150 cache->cached = false; 5151 if (ret) 5152 return ret; 5153 assign: 5154 cache->cached = true; 5155 cache->offset = offset; 5156 cache->phys = phys; 5157 cache->len = len; 5158 cache->flags = flags; 5159 try_submit_last: 5160 if (cache->flags & FIEMAP_EXTENT_LAST) { 5161 ret = fiemap_fill_next_extent(fieinfo, cache->offset, 5162 cache->phys, cache->len, cache->flags); 5163 cache->cached = false; 5164 } 5165 return ret; 5166 } 5167 5168 /* 5169 * Emit last fiemap cache 5170 * 5171 * The last fiemap cache may still be cached in the following case: 5172 * 0 4k 8k 5173 * |<- Fiemap range ->| 5174 * |<------------ First extent ----------->| 5175 * 5176 * In this case, the first extent range will be cached but not emitted. 5177 * So we must emit it before ending extent_fiemap(). 5178 */ 5179 static int emit_last_fiemap_cache(struct fiemap_extent_info *fieinfo, 5180 struct fiemap_cache *cache) 5181 { 5182 int ret; 5183 5184 if (!cache->cached) 5185 return 0; 5186 5187 ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys, 5188 cache->len, cache->flags); 5189 cache->cached = false; 5190 if (ret > 0) 5191 ret = 0; 5192 return ret; 5193 } 5194 5195 int extent_fiemap(struct btrfs_inode *inode, struct fiemap_extent_info *fieinfo, 5196 u64 start, u64 len) 5197 { 5198 int ret = 0; 5199 u64 off = start; 5200 u64 max = start + len; 5201 u32 flags = 0; 5202 u32 found_type; 5203 u64 last; 5204 u64 last_for_get_extent = 0; 5205 u64 disko = 0; 5206 u64 isize = i_size_read(&inode->vfs_inode); 5207 struct btrfs_key found_key; 5208 struct extent_map *em = NULL; 5209 struct extent_state *cached_state = NULL; 5210 struct btrfs_path *path; 5211 struct btrfs_root *root = inode->root; 5212 struct fiemap_cache cache = { 0 }; 5213 struct ulist *roots; 5214 struct ulist *tmp_ulist; 5215 int end = 0; 5216 u64 em_start = 0; 5217 u64 em_len = 0; 5218 u64 em_end = 0; 5219 5220 if (len == 0) 5221 return -EINVAL; 5222 5223 path = btrfs_alloc_path(); 5224 if (!path) 5225 return -ENOMEM; 5226 5227 roots = ulist_alloc(GFP_KERNEL); 5228 tmp_ulist = ulist_alloc(GFP_KERNEL); 5229 if (!roots || !tmp_ulist) { 5230 ret = -ENOMEM; 5231 goto out_free_ulist; 5232 } 5233 5234 start = round_down(start, btrfs_inode_sectorsize(inode)); 5235 len = round_up(max, btrfs_inode_sectorsize(inode)) - start; 5236 5237 /* 5238 * lookup the last file extent. We're not using i_size here 5239 * because there might be preallocation past i_size 5240 */ 5241 ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode), -1, 5242 0); 5243 if (ret < 0) { 5244 goto out_free_ulist; 5245 } else { 5246 WARN_ON(!ret); 5247 if (ret == 1) 5248 ret = 0; 5249 } 5250 5251 path->slots[0]--; 5252 btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]); 5253 found_type = found_key.type; 5254 5255 /* No extents, but there might be delalloc bits */ 5256 if (found_key.objectid != btrfs_ino(inode) || 5257 found_type != BTRFS_EXTENT_DATA_KEY) { 5258 /* have to trust i_size as the end */ 5259 last = (u64)-1; 5260 last_for_get_extent = isize; 5261 } else { 5262 /* 5263 * remember the start of the last extent. There are a 5264 * bunch of different factors that go into the length of the 5265 * extent, so its much less complex to remember where it started 5266 */ 5267 last = found_key.offset; 5268 last_for_get_extent = last + 1; 5269 } 5270 btrfs_release_path(path); 5271 5272 /* 5273 * we might have some extents allocated but more delalloc past those 5274 * extents. so, we trust isize unless the start of the last extent is 5275 * beyond isize 5276 */ 5277 if (last < isize) { 5278 last = (u64)-1; 5279 last_for_get_extent = isize; 5280 } 5281 5282 lock_extent_bits(&inode->io_tree, start, start + len - 1, 5283 &cached_state); 5284 5285 em = get_extent_skip_holes(inode, start, last_for_get_extent); 5286 if (!em) 5287 goto out; 5288 if (IS_ERR(em)) { 5289 ret = PTR_ERR(em); 5290 goto out; 5291 } 5292 5293 while (!end) { 5294 u64 offset_in_extent = 0; 5295 5296 /* break if the extent we found is outside the range */ 5297 if (em->start >= max || extent_map_end(em) < off) 5298 break; 5299 5300 /* 5301 * get_extent may return an extent that starts before our 5302 * requested range. We have to make sure the ranges 5303 * we return to fiemap always move forward and don't 5304 * overlap, so adjust the offsets here 5305 */ 5306 em_start = max(em->start, off); 5307 5308 /* 5309 * record the offset from the start of the extent 5310 * for adjusting the disk offset below. Only do this if the 5311 * extent isn't compressed since our in ram offset may be past 5312 * what we have actually allocated on disk. 5313 */ 5314 if (!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) 5315 offset_in_extent = em_start - em->start; 5316 em_end = extent_map_end(em); 5317 em_len = em_end - em_start; 5318 flags = 0; 5319 if (em->block_start < EXTENT_MAP_LAST_BYTE) 5320 disko = em->block_start + offset_in_extent; 5321 else 5322 disko = 0; 5323 5324 /* 5325 * bump off for our next call to get_extent 5326 */ 5327 off = extent_map_end(em); 5328 if (off >= max) 5329 end = 1; 5330 5331 if (em->block_start == EXTENT_MAP_LAST_BYTE) { 5332 end = 1; 5333 flags |= FIEMAP_EXTENT_LAST; 5334 } else if (em->block_start == EXTENT_MAP_INLINE) { 5335 flags |= (FIEMAP_EXTENT_DATA_INLINE | 5336 FIEMAP_EXTENT_NOT_ALIGNED); 5337 } else if (em->block_start == EXTENT_MAP_DELALLOC) { 5338 flags |= (FIEMAP_EXTENT_DELALLOC | 5339 FIEMAP_EXTENT_UNKNOWN); 5340 } else if (fieinfo->fi_extents_max) { 5341 u64 bytenr = em->block_start - 5342 (em->start - em->orig_start); 5343 5344 /* 5345 * As btrfs supports shared space, this information 5346 * can be exported to userspace tools via 5347 * flag FIEMAP_EXTENT_SHARED. If fi_extents_max == 0 5348 * then we're just getting a count and we can skip the 5349 * lookup stuff. 5350 */ 5351 ret = btrfs_check_shared(root, btrfs_ino(inode), 5352 bytenr, roots, tmp_ulist); 5353 if (ret < 0) 5354 goto out_free; 5355 if (ret) 5356 flags |= FIEMAP_EXTENT_SHARED; 5357 ret = 0; 5358 } 5359 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) 5360 flags |= FIEMAP_EXTENT_ENCODED; 5361 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 5362 flags |= FIEMAP_EXTENT_UNWRITTEN; 5363 5364 free_extent_map(em); 5365 em = NULL; 5366 if ((em_start >= last) || em_len == (u64)-1 || 5367 (last == (u64)-1 && isize <= em_end)) { 5368 flags |= FIEMAP_EXTENT_LAST; 5369 end = 1; 5370 } 5371 5372 /* now scan forward to see if this is really the last extent. */ 5373 em = get_extent_skip_holes(inode, off, last_for_get_extent); 5374 if (IS_ERR(em)) { 5375 ret = PTR_ERR(em); 5376 goto out; 5377 } 5378 if (!em) { 5379 flags |= FIEMAP_EXTENT_LAST; 5380 end = 1; 5381 } 5382 ret = emit_fiemap_extent(fieinfo, &cache, em_start, disko, 5383 em_len, flags); 5384 if (ret) { 5385 if (ret == 1) 5386 ret = 0; 5387 goto out_free; 5388 } 5389 } 5390 out_free: 5391 if (!ret) 5392 ret = emit_last_fiemap_cache(fieinfo, &cache); 5393 free_extent_map(em); 5394 out: 5395 unlock_extent_cached(&inode->io_tree, start, start + len - 1, 5396 &cached_state); 5397 5398 out_free_ulist: 5399 btrfs_free_path(path); 5400 ulist_free(roots); 5401 ulist_free(tmp_ulist); 5402 return ret; 5403 } 5404 5405 static void __free_extent_buffer(struct extent_buffer *eb) 5406 { 5407 kmem_cache_free(extent_buffer_cache, eb); 5408 } 5409 5410 int extent_buffer_under_io(const struct extent_buffer *eb) 5411 { 5412 return (atomic_read(&eb->io_pages) || 5413 test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) || 5414 test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)); 5415 } 5416 5417 static bool page_range_has_eb(struct btrfs_fs_info *fs_info, struct page *page) 5418 { 5419 struct btrfs_subpage *subpage; 5420 5421 lockdep_assert_held(&page->mapping->private_lock); 5422 5423 if (PagePrivate(page)) { 5424 subpage = (struct btrfs_subpage *)page->private; 5425 if (atomic_read(&subpage->eb_refs)) 5426 return true; 5427 } 5428 return false; 5429 } 5430 5431 static void detach_extent_buffer_page(struct extent_buffer *eb, struct page *page) 5432 { 5433 struct btrfs_fs_info *fs_info = eb->fs_info; 5434 const bool mapped = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags); 5435 5436 /* 5437 * For mapped eb, we're going to change the page private, which should 5438 * be done under the private_lock. 5439 */ 5440 if (mapped) 5441 spin_lock(&page->mapping->private_lock); 5442 5443 if (!PagePrivate(page)) { 5444 if (mapped) 5445 spin_unlock(&page->mapping->private_lock); 5446 return; 5447 } 5448 5449 if (fs_info->sectorsize == PAGE_SIZE) { 5450 /* 5451 * We do this since we'll remove the pages after we've 5452 * removed the eb from the radix tree, so we could race 5453 * and have this page now attached to the new eb. So 5454 * only clear page_private if it's still connected to 5455 * this eb. 5456 */ 5457 if (PagePrivate(page) && 5458 page->private == (unsigned long)eb) { 5459 BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)); 5460 BUG_ON(PageDirty(page)); 5461 BUG_ON(PageWriteback(page)); 5462 /* 5463 * We need to make sure we haven't be attached 5464 * to a new eb. 5465 */ 5466 detach_page_private(page); 5467 } 5468 if (mapped) 5469 spin_unlock(&page->mapping->private_lock); 5470 return; 5471 } 5472 5473 /* 5474 * For subpage, we can have dummy eb with page private. In this case, 5475 * we can directly detach the private as such page is only attached to 5476 * one dummy eb, no sharing. 5477 */ 5478 if (!mapped) { 5479 btrfs_detach_subpage(fs_info, page); 5480 return; 5481 } 5482 5483 btrfs_page_dec_eb_refs(fs_info, page); 5484 5485 /* 5486 * We can only detach the page private if there are no other ebs in the 5487 * page range. 5488 */ 5489 if (!page_range_has_eb(fs_info, page)) 5490 btrfs_detach_subpage(fs_info, page); 5491 5492 spin_unlock(&page->mapping->private_lock); 5493 } 5494 5495 /* Release all pages attached to the extent buffer */ 5496 static void btrfs_release_extent_buffer_pages(struct extent_buffer *eb) 5497 { 5498 int i; 5499 int num_pages; 5500 5501 ASSERT(!extent_buffer_under_io(eb)); 5502 5503 num_pages = num_extent_pages(eb); 5504 for (i = 0; i < num_pages; i++) { 5505 struct page *page = eb->pages[i]; 5506 5507 if (!page) 5508 continue; 5509 5510 detach_extent_buffer_page(eb, page); 5511 5512 /* One for when we allocated the page */ 5513 put_page(page); 5514 } 5515 } 5516 5517 /* 5518 * Helper for releasing the extent buffer. 5519 */ 5520 static inline void btrfs_release_extent_buffer(struct extent_buffer *eb) 5521 { 5522 btrfs_release_extent_buffer_pages(eb); 5523 btrfs_leak_debug_del(&eb->fs_info->eb_leak_lock, &eb->leak_list); 5524 __free_extent_buffer(eb); 5525 } 5526 5527 static struct extent_buffer * 5528 __alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start, 5529 unsigned long len) 5530 { 5531 struct extent_buffer *eb = NULL; 5532 5533 eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL); 5534 eb->start = start; 5535 eb->len = len; 5536 eb->fs_info = fs_info; 5537 eb->bflags = 0; 5538 init_rwsem(&eb->lock); 5539 5540 btrfs_leak_debug_add(&fs_info->eb_leak_lock, &eb->leak_list, 5541 &fs_info->allocated_ebs); 5542 INIT_LIST_HEAD(&eb->release_list); 5543 5544 spin_lock_init(&eb->refs_lock); 5545 atomic_set(&eb->refs, 1); 5546 atomic_set(&eb->io_pages, 0); 5547 5548 ASSERT(len <= BTRFS_MAX_METADATA_BLOCKSIZE); 5549 5550 return eb; 5551 } 5552 5553 struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src) 5554 { 5555 int i; 5556 struct page *p; 5557 struct extent_buffer *new; 5558 int num_pages = num_extent_pages(src); 5559 5560 new = __alloc_extent_buffer(src->fs_info, src->start, src->len); 5561 if (new == NULL) 5562 return NULL; 5563 5564 /* 5565 * Set UNMAPPED before calling btrfs_release_extent_buffer(), as 5566 * btrfs_release_extent_buffer() have different behavior for 5567 * UNMAPPED subpage extent buffer. 5568 */ 5569 set_bit(EXTENT_BUFFER_UNMAPPED, &new->bflags); 5570 5571 for (i = 0; i < num_pages; i++) { 5572 int ret; 5573 5574 p = alloc_page(GFP_NOFS); 5575 if (!p) { 5576 btrfs_release_extent_buffer(new); 5577 return NULL; 5578 } 5579 ret = attach_extent_buffer_page(new, p, NULL); 5580 if (ret < 0) { 5581 put_page(p); 5582 btrfs_release_extent_buffer(new); 5583 return NULL; 5584 } 5585 WARN_ON(PageDirty(p)); 5586 new->pages[i] = p; 5587 copy_page(page_address(p), page_address(src->pages[i])); 5588 } 5589 set_extent_buffer_uptodate(new); 5590 5591 return new; 5592 } 5593 5594 struct extent_buffer *__alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info, 5595 u64 start, unsigned long len) 5596 { 5597 struct extent_buffer *eb; 5598 int num_pages; 5599 int i; 5600 5601 eb = __alloc_extent_buffer(fs_info, start, len); 5602 if (!eb) 5603 return NULL; 5604 5605 num_pages = num_extent_pages(eb); 5606 for (i = 0; i < num_pages; i++) { 5607 int ret; 5608 5609 eb->pages[i] = alloc_page(GFP_NOFS); 5610 if (!eb->pages[i]) 5611 goto err; 5612 ret = attach_extent_buffer_page(eb, eb->pages[i], NULL); 5613 if (ret < 0) 5614 goto err; 5615 } 5616 set_extent_buffer_uptodate(eb); 5617 btrfs_set_header_nritems(eb, 0); 5618 set_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags); 5619 5620 return eb; 5621 err: 5622 for (; i > 0; i--) { 5623 detach_extent_buffer_page(eb, eb->pages[i - 1]); 5624 __free_page(eb->pages[i - 1]); 5625 } 5626 __free_extent_buffer(eb); 5627 return NULL; 5628 } 5629 5630 struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info, 5631 u64 start) 5632 { 5633 return __alloc_dummy_extent_buffer(fs_info, start, fs_info->nodesize); 5634 } 5635 5636 static void check_buffer_tree_ref(struct extent_buffer *eb) 5637 { 5638 int refs; 5639 /* 5640 * The TREE_REF bit is first set when the extent_buffer is added 5641 * to the radix tree. It is also reset, if unset, when a new reference 5642 * is created by find_extent_buffer. 5643 * 5644 * It is only cleared in two cases: freeing the last non-tree 5645 * reference to the extent_buffer when its STALE bit is set or 5646 * calling releasepage when the tree reference is the only reference. 5647 * 5648 * In both cases, care is taken to ensure that the extent_buffer's 5649 * pages are not under io. However, releasepage can be concurrently 5650 * called with creating new references, which is prone to race 5651 * conditions between the calls to check_buffer_tree_ref in those 5652 * codepaths and clearing TREE_REF in try_release_extent_buffer. 5653 * 5654 * The actual lifetime of the extent_buffer in the radix tree is 5655 * adequately protected by the refcount, but the TREE_REF bit and 5656 * its corresponding reference are not. To protect against this 5657 * class of races, we call check_buffer_tree_ref from the codepaths 5658 * which trigger io after they set eb->io_pages. Note that once io is 5659 * initiated, TREE_REF can no longer be cleared, so that is the 5660 * moment at which any such race is best fixed. 5661 */ 5662 refs = atomic_read(&eb->refs); 5663 if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) 5664 return; 5665 5666 spin_lock(&eb->refs_lock); 5667 if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) 5668 atomic_inc(&eb->refs); 5669 spin_unlock(&eb->refs_lock); 5670 } 5671 5672 static void mark_extent_buffer_accessed(struct extent_buffer *eb, 5673 struct page *accessed) 5674 { 5675 int num_pages, i; 5676 5677 check_buffer_tree_ref(eb); 5678 5679 num_pages = num_extent_pages(eb); 5680 for (i = 0; i < num_pages; i++) { 5681 struct page *p = eb->pages[i]; 5682 5683 if (p != accessed) 5684 mark_page_accessed(p); 5685 } 5686 } 5687 5688 struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info, 5689 u64 start) 5690 { 5691 struct extent_buffer *eb; 5692 5693 eb = find_extent_buffer_nolock(fs_info, start); 5694 if (!eb) 5695 return NULL; 5696 /* 5697 * Lock our eb's refs_lock to avoid races with free_extent_buffer(). 5698 * When we get our eb it might be flagged with EXTENT_BUFFER_STALE and 5699 * another task running free_extent_buffer() might have seen that flag 5700 * set, eb->refs == 2, that the buffer isn't under IO (dirty and 5701 * writeback flags not set) and it's still in the tree (flag 5702 * EXTENT_BUFFER_TREE_REF set), therefore being in the process of 5703 * decrementing the extent buffer's reference count twice. So here we 5704 * could race and increment the eb's reference count, clear its stale 5705 * flag, mark it as dirty and drop our reference before the other task 5706 * finishes executing free_extent_buffer, which would later result in 5707 * an attempt to free an extent buffer that is dirty. 5708 */ 5709 if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) { 5710 spin_lock(&eb->refs_lock); 5711 spin_unlock(&eb->refs_lock); 5712 } 5713 mark_extent_buffer_accessed(eb, NULL); 5714 return eb; 5715 } 5716 5717 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS 5718 struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info, 5719 u64 start) 5720 { 5721 struct extent_buffer *eb, *exists = NULL; 5722 int ret; 5723 5724 eb = find_extent_buffer(fs_info, start); 5725 if (eb) 5726 return eb; 5727 eb = alloc_dummy_extent_buffer(fs_info, start); 5728 if (!eb) 5729 return ERR_PTR(-ENOMEM); 5730 eb->fs_info = fs_info; 5731 again: 5732 ret = radix_tree_preload(GFP_NOFS); 5733 if (ret) { 5734 exists = ERR_PTR(ret); 5735 goto free_eb; 5736 } 5737 spin_lock(&fs_info->buffer_lock); 5738 ret = radix_tree_insert(&fs_info->buffer_radix, 5739 start >> fs_info->sectorsize_bits, eb); 5740 spin_unlock(&fs_info->buffer_lock); 5741 radix_tree_preload_end(); 5742 if (ret == -EEXIST) { 5743 exists = find_extent_buffer(fs_info, start); 5744 if (exists) 5745 goto free_eb; 5746 else 5747 goto again; 5748 } 5749 check_buffer_tree_ref(eb); 5750 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags); 5751 5752 return eb; 5753 free_eb: 5754 btrfs_release_extent_buffer(eb); 5755 return exists; 5756 } 5757 #endif 5758 5759 static struct extent_buffer *grab_extent_buffer( 5760 struct btrfs_fs_info *fs_info, struct page *page) 5761 { 5762 struct extent_buffer *exists; 5763 5764 /* 5765 * For subpage case, we completely rely on radix tree to ensure we 5766 * don't try to insert two ebs for the same bytenr. So here we always 5767 * return NULL and just continue. 5768 */ 5769 if (fs_info->sectorsize < PAGE_SIZE) 5770 return NULL; 5771 5772 /* Page not yet attached to an extent buffer */ 5773 if (!PagePrivate(page)) 5774 return NULL; 5775 5776 /* 5777 * We could have already allocated an eb for this page and attached one 5778 * so lets see if we can get a ref on the existing eb, and if we can we 5779 * know it's good and we can just return that one, else we know we can 5780 * just overwrite page->private. 5781 */ 5782 exists = (struct extent_buffer *)page->private; 5783 if (atomic_inc_not_zero(&exists->refs)) 5784 return exists; 5785 5786 WARN_ON(PageDirty(page)); 5787 detach_page_private(page); 5788 return NULL; 5789 } 5790 5791 struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, 5792 u64 start, u64 owner_root, int level) 5793 { 5794 unsigned long len = fs_info->nodesize; 5795 int num_pages; 5796 int i; 5797 unsigned long index = start >> PAGE_SHIFT; 5798 struct extent_buffer *eb; 5799 struct extent_buffer *exists = NULL; 5800 struct page *p; 5801 struct address_space *mapping = fs_info->btree_inode->i_mapping; 5802 int uptodate = 1; 5803 int ret; 5804 5805 if (!IS_ALIGNED(start, fs_info->sectorsize)) { 5806 btrfs_err(fs_info, "bad tree block start %llu", start); 5807 return ERR_PTR(-EINVAL); 5808 } 5809 5810 #if BITS_PER_LONG == 32 5811 if (start >= MAX_LFS_FILESIZE) { 5812 btrfs_err_rl(fs_info, 5813 "extent buffer %llu is beyond 32bit page cache limit", start); 5814 btrfs_err_32bit_limit(fs_info); 5815 return ERR_PTR(-EOVERFLOW); 5816 } 5817 if (start >= BTRFS_32BIT_EARLY_WARN_THRESHOLD) 5818 btrfs_warn_32bit_limit(fs_info); 5819 #endif 5820 5821 if (fs_info->sectorsize < PAGE_SIZE && 5822 offset_in_page(start) + len > PAGE_SIZE) { 5823 btrfs_err(fs_info, 5824 "tree block crosses page boundary, start %llu nodesize %lu", 5825 start, len); 5826 return ERR_PTR(-EINVAL); 5827 } 5828 5829 eb = find_extent_buffer(fs_info, start); 5830 if (eb) 5831 return eb; 5832 5833 eb = __alloc_extent_buffer(fs_info, start, len); 5834 if (!eb) 5835 return ERR_PTR(-ENOMEM); 5836 btrfs_set_buffer_lockdep_class(owner_root, eb, level); 5837 5838 num_pages = num_extent_pages(eb); 5839 for (i = 0; i < num_pages; i++, index++) { 5840 struct btrfs_subpage *prealloc = NULL; 5841 5842 p = find_or_create_page(mapping, index, GFP_NOFS|__GFP_NOFAIL); 5843 if (!p) { 5844 exists = ERR_PTR(-ENOMEM); 5845 goto free_eb; 5846 } 5847 5848 /* 5849 * Preallocate page->private for subpage case, so that we won't 5850 * allocate memory with private_lock hold. The memory will be 5851 * freed by attach_extent_buffer_page() or freed manually if 5852 * we exit earlier. 5853 * 5854 * Although we have ensured one subpage eb can only have one 5855 * page, but it may change in the future for 16K page size 5856 * support, so we still preallocate the memory in the loop. 5857 */ 5858 ret = btrfs_alloc_subpage(fs_info, &prealloc, 5859 BTRFS_SUBPAGE_METADATA); 5860 if (ret < 0) { 5861 unlock_page(p); 5862 put_page(p); 5863 exists = ERR_PTR(ret); 5864 goto free_eb; 5865 } 5866 5867 spin_lock(&mapping->private_lock); 5868 exists = grab_extent_buffer(fs_info, p); 5869 if (exists) { 5870 spin_unlock(&mapping->private_lock); 5871 unlock_page(p); 5872 put_page(p); 5873 mark_extent_buffer_accessed(exists, p); 5874 btrfs_free_subpage(prealloc); 5875 goto free_eb; 5876 } 5877 /* Should not fail, as we have preallocated the memory */ 5878 ret = attach_extent_buffer_page(eb, p, prealloc); 5879 ASSERT(!ret); 5880 /* 5881 * To inform we have extra eb under allocation, so that 5882 * detach_extent_buffer_page() won't release the page private 5883 * when the eb hasn't yet been inserted into radix tree. 5884 * 5885 * The ref will be decreased when the eb released the page, in 5886 * detach_extent_buffer_page(). 5887 * Thus needs no special handling in error path. 5888 */ 5889 btrfs_page_inc_eb_refs(fs_info, p); 5890 spin_unlock(&mapping->private_lock); 5891 5892 WARN_ON(btrfs_page_test_dirty(fs_info, p, eb->start, eb->len)); 5893 eb->pages[i] = p; 5894 if (!PageUptodate(p)) 5895 uptodate = 0; 5896 5897 /* 5898 * We can't unlock the pages just yet since the extent buffer 5899 * hasn't been properly inserted in the radix tree, this 5900 * opens a race with btree_releasepage which can free a page 5901 * while we are still filling in all pages for the buffer and 5902 * we could crash. 5903 */ 5904 } 5905 if (uptodate) 5906 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); 5907 again: 5908 ret = radix_tree_preload(GFP_NOFS); 5909 if (ret) { 5910 exists = ERR_PTR(ret); 5911 goto free_eb; 5912 } 5913 5914 spin_lock(&fs_info->buffer_lock); 5915 ret = radix_tree_insert(&fs_info->buffer_radix, 5916 start >> fs_info->sectorsize_bits, eb); 5917 spin_unlock(&fs_info->buffer_lock); 5918 radix_tree_preload_end(); 5919 if (ret == -EEXIST) { 5920 exists = find_extent_buffer(fs_info, start); 5921 if (exists) 5922 goto free_eb; 5923 else 5924 goto again; 5925 } 5926 /* add one reference for the tree */ 5927 check_buffer_tree_ref(eb); 5928 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags); 5929 5930 /* 5931 * Now it's safe to unlock the pages because any calls to 5932 * btree_releasepage will correctly detect that a page belongs to a 5933 * live buffer and won't free them prematurely. 5934 */ 5935 for (i = 0; i < num_pages; i++) 5936 unlock_page(eb->pages[i]); 5937 return eb; 5938 5939 free_eb: 5940 WARN_ON(!atomic_dec_and_test(&eb->refs)); 5941 for (i = 0; i < num_pages; i++) { 5942 if (eb->pages[i]) 5943 unlock_page(eb->pages[i]); 5944 } 5945 5946 btrfs_release_extent_buffer(eb); 5947 return exists; 5948 } 5949 5950 static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head) 5951 { 5952 struct extent_buffer *eb = 5953 container_of(head, struct extent_buffer, rcu_head); 5954 5955 __free_extent_buffer(eb); 5956 } 5957 5958 static int release_extent_buffer(struct extent_buffer *eb) 5959 __releases(&eb->refs_lock) 5960 { 5961 lockdep_assert_held(&eb->refs_lock); 5962 5963 WARN_ON(atomic_read(&eb->refs) == 0); 5964 if (atomic_dec_and_test(&eb->refs)) { 5965 if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags)) { 5966 struct btrfs_fs_info *fs_info = eb->fs_info; 5967 5968 spin_unlock(&eb->refs_lock); 5969 5970 spin_lock(&fs_info->buffer_lock); 5971 radix_tree_delete(&fs_info->buffer_radix, 5972 eb->start >> fs_info->sectorsize_bits); 5973 spin_unlock(&fs_info->buffer_lock); 5974 } else { 5975 spin_unlock(&eb->refs_lock); 5976 } 5977 5978 btrfs_leak_debug_del(&eb->fs_info->eb_leak_lock, &eb->leak_list); 5979 /* Should be safe to release our pages at this point */ 5980 btrfs_release_extent_buffer_pages(eb); 5981 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS 5982 if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags))) { 5983 __free_extent_buffer(eb); 5984 return 1; 5985 } 5986 #endif 5987 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu); 5988 return 1; 5989 } 5990 spin_unlock(&eb->refs_lock); 5991 5992 return 0; 5993 } 5994 5995 void free_extent_buffer(struct extent_buffer *eb) 5996 { 5997 int refs; 5998 int old; 5999 if (!eb) 6000 return; 6001 6002 while (1) { 6003 refs = atomic_read(&eb->refs); 6004 if ((!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) && refs <= 3) 6005 || (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) && 6006 refs == 1)) 6007 break; 6008 old = atomic_cmpxchg(&eb->refs, refs, refs - 1); 6009 if (old == refs) 6010 return; 6011 } 6012 6013 spin_lock(&eb->refs_lock); 6014 if (atomic_read(&eb->refs) == 2 && 6015 test_bit(EXTENT_BUFFER_STALE, &eb->bflags) && 6016 !extent_buffer_under_io(eb) && 6017 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) 6018 atomic_dec(&eb->refs); 6019 6020 /* 6021 * I know this is terrible, but it's temporary until we stop tracking 6022 * the uptodate bits and such for the extent buffers. 6023 */ 6024 release_extent_buffer(eb); 6025 } 6026 6027 void free_extent_buffer_stale(struct extent_buffer *eb) 6028 { 6029 if (!eb) 6030 return; 6031 6032 spin_lock(&eb->refs_lock); 6033 set_bit(EXTENT_BUFFER_STALE, &eb->bflags); 6034 6035 if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) && 6036 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) 6037 atomic_dec(&eb->refs); 6038 release_extent_buffer(eb); 6039 } 6040 6041 static void btree_clear_page_dirty(struct page *page) 6042 { 6043 ASSERT(PageDirty(page)); 6044 ASSERT(PageLocked(page)); 6045 clear_page_dirty_for_io(page); 6046 xa_lock_irq(&page->mapping->i_pages); 6047 if (!PageDirty(page)) 6048 __xa_clear_mark(&page->mapping->i_pages, 6049 page_index(page), PAGECACHE_TAG_DIRTY); 6050 xa_unlock_irq(&page->mapping->i_pages); 6051 } 6052 6053 static void clear_subpage_extent_buffer_dirty(const struct extent_buffer *eb) 6054 { 6055 struct btrfs_fs_info *fs_info = eb->fs_info; 6056 struct page *page = eb->pages[0]; 6057 bool last; 6058 6059 /* btree_clear_page_dirty() needs page locked */ 6060 lock_page(page); 6061 last = btrfs_subpage_clear_and_test_dirty(fs_info, page, eb->start, 6062 eb->len); 6063 if (last) 6064 btree_clear_page_dirty(page); 6065 unlock_page(page); 6066 WARN_ON(atomic_read(&eb->refs) == 0); 6067 } 6068 6069 void clear_extent_buffer_dirty(const struct extent_buffer *eb) 6070 { 6071 int i; 6072 int num_pages; 6073 struct page *page; 6074 6075 if (eb->fs_info->sectorsize < PAGE_SIZE) 6076 return clear_subpage_extent_buffer_dirty(eb); 6077 6078 num_pages = num_extent_pages(eb); 6079 6080 for (i = 0; i < num_pages; i++) { 6081 page = eb->pages[i]; 6082 if (!PageDirty(page)) 6083 continue; 6084 lock_page(page); 6085 btree_clear_page_dirty(page); 6086 ClearPageError(page); 6087 unlock_page(page); 6088 } 6089 WARN_ON(atomic_read(&eb->refs) == 0); 6090 } 6091 6092 bool set_extent_buffer_dirty(struct extent_buffer *eb) 6093 { 6094 int i; 6095 int num_pages; 6096 bool was_dirty; 6097 6098 check_buffer_tree_ref(eb); 6099 6100 was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags); 6101 6102 num_pages = num_extent_pages(eb); 6103 WARN_ON(atomic_read(&eb->refs) == 0); 6104 WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)); 6105 6106 if (!was_dirty) { 6107 bool subpage = eb->fs_info->sectorsize < PAGE_SIZE; 6108 6109 /* 6110 * For subpage case, we can have other extent buffers in the 6111 * same page, and in clear_subpage_extent_buffer_dirty() we 6112 * have to clear page dirty without subpage lock held. 6113 * This can cause race where our page gets dirty cleared after 6114 * we just set it. 6115 * 6116 * Thankfully, clear_subpage_extent_buffer_dirty() has locked 6117 * its page for other reasons, we can use page lock to prevent 6118 * the above race. 6119 */ 6120 if (subpage) 6121 lock_page(eb->pages[0]); 6122 for (i = 0; i < num_pages; i++) 6123 btrfs_page_set_dirty(eb->fs_info, eb->pages[i], 6124 eb->start, eb->len); 6125 if (subpage) 6126 unlock_page(eb->pages[0]); 6127 } 6128 #ifdef CONFIG_BTRFS_DEBUG 6129 for (i = 0; i < num_pages; i++) 6130 ASSERT(PageDirty(eb->pages[i])); 6131 #endif 6132 6133 return was_dirty; 6134 } 6135 6136 void clear_extent_buffer_uptodate(struct extent_buffer *eb) 6137 { 6138 struct btrfs_fs_info *fs_info = eb->fs_info; 6139 struct page *page; 6140 int num_pages; 6141 int i; 6142 6143 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); 6144 num_pages = num_extent_pages(eb); 6145 for (i = 0; i < num_pages; i++) { 6146 page = eb->pages[i]; 6147 if (page) 6148 btrfs_page_clear_uptodate(fs_info, page, 6149 eb->start, eb->len); 6150 } 6151 } 6152 6153 void set_extent_buffer_uptodate(struct extent_buffer *eb) 6154 { 6155 struct btrfs_fs_info *fs_info = eb->fs_info; 6156 struct page *page; 6157 int num_pages; 6158 int i; 6159 6160 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); 6161 num_pages = num_extent_pages(eb); 6162 for (i = 0; i < num_pages; i++) { 6163 page = eb->pages[i]; 6164 btrfs_page_set_uptodate(fs_info, page, eb->start, eb->len); 6165 } 6166 } 6167 6168 static int read_extent_buffer_subpage(struct extent_buffer *eb, int wait, 6169 int mirror_num) 6170 { 6171 struct btrfs_fs_info *fs_info = eb->fs_info; 6172 struct extent_io_tree *io_tree; 6173 struct page *page = eb->pages[0]; 6174 struct bio *bio = NULL; 6175 int ret = 0; 6176 6177 ASSERT(!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags)); 6178 ASSERT(PagePrivate(page)); 6179 io_tree = &BTRFS_I(fs_info->btree_inode)->io_tree; 6180 6181 if (wait == WAIT_NONE) { 6182 ret = try_lock_extent(io_tree, eb->start, 6183 eb->start + eb->len - 1); 6184 if (ret <= 0) 6185 return ret; 6186 } else { 6187 ret = lock_extent(io_tree, eb->start, eb->start + eb->len - 1); 6188 if (ret < 0) 6189 return ret; 6190 } 6191 6192 ret = 0; 6193 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags) || 6194 PageUptodate(page) || 6195 btrfs_subpage_test_uptodate(fs_info, page, eb->start, eb->len)) { 6196 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); 6197 unlock_extent(io_tree, eb->start, eb->start + eb->len - 1); 6198 return ret; 6199 } 6200 6201 clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags); 6202 eb->read_mirror = 0; 6203 atomic_set(&eb->io_pages, 1); 6204 check_buffer_tree_ref(eb); 6205 btrfs_subpage_clear_error(fs_info, page, eb->start, eb->len); 6206 6207 ret = submit_extent_page(REQ_OP_READ | REQ_META, NULL, page, eb->start, 6208 eb->len, eb->start - page_offset(page), &bio, 6209 end_bio_extent_readpage, mirror_num, 0, 0, 6210 true); 6211 if (ret) { 6212 /* 6213 * In the endio function, if we hit something wrong we will 6214 * increase the io_pages, so here we need to decrease it for 6215 * error path. 6216 */ 6217 atomic_dec(&eb->io_pages); 6218 } 6219 if (bio) { 6220 int tmp; 6221 6222 tmp = submit_one_bio(bio, mirror_num, 0); 6223 if (tmp < 0) 6224 return tmp; 6225 } 6226 if (ret || wait != WAIT_COMPLETE) 6227 return ret; 6228 6229 wait_extent_bit(io_tree, eb->start, eb->start + eb->len - 1, EXTENT_LOCKED); 6230 if (!test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags)) 6231 ret = -EIO; 6232 return ret; 6233 } 6234 6235 int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num) 6236 { 6237 int i; 6238 struct page *page; 6239 int err; 6240 int ret = 0; 6241 int locked_pages = 0; 6242 int all_uptodate = 1; 6243 int num_pages; 6244 unsigned long num_reads = 0; 6245 struct bio *bio = NULL; 6246 unsigned long bio_flags = 0; 6247 6248 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags)) 6249 return 0; 6250 6251 if (eb->fs_info->sectorsize < PAGE_SIZE) 6252 return read_extent_buffer_subpage(eb, wait, mirror_num); 6253 6254 num_pages = num_extent_pages(eb); 6255 for (i = 0; i < num_pages; i++) { 6256 page = eb->pages[i]; 6257 if (wait == WAIT_NONE) { 6258 /* 6259 * WAIT_NONE is only utilized by readahead. If we can't 6260 * acquire the lock atomically it means either the eb 6261 * is being read out or under modification. 6262 * Either way the eb will be or has been cached, 6263 * readahead can exit safely. 6264 */ 6265 if (!trylock_page(page)) 6266 goto unlock_exit; 6267 } else { 6268 lock_page(page); 6269 } 6270 locked_pages++; 6271 } 6272 /* 6273 * We need to firstly lock all pages to make sure that 6274 * the uptodate bit of our pages won't be affected by 6275 * clear_extent_buffer_uptodate(). 6276 */ 6277 for (i = 0; i < num_pages; i++) { 6278 page = eb->pages[i]; 6279 if (!PageUptodate(page)) { 6280 num_reads++; 6281 all_uptodate = 0; 6282 } 6283 } 6284 6285 if (all_uptodate) { 6286 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); 6287 goto unlock_exit; 6288 } 6289 6290 clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags); 6291 eb->read_mirror = 0; 6292 atomic_set(&eb->io_pages, num_reads); 6293 /* 6294 * It is possible for releasepage to clear the TREE_REF bit before we 6295 * set io_pages. See check_buffer_tree_ref for a more detailed comment. 6296 */ 6297 check_buffer_tree_ref(eb); 6298 for (i = 0; i < num_pages; i++) { 6299 page = eb->pages[i]; 6300 6301 if (!PageUptodate(page)) { 6302 if (ret) { 6303 atomic_dec(&eb->io_pages); 6304 unlock_page(page); 6305 continue; 6306 } 6307 6308 ClearPageError(page); 6309 err = submit_extent_page(REQ_OP_READ | REQ_META, NULL, 6310 page, page_offset(page), PAGE_SIZE, 0, 6311 &bio, end_bio_extent_readpage, 6312 mirror_num, 0, 0, false); 6313 if (err) { 6314 /* 6315 * We failed to submit the bio so it's the 6316 * caller's responsibility to perform cleanup 6317 * i.e unlock page/set error bit. 6318 */ 6319 ret = err; 6320 SetPageError(page); 6321 unlock_page(page); 6322 atomic_dec(&eb->io_pages); 6323 } 6324 } else { 6325 unlock_page(page); 6326 } 6327 } 6328 6329 if (bio) { 6330 err = submit_one_bio(bio, mirror_num, bio_flags); 6331 if (err) 6332 return err; 6333 } 6334 6335 if (ret || wait != WAIT_COMPLETE) 6336 return ret; 6337 6338 for (i = 0; i < num_pages; i++) { 6339 page = eb->pages[i]; 6340 wait_on_page_locked(page); 6341 if (!PageUptodate(page)) 6342 ret = -EIO; 6343 } 6344 6345 return ret; 6346 6347 unlock_exit: 6348 while (locked_pages > 0) { 6349 locked_pages--; 6350 page = eb->pages[locked_pages]; 6351 unlock_page(page); 6352 } 6353 return ret; 6354 } 6355 6356 static bool report_eb_range(const struct extent_buffer *eb, unsigned long start, 6357 unsigned long len) 6358 { 6359 btrfs_warn(eb->fs_info, 6360 "access to eb bytenr %llu len %lu out of range start %lu len %lu", 6361 eb->start, eb->len, start, len); 6362 WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG)); 6363 6364 return true; 6365 } 6366 6367 /* 6368 * Check if the [start, start + len) range is valid before reading/writing 6369 * the eb. 6370 * NOTE: @start and @len are offset inside the eb, not logical address. 6371 * 6372 * Caller should not touch the dst/src memory if this function returns error. 6373 */ 6374 static inline int check_eb_range(const struct extent_buffer *eb, 6375 unsigned long start, unsigned long len) 6376 { 6377 unsigned long offset; 6378 6379 /* start, start + len should not go beyond eb->len nor overflow */ 6380 if (unlikely(check_add_overflow(start, len, &offset) || offset > eb->len)) 6381 return report_eb_range(eb, start, len); 6382 6383 return false; 6384 } 6385 6386 void read_extent_buffer(const struct extent_buffer *eb, void *dstv, 6387 unsigned long start, unsigned long len) 6388 { 6389 size_t cur; 6390 size_t offset; 6391 struct page *page; 6392 char *kaddr; 6393 char *dst = (char *)dstv; 6394 unsigned long i = get_eb_page_index(start); 6395 6396 if (check_eb_range(eb, start, len)) 6397 return; 6398 6399 offset = get_eb_offset_in_page(eb, start); 6400 6401 while (len > 0) { 6402 page = eb->pages[i]; 6403 6404 cur = min(len, (PAGE_SIZE - offset)); 6405 kaddr = page_address(page); 6406 memcpy(dst, kaddr + offset, cur); 6407 6408 dst += cur; 6409 len -= cur; 6410 offset = 0; 6411 i++; 6412 } 6413 } 6414 6415 int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb, 6416 void __user *dstv, 6417 unsigned long start, unsigned long len) 6418 { 6419 size_t cur; 6420 size_t offset; 6421 struct page *page; 6422 char *kaddr; 6423 char __user *dst = (char __user *)dstv; 6424 unsigned long i = get_eb_page_index(start); 6425 int ret = 0; 6426 6427 WARN_ON(start > eb->len); 6428 WARN_ON(start + len > eb->start + eb->len); 6429 6430 offset = get_eb_offset_in_page(eb, start); 6431 6432 while (len > 0) { 6433 page = eb->pages[i]; 6434 6435 cur = min(len, (PAGE_SIZE - offset)); 6436 kaddr = page_address(page); 6437 if (copy_to_user_nofault(dst, kaddr + offset, cur)) { 6438 ret = -EFAULT; 6439 break; 6440 } 6441 6442 dst += cur; 6443 len -= cur; 6444 offset = 0; 6445 i++; 6446 } 6447 6448 return ret; 6449 } 6450 6451 int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv, 6452 unsigned long start, unsigned long len) 6453 { 6454 size_t cur; 6455 size_t offset; 6456 struct page *page; 6457 char *kaddr; 6458 char *ptr = (char *)ptrv; 6459 unsigned long i = get_eb_page_index(start); 6460 int ret = 0; 6461 6462 if (check_eb_range(eb, start, len)) 6463 return -EINVAL; 6464 6465 offset = get_eb_offset_in_page(eb, start); 6466 6467 while (len > 0) { 6468 page = eb->pages[i]; 6469 6470 cur = min(len, (PAGE_SIZE - offset)); 6471 6472 kaddr = page_address(page); 6473 ret = memcmp(ptr, kaddr + offset, cur); 6474 if (ret) 6475 break; 6476 6477 ptr += cur; 6478 len -= cur; 6479 offset = 0; 6480 i++; 6481 } 6482 return ret; 6483 } 6484 6485 /* 6486 * Check that the extent buffer is uptodate. 6487 * 6488 * For regular sector size == PAGE_SIZE case, check if @page is uptodate. 6489 * For subpage case, check if the range covered by the eb has EXTENT_UPTODATE. 6490 */ 6491 static void assert_eb_page_uptodate(const struct extent_buffer *eb, 6492 struct page *page) 6493 { 6494 struct btrfs_fs_info *fs_info = eb->fs_info; 6495 6496 if (fs_info->sectorsize < PAGE_SIZE) { 6497 bool uptodate; 6498 6499 uptodate = btrfs_subpage_test_uptodate(fs_info, page, 6500 eb->start, eb->len); 6501 WARN_ON(!uptodate); 6502 } else { 6503 WARN_ON(!PageUptodate(page)); 6504 } 6505 } 6506 6507 void write_extent_buffer_chunk_tree_uuid(const struct extent_buffer *eb, 6508 const void *srcv) 6509 { 6510 char *kaddr; 6511 6512 assert_eb_page_uptodate(eb, eb->pages[0]); 6513 kaddr = page_address(eb->pages[0]) + get_eb_offset_in_page(eb, 0); 6514 memcpy(kaddr + offsetof(struct btrfs_header, chunk_tree_uuid), srcv, 6515 BTRFS_FSID_SIZE); 6516 } 6517 6518 void write_extent_buffer_fsid(const struct extent_buffer *eb, const void *srcv) 6519 { 6520 char *kaddr; 6521 6522 assert_eb_page_uptodate(eb, eb->pages[0]); 6523 kaddr = page_address(eb->pages[0]) + get_eb_offset_in_page(eb, 0); 6524 memcpy(kaddr + offsetof(struct btrfs_header, fsid), srcv, 6525 BTRFS_FSID_SIZE); 6526 } 6527 6528 void write_extent_buffer(const struct extent_buffer *eb, const void *srcv, 6529 unsigned long start, unsigned long len) 6530 { 6531 size_t cur; 6532 size_t offset; 6533 struct page *page; 6534 char *kaddr; 6535 char *src = (char *)srcv; 6536 unsigned long i = get_eb_page_index(start); 6537 6538 WARN_ON(test_bit(EXTENT_BUFFER_NO_CHECK, &eb->bflags)); 6539 6540 if (check_eb_range(eb, start, len)) 6541 return; 6542 6543 offset = get_eb_offset_in_page(eb, start); 6544 6545 while (len > 0) { 6546 page = eb->pages[i]; 6547 assert_eb_page_uptodate(eb, page); 6548 6549 cur = min(len, PAGE_SIZE - offset); 6550 kaddr = page_address(page); 6551 memcpy(kaddr + offset, src, cur); 6552 6553 src += cur; 6554 len -= cur; 6555 offset = 0; 6556 i++; 6557 } 6558 } 6559 6560 void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start, 6561 unsigned long len) 6562 { 6563 size_t cur; 6564 size_t offset; 6565 struct page *page; 6566 char *kaddr; 6567 unsigned long i = get_eb_page_index(start); 6568 6569 if (check_eb_range(eb, start, len)) 6570 return; 6571 6572 offset = get_eb_offset_in_page(eb, start); 6573 6574 while (len > 0) { 6575 page = eb->pages[i]; 6576 assert_eb_page_uptodate(eb, page); 6577 6578 cur = min(len, PAGE_SIZE - offset); 6579 kaddr = page_address(page); 6580 memset(kaddr + offset, 0, cur); 6581 6582 len -= cur; 6583 offset = 0; 6584 i++; 6585 } 6586 } 6587 6588 void copy_extent_buffer_full(const struct extent_buffer *dst, 6589 const struct extent_buffer *src) 6590 { 6591 int i; 6592 int num_pages; 6593 6594 ASSERT(dst->len == src->len); 6595 6596 if (dst->fs_info->sectorsize == PAGE_SIZE) { 6597 num_pages = num_extent_pages(dst); 6598 for (i = 0; i < num_pages; i++) 6599 copy_page(page_address(dst->pages[i]), 6600 page_address(src->pages[i])); 6601 } else { 6602 size_t src_offset = get_eb_offset_in_page(src, 0); 6603 size_t dst_offset = get_eb_offset_in_page(dst, 0); 6604 6605 ASSERT(src->fs_info->sectorsize < PAGE_SIZE); 6606 memcpy(page_address(dst->pages[0]) + dst_offset, 6607 page_address(src->pages[0]) + src_offset, 6608 src->len); 6609 } 6610 } 6611 6612 void copy_extent_buffer(const struct extent_buffer *dst, 6613 const struct extent_buffer *src, 6614 unsigned long dst_offset, unsigned long src_offset, 6615 unsigned long len) 6616 { 6617 u64 dst_len = dst->len; 6618 size_t cur; 6619 size_t offset; 6620 struct page *page; 6621 char *kaddr; 6622 unsigned long i = get_eb_page_index(dst_offset); 6623 6624 if (check_eb_range(dst, dst_offset, len) || 6625 check_eb_range(src, src_offset, len)) 6626 return; 6627 6628 WARN_ON(src->len != dst_len); 6629 6630 offset = get_eb_offset_in_page(dst, dst_offset); 6631 6632 while (len > 0) { 6633 page = dst->pages[i]; 6634 assert_eb_page_uptodate(dst, page); 6635 6636 cur = min(len, (unsigned long)(PAGE_SIZE - offset)); 6637 6638 kaddr = page_address(page); 6639 read_extent_buffer(src, kaddr + offset, src_offset, cur); 6640 6641 src_offset += cur; 6642 len -= cur; 6643 offset = 0; 6644 i++; 6645 } 6646 } 6647 6648 /* 6649 * eb_bitmap_offset() - calculate the page and offset of the byte containing the 6650 * given bit number 6651 * @eb: the extent buffer 6652 * @start: offset of the bitmap item in the extent buffer 6653 * @nr: bit number 6654 * @page_index: return index of the page in the extent buffer that contains the 6655 * given bit number 6656 * @page_offset: return offset into the page given by page_index 6657 * 6658 * This helper hides the ugliness of finding the byte in an extent buffer which 6659 * contains a given bit. 6660 */ 6661 static inline void eb_bitmap_offset(const struct extent_buffer *eb, 6662 unsigned long start, unsigned long nr, 6663 unsigned long *page_index, 6664 size_t *page_offset) 6665 { 6666 size_t byte_offset = BIT_BYTE(nr); 6667 size_t offset; 6668 6669 /* 6670 * The byte we want is the offset of the extent buffer + the offset of 6671 * the bitmap item in the extent buffer + the offset of the byte in the 6672 * bitmap item. 6673 */ 6674 offset = start + offset_in_page(eb->start) + byte_offset; 6675 6676 *page_index = offset >> PAGE_SHIFT; 6677 *page_offset = offset_in_page(offset); 6678 } 6679 6680 /** 6681 * extent_buffer_test_bit - determine whether a bit in a bitmap item is set 6682 * @eb: the extent buffer 6683 * @start: offset of the bitmap item in the extent buffer 6684 * @nr: bit number to test 6685 */ 6686 int extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start, 6687 unsigned long nr) 6688 { 6689 u8 *kaddr; 6690 struct page *page; 6691 unsigned long i; 6692 size_t offset; 6693 6694 eb_bitmap_offset(eb, start, nr, &i, &offset); 6695 page = eb->pages[i]; 6696 assert_eb_page_uptodate(eb, page); 6697 kaddr = page_address(page); 6698 return 1U & (kaddr[offset] >> (nr & (BITS_PER_BYTE - 1))); 6699 } 6700 6701 /** 6702 * extent_buffer_bitmap_set - set an area of a bitmap 6703 * @eb: the extent buffer 6704 * @start: offset of the bitmap item in the extent buffer 6705 * @pos: bit number of the first bit 6706 * @len: number of bits to set 6707 */ 6708 void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start, 6709 unsigned long pos, unsigned long len) 6710 { 6711 u8 *kaddr; 6712 struct page *page; 6713 unsigned long i; 6714 size_t offset; 6715 const unsigned int size = pos + len; 6716 int bits_to_set = BITS_PER_BYTE - (pos % BITS_PER_BYTE); 6717 u8 mask_to_set = BITMAP_FIRST_BYTE_MASK(pos); 6718 6719 eb_bitmap_offset(eb, start, pos, &i, &offset); 6720 page = eb->pages[i]; 6721 assert_eb_page_uptodate(eb, page); 6722 kaddr = page_address(page); 6723 6724 while (len >= bits_to_set) { 6725 kaddr[offset] |= mask_to_set; 6726 len -= bits_to_set; 6727 bits_to_set = BITS_PER_BYTE; 6728 mask_to_set = ~0; 6729 if (++offset >= PAGE_SIZE && len > 0) { 6730 offset = 0; 6731 page = eb->pages[++i]; 6732 assert_eb_page_uptodate(eb, page); 6733 kaddr = page_address(page); 6734 } 6735 } 6736 if (len) { 6737 mask_to_set &= BITMAP_LAST_BYTE_MASK(size); 6738 kaddr[offset] |= mask_to_set; 6739 } 6740 } 6741 6742 6743 /** 6744 * extent_buffer_bitmap_clear - clear an area of a bitmap 6745 * @eb: the extent buffer 6746 * @start: offset of the bitmap item in the extent buffer 6747 * @pos: bit number of the first bit 6748 * @len: number of bits to clear 6749 */ 6750 void extent_buffer_bitmap_clear(const struct extent_buffer *eb, 6751 unsigned long start, unsigned long pos, 6752 unsigned long len) 6753 { 6754 u8 *kaddr; 6755 struct page *page; 6756 unsigned long i; 6757 size_t offset; 6758 const unsigned int size = pos + len; 6759 int bits_to_clear = BITS_PER_BYTE - (pos % BITS_PER_BYTE); 6760 u8 mask_to_clear = BITMAP_FIRST_BYTE_MASK(pos); 6761 6762 eb_bitmap_offset(eb, start, pos, &i, &offset); 6763 page = eb->pages[i]; 6764 assert_eb_page_uptodate(eb, page); 6765 kaddr = page_address(page); 6766 6767 while (len >= bits_to_clear) { 6768 kaddr[offset] &= ~mask_to_clear; 6769 len -= bits_to_clear; 6770 bits_to_clear = BITS_PER_BYTE; 6771 mask_to_clear = ~0; 6772 if (++offset >= PAGE_SIZE && len > 0) { 6773 offset = 0; 6774 page = eb->pages[++i]; 6775 assert_eb_page_uptodate(eb, page); 6776 kaddr = page_address(page); 6777 } 6778 } 6779 if (len) { 6780 mask_to_clear &= BITMAP_LAST_BYTE_MASK(size); 6781 kaddr[offset] &= ~mask_to_clear; 6782 } 6783 } 6784 6785 static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len) 6786 { 6787 unsigned long distance = (src > dst) ? src - dst : dst - src; 6788 return distance < len; 6789 } 6790 6791 static void copy_pages(struct page *dst_page, struct page *src_page, 6792 unsigned long dst_off, unsigned long src_off, 6793 unsigned long len) 6794 { 6795 char *dst_kaddr = page_address(dst_page); 6796 char *src_kaddr; 6797 int must_memmove = 0; 6798 6799 if (dst_page != src_page) { 6800 src_kaddr = page_address(src_page); 6801 } else { 6802 src_kaddr = dst_kaddr; 6803 if (areas_overlap(src_off, dst_off, len)) 6804 must_memmove = 1; 6805 } 6806 6807 if (must_memmove) 6808 memmove(dst_kaddr + dst_off, src_kaddr + src_off, len); 6809 else 6810 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len); 6811 } 6812 6813 void memcpy_extent_buffer(const struct extent_buffer *dst, 6814 unsigned long dst_offset, unsigned long src_offset, 6815 unsigned long len) 6816 { 6817 size_t cur; 6818 size_t dst_off_in_page; 6819 size_t src_off_in_page; 6820 unsigned long dst_i; 6821 unsigned long src_i; 6822 6823 if (check_eb_range(dst, dst_offset, len) || 6824 check_eb_range(dst, src_offset, len)) 6825 return; 6826 6827 while (len > 0) { 6828 dst_off_in_page = get_eb_offset_in_page(dst, dst_offset); 6829 src_off_in_page = get_eb_offset_in_page(dst, src_offset); 6830 6831 dst_i = get_eb_page_index(dst_offset); 6832 src_i = get_eb_page_index(src_offset); 6833 6834 cur = min(len, (unsigned long)(PAGE_SIZE - 6835 src_off_in_page)); 6836 cur = min_t(unsigned long, cur, 6837 (unsigned long)(PAGE_SIZE - dst_off_in_page)); 6838 6839 copy_pages(dst->pages[dst_i], dst->pages[src_i], 6840 dst_off_in_page, src_off_in_page, cur); 6841 6842 src_offset += cur; 6843 dst_offset += cur; 6844 len -= cur; 6845 } 6846 } 6847 6848 void memmove_extent_buffer(const struct extent_buffer *dst, 6849 unsigned long dst_offset, unsigned long src_offset, 6850 unsigned long len) 6851 { 6852 size_t cur; 6853 size_t dst_off_in_page; 6854 size_t src_off_in_page; 6855 unsigned long dst_end = dst_offset + len - 1; 6856 unsigned long src_end = src_offset + len - 1; 6857 unsigned long dst_i; 6858 unsigned long src_i; 6859 6860 if (check_eb_range(dst, dst_offset, len) || 6861 check_eb_range(dst, src_offset, len)) 6862 return; 6863 if (dst_offset < src_offset) { 6864 memcpy_extent_buffer(dst, dst_offset, src_offset, len); 6865 return; 6866 } 6867 while (len > 0) { 6868 dst_i = get_eb_page_index(dst_end); 6869 src_i = get_eb_page_index(src_end); 6870 6871 dst_off_in_page = get_eb_offset_in_page(dst, dst_end); 6872 src_off_in_page = get_eb_offset_in_page(dst, src_end); 6873 6874 cur = min_t(unsigned long, len, src_off_in_page + 1); 6875 cur = min(cur, dst_off_in_page + 1); 6876 copy_pages(dst->pages[dst_i], dst->pages[src_i], 6877 dst_off_in_page - cur + 1, 6878 src_off_in_page - cur + 1, cur); 6879 6880 dst_end -= cur; 6881 src_end -= cur; 6882 len -= cur; 6883 } 6884 } 6885 6886 static struct extent_buffer *get_next_extent_buffer( 6887 struct btrfs_fs_info *fs_info, struct page *page, u64 bytenr) 6888 { 6889 struct extent_buffer *gang[BTRFS_SUBPAGE_BITMAP_SIZE]; 6890 struct extent_buffer *found = NULL; 6891 u64 page_start = page_offset(page); 6892 int ret; 6893 int i; 6894 6895 ASSERT(in_range(bytenr, page_start, PAGE_SIZE)); 6896 ASSERT(PAGE_SIZE / fs_info->nodesize <= BTRFS_SUBPAGE_BITMAP_SIZE); 6897 lockdep_assert_held(&fs_info->buffer_lock); 6898 6899 ret = radix_tree_gang_lookup(&fs_info->buffer_radix, (void **)gang, 6900 bytenr >> fs_info->sectorsize_bits, 6901 PAGE_SIZE / fs_info->nodesize); 6902 for (i = 0; i < ret; i++) { 6903 /* Already beyond page end */ 6904 if (gang[i]->start >= page_start + PAGE_SIZE) 6905 break; 6906 /* Found one */ 6907 if (gang[i]->start >= bytenr) { 6908 found = gang[i]; 6909 break; 6910 } 6911 } 6912 return found; 6913 } 6914 6915 static int try_release_subpage_extent_buffer(struct page *page) 6916 { 6917 struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb); 6918 u64 cur = page_offset(page); 6919 const u64 end = page_offset(page) + PAGE_SIZE; 6920 int ret; 6921 6922 while (cur < end) { 6923 struct extent_buffer *eb = NULL; 6924 6925 /* 6926 * Unlike try_release_extent_buffer() which uses page->private 6927 * to grab buffer, for subpage case we rely on radix tree, thus 6928 * we need to ensure radix tree consistency. 6929 * 6930 * We also want an atomic snapshot of the radix tree, thus go 6931 * with spinlock rather than RCU. 6932 */ 6933 spin_lock(&fs_info->buffer_lock); 6934 eb = get_next_extent_buffer(fs_info, page, cur); 6935 if (!eb) { 6936 /* No more eb in the page range after or at cur */ 6937 spin_unlock(&fs_info->buffer_lock); 6938 break; 6939 } 6940 cur = eb->start + eb->len; 6941 6942 /* 6943 * The same as try_release_extent_buffer(), to ensure the eb 6944 * won't disappear out from under us. 6945 */ 6946 spin_lock(&eb->refs_lock); 6947 if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) { 6948 spin_unlock(&eb->refs_lock); 6949 spin_unlock(&fs_info->buffer_lock); 6950 break; 6951 } 6952 spin_unlock(&fs_info->buffer_lock); 6953 6954 /* 6955 * If tree ref isn't set then we know the ref on this eb is a 6956 * real ref, so just return, this eb will likely be freed soon 6957 * anyway. 6958 */ 6959 if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) { 6960 spin_unlock(&eb->refs_lock); 6961 break; 6962 } 6963 6964 /* 6965 * Here we don't care about the return value, we will always 6966 * check the page private at the end. And 6967 * release_extent_buffer() will release the refs_lock. 6968 */ 6969 release_extent_buffer(eb); 6970 } 6971 /* 6972 * Finally to check if we have cleared page private, as if we have 6973 * released all ebs in the page, the page private should be cleared now. 6974 */ 6975 spin_lock(&page->mapping->private_lock); 6976 if (!PagePrivate(page)) 6977 ret = 1; 6978 else 6979 ret = 0; 6980 spin_unlock(&page->mapping->private_lock); 6981 return ret; 6982 6983 } 6984 6985 int try_release_extent_buffer(struct page *page) 6986 { 6987 struct extent_buffer *eb; 6988 6989 if (btrfs_sb(page->mapping->host->i_sb)->sectorsize < PAGE_SIZE) 6990 return try_release_subpage_extent_buffer(page); 6991 6992 /* 6993 * We need to make sure nobody is changing page->private, as we rely on 6994 * page->private as the pointer to extent buffer. 6995 */ 6996 spin_lock(&page->mapping->private_lock); 6997 if (!PagePrivate(page)) { 6998 spin_unlock(&page->mapping->private_lock); 6999 return 1; 7000 } 7001 7002 eb = (struct extent_buffer *)page->private; 7003 BUG_ON(!eb); 7004 7005 /* 7006 * This is a little awful but should be ok, we need to make sure that 7007 * the eb doesn't disappear out from under us while we're looking at 7008 * this page. 7009 */ 7010 spin_lock(&eb->refs_lock); 7011 if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) { 7012 spin_unlock(&eb->refs_lock); 7013 spin_unlock(&page->mapping->private_lock); 7014 return 0; 7015 } 7016 spin_unlock(&page->mapping->private_lock); 7017 7018 /* 7019 * If tree ref isn't set then we know the ref on this eb is a real ref, 7020 * so just return, this page will likely be freed soon anyway. 7021 */ 7022 if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) { 7023 spin_unlock(&eb->refs_lock); 7024 return 0; 7025 } 7026 7027 return release_extent_buffer(eb); 7028 } 7029 7030 /* 7031 * btrfs_readahead_tree_block - attempt to readahead a child block 7032 * @fs_info: the fs_info 7033 * @bytenr: bytenr to read 7034 * @owner_root: objectid of the root that owns this eb 7035 * @gen: generation for the uptodate check, can be 0 7036 * @level: level for the eb 7037 * 7038 * Attempt to readahead a tree block at @bytenr. If @gen is 0 then we do a 7039 * normal uptodate check of the eb, without checking the generation. If we have 7040 * to read the block we will not block on anything. 7041 */ 7042 void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info, 7043 u64 bytenr, u64 owner_root, u64 gen, int level) 7044 { 7045 struct extent_buffer *eb; 7046 int ret; 7047 7048 eb = btrfs_find_create_tree_block(fs_info, bytenr, owner_root, level); 7049 if (IS_ERR(eb)) 7050 return; 7051 7052 if (btrfs_buffer_uptodate(eb, gen, 1)) { 7053 free_extent_buffer(eb); 7054 return; 7055 } 7056 7057 ret = read_extent_buffer_pages(eb, WAIT_NONE, 0); 7058 if (ret < 0) 7059 free_extent_buffer_stale(eb); 7060 else 7061 free_extent_buffer(eb); 7062 } 7063 7064 /* 7065 * btrfs_readahead_node_child - readahead a node's child block 7066 * @node: parent node we're reading from 7067 * @slot: slot in the parent node for the child we want to read 7068 * 7069 * A helper for btrfs_readahead_tree_block, we simply read the bytenr pointed at 7070 * the slot in the node provided. 7071 */ 7072 void btrfs_readahead_node_child(struct extent_buffer *node, int slot) 7073 { 7074 btrfs_readahead_tree_block(node->fs_info, 7075 btrfs_node_blockptr(node, slot), 7076 btrfs_header_owner(node), 7077 btrfs_node_ptr_generation(node, slot), 7078 btrfs_header_level(node) - 1); 7079 } 7080